diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/AUTHORS b/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..1062f3bf743c0a37bcde05b1afc1ba5e98a95822 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/AUTHORS @@ -0,0 +1,1379 @@ +All people who contributed to SymPy by sending at least a patch or +more (in the order of the date of their first contribution), except +those who explicitly didn't want to be mentioned. People with a * next +to their names are not found in the metadata of the git history. This +file is generated automatically by running `./bin/authors_update.py`. + +There are a total of 1371 authors. + +Ondřej Čertík +Fabian Pedregosa +Jurjen N.E. Bos +Mateusz Paprocki +*Marc-Etienne M.Leveille +Brian Jorgensen +Jason Gedge +Robert Schwarz +Pearu Peterson +Fredrik Johansson +Chris Wu +*Ulrich Hecht +Goutham Lakshminarayan +David Lawrence +Jaroslaw Tworek +David Marek +Bernhard R. Link +Andrej Tokarčík +Or Dvory +Saroj Adhikari +Pauli Virtanen +Robert Kern +James Aspnes +Nimish Telang +Abderrahim Kitouni +Pan Peng +Friedrich Hagedorn +Elrond der Elbenfuerst +Rizgar Mella +Felix Kaiser +Roberto Nobrega +David Roberts +Sebastian Krämer +Vinzent Steinberg +Riccardo Gori +Case Van Horsen +Stepan Roucka +Ali Raza Syed +Stefano Maggiolo +Robert Cimrman +Bastian Weber +Sebastian Krause +Sebastian Kreft +*Dan +Alan Bromborsky +Boris Timokhin +Robert +Andy R. Terrel +Hubert Tsang +Konrad Meyer +Henrik Johansson +Priit Laes +Freddie Witherden +Brian E. Granger +Andrew Straw +Kaifeng Zhu +Ted Horst +Andrew Docherty +Akshay Srinivasan +Aaron Meurer +Barry Wardell +Tomasz Buchert +Vinay Kumar +Johann Cohen-Tanugi +Jochen Voss +Luke Peterson +Chris Smith +Thomas Sidoti +Florian Mickler +Nicolas Pourcelot +Ben Goodrich +Toon Verstraelen +Ronan Lamy +James Abbatiello +Ryan Krauss +Bill Flynn +Kevin Goodsell +Jorn Baayen +Eh Tan +Renato Coutinho +Oscar Benjamin +Øyvind Jensen +Julio Idichekop Filho +Łukasz Pankowski +*Chu-Ching Huang +Fernando Perez +Raffaele De Feo +Christian Muise +Matt Curry +Kazuo Thow +Christian Schubert +Jezreel Ng +James Pearson +Matthew Brett +Addison Cugini +Nicholas J.S. Kinar +Harold Erbin +Thomas Dixon +Cristóvão Sousa +Andre de Fortier Smit +Mark Dewing +Alexey U. Gudchenko +Gary Kerr +Sherjil Ozair +Oleksandr Gituliar +Sean Vig +Prafullkumar P. Tale +Vladimir Perić +Tom Bachmann +Yuri Karadzhov +Vladimir Lagunov +Matthew Rocklin +Saptarshi Mandal +Gilbert Gede +Anatolii Koval +Tomo Lazovich +Pavel Fedotov +Jack McCaffery +Jeremias Yehdegho +Kibeom Kim +Gregory Ksionda +Tomáš Bambas +Raymond Wong +Luca Weihs +Shai 'Deshe' Wyborski +Thomas Wiecki +Óscar Nájera +Mario Pernici +Benjamin McDonald +Sam Magura +Stefan Krastanov +Bradley Froehle +Min Ragan-Kelley +Emma Hogan +Nikhil Sarda +Julien Rioux +Roberto Colistete, Jr. +Raoul Bourquin +Gert-Ludwig Ingold +Srinivas Vasudevan +Jason Moore +Miha Marolt +Tim Lahey +Luis Garcia +Matt Rajca +David Li +Alexandr Gudulin +Bilal Akhtar +Grzegorz Świrski +Matt Habel +David Ju +Nichita Utiu +Nikolay Lazarov +Steve Anton +Imran Ahmed Manzoor +Ljubiša Moćić <3rdslasher@gmail.com> +Piotr Korgul +Jim Zhang +Sam Sleight +tborisova +Chancellor Arkantos +Stepan Simsa +Tobias Lenz +Siddhanathan Shanmugam +Tiffany Zhu +Tristan Hume +Alexey Subach +Joan Creus +Geoffry Song +Puneeth Chaganti +Marcin Kostrzewa <> +Natalia Nawara +vishal +Shruti Mangipudi +Davy Mao +Swapnil Agarwal +Dhia Kennouche +jerryma1121 +Joachim Durchholz +Martin Povišer +Siddhant Jain +Kevin Hunter +Michael Mayorov +Nathan Alison +Christian Bühler +Carsten Knoll +Bharath M R +Matthias Toews +Sergiu Ivanov +Jorge E. Cardona +Sanket Agarwal +Manoj Babu K. +Sai Nikhil +Aleksandar Makelov +Sachin Irukula +Raphael Michel +Ashwini Oruganti +Andreas Klöckner +Prateek Papriwal +Arpit Goyal +Angadh Nanjangud +Comer Duncan +Jens H. Nielsen +Joseph Dougherty +Elliot Marshall +Guru Devanla +George Waksman +Alexandr Popov +Tarun Gaba +Takafumi Arakaki +Saurabh Jha +Rom le Clair +Angus Griffith <16sn6uv@gmail.com> +Timothy Reluga +Brian Stephanik +Alexander Eberspächer +Sachin Joglekar +Tyler Pirtle +Vasily Povalyaev +Colleen Lee +Matthew Hoff +Niklas Thörne +Huijun Mai +Marek Šuppa +Ramana Venkata +Prasoon Shukla +Stefen Yin +Thomas Hisch +Madeleine Ball +Mary Clark +Rishabh Dixit +Manoj Kumar +Akshit Agarwal +CJ Carey +Patrick Lacasse +Ananya H +Tarang Patel +Christopher Dembia +Benjamin Fishbein +Sean Ge +Amit Jamadagni +Ankit Agrawal +Björn Dahlgren +Christophe Saint-Jean +Demian Wassermann +Khagesh Patel +Stephen Loo +hm +Patrick Poitras +Katja Sophie Hotz +Varun Joshi +Chetna Gupta +Thilina Rathnayake +Max Hutchinson +Shravas K Rao +Matthew Tadd +Alexander Hirzel +Randy Heydon +Oliver Lee +Seshagiri Prabhu +Pradyumna +Erik Welch +Eric Nelson +Roland Puntaier +Chris Conley +Tim Swast +Dmitry Batkovich +Francesco Bonazzi +Yuriy Demidov +Rick Muller +Manish Gill +Markus Müller +Amit Saha +Jeremy +QuaBoo +Stefan van der Walt +David Joyner +Lars Buitinck +Alkiviadis G. Akritas +Vinit Ravishankar +Michael Boyle +Heiner Kirchhoffer +Pablo Puente +James Fiedler +Harsh Gupta +Tuomas Airaksinen +Paul Strickland +James Goppert +rathmann +Avichal Dayal +Paul Scott +Shipra Banga +Pramod Ch +Akshay +Buck Shlegeris +Jonathan Miller +Edward Schembor +Rajath Shashidhara +Zamrath Nizam +Aditya Shah +Rajat Aggarwal +Sambuddha Basu +Zeel Shah +Abhinav Chanda +Jim Crist +Sudhanshu Mishra +Anurag Sharma +Soumya Dipta Biswas +Sushant Hiray +Ben Lucato +Kunal Arora +Henry Gebhardt +Dammina Sahabandu +Manish Shukla +Ralph Bean +richierichrawr +John Connor +Juan Luis Cano Rodríguez +Sahil Shekhawat +Kundan Kumar +Stas Kelvich +sevaader +Dhruvesh Vijay Parikh +Venkatesh Halli +Lennart Fricke +Vlad Seghete +Shashank Agarwal +carstimon +Pierre Haessig +Maciej Baranski +Benjamin Gudehus +Faisal Anees +Mark Shoulson +Robert Johansson +Kalevi Suominen +Kaushik Varanasi +Fawaz Alazemi +Ambar Mehrotra +David P. Sanders +Peter Brady +John V. Siratt +Sarwar Chahal +Nathan Woods +Colin B. Macdonald +Marcus Näslund +Clemens Novak +Mridul Seth +Craig A. Stoudt +Raj +Mihai A. Ionescu +immerrr +Chai Wah Wu +Leonid Blouvshtein +Peleg Michaeli +ck Lux +zsc347 +Hamish Dickson +Michael Gallaspy +Roman Inflianskas +Duane Nykamp +Ted Dokos +Sunny Aggarwal +Victor Brebenar +Akshat Jain +Shivam Vats +Longqi Wang +Juan Felipe Osorio +Ray Cathcart +Lukas Zorich +Eric Miller +Cody Herbst +Nishith Shah +Amit Kumar +Yury G. Kudryashov +Guillaume Gay +Mihir Wadwekar +Tuan Manh Lai +Asish Panda +Darshan Chaudhary +Alec Kalinin +Ralf Stephan +Aaditya Nair +Jayesh Lahori +Harshil Goel +Luv Agarwal +Jason Ly +Lokesh Sharma +Sartaj Singh +Chris Swierczewski +Konstantin Togoi +Param Singh +Sumith Kulal +Juha Remes +Philippe Bouafia +Peter Schmidt +Jiaxing Liang +Lucas Jones +Gregory Ashton +Jennifer White +Renato Orsino +Alistair Lynn +Govind Sahai +Adam Bloomston +Kyle McDaniel +Nguyen Truong Duy +Alex Lindsay +Mathew Chong +Jason Siefken +Gaurav Dhingra +Gao, Xiang +Kevin Ventullo +mao8 +Isuru Fernando +Shivam Tyagi +Richard Otis +Rich LaSota +dustyrockpyle +Anton Akhmerov +Michael Zingale +Chak-Pong Chung +David T +Phil Ruffwind +Sebastian Koslowski +Kumar Krishna Agrawal +Dustin Gadal +João Moura +Yu Kobayashi +Shashank Kumar +Timothy Cyrus +Devyani Kota +Keval Shah +Dzhelil Rufat +Pastafarianist +Sourav Singh +Jacob Garber +Vinay Singh +GolimarOurHero +Prashant Tyagi +Matthew Davis +Tschijnmo TSCHAU +Alexander Bentkamp +Jack Kemp +Kshitij Saraogi +Thomas Baruchel +Nicolás Guarín-Zapata +Jens Jørgen Mortensen +Sampad Kumar Saha +Eva Charlotte Mayer +Laura Domine +Justin Blythe +Meghana Madhyastha +Tanu Hari Dixit +Shekhar Prasad Rajak +Aqnouch Mohammed +Arafat Dad Khan +Boris Atamanovskiy +Sam Tygier +Jai Luthra +Guo Xingjian +Sandeep Veethu +Archit Verma +Shubham Tibra +Ashutosh Saboo +Michael S. Hansen +Anish Shah +Guillaume Jacquenot +Bhautik Mavani +Michał Radwański +Jerry Li +Pablo Zubieta +Shivam Agarwal +Chaitanya Sai Alaparthi +Arihant Parsoya +Ruslan Pisarev +Akash Trehan +Nishant Nikhil +Vladimir Poluhsin +Akshay Nagar +James Brandon Milam +Abhinav Agarwal +Rishabh Daal +Sanya Khurana +Aman Deep +Aravind Reddy +Abhishek Verma +Matthew Parnell +Thomas Hickman +Akshay Siramdas +YiDing Jiang +Jatin Yadav +Matthew Thomas +Rehas Sachdeva +Michael Mueller +Srajan Garg +Prabhjot Singh +Haruki Moriguchi +Tom Gijselinck +Nitin Chaudhary +Alex Argunov +Nathan Musoke +Abhishek Garg +Dana Jacobsen +Vasiliy Dommes +Phillip Berndt +Haimo Zhang +Anthony Scopatz +bluebrook +Leonid Kovalev +Josh Burkart +Dimitra Konomi +Christina Zografou +Fiach Antaw +Langston Barrett +Krit Karan +G. D. McBain +Prempal Singh +Gabriel Orisaka +Matthias Bussonnier +rahuldan +Colin Marquardt +Andrew Taber +Yash Reddy +Peter Stangl +elvis-sik +Nikos Karagiannakis +Jainul Vaghasia +Dennis Meckel +Harshil Meena +Micky +Nick Curtis +Michele Zaffalon +Martha Giannoudovardi +Devang Kulshreshtha +Steph Papanik +Mohammad Sadeq Dousti +Arif Ahmed +Abdullah Javed Nesar +Lakshya Agrawal +shruti +Rohit Rango +Hong Xu +Ivan Petuhov +Alsheh +Marcel Stimberg +Alexey Pakhocmhik +Tommy Olofsson +Zulfikar +Blair Azzopardi +Danny Hermes +Sergey Pestov +Mohit Chandra +Karthik Chintapalli +Marcin Briański +andreo +Flamy Owl +Yicong Guo +Varun Garg +Rishabh Madan +Aditya Kapoor +Karan Sharma +Vedant Rathore +Johan Blåbäck +Pranjal Tale +Jason Tokayer +Raghav Jajodia +Rajat Thakur +Dhruv Bhanushali +Anjul Kumar Tyagi +Barun Parruck +Bao Chau +Tanay Agrawal +Ranjith Kumar +Shikhar Makhija +Yathartha Joshi +Valeriia Gladkova +Sagar Bharadwaj +Daniel Mahler +Ka Yi +Rishat Iskhakov +Szymon Mieszczak +Sachin Agarwal +Priyank Patel +Satya Prakash Dwibedi +tools4origins +Nico Schlömer +Fermi Paradox +Ekansh Purohit +Vedarth Sharma +Peeyush Kushwaha +Jayjayyy +Christopher J. Wright +Jakub Wilk +Mauro Garavello +Chris Tefer +Shikhar Jaiswal +Chiu-Hsiang Hsu +Carlos Cordoba +Fabian Ball +Yerniyaz +Christiano Anderson +Robin Neatherway +Thomas Hunt +Theodore Han +Duc-Minh Phan +Lejla Metohajrova +Samyak Jain +Aditya Rohan +Vincent Delecroix +Michael Sparapany +Harsh Jain +Nathan Goldbaum +latot +Kenneth Lyons +Stan Schymanski +David Daly +Ayush Shridhar +Javed Nissar +Jiri Kuncar +vedantc98 +Rupesh Harode +Rob Zinkov +James Harrop +James Taylor +Ishan Joshi +Marco Mancini +Boris Ettinger +Micah Fitch +Daniel Wennberg +ylemkimon +Akash Vaish +Peter Enenkel +Waldir Pimenta +Jithin D. George +Lev Chelyadinov +Lucas Wiman +Rhea Parekh +James Cotton +Robert Pollak +anca-mc +Sourav Ghosh +Jonathan Allan +Nikhil Pappu +Ethan Ward +Cezary Marczak +dps7ud +Nilabja Bhattacharya +Itay4 <31018228+Itay4@users.noreply.github.com> +Poom Chiarawongse +Yang Yang +Cavendish McKay +Bradley Gannon +B McG +Rob Drynkin +Seth Ebner +Akash Kundu +Mark Jeromin +Roberto Díaz Pérez +Gleb Siroki +Segev Finer +Alex Lubbock +Ayodeji Ige +Matthew Wardrop +Hugo van Kemenade +Austin Palmer +der-blaue-elefant +Filip Gokstorp +Yuki Matsuda +Aaron Miller +Salil Vishnu Kapur +Atharva Khare +Shubham Maheshwari +Pavel Tkachenko +Ashish Kumar Gaurav +Rajeev Singh +Keno Goertz +Lucas Gallindo +Himanshu +David Menéndez Hurtado +Amit Manchanda +Rohit Jain +Jonathan A. Gross +Unknown +Sayan Goswami +Subhash Saurabh +Rastislav Rabatin +Vishal +Jeremey Gluck +Akshat Maheshwari +symbolique +Saloni Jain +Arighna Chakrabarty +Abhigyan Khaund +Jashanpreet Singh +Saurabh Agarwal +luzpaz +P. Sai Prasanth +Nirmal Sarswat +Cristian Di Pietrantonio +Ravi charan +Nityananda Gohain +Cédric Travelletti +Nicholas Bollweg +Himanshu Ladia +Adwait Baokar +Mihail Tarigradschi +Saketh +rushyam +sfoo +Rahil Hastu +Zach Raines +Sidhant Nagpal +Gagandeep Singh +Rishav Chakraborty +Malkhan Singh +Joaquim Monserrat +Mayank Singh +Rémy Léone +Maxence Mayrand <35958639+maxencemayrand@users.noreply.github.com> +Nikoleta Glynatsi +helo9 +Ken Wakita +Carl Sandrock +Fredrik Eriksson +Ian Swire +Bulat +Ehren Metcalfe +Dmitry Savransky +Kiyohito Yamazaki +Caley Finn +Zhi-Qiang Zhou +Alexander Pozdneev +Wes Turner <50891+westurner@users.noreply.github.com> +JMSS-Unknown <31131631+JMSS-Unknown@users.noreply.github.com> +Arshdeep Singh +cym1 <16437732+cym1@users.noreply.github.com> +Stewart Wadsworth +Jared Lumpe +Avi Shrivastava +ramvenkat98 +Bilal Ahmed +Dimas Abreu Archanjo Dutra +Yatna Verma +S.Y. Lee +Miro Hrončok +Sudarshan Kamath +Ayushman Koul +Robert Dougherty-Bliss +Andrey Grozin +Bavish Kulur +Arun Singh +sirnicolaf <43586954+sirnicolaf@users.noreply.github.com> +Zachariah Etienne +Prayush Dawda <35144226+iamprayush@users.noreply.github.com> +2torus +Faisal Riyaz +Martin Roelfs +SirJohnFranklin +Anthony Sottile +ViacheslavP +Safiya03 +Alexander Dunlap +Rohit Sharma <31184621+rohitx007@users.noreply.github.com> +Jonathan Warner +Mohit Balwani +Marduk Bolaños +amsuhane +Matthias Geier +klaasvanaarsen <44929042+klaasvanaarsen@users.noreply.github.com> +Shubham Kumar Jha +rationa-kunal +Animesh Sinha +Gaurang Tandon <1gaurangtandon@gmail.com> +Matthew Craven +Daniel Ingram +Jogi Miglani +Takumasa Nakamura +Ritu Raj Singh +Rajiv Ranjan Singh +Vera Lozhkina +adhoc-king <46354827+adhoc-king@users.noreply.github.com> +Mikel Rouco +Oscar Gustafsson +damianos +Supreet Agrawal +shiksha11 +Martin Ueding +sharma-kunal +Divyanshu Thakur +Susumu Ishizuka +Samnan Rahee +Fredrik Andersson +Bhavya Srivastava +Alpesh Jamgade +Shubham Abhang +Vishesh Mangla +Nicko van Someren +dandiez <47832466+dandiez@users.noreply.github.com> +Frédéric Chapoton +jhanwar +Noumbissi valere Gille Geovan +Salmista-94 +Shivani Kohli +Parker Berry +Pragyan Mehrotra +Nabanita Dash +Gaetano Guerriero +Ankit Raj Pandey +Ritesh Kumar +kangzhiq <709563092@qq.com> +Jun Lin +Petr Kungurtsev +Anway De +znxftw +Denis Ivanenko +Orestis Vaggelis +Nikhil Maan +Abhinav Anand +Qingsha Shi +Juan Barbosa +Prionti Nasir +Bharat Raghunathan +arooshiverma +Christoph Gohle +Charalampos Tsiagkalis +Daniel Sears +Megan Ly +Sean P. Cornelius +Erik R. Gomez +Riccardo Magliocchetti +Henry Metlov +pekochun +Bendik Samseth +Vighnesh Shenoy +Versus Void +Denys Rybalka +Mark Dickinson +Rimi +rimibis <33387803+rimibis@users.noreply.github.com> +Steven Lee +Gilles Schintgen +Abhi58 +Tomasz Pytel +Aadit Kamat +Samesh +Velibor Zeli +Gabriel Bernardino +Joseph Redfern +Evelyn King +Miguel Marco +David Hagen +Hannah Kari +Soniya Nayak +Harsh Agarwal +Enric Florit +Yogesh Mishra +Denis Rykov +Ivan Tkachenko +Kenneth Emeka Odoh +Stephan Seitz +Yeshwanth N +Oscar Gerardo Lazo Arjona +Srinivasa Arun Yeragudipati +Kirtan Mali +TitanSnow +Pengning Chao <8857165+PengningChao@users.noreply.github.com> +Louis Abraham +Morten Olsen Lysgaard +Akash Nagaraj (akasnaga) +Akash Nagaraj +Lauren Glattly +Hou-Rui +George Korepanov +dranknight09 +aditisingh2362 +Gina +gregmedlock +Georgios Giapitzakis Tzintanos +Eric Wieser +Bradley Dowling <34559056+btdow@users.noreply.github.com> +Maria Marginean <33810762+mmargin@users.noreply.github.com> +Akash Agrawall +jgulian +Sourav Goyal +Zlatan Vasović +Alex Meiburg +Smit Lunagariya +Naman Gera +Julien Palard +Dhruv Mendiratta +erdOne <36414270+erdOne@users.noreply.github.com> +risubaba +abhinav28071999 <41710346+abhinav28071999@users.noreply.github.com> +Jisoo Song +Jaime R <38530589+Jaime02@users.noreply.github.com> +Vikrant Malik +Hardik Saini <43683678+Guardianofgotham@users.noreply.github.com> +Abhishek +Johannes Hartung +Milan Jolly +faizan2700 +mohit <39158356+mohitacecode@users.noreply.github.com> +Mohit Gupta +Psycho-Pirate +Chanakya-Ekbote +Rashmi Shehana +Jonty16117 +Anubhav Gupta +Michal Grňo +vezeli <37907135+vezeli@users.noreply.github.com> +Tim Gates +Sandeep Murthy +Neil +V1krant <46847915+V1krant@users.noreply.github.com> +alejandro +Riyan Dhiman +sbt4104 +Seth Troisi +Bhaskar Gupta +Smit Gajjar +rbl +Ilya Pchelintsev +Omar Wagih +prshnt19 +Johan Guzman +Vasileios Kalos +BasileiosKal <61801875+BasileiosKal@users.noreply.github.com> +Shubham Thorat <37049710+sbt4104@users.noreply.github.com> +Arpan Chattopadhyay +Ashutosh Hathidara +Moses Paul R +Saanidhya vats +tnzl +Vatsal Srivastava +Jean-Luc Herren +Dhruv Kothari +seadavis <45022599+seadavis@users.noreply.github.com> +kamimura +slacker404 +Jaime Resano +Ebrahim Byagowi +wuyudi +Akira Kyle +Calvin Jay Ross +Martin Thoma +Thomas A Caswell +Lagaras Stelios +Jerry James +Jan Kruse +Nathan Taylor +Vaishnav Damani +Mohit Shah +Mathias Louboutin +Marijan Smetko +Dave Witte Morris +soumi7 +Zhongshi +Wes Galbraith +KaustubhDamania +w495 +Akhil Rajput +Markus Mohrhard +Benjamin Wolba +彭于斌 <1931127624@qq.com> +Rudr Tiwari +Aaryan Dewan +Benedikt Placke +Sneha Goddu +goddus <39923708+goddus@users.noreply.github.com> +Shivang Dubey +Michael Greminger +Peter Cock +Willem Melching +Elias Basler +Brandon David +Abhay_Dhiman +Tasha Kim +Ayush Malik +Devesh Sawant +Wolfgang Stöcher +Sudeep Sidhu +foice +Ben Payne +Muskan Kumar <31043527+muskanvk@users.noreply.github.com> +noam simcha finkelstein +Garrett Folbe +Islam Mansour +Sayandip Halder +Shubham Agrawal +numbermaniac <5206120+numbermaniac@users.noreply.github.com> +Sakirul Alam +Mohammed Bilal +Chris du Plessis +Coder-RG +Ansh Mishra +Alex Malins +Lorenzo Contento +Naveen Sai +Shital Mule +Amanda Dsouza +Nijso Beishuizen +Harry Zheng +Felix Yan +Constantin Mateescu +Eva Tiwari +Aditya Kumar Sinha +Soumi Bardhan <51290447+Soumi7@users.noreply.github.com> +Kaustubh Chaudhari +Kristian Brünn +Neel Gorasiya +Akshat Sood <68052998+akshatsood2249@users.noreply.github.com> +Jose M. Gomez +Stefan Petrea +Praveen Sahu +Mark Bell +AlexCQY +Fabian Froehlich +Nikhil Gopalam +Kartik Sethi +Muhammed Abdul Quadir Owais +Harshit Yadav +Sidharth Mundhra +Suryam Arnav Kalra +Prince Gupta +Kunal Singh +Mayank Raj +Achal Jain <2achaljain@gmail.com> +Mario Maio +Aaron Stiff <69512633+AaronStiff@users.noreply.github.com> +Wyatt Peak +Bhaskar Joshi +Aditya Jindal +Vaibhav Bhat +Priyansh Rathi +Saket Kumar Singh +Yukai Chou +Qijia Liu +Paul Mandel +Nisarg Chaudhari <54911392+Nisarg-Chaudhari@users.noreply.github.com> +Dominik Stańczak +Rodrigo Luger +Marco Antônio Habitzreuter +Ayush Bisht +Akshansh Bhatt +Brandon T. Willard +Thomas Aarholt +Hiren Chalodiya +Roland Dixon +dimasvq +Sagar231 +Michael Chu +Abby Ng +Angad Sandhu <55819847+angadsinghsandhu@users.noreply.github.com> +Alexander Cockburn +Yaser AlOsh +Davide Sandonà +Jonathan Gutow +Nihir Agarwal +Lee Johnston +Zach Carmichael <20629897+craymichael@users.noreply.github.com> +Vijairam Ganesh Moorthy +Hanspeter Schmid +Ben Oostendorp +Nikita +Aman +Shashank KS +Aman Sharma +Anup Parikh +Lucy Mountain +Miguel Torres Costa +Rikard Nordgren +Arun sanganal <74652697+ArunSanganal@users.noreply.github.com> +Kamlesh Joshi <72374645+kamleshjoshi8102@users.noreply.github.com> +Joseph Rance <56409230+Joseph-Rance@users.noreply.github.com> +Huangduirong +Nils Schulte <47043622+Schnilz@users.noreply.github.com> +Matt Bogosian +Elisha Hollander +Aditya Ravuri +Mamidi Ratna Praneeth +Jeffrey Ryan +Jonathan Daniel <36337649+jond01@users.noreply.github.com> +Robin Richard +Gautam Menghani +Remco de Boer <29308176+redeboer@users.noreply.github.com> +Sebastian East +Evani Balasubramanyam +Rahil Parikh +Jason Ross +Joannah Nanjekye +Ayush Kumar +Kshitij +Daniel Hyams +alijosephine +Matthias Köppe +mohajain +Anibal M. Medina-Mardones +Travis Ens +Evgenia Karunus +Risiraj Dey +lastcodestanding +Andrey Lekar +Abbas Mohammed <42001049+iam-abbas@users.noreply.github.com> +Anutosh Bhat +Steve Kieffer +Paul Spiering +Pieter Gijsbers +Wang Ran (汪然) +naelsondouglas +Aman Thakur +S. Hanko +Dennis Sweeney +Gurpartap Singh +Hampus Malmberg +scimax +Nikhil Date +Kuldeep Borkar Jr +AkuBrain <76952313+Franck2111@users.noreply.github.com> +Leo Battle +Advait Pote +Anurag Bhat +Jeremy Monat +Diane Tchuindjo +Tom Fryers <61272761+TomFryers@users.noreply.github.com> +Zouhair +zzj <29055749+zjzh@users.noreply.github.com> +shubhayu09 +Siddhant Jain +Tirthankar Mazumder <63574588+wermos@users.noreply.github.com> +Sumit Kumar +Shivam Sagar +Gaurav Jain +Andrii Oriekhov +Luis Talavera +Arie Bovenberg +Carson McManus +Jack Schmidt <1107865+jackschmidt@users.noreply.github.com> +Riley Britten +Georges Khaznadar +Donald Wilson +Timo Stienstra +dispasha +Saksham Alok +Varenyam Bhardwaj +oittaa <8972248+oittaa@users.noreply.github.com> +Omkaar <79257339+Pysics@users.noreply.github.com> +Islem BOUZENIA +extraymond +Alexander Behrens +user202729 <25191436+user202729@users.noreply.github.com> +Pieter Eendebak +Zaz Brown +ritikBhandari +viocha <66580331+viocha@users.noreply.github.com> +Arthur Ryman +Xiang Wu +tttc3 +Seth Poulsen +cocolato +Anton Golovanov +Gareth Ma +Clément M.T. Robert +Glenn Horton-Smith +Karan +Stefan Behnle <84378403+behnle@users.noreply.github.com> +Shreyash Mishra <72146041+Shreyash-cyber@users.noreply.github.com> +Arthur Milchior +NotWearingPants <26556598+NotWearingPants@users.noreply.github.com> +Ishan Pandhare +Carlos García Montoro +Parcly Taxel +Saicharan +Kunal Sheth +Biswadeep Purkayastha <98874428+metabiswadeep@users.noreply.github.com> +Jyn Spring 琴春 +Phil LeMaitre +Chris Kerr +José Senart +Uwe L. Korn +ForeverHaibara <69423537+ForeverHaibara@users.noreply.github.com> +Yves Tumushimire +wookie184 +Costor +Klaus Rettinghaus +Sam Brockie +Abhishek Patidar <1e9abhi1e10@gmail.com> +Eric Demer +Pontus von Brömssen +Victor Immanuel +Evandro Bernardes +Michele Ceccacci +Ayush Aryan +Kishore Gopalakrishnan +Jan-Philipp Hoffmann +Daiki Takahashi +Sayan Mitra +Aman Kumar Shukla +Zoufiné Lauer-Baré +Charles Harris +Tejaswini Sanapathi +Devansh +Aaron Gokaslan +Daan Koning (he/him) +Steven Burns +Jay Patankar +Vivek Soni +Le Cong Minh Hieu +Sam Ritchie +Maciej Skórski +Tilo Reneau-Cardoso +Laurence Warne +Lukas Molleman +Konstantinos Riganas +Grace Su +Pedro Rosa +Abhinav Cillanki +Baiyuan Qiu <1061688677@qq.com> +Liwei Cai +Daniel Weindl +Isidora Araya +Seb Tiburzio +Victory Omole +Abhishek Chaudhary +Alexander Zhura +Shuai Zhou +Martin Manns +John Möller +zzc <1378113190@qq.com> +Pablo Galindo Salgado +Johannes Kasimir +Theodore Dias +Kaustubh <90597818+kaustubh-765@users.noreply.github.com> +Idan Pazi +Bobby Palmer +Saikat Das +Suman mondal +Taylan Sahin +Fabio Luporini +Oriel Malihi +Geetika Vadali +Matthias Rettl +Mikhail Remnev +philwillnyc <56197213+philwillnyc@users.noreply.github.com> +Raphael Lehner +Harry Mountain +Bhavik Sachdev +袁野 (Yuan Ye) +fazledyn-or +mohammedouahman +K. Kraus +Zac Hatfield-Dodds +platypus +codecruisader +James Whitehead +atharvParlikar +Ivan Petukhov +Augusto Borges +Han Wei Ang +Congxu Yang +Saicharan <62512681+saicharan2804@users.noreply.github.com> +Arnab Nandi +Harrison Oates <48871176+HarrisonOates@users.noreply.github.com> +Corey Cerovsek +Harsh Kasat +omahs <73983677+omahs@users.noreply.github.com> +Pascal Gitz +Ravindu-Hirimuthugoda +Sophia Pustova +George Pittock +Warren Jacinto +Sachin Singh +Zedmat <104870914+harshkasat@users.noreply.github.com> +Soumendra Ganguly +Samith Karunathilake <55777141+samithkavishke@users.noreply.github.com> +Viraj Vekaria +Shishir Kushwaha +Ankit Kumar Singh +Abhishek Kumar +Mohak Malviya +Matthias Liesenfeld <116307294+maliesen@users.noreply.github.com> +dodo +Mohamed Rezk +Tommaso Vaccari <05-gesto-follemente@icloud.com> +Alexis Schotte +Lauren Yim <31467609+cherryblossom000@users.noreply.github.com> +Prey Patel +Riccardo Di Girolamo +Abhishek kumar +Sam Lubelsky +Henrique Soares +Vladimir Sereda +Hwayeon Kang +Raj Sapale +Gerald Teschl +Richard Samuel <98638849+samuelard7@users.noreply.github.com> +HeeJae Chang +Nick Harder +Ethan DeGuire +Lorenz Winkler +Richard Rodenbusch +Zhenxu Zhu +Mark van Gelder +Mark van Gelder +Ishan Pandhare <91841626+Ishanned@users.noreply.github.com> +James A. Preiss +Emile Fourcini +Alberto Jiménez Ruiz +João Bravo +Dean Price +Edward Z. Yang +James Titus +Zhuoyuan Li +Hugo Kerstens +Jan Jancar +Andrew Mosson +Marek Madejski +Gonzalo Tornaría +Peter Stahlecker +Jean-François B <2589111+jfbu@users.noreply.github.com> +Zexuan Zhou (Bruce) +George Frolov +Corbet Elkins +Håkon Kvernmoen +Muhammad Maaz +Shishir Kushwaha <138311586+shishir-11@users.noreply.github.com> +Matt Wang +bharatAmeria <21001019007@jcboseust.ac.in> +Amir Ebrahimi +Steven Esquea +Rishabh Kamboj <111004091+VectorNd@users.noreply.github.com> +Aasim Ali +Ivan A. Melnikov +Borek Saheli +Guido Roncarolo +Quek Zi Yao +Roelof Rietbroek +MostafaGalal1 +Au Huishan +Kris Katterjohn +Shiyao Guo +Rushabh Mehta +Temiloluwa Yusuf ytemiloluwa@gmail.com ytemiloluwa +Davi Laerte +Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> +Harshit Gupta +Praveen Perumal +Kevin McWhirter +Prayag V +Lucas Kletzander +Pratyksh Gupta +Leonardo Mangani +Karan Anand +Gagan Mishra +Krishnav Bajoria +Matt Ord +Jatin Bhardwaj +Prashant Tandon +Paramjit Singh +João Rodrigues +Alejandro García Prada <114813960+AlexGarciaPrada@users.noreply.github.com> +Matthew Treinish +Clayton Rabideau +Victoria Koval +Voaides Negustor Robert <134785947+voaidesr@users.noreply.github.com> +Ovsk Mendov +David Brooks +Nicholas Laustrup <124007393+nicklaustrup@users.noreply.github.com> +Harikrishna Srinivasan +Mathis Cros +Arnav Mummineni <45217840+RCoder01@users.noreply.github.com> +Thangaraju Sibiraj <85477603+t-sibiraj@users.noreply.github.com> +KJaybhaye diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/LICENSE b/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..0744f229d697ca3ed1b1b257bfdb70e3eecf0b9e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy-1.14.0.dist-info/licenses/LICENSE @@ -0,0 +1,153 @@ +Copyright (c) 2006-2023 SymPy Development Team + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of SymPy nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +-------------------------------------------------------------------------------- + +Patches that were taken from the Diofant project (https://github.com/diofant/diofant) +are licensed as: + +Copyright (c) 2006-2018 SymPy Development Team, + 2013-2023 Sergey B Kirpichev + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of Diofant or SymPy nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +-------------------------------------------------------------------------------- + +Submodules taken from the multipledispatch project (https://github.com/mrocklin/multipledispatch) +are licensed as: + +Copyright (c) 2014 Matthew Rocklin + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of multipledispatch nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +-------------------------------------------------------------------------------- + +The files under the directory sympy/parsing/autolev/tests/pydy-example-repo +are directly copied from PyDy project and are licensed as: + +Copyright (c) 2009-2023, PyDy Authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of this project nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL PYDY AUTHORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under the directory sympy/parsing/latex +are directly copied from latex2sympy project and are licensed as: + +Copyright 2016, latex2sympy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5993bf0bdfc428d878a55fdaf05de15b599e58e3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61a802bc8816e3f5de299f736e824504ebceaedd Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix_models.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix_models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a98cc5ea7384909e785390a6b25e1a0e7fcb34b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/random_matrix_models.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a803193fab90c9292c3b404657061d5c247dc46 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv_interface.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv_interface.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d951a98b0813d2a0f4e08b6a2dd0817ab0aa56a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/rv_interface.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/stochastic_process.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/stochastic_process.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a627be495235a0c22b36f2e7136afce28e1f9100 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/stochastic_process.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_multivariate_probability.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_multivariate_probability.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2af4b0bba420c8d3e811d222b320f2cf2ea859b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_multivariate_probability.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_probability.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_probability.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19e1a55144bdb2a8fec83166fd4bbc4653d36ce9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/__pycache__/symbolic_probability.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42ae11a21155dbca3fa3efe9d8e943b9f394b850 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_numpy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_numpy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..125b2406f68affbb85504de68cdab42ff9d62102 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_numpy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_pymc.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_pymc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..085d4bec4f1cf48924e3256b24190555157e33f3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_pymc.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_scipy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_scipy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1101a594406abeeaaaa614eb8a596f7bd3178852 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/__pycache__/sample_scipy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f60cb4280b262a480059568a8ca88e8f536f1245 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_continuous_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_continuous_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..755a65e6efca142ccdfc7157782b0815b7041cb7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_continuous_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_discrete_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_discrete_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc421aa8ec0b69cf1ed60ade6dadac13073925a9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_discrete_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_finite_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_finite_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29fa812deda09cc14041292fbb9b09e37f69b1f4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/__pycache__/test_sample_finite_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_continuous_rv.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_continuous_rv.py new file mode 100644 index 0000000000000000000000000000000000000000..953bb602df5e63da2882ee118de9dbf24b6f7804 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_continuous_rv.py @@ -0,0 +1,181 @@ +from sympy.core.numbers import oo +from sympy.core.symbol import Symbol +from sympy.functions.elementary.exponential import exp +from sympy.sets.sets import Interval +from sympy.external import import_module +from sympy.stats import Beta, Chi, Normal, Gamma, Exponential, LogNormal, Pareto, ChiSquared, Uniform, sample, \ + BetaPrime, Cauchy, GammaInverse, GaussianInverse, StudentT, Weibull, density, ContinuousRV, FDistribution, \ + Gumbel, Laplace, Logistic, Rayleigh, Triangular +from sympy.testing.pytest import skip, raises + + +def test_sample_numpy(): + distribs_numpy = [ + Beta("B", 1, 1), + Normal("N", 0, 1), + Gamma("G", 2, 7), + Exponential("E", 2), + LogNormal("LN", 0, 1), + Pareto("P", 1, 1), + ChiSquared("CS", 2), + Uniform("U", 0, 1), + FDistribution("FD", 1, 2), + Gumbel("GB", 1, 2), + Laplace("L", 1, 2), + Logistic("LO", 1, 2), + Rayleigh("R", 1), + Triangular("T", 1, 2, 2), + ] + size = 3 + numpy = import_module('numpy') + if not numpy: + skip('Numpy is not installed. Abort tests for _sample_numpy.') + else: + for X in distribs_numpy: + samps = sample(X, size=size, library='numpy') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: sample(Chi("C", 1), library='numpy')) + raises(NotImplementedError, + lambda: Chi("C", 1).pspace.distribution.sample(library='tensorflow')) + + +def test_sample_scipy(): + distribs_scipy = [ + Beta("B", 1, 1), + BetaPrime("BP", 1, 1), + Cauchy("C", 1, 1), + Chi("C", 1), + Normal("N", 0, 1), + Gamma("G", 2, 7), + GammaInverse("GI", 1, 1), + GaussianInverse("GUI", 1, 1), + Exponential("E", 2), + LogNormal("LN", 0, 1), + Pareto("P", 1, 1), + StudentT("S", 2), + ChiSquared("CS", 2), + Uniform("U", 0, 1) + ] + size = 3 + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests for _sample_scipy.') + else: + for X in distribs_scipy: + samps = sample(X, size=size, library='scipy') + samps2 = sample(X, size=(2, 2), library='scipy') + for sam in samps: + assert sam in X.pspace.domain.set + for i in range(2): + for j in range(2): + assert samps2[i][j] in X.pspace.domain.set + + +def test_sample_pymc(): + distribs_pymc = [ + Beta("B", 1, 1), + Cauchy("C", 1, 1), + Normal("N", 0, 1), + Gamma("G", 2, 7), + GaussianInverse("GI", 1, 1), + Exponential("E", 2), + LogNormal("LN", 0, 1), + Pareto("P", 1, 1), + ChiSquared("CS", 2), + Uniform("U", 0, 1) + ] + size = 3 + pymc = import_module('pymc') + if not pymc: + skip('PyMC is not installed. Abort tests for _sample_pymc.') + else: + for X in distribs_pymc: + samps = sample(X, size=size, library='pymc') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: sample(Chi("C", 1), library='pymc')) + + +def test_sampling_gamma_inverse(): + scipy = import_module('scipy') + if not scipy: + skip('Scipy not installed. Abort tests for sampling of gamma inverse.') + X = GammaInverse("x", 1, 1) + assert sample(X) in X.pspace.domain.set + + +def test_lognormal_sampling(): + # Right now, only density function and sampling works + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests') + for i in range(3): + X = LogNormal('x', i, 1) + assert sample(X) in X.pspace.domain.set + + size = 5 + samps = sample(X, size=size) + for samp in samps: + assert samp in X.pspace.domain.set + + +def test_sampling_gaussian_inverse(): + scipy = import_module('scipy') + if not scipy: + skip('Scipy not installed. Abort tests for sampling of Gaussian inverse.') + X = GaussianInverse("x", 1, 1) + assert sample(X, library='scipy') in X.pspace.domain.set + + +def test_prefab_sampling(): + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests') + N = Normal('X', 0, 1) + L = LogNormal('L', 0, 1) + E = Exponential('Ex', 1) + P = Pareto('P', 1, 3) + W = Weibull('W', 1, 1) + U = Uniform('U', 0, 1) + B = Beta('B', 2, 5) + G = Gamma('G', 1, 3) + + variables = [N, L, E, P, W, U, B, G] + niter = 10 + size = 5 + for var in variables: + for _ in range(niter): + assert sample(var) in var.pspace.domain.set + samps = sample(var, size=size) + for samp in samps: + assert samp in var.pspace.domain.set + + +def test_sample_continuous(): + z = Symbol('z') + Z = ContinuousRV(z, exp(-z), set=Interval(0, oo)) + assert density(Z)(-1) == 0 + + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests') + assert sample(Z) in Z.pspace.domain.set + sym, val = list(Z.pspace.sample().items())[0] + assert sym == Z and val in Interval(0, oo) + + libraries = ['scipy', 'numpy', 'pymc'] + for lib in libraries: + try: + imported_lib = import_module(lib) + if imported_lib: + s0, s1, s2 = [], [], [] + s0 = sample(Z, size=10, library=lib, seed=0) + s1 = sample(Z, size=10, library=lib, seed=0) + s2 = sample(Z, size=10, library=lib, seed=1) + assert all(s0 == s1) + assert all(s1 != s2) + except NotImplementedError: + continue diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_discrete_rv.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_discrete_rv.py new file mode 100644 index 0000000000000000000000000000000000000000..90d385cd599222fd7da7c1559b619bafbeb01831 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_discrete_rv.py @@ -0,0 +1,109 @@ +from sympy.core.singleton import S +from sympy.core.symbol import Symbol +from sympy.external import import_module +from sympy.stats import ( + Geometric, + Poisson, + Zeta, + sample, + Skellam, + Logarithmic, + NegativeBinomial, + YuleSimon, + DiscreteRV, +) +from sympy.testing.pytest import skip, raises, slow + + +def test_sample_numpy(): + distribs_numpy = [ + Geometric('G', 0.5), + Poisson('P', 1), + Zeta('Z', 2) + ] + size = 3 + numpy = import_module('numpy') + if not numpy: + skip('Numpy is not installed. Abort tests for _sample_numpy.') + else: + for X in distribs_numpy: + samps = sample(X, size=size, library='numpy') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: sample(Skellam('S', 1, 1), library='numpy')) + raises(NotImplementedError, + lambda: Skellam('S', 1, 1).pspace.distribution.sample(library='tensorflow')) + + +def test_sample_scipy(): + p = S(2)/3 + x = Symbol('x', integer=True, positive=True) + pdf = p*(1 - p)**(x - 1) # pdf of Geometric Distribution + distribs_scipy = [ + DiscreteRV(x, pdf, set=S.Naturals), + Geometric('G', 0.5), + Logarithmic('L', 0.5), + NegativeBinomial('N', 5, 0.4), + Poisson('P', 1), + Skellam('S', 1, 1), + YuleSimon('Y', 1), + Zeta('Z', 2) + ] + size = 3 + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests for _sample_scipy.') + else: + for X in distribs_scipy: + samps = sample(X, size=size, library='scipy') + samps2 = sample(X, size=(2, 2), library='scipy') + for sam in samps: + assert sam in X.pspace.domain.set + for i in range(2): + for j in range(2): + assert samps2[i][j] in X.pspace.domain.set + + +def test_sample_pymc(): + distribs_pymc = [ + Geometric('G', 0.5), + Poisson('P', 1), + NegativeBinomial('N', 5, 0.4) + ] + size = 3 + pymc = import_module('pymc') + if not pymc: + skip('PyMC is not installed. Abort tests for _sample_pymc.') + else: + for X in distribs_pymc: + samps = sample(X, size=size, library='pymc') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: sample(Skellam('S', 1, 1), library='pymc')) + +@slow +def test_sample_discrete(): + X = Geometric('X', S.Half) + scipy = import_module('scipy') + if not scipy: + skip('Scipy not installed. Abort tests') + assert sample(X) in X.pspace.domain.set + samps = sample(X, size=2) # This takes long time if ran without scipy + for samp in samps: + assert samp in X.pspace.domain.set + + libraries = ['scipy', 'numpy', 'pymc'] + for lib in libraries: + try: + imported_lib = import_module(lib) + if imported_lib: + s0, s1, s2 = [], [], [] + s0 = sample(X, size=10, library=lib, seed=0) + s1 = sample(X, size=10, library=lib, seed=0) + s2 = sample(X, size=10, library=lib, seed=1) + assert all(s0 == s1) + assert not all(s1 == s2) + except NotImplementedError: + continue diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_finite_rv.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_finite_rv.py new file mode 100644 index 0000000000000000000000000000000000000000..96cabe0ff4aaa5977e16600217fbbdeb08b962ae --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/sampling/tests/test_sample_finite_rv.py @@ -0,0 +1,94 @@ +from sympy.core.numbers import Rational +from sympy.core.singleton import S +from sympy.external import import_module +from sympy.stats import Binomial, sample, Die, FiniteRV, DiscreteUniform, Bernoulli, BetaBinomial, Hypergeometric, \ + Rademacher +from sympy.testing.pytest import skip, raises + +def test_given_sample(): + X = Die('X', 6) + scipy = import_module('scipy') + if not scipy: + skip('Scipy is not installed. Abort tests') + assert sample(X, X > 5) == 6 + +def test_sample_numpy(): + distribs_numpy = [ + Binomial("B", 5, 0.4), + Hypergeometric("H", 2, 1, 1) + ] + size = 3 + numpy = import_module('numpy') + if not numpy: + skip('Numpy is not installed. Abort tests for _sample_numpy.') + else: + for X in distribs_numpy: + samps = sample(X, size=size, library='numpy') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: sample(Die("D"), library='numpy')) + raises(NotImplementedError, + lambda: Die("D").pspace.sample(library='tensorflow')) + + +def test_sample_scipy(): + distribs_scipy = [ + FiniteRV('F', {1: S.Half, 2: Rational(1, 4), 3: Rational(1, 4)}), + DiscreteUniform("Y", list(range(5))), + Die("D"), + Bernoulli("Be", 0.3), + Binomial("Bi", 5, 0.4), + BetaBinomial("Bb", 2, 1, 1), + Hypergeometric("H", 1, 1, 1), + Rademacher("R") + ] + + size = 3 + scipy = import_module('scipy') + if not scipy: + skip('Scipy not installed. Abort tests for _sample_scipy.') + else: + for X in distribs_scipy: + samps = sample(X, size=size) + samps2 = sample(X, size=(2, 2)) + for sam in samps: + assert sam in X.pspace.domain.set + for i in range(2): + for j in range(2): + assert samps2[i][j] in X.pspace.domain.set + + +def test_sample_pymc(): + distribs_pymc = [ + Bernoulli('B', 0.2), + Binomial('N', 5, 0.4) + ] + size = 3 + pymc = import_module('pymc') + if not pymc: + skip('PyMC is not installed. Abort tests for _sample_pymc.') + else: + for X in distribs_pymc: + samps = sample(X, size=size, library='pymc') + for sam in samps: + assert sam in X.pspace.domain.set + raises(NotImplementedError, + lambda: (sample(Die("D"), library='pymc'))) + + +def test_sample_seed(): + F = FiniteRV('F', {1: S.Half, 2: Rational(1, 4), 3: Rational(1, 4)}) + size = 10 + libraries = ['scipy', 'numpy', 'pymc'] + for lib in libraries: + try: + imported_lib = import_module(lib) + if imported_lib: + s0 = sample(F, size=size, library=lib, seed=0) + s1 = sample(F, size=size, library=lib, seed=0) + s2 = sample(F, size=size, library=lib, seed=1) + assert all(s0 == s1) + assert not all(s1 == s2) + except NotImplementedError: + continue diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b9f61ece7c41f52394a1c730652e70926dedb40 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_compound_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_compound_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bada760f8de84ebd390056736ab17baaf86d5af Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_compound_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_discrete_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_discrete_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2c3d9e85fb54791c9011a7d6742be0a8c3d53f0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_discrete_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_error_prop.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_error_prop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4fd60e9ed915f450e49be037c38cb831f4708a2e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_error_prop.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_finite_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_finite_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..196dc8249cd06dc98e5de5879c464f4f81f43c7d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_finite_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_joint_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_joint_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c28405c3323036bb4936eb5c7d0df95dd8a1e3b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_joint_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_matrix_distributions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_matrix_distributions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1089ecdd0c9719d210045040d937fd170fd137de Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_matrix_distributions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_mix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_mix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e2521d0997b315bdce031abe64ff07ae23f8844 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_mix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_random_matrix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_random_matrix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c857ac967098cd1884bce990d887a9d89ab3a5a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_random_matrix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_rv.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_rv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8d28c44ec186fb06d31957d079465ac8613c078 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_rv.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_stochastic_process.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_stochastic_process.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..430d0abd8c2f7de6e8c0847dfb04c4e0b654863b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_stochastic_process.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_multivariate.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_multivariate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfe751ffa0f8b2b9e819d621adf3a3ba2c107589 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_multivariate.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_probability.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_probability.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a160dd8b884bac878935fad4516c78fe976395e3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/stats/tests/__pycache__/test_symbolic_probability.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b2c35fc92e3ff0e58549391f7008ec5f7a98657 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/core.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93c0c6aded2a47699d33f9017f1bfad7de454be9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/core.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/rl.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/rl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c68677184d334dc12cd3f9494897c13b1f76118a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/rl.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tools.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa5b9bca4b31e76ad0ffb01ab18ba3afc3ff3d52 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tools.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/traverse.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/traverse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dee220e5ca9458464773587e1ba341c1240baa85 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/traverse.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tree.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tree.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd533bbe4b9f8775f9ea9ebff4a9aa770fbba025 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/tree.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/util.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2333f2ee094b22e1d71ed161460f44022e3529ca Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/__pycache__/util.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fec5afe84a58f3d887a8c762692a3673a2b6d4c8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__init__.py @@ -0,0 +1,14 @@ +from . import traverse +from .core import ( + condition, debug, multiplex, exhaust, notempty, + chain, onaction, sfilter, yieldify, do_one, identity) +from .tools import canon + +__all__ = [ + 'traverse', + + 'condition', 'debug', 'multiplex', 'exhaust', 'notempty', 'chain', + 'onaction', 'sfilter', 'yieldify', 'do_one', 'identity', + + 'canon', +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30d874072a818ebc98184d9912bbfbe0e020ef26 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/core.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c241c907c39bfa1b1a845200283664ffde105c04 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/core.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/tools.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/tools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..717ba49990ed92ba74aabc43172c21401ad2f5c9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/tools.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/traverse.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/traverse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23f0fe6393bcf7ca46e96d8fb21392f014638d20 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/__pycache__/traverse.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/core.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/core.py new file mode 100644 index 0000000000000000000000000000000000000000..2dabaef69b60d994799f71414699223f84e1809b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/core.py @@ -0,0 +1,116 @@ +""" Generic SymPy-Independent Strategies """ + + +def identity(x): + yield x + + +def exhaust(brule): + """ Apply a branching rule repeatedly until it has no effect """ + def exhaust_brl(expr): + seen = {expr} + for nexpr in brule(expr): + if nexpr not in seen: + seen.add(nexpr) + yield from exhaust_brl(nexpr) + if seen == {expr}: + yield expr + return exhaust_brl + + +def onaction(brule, fn): + def onaction_brl(expr): + for result in brule(expr): + if result != expr: + fn(brule, expr, result) + yield result + return onaction_brl + + +def debug(brule, file=None): + """ Print the input and output expressions at each rule application """ + if not file: + from sys import stdout + file = stdout + + def write(brl, expr, result): + file.write("Rule: %s\n" % brl.__name__) + file.write("In: %s\nOut: %s\n\n" % (expr, result)) + + return onaction(brule, write) + + +def multiplex(*brules): + """ Multiplex many branching rules into one """ + def multiplex_brl(expr): + seen = set() + for brl in brules: + for nexpr in brl(expr): + if nexpr not in seen: + seen.add(nexpr) + yield nexpr + return multiplex_brl + + +def condition(cond, brule): + """ Only apply branching rule if condition is true """ + def conditioned_brl(expr): + if cond(expr): + yield from brule(expr) + else: + pass + return conditioned_brl + + +def sfilter(pred, brule): + """ Yield only those results which satisfy the predicate """ + def filtered_brl(expr): + yield from filter(pred, brule(expr)) + return filtered_brl + + +def notempty(brule): + def notempty_brl(expr): + yielded = False + for nexpr in brule(expr): + yielded = True + yield nexpr + if not yielded: + yield expr + return notempty_brl + + +def do_one(*brules): + """ Execute one of the branching rules """ + def do_one_brl(expr): + yielded = False + for brl in brules: + for nexpr in brl(expr): + yielded = True + yield nexpr + if yielded: + return + return do_one_brl + + +def chain(*brules): + """ + Compose a sequence of brules so that they apply to the expr sequentially + """ + def chain_brl(expr): + if not brules: + yield expr + return + + head, tail = brules[0], brules[1:] + for nexpr in head(expr): + yield from chain(*tail)(nexpr) + + return chain_brl + + +def yieldify(rl): + """ Turn a rule into a branching rule """ + def brl(expr): + yield rl(expr) + return brl diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a91329bc49ce8749dad5e18954a2298ad08aa6e3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_core.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b54653c30c83ce01a9c8ce595d072543c1e5f45 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_core.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_tools.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_tools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f107cd6d1c4459565b7cb866bec967cf3a99484f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_tools.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_traverse.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_traverse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7ddb7932565a53849c184f10f1da038d72e7410 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/__pycache__/test_traverse.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_core.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_core.py new file mode 100644 index 0000000000000000000000000000000000000000..ac620b0afb6dbadc4d97b29ddbb341cd920b6588 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_core.py @@ -0,0 +1,117 @@ +from sympy.strategies.branch.core import ( + exhaust, debug, multiplex, condition, notempty, chain, onaction, sfilter, + yieldify, do_one, identity) + + +def posdec(x): + if x > 0: + yield x - 1 + else: + yield x + + +def branch5(x): + if 0 < x < 5: + yield x - 1 + elif 5 < x < 10: + yield x + 1 + elif x == 5: + yield x + 1 + yield x - 1 + else: + yield x + + +def even(x): + return x % 2 == 0 + + +def inc(x): + yield x + 1 + + +def one_to_n(n): + yield from range(n) + + +def test_exhaust(): + brl = exhaust(branch5) + assert set(brl(3)) == {0} + assert set(brl(7)) == {10} + assert set(brl(5)) == {0, 10} + + +def test_debug(): + from io import StringIO + file = StringIO() + rl = debug(posdec, file) + list(rl(5)) + log = file.getvalue() + file.close() + + assert posdec.__name__ in log + assert '5' in log + assert '4' in log + + +def test_multiplex(): + brl = multiplex(posdec, branch5) + assert set(brl(3)) == {2} + assert set(brl(7)) == {6, 8} + assert set(brl(5)) == {4, 6} + + +def test_condition(): + brl = condition(even, branch5) + assert set(brl(4)) == set(branch5(4)) + assert set(brl(5)) == set() + + +def test_sfilter(): + brl = sfilter(even, one_to_n) + assert set(brl(10)) == {0, 2, 4, 6, 8} + + +def test_notempty(): + def ident_if_even(x): + if even(x): + yield x + + brl = notempty(ident_if_even) + assert set(brl(4)) == {4} + assert set(brl(5)) == {5} + + +def test_chain(): + assert list(chain()(2)) == [2] # identity + assert list(chain(inc, inc)(2)) == [4] + assert list(chain(branch5, inc)(4)) == [4] + assert set(chain(branch5, inc)(5)) == {5, 7} + assert list(chain(inc, branch5)(5)) == [7] + + +def test_onaction(): + L = [] + + def record(fn, input, output): + L.append((input, output)) + + list(onaction(inc, record)(2)) + assert L == [(2, 3)] + + list(onaction(identity, record)(2)) + assert L == [(2, 3)] + + +def test_yieldify(): + yinc = yieldify(lambda x: x + 1) + assert list(yinc(3)) == [4] + + +def test_do_one(): + def bad(expr): + raise ValueError + + assert list(do_one(inc)(3)) == [4] + assert list(do_one(inc, bad)(3)) == [4] + assert list(do_one(inc, posdec)(3)) == [4] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_tools.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..c2bd224030c337f0a000d94f6e7e65f3b8bd118f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_tools.py @@ -0,0 +1,42 @@ +from sympy.strategies.branch.tools import canon +from sympy.core.basic import Basic +from sympy.core.numbers import Integer +from sympy.core.singleton import S + + +def posdec(x): + if isinstance(x, Integer) and x > 0: + yield x - 1 + else: + yield x + + +def branch5(x): + if isinstance(x, Integer): + if 0 < x < 5: + yield x - 1 + elif 5 < x < 10: + yield x + 1 + elif x == 5: + yield x + 1 + yield x - 1 + else: + yield x + + +def test_zero_ints(): + expr = Basic(S(2), Basic(S(5), S(3)), S(8)) + expected = {Basic(S(0), Basic(S(0), S(0)), S(0))} + + brl = canon(posdec) + assert set(brl(expr)) == expected + + +def test_split5(): + expr = Basic(S(2), Basic(S(5), S(3)), S(8)) + expected = { + Basic(S(0), Basic(S(0), S(0)), S(10)), + Basic(S(0), Basic(S(10), S(0)), S(10))} + + brl = canon(branch5) + assert set(brl(expr)) == expected diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_traverse.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_traverse.py new file mode 100644 index 0000000000000000000000000000000000000000..e051928210981223004de28b8c617d0438e11ac6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tests/test_traverse.py @@ -0,0 +1,53 @@ +from sympy.core.basic import Basic +from sympy.core.numbers import Integer +from sympy.core.singleton import S +from sympy.strategies.branch.traverse import top_down, sall +from sympy.strategies.branch.core import do_one, identity + + +def inc(x): + if isinstance(x, Integer): + yield x + 1 + + +def test_top_down_easy(): + expr = Basic(S(1), S(2)) + expected = Basic(S(2), S(3)) + brl = top_down(inc) + + assert set(brl(expr)) == {expected} + + +def test_top_down_big_tree(): + expr = Basic(S(1), Basic(S(2)), Basic(S(3), Basic(S(4)), S(5))) + expected = Basic(S(2), Basic(S(3)), Basic(S(4), Basic(S(5)), S(6))) + brl = top_down(inc) + + assert set(brl(expr)) == {expected} + + +def test_top_down_harder_function(): + def split5(x): + if x == 5: + yield x - 1 + yield x + 1 + + expr = Basic(Basic(S(5), S(6)), S(1)) + expected = {Basic(Basic(S(4), S(6)), S(1)), Basic(Basic(S(6), S(6)), S(1))} + brl = top_down(split5) + + assert set(brl(expr)) == expected + + +def test_sall(): + expr = Basic(S(1), S(2)) + expected = Basic(S(2), S(3)) + brl = sall(inc) + + assert list(brl(expr)) == [expected] + + expr = Basic(S(1), S(2), Basic(S(3), S(4))) + expected = Basic(S(2), S(3), Basic(S(3), S(4))) + brl = sall(do_one(inc, identity)) + + assert list(brl(expr)) == [expected] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tools.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..a6c9097323a7962080ae4497ead976818e386518 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/tools.py @@ -0,0 +1,12 @@ +from .core import exhaust, multiplex +from .traverse import top_down + + +def canon(*rules): + """ Strategy for canonicalization + + Apply each branching rule in a top-down fashion through the tree. + Multiplex through all branching rule traversals + Keep doing this until there is no change. + """ + return exhaust(multiplex(*map(top_down, rules))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/traverse.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/traverse.py new file mode 100644 index 0000000000000000000000000000000000000000..28b7098dbda401fc0f0b6d27988d8c37e2f231ae --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/branch/traverse.py @@ -0,0 +1,25 @@ +""" Branching Strategies to Traverse a Tree """ +from itertools import product +from sympy.strategies.util import basic_fns +from .core import chain, identity, do_one + + +def top_down(brule, fns=basic_fns): + """ Apply a rule down a tree running it on the top nodes first """ + return chain(do_one(brule, identity), + lambda expr: sall(top_down(brule, fns), fns)(expr)) + + +def sall(brule, fns=basic_fns): + """ Strategic all - apply rule to args """ + op, new, children, leaf = map(fns.get, ('op', 'new', 'children', 'leaf')) + + def all_rl(expr): + if leaf(expr): + yield expr + else: + myop = op(expr) + argss = product(*map(brule, children(expr))) + for args in argss: + yield new(myop, *args) + return all_rl diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccf2ba54fae034c4c9f6aee7a6a01a7c719b9ac2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_core.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b22f01b58338dde992300664d89a980cdf4e3334 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_core.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_rl.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_rl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..146d51a3a0ded5832b849e18632b8de778aa5ccb Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_rl.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tools.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..832b09e51c6b0ead63db99a7396dbbe139cdadb4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tools.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_traverse.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_traverse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..771f7e6f54be8e9bdc26fc4db2d385a0be791a38 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_traverse.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tree.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tree.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..922c16579444a9653769bef010910685328229bf Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/__pycache__/test_tree.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_core.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_core.py new file mode 100644 index 0000000000000000000000000000000000000000..1e19bcab1940c476a8996b7ba92e7645a6230034 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_core.py @@ -0,0 +1,118 @@ +from __future__ import annotations +from sympy.core.singleton import S +from sympy.core.basic import Basic +from sympy.strategies.core import ( + null_safe, exhaust, memoize, condition, + chain, tryit, do_one, debug, switch, minimize) +from io import StringIO + + +def posdec(x: int) -> int: + if x > 0: + return x - 1 + return x + + +def inc(x: int) -> int: + return x + 1 + + +def dec(x: int) -> int: + return x - 1 + + +def test_null_safe(): + def rl(expr: int) -> int | None: + if expr == 1: + return 2 + return None + + safe_rl = null_safe(rl) + assert rl(1) == safe_rl(1) + assert rl(3) is None + assert safe_rl(3) == 3 + + +def test_exhaust(): + sink = exhaust(posdec) + assert sink(5) == 0 + assert sink(10) == 0 + + +def test_memoize(): + rl = memoize(posdec) + assert rl(5) == posdec(5) + assert rl(5) == posdec(5) + assert rl(-2) == posdec(-2) + + +def test_condition(): + rl = condition(lambda x: x % 2 == 0, posdec) + assert rl(5) == 5 + assert rl(4) == 3 + + +def test_chain(): + rl = chain(posdec, posdec) + assert rl(5) == 3 + assert rl(1) == 0 + + +def test_tryit(): + def rl(expr: Basic) -> Basic: + assert False + + safe_rl = tryit(rl, AssertionError) + assert safe_rl(S(1)) == S(1) + + +def test_do_one(): + rl = do_one(posdec, posdec) + assert rl(5) == 4 + + def rl1(x: int) -> int: + if x == 1: + return 2 + return x + + def rl2(x: int) -> int: + if x == 2: + return 3 + return x + + rule = do_one(rl1, rl2) + assert rule(1) == 2 + assert rule(rule(1)) == 3 + + +def test_debug(): + file = StringIO() + rl = debug(posdec, file) + rl(5) + log = file.getvalue() + file.close() + + assert posdec.__name__ in log + assert '5' in log + assert '4' in log + + +def test_switch(): + def key(x: int) -> int: + return x % 3 + + rl = switch(key, {0: inc, 1: dec}) + assert rl(3) == 4 + assert rl(4) == 3 + assert rl(5) == 5 + + +def test_minimize(): + def key(x: int) -> int: + return -x + + rl = minimize(inc, dec) + assert rl(4) == 3 + + rl = minimize(inc, dec, objective=key) + assert rl(4) == 5 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_rl.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_rl.py new file mode 100644 index 0000000000000000000000000000000000000000..8bfa90ad4d970b21396e0e1b6427b5a5c68fe381 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_rl.py @@ -0,0 +1,78 @@ +from sympy.core.singleton import S +from sympy.strategies.rl import ( + rm_id, glom, flatten, unpack, sort, distribute, subs, rebuild) +from sympy.core.basic import Basic +from sympy.core.add import Add +from sympy.core.mul import Mul +from sympy.core.symbol import symbols +from sympy.abc import x + + +def test_rm_id(): + rmzeros = rm_id(lambda x: x == 0) + assert rmzeros(Basic(S(0), S(1))) == Basic(S(1)) + assert rmzeros(Basic(S(0), S(0))) == Basic(S(0)) + assert rmzeros(Basic(S(2), S(1))) == Basic(S(2), S(1)) + + +def test_glom(): + def key(x): + return x.as_coeff_Mul()[1] + + def count(x): + return x.as_coeff_Mul()[0] + + def newargs(cnt, arg): + return cnt * arg + + rl = glom(key, count, newargs) + + result = rl(Add(x, -x, 3 * x, 2, 3, evaluate=False)) + expected = Add(3 * x, 5) + assert set(result.args) == set(expected.args) + + +def test_flatten(): + assert flatten(Basic(S(1), S(2), Basic(S(3), S(4)))) == \ + Basic(S(1), S(2), S(3), S(4)) + + +def test_unpack(): + assert unpack(Basic(S(2))) == 2 + assert unpack(Basic(S(2), S(3))) == Basic(S(2), S(3)) + + +def test_sort(): + assert sort(str)(Basic(S(3), S(1), S(2))) == Basic(S(1), S(2), S(3)) + + +def test_distribute(): + class T1(Basic): + pass + + class T2(Basic): + pass + + distribute_t12 = distribute(T1, T2) + assert distribute_t12(T1(S(1), S(2), T2(S(3), S(4)), S(5))) == \ + T2(T1(S(1), S(2), S(3), S(5)), T1(S(1), S(2), S(4), S(5))) + assert distribute_t12(T1(S(1), S(2), S(3))) == T1(S(1), S(2), S(3)) + + +def test_distribute_add_mul(): + x, y = symbols('x, y') + expr = Mul(2, Add(x, y), evaluate=False) + expected = Add(Mul(2, x), Mul(2, y)) + distribute_mul = distribute(Mul, Add) + assert distribute_mul(expr) == expected + + +def test_subs(): + rl = subs(1, 2) + assert rl(1) == 2 + assert rl(3) == 3 + + +def test_rebuild(): + expr = Basic.__new__(Add, S(1), S(2)) + assert rebuild(expr) == 3 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tools.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..89774aeb92ead5781966e4f48ad32dc63e1bf7e2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tools.py @@ -0,0 +1,32 @@ +from sympy.strategies.tools import subs, typed +from sympy.strategies.rl import rm_id +from sympy.core.basic import Basic +from sympy.core.singleton import S + + +def test_subs(): + from sympy.core.symbol import symbols + a, b, c, d, e, f = symbols('a,b,c,d,e,f') + mapping = {a: d, d: a, Basic(e): Basic(f)} + expr = Basic(a, Basic(b, c), Basic(d, Basic(e))) + result = Basic(d, Basic(b, c), Basic(a, Basic(f))) + assert subs(mapping)(expr) == result + + +def test_subs_empty(): + assert subs({})(Basic(S(1), S(2))) == Basic(S(1), S(2)) + + +def test_typed(): + class A(Basic): + pass + + class B(Basic): + pass + + rmzeros = rm_id(lambda x: x == S(0)) + rmones = rm_id(lambda x: x == S(1)) + remove_something = typed({A: rmzeros, B: rmones}) + + assert remove_something(A(S(0), S(1))) == A(S(1)) + assert remove_something(B(S(0), S(1))) == B(S(0)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_traverse.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_traverse.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2409616a8b4f750af8baea149b2ea52c56be9d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_traverse.py @@ -0,0 +1,84 @@ +from sympy.strategies.traverse import ( + top_down, bottom_up, sall, top_down_once, bottom_up_once, basic_fns) +from sympy.strategies.rl import rebuild +from sympy.strategies.util import expr_fns +from sympy.core.add import Add +from sympy.core.basic import Basic +from sympy.core.numbers import Integer +from sympy.core.singleton import S +from sympy.core.symbol import Str, Symbol +from sympy.abc import x, y, z + + +def zero_symbols(expression): + return S.Zero if isinstance(expression, Symbol) else expression + + +def test_sall(): + zero_onelevel = sall(zero_symbols) + + assert zero_onelevel(Basic(x, y, Basic(x, z))) == \ + Basic(S(0), S(0), Basic(x, z)) + + +def test_bottom_up(): + _test_global_traversal(bottom_up) + _test_stop_on_non_basics(bottom_up) + + +def test_top_down(): + _test_global_traversal(top_down) + _test_stop_on_non_basics(top_down) + + +def _test_global_traversal(trav): + zero_all_symbols = trav(zero_symbols) + + assert zero_all_symbols(Basic(x, y, Basic(x, z))) == \ + Basic(S(0), S(0), Basic(S(0), S(0))) + + +def _test_stop_on_non_basics(trav): + def add_one_if_can(expr): + try: + return expr + 1 + except TypeError: + return expr + + expr = Basic(S(1), Str('a'), Basic(S(2), Str('b'))) + expected = Basic(S(2), Str('a'), Basic(S(3), Str('b'))) + rl = trav(add_one_if_can) + + assert rl(expr) == expected + + +class Basic2(Basic): + pass + + +def rl(x): + if x.args and not isinstance(x.args[0], Integer): + return Basic2(*x.args) + return x + + +def test_top_down_once(): + top_rl = top_down_once(rl) + + assert top_rl(Basic(S(1.0), S(2.0), Basic(S(3), S(4)))) == \ + Basic2(S(1.0), S(2.0), Basic(S(3), S(4))) + + +def test_bottom_up_once(): + bottom_rl = bottom_up_once(rl) + + assert bottom_rl(Basic(S(1), S(2), Basic(S(3.0), S(4.0)))) == \ + Basic(S(1), S(2), Basic2(S(3.0), S(4.0))) + + +def test_expr_fns(): + expr = x + y**3 + e = bottom_up(lambda v: v + 1, expr_fns)(expr) + b = bottom_up(lambda v: Basic.__new__(Add, v, S(1)), basic_fns)(expr) + + assert rebuild(b) == e diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tree.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tree.py new file mode 100644 index 0000000000000000000000000000000000000000..d5cdde747fe3ab90c8fd181701194403bc526067 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/strategies/tests/test_tree.py @@ -0,0 +1,92 @@ +from sympy.strategies.tree import treeapply, greedy, allresults, brute +from functools import partial, reduce + + +def inc(x): + return x + 1 + + +def dec(x): + return x - 1 + + +def double(x): + return 2 * x + + +def square(x): + return x**2 + + +def add(*args): + return sum(args) + + +def mul(*args): + return reduce(lambda a, b: a * b, args, 1) + + +def test_treeapply(): + tree = ([3, 3], [4, 1], 2) + assert treeapply(tree, {list: min, tuple: max}) == 3 + assert treeapply(tree, {list: add, tuple: mul}) == 60 + + +def test_treeapply_leaf(): + assert treeapply(3, {}, leaf=lambda x: x**2) == 9 + tree = ([3, 3], [4, 1], 2) + treep1 = ([4, 4], [5, 2], 3) + assert treeapply(tree, {list: min, tuple: max}, leaf=lambda x: x + 1) == \ + treeapply(treep1, {list: min, tuple: max}) + + +def test_treeapply_strategies(): + from sympy.strategies import chain, minimize + join = {list: chain, tuple: minimize} + + assert treeapply(inc, join) == inc + assert treeapply((inc, dec), join)(5) == minimize(inc, dec)(5) + assert treeapply([inc, dec], join)(5) == chain(inc, dec)(5) + tree = (inc, [dec, double]) # either inc or dec-then-double + assert treeapply(tree, join)(5) == 6 + assert treeapply(tree, join)(1) == 0 + + maximize = partial(minimize, objective=lambda x: -x) + join = {list: chain, tuple: maximize} + fn = treeapply(tree, join) + assert fn(4) == 6 # highest value comes from the dec then double + assert fn(1) == 2 # highest value comes from the inc + + +def test_greedy(): + tree = [inc, (dec, double)] # either inc or dec-then-double + + fn = greedy(tree, objective=lambda x: -x) + assert fn(4) == 6 # highest value comes from the dec then double + assert fn(1) == 2 # highest value comes from the inc + + tree = [inc, dec, [inc, dec, [(inc, inc), (dec, dec)]]] + lowest = greedy(tree) + assert lowest(10) == 8 + + highest = greedy(tree, objective=lambda x: -x) + assert highest(10) == 12 + + +def test_allresults(): + # square = lambda x: x**2 + + assert set(allresults(inc)(3)) == {inc(3)} + assert set(allresults([inc, dec])(3)) == {2, 4} + assert set(allresults((inc, dec))(3)) == {3} + assert set(allresults([inc, (dec, double)])(4)) == {5, 6} + + +def test_brute(): + tree = ([inc, dec], square) + fn = brute(tree, lambda x: -x) + + assert fn(2) == (2 + 1)**2 + assert fn(-2) == (-2 - 1)**2 + + assert brute(inc)(1) == 2 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c16eaa09b95441f4bd51bc043a2bc2bbe18fbcdf Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/functions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..491d4d10f181024e504b3c5051d0284561b3eace Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/functions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/index_methods.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/index_methods.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13b05394ebe2ae7dc9f5a50b8adc62ba6f6a64f1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/index_methods.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/indexed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/indexed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a71857f91001699044e1b8bd671ce2b0b5b993ed Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/indexed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/toperators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/toperators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b4b1b6301edc3f86dad685de03ca44aef89a6f9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/__pycache__/toperators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eca2eb4c6c58cb113517b6e41737e9d97abbb84e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__init__.py @@ -0,0 +1,271 @@ +r""" +N-dim array module for SymPy. + +Four classes are provided to handle N-dim arrays, given by the combinations +dense/sparse (i.e. whether to store all elements or only the non-zero ones in +memory) and mutable/immutable (immutable classes are SymPy objects, but cannot +change after they have been created). + +Examples +======== + +The following examples show the usage of ``Array``. This is an abbreviation for +``ImmutableDenseNDimArray``, that is an immutable and dense N-dim array, the +other classes are analogous. For mutable classes it is also possible to change +element values after the object has been constructed. + +Array construction can detect the shape of nested lists and tuples: + +>>> from sympy import Array +>>> a1 = Array([[1, 2], [3, 4], [5, 6]]) +>>> a1 +[[1, 2], [3, 4], [5, 6]] +>>> a1.shape +(3, 2) +>>> a1.rank() +2 +>>> from sympy.abc import x, y, z +>>> a2 = Array([[[x, y], [z, x*z]], [[1, x*y], [1/x, x/y]]]) +>>> a2 +[[[x, y], [z, x*z]], [[1, x*y], [1/x, x/y]]] +>>> a2.shape +(2, 2, 2) +>>> a2.rank() +3 + +Otherwise one could pass a 1-dim array followed by a shape tuple: + +>>> m1 = Array(range(12), (3, 4)) +>>> m1 +[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]] +>>> m2 = Array(range(12), (3, 2, 2)) +>>> m2 +[[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]] +>>> m2[1,1,1] +7 +>>> m2.reshape(4, 3) +[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + +Slice support: + +>>> m2[:, 1, 1] +[3, 7, 11] + +Elementwise derivative: + +>>> from sympy.abc import x, y, z +>>> m3 = Array([x**3, x*y, z]) +>>> m3.diff(x) +[3*x**2, y, 0] +>>> m3.diff(z) +[0, 0, 1] + +Multiplication with other SymPy expressions is applied elementwisely: + +>>> (1+x)*m3 +[x**3*(x + 1), x*y*(x + 1), z*(x + 1)] + +To apply a function to each element of the N-dim array, use ``applyfunc``: + +>>> m3.applyfunc(lambda x: x/2) +[x**3/2, x*y/2, z/2] + +N-dim arrays can be converted to nested lists by the ``tolist()`` method: + +>>> m2.tolist() +[[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]] +>>> isinstance(m2.tolist(), list) +True + +If the rank is 2, it is possible to convert them to matrices with ``tomatrix()``: + +>>> m1.tomatrix() +Matrix([ +[0, 1, 2, 3], +[4, 5, 6, 7], +[8, 9, 10, 11]]) + +Products and contractions +------------------------- + +Tensor product between arrays `A_{i_1,\ldots,i_n}` and `B_{j_1,\ldots,j_m}` +creates the combined array `P = A \otimes B` defined as + +`P_{i_1,\ldots,i_n,j_1,\ldots,j_m} := A_{i_1,\ldots,i_n}\cdot B_{j_1,\ldots,j_m}.` + +It is available through ``tensorproduct(...)``: + +>>> from sympy import Array, tensorproduct +>>> from sympy.abc import x,y,z,t +>>> A = Array([x, y, z, t]) +>>> B = Array([1, 2, 3, 4]) +>>> tensorproduct(A, B) +[[x, 2*x, 3*x, 4*x], [y, 2*y, 3*y, 4*y], [z, 2*z, 3*z, 4*z], [t, 2*t, 3*t, 4*t]] + +In case you don't want to evaluate the tensor product immediately, you can use +``ArrayTensorProduct``, which creates an unevaluated tensor product expression: + +>>> from sympy.tensor.array.expressions import ArrayTensorProduct +>>> ArrayTensorProduct(A, B) +ArrayTensorProduct([x, y, z, t], [1, 2, 3, 4]) + +Calling ``.as_explicit()`` on ``ArrayTensorProduct`` is equivalent to just calling +``tensorproduct(...)``: + +>>> ArrayTensorProduct(A, B).as_explicit() +[[x, 2*x, 3*x, 4*x], [y, 2*y, 3*y, 4*y], [z, 2*z, 3*z, 4*z], [t, 2*t, 3*t, 4*t]] + +Tensor product between a rank-1 array and a matrix creates a rank-3 array: + +>>> from sympy import eye +>>> p1 = tensorproduct(A, eye(4)) +>>> p1 +[[[x, 0, 0, 0], [0, x, 0, 0], [0, 0, x, 0], [0, 0, 0, x]], [[y, 0, 0, 0], [0, y, 0, 0], [0, 0, y, 0], [0, 0, 0, y]], [[z, 0, 0, 0], [0, z, 0, 0], [0, 0, z, 0], [0, 0, 0, z]], [[t, 0, 0, 0], [0, t, 0, 0], [0, 0, t, 0], [0, 0, 0, t]]] + +Now, to get back `A_0 \otimes \mathbf{1}` one can access `p_{0,m,n}` by slicing: + +>>> p1[0,:,:] +[[x, 0, 0, 0], [0, x, 0, 0], [0, 0, x, 0], [0, 0, 0, x]] + +Tensor contraction sums over the specified axes, for example contracting +positions `a` and `b` means + +`A_{i_1,\ldots,i_a,\ldots,i_b,\ldots,i_n} \implies \sum_k A_{i_1,\ldots,k,\ldots,k,\ldots,i_n}` + +Remember that Python indexing is zero starting, to contract the a-th and b-th +axes it is therefore necessary to specify `a-1` and `b-1` + +>>> from sympy import tensorcontraction +>>> C = Array([[x, y], [z, t]]) + +The matrix trace is equivalent to the contraction of a rank-2 array: + +`A_{m,n} \implies \sum_k A_{k,k}` + +>>> tensorcontraction(C, (0, 1)) +t + x + +To create an expression representing a tensor contraction that does not get +evaluated immediately, use ``ArrayContraction``, which is equivalent to +``tensorcontraction(...)`` if it is followed by ``.as_explicit()``: + +>>> from sympy.tensor.array.expressions import ArrayContraction +>>> ArrayContraction(C, (0, 1)) +ArrayContraction([[x, y], [z, t]], (0, 1)) +>>> ArrayContraction(C, (0, 1)).as_explicit() +t + x + +Matrix product is equivalent to a tensor product of two rank-2 arrays, followed +by a contraction of the 2nd and 3rd axes (in Python indexing axes number 1, 2). + +`A_{m,n}\cdot B_{i,j} \implies \sum_k A_{m, k}\cdot B_{k, j}` + +>>> D = Array([[2, 1], [0, -1]]) +>>> tensorcontraction(tensorproduct(C, D), (1, 2)) +[[2*x, x - y], [2*z, -t + z]] + +One may verify that the matrix product is equivalent: + +>>> from sympy import Matrix +>>> Matrix([[x, y], [z, t]])*Matrix([[2, 1], [0, -1]]) +Matrix([ +[2*x, x - y], +[2*z, -t + z]]) + +or equivalently + +>>> C.tomatrix()*D.tomatrix() +Matrix([ +[2*x, x - y], +[2*z, -t + z]]) + +Diagonal operator +----------------- + +The ``tensordiagonal`` function acts in a similar manner as ``tensorcontraction``, +but the joined indices are not summed over, for example diagonalizing +positions `a` and `b` means + +`A_{i_1,\ldots,i_a,\ldots,i_b,\ldots,i_n} \implies A_{i_1,\ldots,k,\ldots,k,\ldots,i_n} +\implies \tilde{A}_{i_1,\ldots,i_{a-1},i_{a+1},\ldots,i_{b-1},i_{b+1},\ldots,i_n,k}` + +where `\tilde{A}` is the array equivalent to the diagonal of `A` at positions +`a` and `b` moved to the last index slot. + +Compare the difference between contraction and diagonal operators: + +>>> from sympy import tensordiagonal +>>> from sympy.abc import a, b, c, d +>>> m = Matrix([[a, b], [c, d]]) +>>> tensorcontraction(m, [0, 1]) +a + d +>>> tensordiagonal(m, [0, 1]) +[a, d] + +In short, no summation occurs with ``tensordiagonal``. + + +Derivatives by array +-------------------- + +The usual derivative operation may be extended to support derivation with +respect to arrays, provided that all elements in the that array are symbols or +expressions suitable for derivations. + +The definition of a derivative by an array is as follows: given the array +`A_{i_1, \ldots, i_N}` and the array `X_{j_1, \ldots, j_M}` +the derivative of arrays will return a new array `B` defined by + +`B_{j_1,\ldots,j_M,i_1,\ldots,i_N} := \frac{\partial A_{i_1,\ldots,i_N}}{\partial X_{j_1,\ldots,j_M}}` + +The function ``derive_by_array`` performs such an operation: + +>>> from sympy import derive_by_array +>>> from sympy.abc import x, y, z, t +>>> from sympy import sin, exp + +With scalars, it behaves exactly as the ordinary derivative: + +>>> derive_by_array(sin(x*y), x) +y*cos(x*y) + +Scalar derived by an array basis: + +>>> derive_by_array(sin(x*y), [x, y, z]) +[y*cos(x*y), x*cos(x*y), 0] + +Deriving array by an array basis: `B^{nm} := \frac{\partial A^m}{\partial x^n}` + +>>> basis = [x, y, z] +>>> ax = derive_by_array([exp(x), sin(y*z), t], basis) +>>> ax +[[exp(x), 0, 0], [0, z*cos(y*z), 0], [0, y*cos(y*z), 0]] + +Contraction of the resulting array: `\sum_m \frac{\partial A^m}{\partial x^m}` + +>>> tensorcontraction(ax, (0, 1)) +z*cos(y*z) + exp(x) + +""" + +from .dense_ndim_array import MutableDenseNDimArray, ImmutableDenseNDimArray, DenseNDimArray +from .sparse_ndim_array import MutableSparseNDimArray, ImmutableSparseNDimArray, SparseNDimArray +from .ndim_array import NDimArray, ArrayKind +from .arrayop import tensorproduct, tensorcontraction, tensordiagonal, derive_by_array, permutedims +from .array_comprehension import ArrayComprehension, ArrayComprehensionMap + +Array = ImmutableDenseNDimArray + +__all__ = [ + 'MutableDenseNDimArray', 'ImmutableDenseNDimArray', 'DenseNDimArray', + + 'MutableSparseNDimArray', 'ImmutableSparseNDimArray', 'SparseNDimArray', + + 'NDimArray', 'ArrayKind', + + 'tensorproduct', 'tensorcontraction', 'tensordiagonal', 'derive_by_array', + + 'permutedims', 'ArrayComprehension', 'ArrayComprehensionMap', + + 'Array', +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f66f2d93d74a587e49b723c25b1eb01bb127563 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_comprehension.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_comprehension.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17bc820e0e96c6e7662fb9a55364e9ad1557d927 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_comprehension.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_derivatives.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_derivatives.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d73b7ba1daae723db8d85a15ea71a5162bf587d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/array_derivatives.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/arrayop.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/arrayop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2532f1ae8f710ad222d424c104145b62eae4e37f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/arrayop.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/dense_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/dense_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62602aed61260bb7b4204170f3e838fb8336a647 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/dense_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/mutable_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/mutable_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..446602b1fbb150e4f777103994750ff5a3d8707d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/mutable_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c1dc7488577aaa5925a9f3f1abad87b0f49d200 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/sparse_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/sparse_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e41aff9fd335f9ebf0e29916f3667d3e56fe21b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/__pycache__/sparse_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_comprehension.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_comprehension.py new file mode 100644 index 0000000000000000000000000000000000000000..95702f499f3e40597fd0144929138ac1329962ee --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_comprehension.py @@ -0,0 +1,399 @@ +import functools, itertools +from sympy.core.sympify import _sympify, sympify +from sympy.core.expr import Expr +from sympy.core import Basic, Tuple +from sympy.tensor.array import ImmutableDenseNDimArray +from sympy.core.symbol import Symbol +from sympy.core.numbers import Integer + + +class ArrayComprehension(Basic): + """ + Generate a list comprehension. + + Explanation + =========== + + If there is a symbolic dimension, for example, say [i for i in range(1, N)] where + N is a Symbol, then the expression will not be expanded to an array. Otherwise, + calling the doit() function will launch the expansion. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a + ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.doit() + [[11, 12, 13], [21, 22, 23], [31, 32, 33], [41, 42, 43]] + >>> b = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, k)) + >>> b.doit() + ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, k)) + """ + def __new__(cls, function, *symbols, **assumptions): + if any(len(l) != 3 or None for l in symbols): + raise ValueError('ArrayComprehension requires values lower and upper bound' + ' for the expression') + arglist = [sympify(function)] + arglist.extend(cls._check_limits_validity(function, symbols)) + obj = Basic.__new__(cls, *arglist, **assumptions) + obj._limits = obj._args[1:] + obj._shape = cls._calculate_shape_from_limits(obj._limits) + obj._rank = len(obj._shape) + obj._loop_size = cls._calculate_loop_size(obj._shape) + return obj + + @property + def function(self): + """The function applied across limits. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j = symbols('i j') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.function + 10*i + j + """ + return self._args[0] + + @property + def limits(self): + """ + The list of limits that will be applied while expanding the array. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j = symbols('i j') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.limits + ((i, 1, 4), (j, 1, 3)) + """ + return self._limits + + @property + def free_symbols(self): + """ + The set of the free_symbols in the array. + Variables appeared in the bounds are supposed to be excluded + from the free symbol set. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.free_symbols + set() + >>> b = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, k+3)) + >>> b.free_symbols + {k} + """ + expr_free_sym = self.function.free_symbols + for var, inf, sup in self._limits: + expr_free_sym.discard(var) + curr_free_syms = inf.free_symbols.union(sup.free_symbols) + expr_free_sym = expr_free_sym.union(curr_free_syms) + return expr_free_sym + + @property + def variables(self): + """The tuples of the variables in the limits. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.variables + [i, j] + """ + return [l[0] for l in self._limits] + + @property + def bound_symbols(self): + """The list of dummy variables. + + Note + ==== + + Note that all variables are dummy variables since a limit without + lower bound or upper bound is not accepted. + """ + return [l[0] for l in self._limits if len(l) != 1] + + @property + def shape(self): + """ + The shape of the expanded array, which may have symbols. + + Note + ==== + + Both the lower and the upper bounds are included while + calculating the shape. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.shape + (4, 3) + >>> b = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, k+3)) + >>> b.shape + (4, k + 3) + """ + return self._shape + + @property + def is_shape_numeric(self): + """ + Test if the array is shape-numeric which means there is no symbolic + dimension. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.is_shape_numeric + True + >>> b = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, k+3)) + >>> b.is_shape_numeric + False + """ + for _, inf, sup in self._limits: + if Basic(inf, sup).atoms(Symbol): + return False + return True + + def rank(self): + """The rank of the expanded array. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.rank() + 2 + """ + return self._rank + + def __len__(self): + """ + The length of the expanded array which means the number + of elements in the array. + + Raises + ====== + + ValueError : When the length of the array is symbolic + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j = symbols('i j') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> len(a) + 12 + """ + if self._loop_size.free_symbols: + raise ValueError('Symbolic length is not supported') + return self._loop_size + + @classmethod + def _check_limits_validity(cls, function, limits): + #limits = sympify(limits) + new_limits = [] + for var, inf, sup in limits: + var = _sympify(var) + inf = _sympify(inf) + #since this is stored as an argument, it should be + #a Tuple + if isinstance(sup, list): + sup = Tuple(*sup) + else: + sup = _sympify(sup) + new_limits.append(Tuple(var, inf, sup)) + if any((not isinstance(i, Expr)) or i.atoms(Symbol, Integer) != i.atoms() + for i in [inf, sup]): + raise TypeError('Bounds should be an Expression(combination of Integer and Symbol)') + if (inf > sup) == True: + raise ValueError('Lower bound should be inferior to upper bound') + if var in inf.free_symbols or var in sup.free_symbols: + raise ValueError('Variable should not be part of its bounds') + return new_limits + + @classmethod + def _calculate_shape_from_limits(cls, limits): + return tuple([sup - inf + 1 for _, inf, sup in limits]) + + @classmethod + def _calculate_loop_size(cls, shape): + if not shape: + return 0 + loop_size = 1 + for l in shape: + loop_size = loop_size * l + + return loop_size + + def doit(self, **hints): + if not self.is_shape_numeric: + return self + + return self._expand_array() + + def _expand_array(self): + res = [] + for values in itertools.product(*[range(inf, sup+1) + for var, inf, sup + in self._limits]): + res.append(self._get_element(values)) + + return ImmutableDenseNDimArray(res, self.shape) + + def _get_element(self, values): + temp = self.function + for var, val in zip(self.variables, values): + temp = temp.subs(var, val) + return temp + + def tolist(self): + """Transform the expanded array to a list. + + Raises + ====== + + ValueError : When there is a symbolic dimension + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j = symbols('i j') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.tolist() + [[11, 12, 13], [21, 22, 23], [31, 32, 33], [41, 42, 43]] + """ + if self.is_shape_numeric: + return self._expand_array().tolist() + + raise ValueError("A symbolic array cannot be expanded to a list") + + def tomatrix(self): + """Transform the expanded array to a matrix. + + Raises + ====== + + ValueError : When there is a symbolic dimension + ValueError : When the rank of the expanded array is not equal to 2 + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehension + >>> from sympy import symbols + >>> i, j = symbols('i j') + >>> a = ArrayComprehension(10*i + j, (i, 1, 4), (j, 1, 3)) + >>> a.tomatrix() + Matrix([ + [11, 12, 13], + [21, 22, 23], + [31, 32, 33], + [41, 42, 43]]) + """ + from sympy.matrices import Matrix + + if not self.is_shape_numeric: + raise ValueError("A symbolic array cannot be expanded to a matrix") + if self._rank != 2: + raise ValueError('Dimensions must be of size of 2') + + return Matrix(self._expand_array().tomatrix()) + + +def isLambda(v): + LAMBDA = lambda: 0 + return isinstance(v, type(LAMBDA)) and v.__name__ == LAMBDA.__name__ + +class ArrayComprehensionMap(ArrayComprehension): + ''' + A subclass of ArrayComprehension dedicated to map external function lambda. + + Notes + ===== + + Only the lambda function is considered. + At most one argument in lambda function is accepted in order to avoid ambiguity + in value assignment. + + Examples + ======== + + >>> from sympy.tensor.array import ArrayComprehensionMap + >>> from sympy import symbols + >>> i, j, k = symbols('i j k') + >>> a = ArrayComprehensionMap(lambda: 1, (i, 1, 4)) + >>> a.doit() + [1, 1, 1, 1] + >>> b = ArrayComprehensionMap(lambda a: a+1, (j, 1, 4)) + >>> b.doit() + [2, 3, 4, 5] + + ''' + def __new__(cls, function, *symbols, **assumptions): + if any(len(l) != 3 or None for l in symbols): + raise ValueError('ArrayComprehension requires values lower and upper bound' + ' for the expression') + + if not isLambda(function): + raise ValueError('Data type not supported') + + arglist = cls._check_limits_validity(function, symbols) + obj = Basic.__new__(cls, *arglist, **assumptions) + obj._limits = obj._args + obj._shape = cls._calculate_shape_from_limits(obj._limits) + obj._rank = len(obj._shape) + obj._loop_size = cls._calculate_loop_size(obj._shape) + obj._lambda = function + return obj + + @property + def func(self): + class _(ArrayComprehensionMap): + def __new__(cls, *args, **kwargs): + return ArrayComprehensionMap(self._lambda, *args, **kwargs) + return _ + + def _get_element(self, values): + temp = self._lambda + if self._lambda.__code__.co_argcount == 0: + temp = temp() + elif self._lambda.__code__.co_argcount == 1: + temp = temp(functools.reduce(lambda a, b: a*b, values)) + return temp diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_derivatives.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_derivatives.py new file mode 100644 index 0000000000000000000000000000000000000000..a38db6caefe256a8c7e1f3415b78351b3787fee9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/array_derivatives.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +from sympy.core.expr import Expr +from sympy.core.function import Derivative +from sympy.core.numbers import Integer +from sympy.matrices.matrixbase import MatrixBase +from .ndim_array import NDimArray +from .arrayop import derive_by_array +from sympy.matrices.expressions.matexpr import MatrixExpr +from sympy.matrices.expressions.special import ZeroMatrix +from sympy.matrices.expressions.matexpr import _matrix_derivative + + +class ArrayDerivative(Derivative): + + is_scalar = False + + def __new__(cls, expr, *variables, **kwargs): + obj = super().__new__(cls, expr, *variables, **kwargs) + if isinstance(obj, ArrayDerivative): + obj._shape = obj._get_shape() + return obj + + def _get_shape(self): + shape = () + for v, count in self.variable_count: + if hasattr(v, "shape"): + for i in range(count): + shape += v.shape + if hasattr(self.expr, "shape"): + shape += self.expr.shape + return shape + + @property + def shape(self): + return self._shape + + @classmethod + def _get_zero_with_shape_like(cls, expr): + if isinstance(expr, (MatrixBase, NDimArray)): + return expr.zeros(*expr.shape) + elif isinstance(expr, MatrixExpr): + return ZeroMatrix(*expr.shape) + else: + raise RuntimeError("Unable to determine shape of array-derivative.") + + @staticmethod + def _call_derive_scalar_by_matrix(expr: Expr, v: MatrixBase) -> Expr: + return v.applyfunc(lambda x: expr.diff(x)) + + @staticmethod + def _call_derive_scalar_by_matexpr(expr: Expr, v: MatrixExpr) -> Expr: + if expr.has(v): + return _matrix_derivative(expr, v) + else: + return ZeroMatrix(*v.shape) + + @staticmethod + def _call_derive_scalar_by_array(expr: Expr, v: NDimArray) -> Expr: + return v.applyfunc(lambda x: expr.diff(x)) + + @staticmethod + def _call_derive_matrix_by_scalar(expr: MatrixBase, v: Expr) -> Expr: + return _matrix_derivative(expr, v) + + @staticmethod + def _call_derive_matexpr_by_scalar(expr: MatrixExpr, v: Expr) -> Expr: + return expr._eval_derivative(v) + + @staticmethod + def _call_derive_array_by_scalar(expr: NDimArray, v: Expr) -> Expr: + return expr.applyfunc(lambda x: x.diff(v)) + + @staticmethod + def _call_derive_default(expr: Expr, v: Expr) -> Expr | None: + if expr.has(v): + return _matrix_derivative(expr, v) + else: + return None + + @classmethod + def _dispatch_eval_derivative_n_times(cls, expr, v, count): + # Evaluate the derivative `n` times. If + # `_eval_derivative_n_times` is not overridden by the current + # object, the default in `Basic` will call a loop over + # `_eval_derivative`: + + if not isinstance(count, (int, Integer)) or ((count <= 0) == True): + return None + + # TODO: this could be done with multiple-dispatching: + if expr.is_scalar: + if isinstance(v, MatrixBase): + result = cls._call_derive_scalar_by_matrix(expr, v) + elif isinstance(v, MatrixExpr): + result = cls._call_derive_scalar_by_matexpr(expr, v) + elif isinstance(v, NDimArray): + result = cls._call_derive_scalar_by_array(expr, v) + elif v.is_scalar: + # scalar by scalar has a special + return super()._dispatch_eval_derivative_n_times(expr, v, count) + else: + return None + elif v.is_scalar: + if isinstance(expr, MatrixBase): + result = cls._call_derive_matrix_by_scalar(expr, v) + elif isinstance(expr, MatrixExpr): + result = cls._call_derive_matexpr_by_scalar(expr, v) + elif isinstance(expr, NDimArray): + result = cls._call_derive_array_by_scalar(expr, v) + else: + return None + else: + # Both `expr` and `v` are some array/matrix type: + if isinstance(expr, MatrixBase) or isinstance(v, MatrixBase): + result = derive_by_array(expr, v) + elif isinstance(expr, MatrixExpr) and isinstance(v, MatrixExpr): + result = cls._call_derive_default(expr, v) + elif isinstance(expr, MatrixExpr) or isinstance(v, MatrixExpr): + # if one expression is a symbolic matrix expression while the other isn't, don't evaluate: + return None + else: + result = derive_by_array(expr, v) + if result is None: + return None + if count == 1: + return result + else: + return cls._dispatch_eval_derivative_n_times(result, v, count - 1) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/arrayop.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/arrayop.py new file mode 100644 index 0000000000000000000000000000000000000000..a81e6b381a8a93f0cd585278a4be0259b06406dd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/arrayop.py @@ -0,0 +1,528 @@ +import itertools +from collections.abc import Iterable + +from sympy.core._print_helpers import Printable +from sympy.core.containers import Tuple +from sympy.core.function import diff +from sympy.core.singleton import S +from sympy.core.sympify import _sympify + +from sympy.tensor.array.ndim_array import NDimArray +from sympy.tensor.array.dense_ndim_array import DenseNDimArray, ImmutableDenseNDimArray +from sympy.tensor.array.sparse_ndim_array import SparseNDimArray + + +def _arrayfy(a): + from sympy.matrices import MatrixBase + + if isinstance(a, NDimArray): + return a + if isinstance(a, (MatrixBase, list, tuple, Tuple)): + return ImmutableDenseNDimArray(a) + return a + + +def tensorproduct(*args): + """ + Tensor product among scalars or array-like objects. + + The equivalent operator for array expressions is ``ArrayTensorProduct``, + which can be used to keep the expression unevaluated. + + Examples + ======== + + >>> from sympy.tensor.array import tensorproduct, Array + >>> from sympy.abc import x, y, z, t + >>> A = Array([[1, 2], [3, 4]]) + >>> B = Array([x, y]) + >>> tensorproduct(A, B) + [[[x, y], [2*x, 2*y]], [[3*x, 3*y], [4*x, 4*y]]] + >>> tensorproduct(A, x) + [[x, 2*x], [3*x, 4*x]] + >>> tensorproduct(A, B, B) + [[[[x**2, x*y], [x*y, y**2]], [[2*x**2, 2*x*y], [2*x*y, 2*y**2]]], [[[3*x**2, 3*x*y], [3*x*y, 3*y**2]], [[4*x**2, 4*x*y], [4*x*y, 4*y**2]]]] + + Applying this function on two matrices will result in a rank 4 array. + + >>> from sympy import Matrix, eye + >>> m = Matrix([[x, y], [z, t]]) + >>> p = tensorproduct(eye(3), m) + >>> p + [[[[x, y], [z, t]], [[0, 0], [0, 0]], [[0, 0], [0, 0]]], [[[0, 0], [0, 0]], [[x, y], [z, t]], [[0, 0], [0, 0]]], [[[0, 0], [0, 0]], [[0, 0], [0, 0]], [[x, y], [z, t]]]] + + See Also + ======== + + sympy.tensor.array.expressions.array_expressions.ArrayTensorProduct + + """ + from sympy.tensor.array import SparseNDimArray, ImmutableSparseNDimArray + + if len(args) == 0: + return S.One + if len(args) == 1: + return _arrayfy(args[0]) + from sympy.tensor.array.expressions.array_expressions import _CodegenArrayAbstract + from sympy.tensor.array.expressions.array_expressions import ArrayTensorProduct + from sympy.tensor.array.expressions.array_expressions import _ArrayExpr + from sympy.matrices.expressions.matexpr import MatrixSymbol + if any(isinstance(arg, (_ArrayExpr, _CodegenArrayAbstract, MatrixSymbol)) for arg in args): + return ArrayTensorProduct(*args) + if len(args) > 2: + return tensorproduct(tensorproduct(args[0], args[1]), *args[2:]) + + # length of args is 2: + a, b = map(_arrayfy, args) + + if not isinstance(a, NDimArray) or not isinstance(b, NDimArray): + return a*b + + if isinstance(a, SparseNDimArray) and isinstance(b, SparseNDimArray): + lp = len(b) + new_array = {k1*lp + k2: v1*v2 for k1, v1 in a._sparse_array.items() for k2, v2 in b._sparse_array.items()} + return ImmutableSparseNDimArray(new_array, a.shape + b.shape) + + product_list = [i*j for i in Flatten(a) for j in Flatten(b)] + return ImmutableDenseNDimArray(product_list, a.shape + b.shape) + + +def _util_contraction_diagonal(array, *contraction_or_diagonal_axes): + array = _arrayfy(array) + + # Verify contraction_axes: + taken_dims = set() + for axes_group in contraction_or_diagonal_axes: + if not isinstance(axes_group, Iterable): + raise ValueError("collections of contraction/diagonal axes expected") + + dim = array.shape[axes_group[0]] + + for d in axes_group: + if d in taken_dims: + raise ValueError("dimension specified more than once") + if dim != array.shape[d]: + raise ValueError("cannot contract or diagonalize between axes of different dimension") + taken_dims.add(d) + + rank = array.rank() + + remaining_shape = [dim for i, dim in enumerate(array.shape) if i not in taken_dims] + cum_shape = [0]*rank + _cumul = 1 + for i in range(rank): + cum_shape[rank - i - 1] = _cumul + _cumul *= int(array.shape[rank - i - 1]) + + # DEFINITION: by absolute position it is meant the position along the one + # dimensional array containing all the tensor components. + + # Possible future work on this module: move computation of absolute + # positions to a class method. + + # Determine absolute positions of the uncontracted indices: + remaining_indices = [[cum_shape[i]*j for j in range(array.shape[i])] + for i in range(rank) if i not in taken_dims] + + # Determine absolute positions of the contracted indices: + summed_deltas = [] + for axes_group in contraction_or_diagonal_axes: + lidx = [] + for js in range(array.shape[axes_group[0]]): + lidx.append(sum(cum_shape[ig] * js for ig in axes_group)) + summed_deltas.append(lidx) + + return array, remaining_indices, remaining_shape, summed_deltas + + +def tensorcontraction(array, *contraction_axes): + """ + Contraction of an array-like object on the specified axes. + + The equivalent operator for array expressions is ``ArrayContraction``, + which can be used to keep the expression unevaluated. + + Examples + ======== + + >>> from sympy import Array, tensorcontraction + >>> from sympy import Matrix, eye + >>> tensorcontraction(eye(3), (0, 1)) + 3 + >>> A = Array(range(18), (3, 2, 3)) + >>> A + [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]], [[12, 13, 14], [15, 16, 17]]] + >>> tensorcontraction(A, (0, 2)) + [21, 30] + + Matrix multiplication may be emulated with a proper combination of + ``tensorcontraction`` and ``tensorproduct`` + + >>> from sympy import tensorproduct + >>> from sympy.abc import a,b,c,d,e,f,g,h + >>> m1 = Matrix([[a, b], [c, d]]) + >>> m2 = Matrix([[e, f], [g, h]]) + >>> p = tensorproduct(m1, m2) + >>> p + [[[[a*e, a*f], [a*g, a*h]], [[b*e, b*f], [b*g, b*h]]], [[[c*e, c*f], [c*g, c*h]], [[d*e, d*f], [d*g, d*h]]]] + >>> tensorcontraction(p, (1, 2)) + [[a*e + b*g, a*f + b*h], [c*e + d*g, c*f + d*h]] + >>> m1*m2 + Matrix([ + [a*e + b*g, a*f + b*h], + [c*e + d*g, c*f + d*h]]) + + See Also + ======== + + sympy.tensor.array.expressions.array_expressions.ArrayContraction + + """ + from sympy.tensor.array.expressions.array_expressions import _array_contraction + from sympy.tensor.array.expressions.array_expressions import _CodegenArrayAbstract + from sympy.tensor.array.expressions.array_expressions import _ArrayExpr + from sympy.matrices.expressions.matexpr import MatrixSymbol + if isinstance(array, (_ArrayExpr, _CodegenArrayAbstract, MatrixSymbol)): + return _array_contraction(array, *contraction_axes) + + array, remaining_indices, remaining_shape, summed_deltas = _util_contraction_diagonal(array, *contraction_axes) + + # Compute the contracted array: + # + # 1. external for loops on all uncontracted indices. + # Uncontracted indices are determined by the combinatorial product of + # the absolute positions of the remaining indices. + # 2. internal loop on all contracted indices. + # It sums the values of the absolute contracted index and the absolute + # uncontracted index for the external loop. + contracted_array = [] + for icontrib in itertools.product(*remaining_indices): + index_base_position = sum(icontrib) + isum = S.Zero + for sum_to_index in itertools.product(*summed_deltas): + idx = array._get_tuple_index(index_base_position + sum(sum_to_index)) + isum += array[idx] + + contracted_array.append(isum) + + if len(remaining_indices) == 0: + assert len(contracted_array) == 1 + return contracted_array[0] + + return type(array)(contracted_array, remaining_shape) + + +def tensordiagonal(array, *diagonal_axes): + """ + Diagonalization of an array-like object on the specified axes. + + This is equivalent to multiplying the expression by Kronecker deltas + uniting the axes. + + The diagonal indices are put at the end of the axes. + + The equivalent operator for array expressions is ``ArrayDiagonal``, which + can be used to keep the expression unevaluated. + + Examples + ======== + + ``tensordiagonal`` acting on a 2-dimensional array by axes 0 and 1 is + equivalent to the diagonal of the matrix: + + >>> from sympy import Array, tensordiagonal + >>> from sympy import Matrix, eye + >>> tensordiagonal(eye(3), (0, 1)) + [1, 1, 1] + + >>> from sympy.abc import a,b,c,d + >>> m1 = Matrix([[a, b], [c, d]]) + >>> tensordiagonal(m1, [0, 1]) + [a, d] + + In case of higher dimensional arrays, the diagonalized out dimensions + are appended removed and appended as a single dimension at the end: + + >>> A = Array(range(18), (3, 2, 3)) + >>> A + [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]], [[12, 13, 14], [15, 16, 17]]] + >>> tensordiagonal(A, (0, 2)) + [[0, 7, 14], [3, 10, 17]] + >>> from sympy import permutedims + >>> tensordiagonal(A, (0, 2)) == permutedims(Array([A[0, :, 0], A[1, :, 1], A[2, :, 2]]), [1, 0]) + True + + See Also + ======== + + sympy.tensor.array.expressions.array_expressions.ArrayDiagonal + + """ + if any(len(i) <= 1 for i in diagonal_axes): + raise ValueError("need at least two axes to diagonalize") + + from sympy.tensor.array.expressions.array_expressions import _ArrayExpr + from sympy.tensor.array.expressions.array_expressions import _CodegenArrayAbstract + from sympy.tensor.array.expressions.array_expressions import ArrayDiagonal, _array_diagonal + from sympy.matrices.expressions.matexpr import MatrixSymbol + if isinstance(array, (_ArrayExpr, _CodegenArrayAbstract, MatrixSymbol)): + return _array_diagonal(array, *diagonal_axes) + + ArrayDiagonal._validate(array, *diagonal_axes) + + array, remaining_indices, remaining_shape, diagonal_deltas = _util_contraction_diagonal(array, *diagonal_axes) + + # Compute the diagonalized array: + # + # 1. external for loops on all undiagonalized indices. + # Undiagonalized indices are determined by the combinatorial product of + # the absolute positions of the remaining indices. + # 2. internal loop on all diagonal indices. + # It appends the values of the absolute diagonalized index and the absolute + # undiagonalized index for the external loop. + diagonalized_array = [] + diagonal_shape = [len(i) for i in diagonal_deltas] + for icontrib in itertools.product(*remaining_indices): + index_base_position = sum(icontrib) + isum = [] + for sum_to_index in itertools.product(*diagonal_deltas): + idx = array._get_tuple_index(index_base_position + sum(sum_to_index)) + isum.append(array[idx]) + + isum = type(array)(isum).reshape(*diagonal_shape) + diagonalized_array.append(isum) + + return type(array)(diagonalized_array, remaining_shape + diagonal_shape) + + +def derive_by_array(expr, dx): + r""" + Derivative by arrays. Supports both arrays and scalars. + + The equivalent operator for array expressions is ``array_derive``. + + Explanation + =========== + + Given the array `A_{i_1, \ldots, i_N}` and the array `X_{j_1, \ldots, j_M}` + this function will return a new array `B` defined by + + `B_{j_1,\ldots,j_M,i_1,\ldots,i_N} := \frac{\partial A_{i_1,\ldots,i_N}}{\partial X_{j_1,\ldots,j_M}}` + + Examples + ======== + + >>> from sympy import derive_by_array + >>> from sympy.abc import x, y, z, t + >>> from sympy import cos + >>> derive_by_array(cos(x*t), x) + -t*sin(t*x) + >>> derive_by_array(cos(x*t), [x, y, z, t]) + [-t*sin(t*x), 0, 0, -x*sin(t*x)] + >>> derive_by_array([x, y**2*z], [[x, y], [z, t]]) + [[[1, 0], [0, 2*y*z]], [[0, y**2], [0, 0]]] + + """ + from sympy.matrices import MatrixBase + from sympy.tensor.array import SparseNDimArray + array_types = (Iterable, MatrixBase, NDimArray) + + if isinstance(dx, array_types): + dx = ImmutableDenseNDimArray(dx) + for i in dx: + if not i._diff_wrt: + raise ValueError("cannot derive by this array") + + if isinstance(expr, array_types): + if isinstance(expr, NDimArray): + expr = expr.as_immutable() + else: + expr = ImmutableDenseNDimArray(expr) + + if isinstance(dx, array_types): + if isinstance(expr, SparseNDimArray): + lp = len(expr) + new_array = {k + i*lp: v + for i, x in enumerate(Flatten(dx)) + for k, v in expr.diff(x)._sparse_array.items()} + else: + new_array = [[y.diff(x) for y in Flatten(expr)] for x in Flatten(dx)] + return type(expr)(new_array, dx.shape + expr.shape) + else: + return expr.diff(dx) + else: + expr = _sympify(expr) + if isinstance(dx, array_types): + return ImmutableDenseNDimArray([expr.diff(i) for i in Flatten(dx)], dx.shape) + else: + dx = _sympify(dx) + return diff(expr, dx) + + +def permutedims(expr, perm=None, index_order_old=None, index_order_new=None): + """ + Permutes the indices of an array. + + Parameter specifies the permutation of the indices. + + The equivalent operator for array expressions is ``PermuteDims``, which can + be used to keep the expression unevaluated. + + Examples + ======== + + >>> from sympy.abc import x, y, z, t + >>> from sympy import sin + >>> from sympy import Array, permutedims + >>> a = Array([[x, y, z], [t, sin(x), 0]]) + >>> a + [[x, y, z], [t, sin(x), 0]] + >>> permutedims(a, (1, 0)) + [[x, t], [y, sin(x)], [z, 0]] + + If the array is of second order, ``transpose`` can be used: + + >>> from sympy import transpose + >>> transpose(a) + [[x, t], [y, sin(x)], [z, 0]] + + Examples on higher dimensions: + + >>> b = Array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + >>> permutedims(b, (2, 1, 0)) + [[[1, 5], [3, 7]], [[2, 6], [4, 8]]] + >>> permutedims(b, (1, 2, 0)) + [[[1, 5], [2, 6]], [[3, 7], [4, 8]]] + + An alternative way to specify the same permutations as in the previous + lines involves passing the *old* and *new* indices, either as a list or as + a string: + + >>> permutedims(b, index_order_old="cba", index_order_new="abc") + [[[1, 5], [3, 7]], [[2, 6], [4, 8]]] + >>> permutedims(b, index_order_old="cab", index_order_new="abc") + [[[1, 5], [2, 6]], [[3, 7], [4, 8]]] + + ``Permutation`` objects are also allowed: + + >>> from sympy.combinatorics import Permutation + >>> permutedims(b, Permutation([1, 2, 0])) + [[[1, 5], [2, 6]], [[3, 7], [4, 8]]] + + See Also + ======== + + sympy.tensor.array.expressions.array_expressions.PermuteDims + + """ + from sympy.tensor.array import SparseNDimArray + + from sympy.tensor.array.expressions.array_expressions import _ArrayExpr + from sympy.tensor.array.expressions.array_expressions import _CodegenArrayAbstract + from sympy.tensor.array.expressions.array_expressions import _permute_dims + from sympy.matrices.expressions.matexpr import MatrixSymbol + from sympy.tensor.array.expressions import PermuteDims + from sympy.tensor.array.expressions.array_expressions import get_rank + perm = PermuteDims._get_permutation_from_arguments(perm, index_order_old, index_order_new, get_rank(expr)) + if isinstance(expr, (_ArrayExpr, _CodegenArrayAbstract, MatrixSymbol)): + return _permute_dims(expr, perm) + + if not isinstance(expr, NDimArray): + expr = ImmutableDenseNDimArray(expr) + + from sympy.combinatorics import Permutation + if not isinstance(perm, Permutation): + perm = Permutation(list(perm)) + + if perm.size != expr.rank(): + raise ValueError("wrong permutation size") + + # Get the inverse permutation: + iperm = ~perm + new_shape = perm(expr.shape) + + if isinstance(expr, SparseNDimArray): + return type(expr)({tuple(perm(expr._get_tuple_index(k))): v + for k, v in expr._sparse_array.items()}, new_shape) + + indices_span = perm([range(i) for i in expr.shape]) + + new_array = [None]*len(expr) + for i, idx in enumerate(itertools.product(*indices_span)): + t = iperm(idx) + new_array[i] = expr[t] + + return type(expr)(new_array, new_shape) + + +class Flatten(Printable): + """ + Flatten an iterable object to a list in a lazy-evaluation way. + + Notes + ===== + + This class is an iterator with which the memory cost can be economised. + Optimisation has been considered to ameliorate the performance for some + specific data types like DenseNDimArray and SparseNDimArray. + + Examples + ======== + + >>> from sympy.tensor.array.arrayop import Flatten + >>> from sympy.tensor.array import Array + >>> A = Array(range(6)).reshape(2, 3) + >>> Flatten(A) + Flatten([[0, 1, 2], [3, 4, 5]]) + >>> [i for i in Flatten(A)] + [0, 1, 2, 3, 4, 5] + """ + def __init__(self, iterable): + from sympy.matrices.matrixbase import MatrixBase + from sympy.tensor.array import NDimArray + + if not isinstance(iterable, (Iterable, MatrixBase)): + raise NotImplementedError("Data type not yet supported") + + if isinstance(iterable, list): + iterable = NDimArray(iterable) + + self._iter = iterable + self._idx = 0 + + def __iter__(self): + return self + + def __next__(self): + from sympy.matrices.matrixbase import MatrixBase + + if len(self._iter) > self._idx: + if isinstance(self._iter, DenseNDimArray): + result = self._iter._array[self._idx] + + elif isinstance(self._iter, SparseNDimArray): + if self._idx in self._iter._sparse_array: + result = self._iter._sparse_array[self._idx] + else: + result = 0 + + elif isinstance(self._iter, MatrixBase): + result = self._iter[self._idx] + + elif hasattr(self._iter, '__next__'): + result = next(self._iter) + + else: + result = self._iter[self._idx] + + else: + raise StopIteration + + self._idx += 1 + return result + + def next(self): + return self.__next__() + + def _sympystr(self, printer): + return type(self).__name__ + '(' + printer._print(self._iter) + ')' diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/dense_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/dense_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..576e452c55d8d374ca1f72c553f3a64de7227d43 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/dense_ndim_array.py @@ -0,0 +1,206 @@ +import functools +from typing import List + +from sympy.core.basic import Basic +from sympy.core.containers import Tuple +from sympy.core.singleton import S +from sympy.core.sympify import _sympify +from sympy.tensor.array.mutable_ndim_array import MutableNDimArray +from sympy.tensor.array.ndim_array import NDimArray, ImmutableNDimArray, ArrayKind +from sympy.utilities.iterables import flatten + + +class DenseNDimArray(NDimArray): + + _array: List[Basic] + + def __new__(self, *args, **kwargs): + return ImmutableDenseNDimArray(*args, **kwargs) + + @property + def kind(self) -> ArrayKind: + return ArrayKind._union(self._array) + + def __getitem__(self, index): + """ + Allows to get items from N-dim array. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray([0, 1, 2, 3], (2, 2)) + >>> a + [[0, 1], [2, 3]] + >>> a[0, 0] + 0 + >>> a[1, 1] + 3 + >>> a[0] + [0, 1] + >>> a[1] + [2, 3] + + + Symbolic index: + + >>> from sympy.abc import i, j + >>> a[i, j] + [[0, 1], [2, 3]][i, j] + + Replace `i` and `j` to get element `(1, 1)`: + + >>> a[i, j].subs({i: 1, j: 1}) + 3 + + """ + syindex = self._check_symbolic_index(index) + if syindex is not None: + return syindex + + index = self._check_index_for_getitem(index) + + if isinstance(index, tuple) and any(isinstance(i, slice) for i in index): + sl_factors, eindices = self._get_slice_data_for_array_access(index) + array = [self._array[self._parse_index(i)] for i in eindices] + nshape = [len(el) for i, el in enumerate(sl_factors) if isinstance(index[i], slice)] + return type(self)(array, nshape) + else: + index = self._parse_index(index) + return self._array[index] + + @classmethod + def zeros(cls, *shape): + list_length = functools.reduce(lambda x, y: x*y, shape, S.One) + return cls._new(([0]*list_length,), shape) + + def tomatrix(self): + """ + Converts MutableDenseNDimArray to Matrix. Can convert only 2-dim array, else will raise error. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray([1 for i in range(9)], (3, 3)) + >>> b = a.tomatrix() + >>> b + Matrix([ + [1, 1, 1], + [1, 1, 1], + [1, 1, 1]]) + + """ + from sympy.matrices import Matrix + + if self.rank() != 2: + raise ValueError('Dimensions must be of size of 2') + + return Matrix(self.shape[0], self.shape[1], self._array) + + def reshape(self, *newshape): + """ + Returns MutableDenseNDimArray instance with new shape. Elements number + must be suitable to new shape. The only argument of method sets + new shape. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray([1, 2, 3, 4, 5, 6], (2, 3)) + >>> a.shape + (2, 3) + >>> a + [[1, 2, 3], [4, 5, 6]] + >>> b = a.reshape(3, 2) + >>> b.shape + (3, 2) + >>> b + [[1, 2], [3, 4], [5, 6]] + + """ + new_total_size = functools.reduce(lambda x,y: x*y, newshape) + if new_total_size != self._loop_size: + raise ValueError('Expecting reshape size to %d but got prod(%s) = %d' % ( + self._loop_size, str(newshape), new_total_size)) + + # there is no `.func` as this class does not subtype `Basic`: + return type(self)(self._array, newshape) + + +class ImmutableDenseNDimArray(DenseNDimArray, ImmutableNDimArray): # type: ignore + def __new__(cls, iterable, shape=None, **kwargs): + return cls._new(iterable, shape, **kwargs) + + @classmethod + def _new(cls, iterable, shape, **kwargs): + shape, flat_list = cls._handle_ndarray_creation_inputs(iterable, shape, **kwargs) + shape = Tuple(*map(_sympify, shape)) + cls._check_special_bounds(flat_list, shape) + flat_list = flatten(flat_list) + flat_list = Tuple(*flat_list) + self = Basic.__new__(cls, flat_list, shape, **kwargs) + self._shape = shape + self._array = list(flat_list) + self._rank = len(shape) + self._loop_size = functools.reduce(lambda x,y: x*y, shape, 1) + return self + + def __setitem__(self, index, value): + raise TypeError('immutable N-dim array') + + def as_mutable(self): + return MutableDenseNDimArray(self) + + def _eval_simplify(self, **kwargs): + from sympy.simplify.simplify import simplify + return self.applyfunc(simplify) + +class MutableDenseNDimArray(DenseNDimArray, MutableNDimArray): + + def __new__(cls, iterable=None, shape=None, **kwargs): + return cls._new(iterable, shape, **kwargs) + + @classmethod + def _new(cls, iterable, shape, **kwargs): + shape, flat_list = cls._handle_ndarray_creation_inputs(iterable, shape, **kwargs) + flat_list = flatten(flat_list) + self = object.__new__(cls) + self._shape = shape + self._array = list(flat_list) + self._rank = len(shape) + self._loop_size = functools.reduce(lambda x,y: x*y, shape) if shape else len(flat_list) + return self + + def __setitem__(self, index, value): + """Allows to set items to MutableDenseNDimArray. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(2, 2) + >>> a[0,0] = 1 + >>> a[1,1] = 1 + >>> a + [[1, 0], [0, 1]] + + """ + if isinstance(index, tuple) and any(isinstance(i, slice) for i in index): + value, eindices, slice_offsets = self._get_slice_data_for_array_assignment(index, value) + for i in eindices: + other_i = [ind - j for ind, j in zip(i, slice_offsets) if j is not None] + self._array[self._parse_index(i)] = value[other_i] + else: + index = self._parse_index(index) + self._setter_iterable_check(value) + value = _sympify(value) + self._array[index] = value + + def as_immutable(self): + return ImmutableDenseNDimArray(self) + + @property + def free_symbols(self): + return {i for j in self._array for i in j.free_symbols} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f1658241782cdf0e38a30c43a6d67f9811297f4c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__init__.py @@ -0,0 +1,178 @@ +r""" +Array expressions are expressions representing N-dimensional arrays, without +evaluating them. These expressions represent in a certain way abstract syntax +trees of operations on N-dimensional arrays. + +Every N-dimensional array operator has a corresponding array expression object. + +Table of correspondences: + +=============================== ============================= + Array operator Array expression operator +=============================== ============================= + tensorproduct ArrayTensorProduct + tensorcontraction ArrayContraction + tensordiagonal ArrayDiagonal + permutedims PermuteDims +=============================== ============================= + +Examples +======== + +``ArraySymbol`` objects are the N-dimensional equivalent of ``MatrixSymbol`` +objects in the matrix module: + +>>> from sympy.tensor.array.expressions import ArraySymbol +>>> from sympy.abc import i, j, k +>>> A = ArraySymbol("A", (3, 2, 4)) +>>> A.shape +(3, 2, 4) +>>> A[i, j, k] +A[i, j, k] +>>> A.as_explicit() +[[[A[0, 0, 0], A[0, 0, 1], A[0, 0, 2], A[0, 0, 3]], + [A[0, 1, 0], A[0, 1, 1], A[0, 1, 2], A[0, 1, 3]]], + [[A[1, 0, 0], A[1, 0, 1], A[1, 0, 2], A[1, 0, 3]], + [A[1, 1, 0], A[1, 1, 1], A[1, 1, 2], A[1, 1, 3]]], + [[A[2, 0, 0], A[2, 0, 1], A[2, 0, 2], A[2, 0, 3]], + [A[2, 1, 0], A[2, 1, 1], A[2, 1, 2], A[2, 1, 3]]]] + +Component-explicit arrays can be added inside array expressions: + +>>> from sympy import Array +>>> from sympy import tensorproduct +>>> from sympy.tensor.array.expressions import ArrayTensorProduct +>>> a = Array([1, 2, 3]) +>>> b = Array([i, j, k]) +>>> expr = ArrayTensorProduct(a, b, b) +>>> expr +ArrayTensorProduct([1, 2, 3], [i, j, k], [i, j, k]) +>>> expr.as_explicit() == tensorproduct(a, b, b) +True + +Constructing array expressions from index-explicit forms +-------------------------------------------------------- + +Array expressions are index-implicit. This means they do not use any indices to +represent array operations. The function ``convert_indexed_to_array( ... )`` +may be used to convert index-explicit expressions to array expressions. +It takes as input two parameters: the index-explicit expression and the order +of the indices: + +>>> from sympy.tensor.array.expressions import convert_indexed_to_array +>>> from sympy import Sum +>>> A = ArraySymbol("A", (3, 3)) +>>> B = ArraySymbol("B", (3, 3)) +>>> convert_indexed_to_array(A[i, j], [i, j]) +A +>>> convert_indexed_to_array(A[i, j], [j, i]) +PermuteDims(A, (0 1)) +>>> convert_indexed_to_array(A[i, j] + B[j, i], [i, j]) +ArrayAdd(A, PermuteDims(B, (0 1))) +>>> convert_indexed_to_array(Sum(A[i, j]*B[j, k], (j, 0, 2)), [i, k]) +ArrayContraction(ArrayTensorProduct(A, B), (1, 2)) + +The diagonal of a matrix in the array expression form: + +>>> convert_indexed_to_array(A[i, i], [i]) +ArrayDiagonal(A, (0, 1)) + +The trace of a matrix in the array expression form: + +>>> convert_indexed_to_array(Sum(A[i, i], (i, 0, 2)), [i]) +ArrayContraction(A, (0, 1)) + +Compatibility with matrices +--------------------------- + +Array expressions can be mixed with objects from the matrix module: + +>>> from sympy import MatrixSymbol +>>> from sympy.tensor.array.expressions import ArrayContraction +>>> M = MatrixSymbol("M", 3, 3) +>>> N = MatrixSymbol("N", 3, 3) + +Express the matrix product in the array expression form: + +>>> from sympy.tensor.array.expressions import convert_matrix_to_array +>>> expr = convert_matrix_to_array(M*N) +>>> expr +ArrayContraction(ArrayTensorProduct(M, N), (1, 2)) + +The expression can be converted back to matrix form: + +>>> from sympy.tensor.array.expressions import convert_array_to_matrix +>>> convert_array_to_matrix(expr) +M*N + +Add a second contraction on the remaining axes in order to get the trace of `M \cdot N`: + +>>> expr_tr = ArrayContraction(expr, (0, 1)) +>>> expr_tr +ArrayContraction(ArrayContraction(ArrayTensorProduct(M, N), (1, 2)), (0, 1)) + +Flatten the expression by calling ``.doit()`` and remove the nested array contraction operations: + +>>> expr_tr.doit() +ArrayContraction(ArrayTensorProduct(M, N), (0, 3), (1, 2)) + +Get the explicit form of the array expression: + +>>> expr.as_explicit() +[[M[0, 0]*N[0, 0] + M[0, 1]*N[1, 0] + M[0, 2]*N[2, 0], M[0, 0]*N[0, 1] + M[0, 1]*N[1, 1] + M[0, 2]*N[2, 1], M[0, 0]*N[0, 2] + M[0, 1]*N[1, 2] + M[0, 2]*N[2, 2]], + [M[1, 0]*N[0, 0] + M[1, 1]*N[1, 0] + M[1, 2]*N[2, 0], M[1, 0]*N[0, 1] + M[1, 1]*N[1, 1] + M[1, 2]*N[2, 1], M[1, 0]*N[0, 2] + M[1, 1]*N[1, 2] + M[1, 2]*N[2, 2]], + [M[2, 0]*N[0, 0] + M[2, 1]*N[1, 0] + M[2, 2]*N[2, 0], M[2, 0]*N[0, 1] + M[2, 1]*N[1, 1] + M[2, 2]*N[2, 1], M[2, 0]*N[0, 2] + M[2, 1]*N[1, 2] + M[2, 2]*N[2, 2]]] + +Express the trace of a matrix: + +>>> from sympy import Trace +>>> convert_matrix_to_array(Trace(M)) +ArrayContraction(M, (0, 1)) +>>> convert_matrix_to_array(Trace(M*N)) +ArrayContraction(ArrayTensorProduct(M, N), (0, 3), (1, 2)) + +Express the transposition of a matrix (will be expressed as a permutation of the axes: + +>>> convert_matrix_to_array(M.T) +PermuteDims(M, (0 1)) + +Compute the derivative array expressions: + +>>> from sympy.tensor.array.expressions import array_derive +>>> d = array_derive(M, M) +>>> d +PermuteDims(ArrayTensorProduct(I, I), (3)(1 2)) + +Verify that the derivative corresponds to the form computed with explicit matrices: + +>>> d.as_explicit() +[[[[1, 0, 0], [0, 0, 0], [0, 0, 0]], [[0, 1, 0], [0, 0, 0], [0, 0, 0]], [[0, 0, 1], [0, 0, 0], [0, 0, 0]]], [[[0, 0, 0], [1, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 1, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 1], [0, 0, 0]]], [[[0, 0, 0], [0, 0, 0], [1, 0, 0]], [[0, 0, 0], [0, 0, 0], [0, 1, 0]], [[0, 0, 0], [0, 0, 0], [0, 0, 1]]]] +>>> Me = M.as_explicit() +>>> Me.diff(Me) +[[[[1, 0, 0], [0, 0, 0], [0, 0, 0]], [[0, 1, 0], [0, 0, 0], [0, 0, 0]], [[0, 0, 1], [0, 0, 0], [0, 0, 0]]], [[[0, 0, 0], [1, 0, 0], [0, 0, 0]], [[0, 0, 0], [0, 1, 0], [0, 0, 0]], [[0, 0, 0], [0, 0, 1], [0, 0, 0]]], [[[0, 0, 0], [0, 0, 0], [1, 0, 0]], [[0, 0, 0], [0, 0, 0], [0, 1, 0]], [[0, 0, 0], [0, 0, 0], [0, 0, 1]]]] + +""" + +__all__ = [ + "ArraySymbol", "ArrayElement", "ZeroArray", "OneArray", + "ArrayTensorProduct", + "ArrayContraction", + "ArrayDiagonal", + "PermuteDims", + "ArrayAdd", + "ArrayElementwiseApplyFunc", + "Reshape", + "convert_array_to_matrix", + "convert_matrix_to_array", + "convert_array_to_indexed", + "convert_indexed_to_array", + "array_derive", +] + +from sympy.tensor.array.expressions.array_expressions import ArrayTensorProduct, ArrayAdd, PermuteDims, ArrayDiagonal, \ + ArrayContraction, Reshape, ArraySymbol, ArrayElement, ZeroArray, OneArray, ArrayElementwiseApplyFunc +from sympy.tensor.array.expressions.arrayexpr_derivatives import array_derive +from sympy.tensor.array.expressions.from_array_to_indexed import convert_array_to_indexed +from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix +from sympy.tensor.array.expressions.from_indexed_to_array import convert_indexed_to_array +from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..207b38a57d1ec687307a68b953117964fdd3b4d9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/arrayexpr_derivatives.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/arrayexpr_derivatives.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b5ae8db46867f631c0ad35bcdfbd154c7af1e79 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/arrayexpr_derivatives.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_indexed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_indexed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ae0ec21ead2227d76ca158a048b17af25a992b6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_indexed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_matrix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_matrix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3740d36ba97f29dc1e9d64b66142bb620a9fa536 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_array_to_matrix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_indexed_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_indexed_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd0487fddac6d91e9cd9148129d32223756b6642 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_indexed_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_matrix_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_matrix_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12cdc9389563c02c54f86a3362b87ac5a59fc4bc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/conv_matrix_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_indexed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_indexed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfa4079c9c8177eab6da80184bed42ef20299101 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_indexed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_matrix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_matrix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1378c246926a8e80746e27edc82e17ef7975650b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_array_to_matrix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_indexed_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_indexed_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6d31266b9e4c93c5a9ecd4bbc391ac1daa4608b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_indexed_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_matrix_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_matrix_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0462b48cba6a62982227543e1eb60f3df02fa18a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/from_matrix_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/utils.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea844121fc6bd26001dcad1d71905e3fd22c5a1d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/__pycache__/utils.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/array_expressions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/array_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..f062e3de4c24987d62ba0b3a19fe474fb4687940 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/array_expressions.py @@ -0,0 +1,1969 @@ +from __future__ import annotations +import collections.abc +import operator +from collections import defaultdict, Counter +from functools import reduce +import itertools +from itertools import accumulate + +import typing + +from sympy.core.numbers import Integer +from sympy.core.relational import Equality +from sympy.functions.special.tensor_functions import KroneckerDelta +from sympy.core.basic import Basic +from sympy.core.containers import Tuple +from sympy.core.expr import Expr +from sympy.core.function import (Function, Lambda) +from sympy.core.mul import Mul +from sympy.core.singleton import S +from sympy.core.sorting import default_sort_key +from sympy.core.symbol import (Dummy, Symbol) +from sympy.matrices.matrixbase import MatrixBase +from sympy.matrices.expressions.diagonal import diagonalize_vector +from sympy.matrices.expressions.matexpr import MatrixExpr +from sympy.matrices.expressions.special import ZeroMatrix +from sympy.tensor.array.arrayop import (permutedims, tensorcontraction, tensordiagonal, tensorproduct) +from sympy.tensor.array.dense_ndim_array import ImmutableDenseNDimArray +from sympy.tensor.array.ndim_array import NDimArray +from sympy.tensor.indexed import (Indexed, IndexedBase) +from sympy.matrices.expressions.matexpr import MatrixElement +from sympy.tensor.array.expressions.utils import _apply_recursively_over_nested_lists, _sort_contraction_indices, \ + _get_mapping_from_subranks, _build_push_indices_up_func_transformation, _get_contraction_links, \ + _build_push_indices_down_func_transformation +from sympy.combinatorics import Permutation +from sympy.combinatorics.permutations import _af_invert +from sympy.core.sympify import _sympify + + +class _ArrayExpr(Expr): + shape: tuple[Expr, ...] + + def __getitem__(self, item): + if not isinstance(item, collections.abc.Iterable): + item = (item,) + ArrayElement._check_shape(self, item) + return self._get(item) + + def _get(self, item): + return _get_array_element_or_slice(self, item) + + +class ArraySymbol(_ArrayExpr): + """ + Symbol representing an array expression + """ + + _iterable = False + + def __new__(cls, symbol, shape: typing.Iterable) -> "ArraySymbol": + if isinstance(symbol, str): + symbol = Symbol(symbol) + # symbol = _sympify(symbol) + shape = Tuple(*map(_sympify, shape)) + obj = Expr.__new__(cls, symbol, shape) + return obj + + @property + def name(self): + return self._args[0] + + @property + def shape(self): + return self._args[1] + + def as_explicit(self): + if not all(i.is_Integer for i in self.shape): + raise ValueError("cannot express explicit array with symbolic shape") + data = [self[i] for i in itertools.product(*[range(j) for j in self.shape])] + return ImmutableDenseNDimArray(data).reshape(*self.shape) + + +class ArrayElement(Expr): + """ + An element of an array. + """ + + _diff_wrt = True + is_symbol = True + is_commutative = True + + def __new__(cls, name, indices): + if isinstance(name, str): + name = Symbol(name) + name = _sympify(name) + if not isinstance(indices, collections.abc.Iterable): + indices = (indices,) + indices = _sympify(tuple(indices)) + cls._check_shape(name, indices) + obj = Expr.__new__(cls, name, indices) + return obj + + @classmethod + def _check_shape(cls, name, indices): + indices = tuple(indices) + if hasattr(name, "shape"): + index_error = IndexError("number of indices does not match shape of the array") + if len(indices) != len(name.shape): + raise index_error + if any((i >= s) == True for i, s in zip(indices, name.shape)): + raise ValueError("shape is out of bounds") + if any((i < 0) == True for i in indices): + raise ValueError("shape contains negative values") + + @property + def name(self): + return self._args[0] + + @property + def indices(self): + return self._args[1] + + def _eval_derivative(self, s): + if not isinstance(s, ArrayElement): + return S.Zero + + if s == self: + return S.One + + if s.name != self.name: + return S.Zero + + return Mul.fromiter(KroneckerDelta(i, j) for i, j in zip(self.indices, s.indices)) + + +class ZeroArray(_ArrayExpr): + """ + Symbolic array of zeros. Equivalent to ``ZeroMatrix`` for matrices. + """ + + def __new__(cls, *shape): + if len(shape) == 0: + return S.Zero + shape = map(_sympify, shape) + obj = Expr.__new__(cls, *shape) + return obj + + @property + def shape(self): + return self._args + + def as_explicit(self): + if not all(i.is_Integer for i in self.shape): + raise ValueError("Cannot return explicit form for symbolic shape.") + return ImmutableDenseNDimArray.zeros(*self.shape) + + def _get(self, item): + return S.Zero + + +class OneArray(_ArrayExpr): + """ + Symbolic array of ones. + """ + + def __new__(cls, *shape): + if len(shape) == 0: + return S.One + shape = map(_sympify, shape) + obj = Expr.__new__(cls, *shape) + return obj + + @property + def shape(self): + return self._args + + def as_explicit(self): + if not all(i.is_Integer for i in self.shape): + raise ValueError("Cannot return explicit form for symbolic shape.") + return ImmutableDenseNDimArray([S.One for i in range(reduce(operator.mul, self.shape))]).reshape(*self.shape) + + def _get(self, item): + return S.One + + +class _CodegenArrayAbstract(Basic): + + @property + def subranks(self): + """ + Returns the ranks of the objects in the uppermost tensor product inside + the current object. In case no tensor products are contained, return + the atomic ranks. + + Examples + ======== + + >>> from sympy.tensor.array import tensorproduct, tensorcontraction + >>> from sympy import MatrixSymbol + >>> M = MatrixSymbol("M", 3, 3) + >>> N = MatrixSymbol("N", 3, 3) + >>> P = MatrixSymbol("P", 3, 3) + + Important: do not confuse the rank of the matrix with the rank of an array. + + >>> tp = tensorproduct(M, N, P) + >>> tp.subranks + [2, 2, 2] + + >>> co = tensorcontraction(tp, (1, 2), (3, 4)) + >>> co.subranks + [2, 2, 2] + """ + return self._subranks[:] + + def subrank(self): + """ + The sum of ``subranks``. + """ + return sum(self.subranks) + + @property + def shape(self): + return self._shape + + def doit(self, **hints): + deep = hints.get("deep", True) + if deep: + return self.func(*[arg.doit(**hints) for arg in self.args])._canonicalize() + else: + return self._canonicalize() + +class ArrayTensorProduct(_CodegenArrayAbstract): + r""" + Class to represent the tensor product of array-like objects. + """ + + def __new__(cls, *args, **kwargs): + args = [_sympify(arg) for arg in args] + + canonicalize = kwargs.pop("canonicalize", False) + + ranks = [get_rank(arg) for arg in args] + + obj = Basic.__new__(cls, *args) + obj._subranks = ranks + shapes = [get_shape(i) for i in args] + + if any(i is None for i in shapes): + obj._shape = None + else: + obj._shape = tuple(j for i in shapes for j in i) + if canonicalize: + return obj._canonicalize() + return obj + + def _canonicalize(self): + args = self.args + args = self._flatten(args) + + ranks = [get_rank(arg) for arg in args] + + # Check if there are nested permutation and lift them up: + permutation_cycles = [] + for i, arg in enumerate(args): + if not isinstance(arg, PermuteDims): + continue + permutation_cycles.extend([[k + sum(ranks[:i]) for k in j] for j in arg.permutation.cyclic_form]) + args[i] = arg.expr + if permutation_cycles: + return _permute_dims(_array_tensor_product(*args), Permutation(sum(ranks)-1)*Permutation(permutation_cycles)) + + if len(args) == 1: + return args[0] + + # If any object is a ZeroArray, return a ZeroArray: + if any(isinstance(arg, (ZeroArray, ZeroMatrix)) for arg in args): + shapes = reduce(operator.add, [get_shape(i) for i in args], ()) + return ZeroArray(*shapes) + + # If there are contraction objects inside, transform the whole + # expression into `ArrayContraction`: + contractions = {i: arg for i, arg in enumerate(args) if isinstance(arg, ArrayContraction)} + if contractions: + ranks = [_get_subrank(arg) if isinstance(arg, ArrayContraction) else get_rank(arg) for arg in args] + cumulative_ranks = list(accumulate([0] + ranks))[:-1] + tp = _array_tensor_product(*[arg.expr if isinstance(arg, ArrayContraction) else arg for arg in args]) + contraction_indices = [tuple(cumulative_ranks[i] + k for k in j) for i, arg in contractions.items() for j in arg.contraction_indices] + return _array_contraction(tp, *contraction_indices) + + diagonals = {i: arg for i, arg in enumerate(args) if isinstance(arg, ArrayDiagonal)} + if diagonals: + inverse_permutation = [] + last_perm = [] + ranks = [get_rank(arg) for arg in args] + cumulative_ranks = list(accumulate([0] + ranks))[:-1] + for i, arg in enumerate(args): + if isinstance(arg, ArrayDiagonal): + i1 = get_rank(arg) - len(arg.diagonal_indices) + i2 = len(arg.diagonal_indices) + inverse_permutation.extend([cumulative_ranks[i] + j for j in range(i1)]) + last_perm.extend([cumulative_ranks[i] + j for j in range(i1, i1 + i2)]) + else: + inverse_permutation.extend([cumulative_ranks[i] + j for j in range(get_rank(arg))]) + inverse_permutation.extend(last_perm) + tp = _array_tensor_product(*[arg.expr if isinstance(arg, ArrayDiagonal) else arg for arg in args]) + ranks2 = [_get_subrank(arg) if isinstance(arg, ArrayDiagonal) else get_rank(arg) for arg in args] + cumulative_ranks2 = list(accumulate([0] + ranks2))[:-1] + diagonal_indices = [tuple(cumulative_ranks2[i] + k for k in j) for i, arg in diagonals.items() for j in arg.diagonal_indices] + return _permute_dims(_array_diagonal(tp, *diagonal_indices), _af_invert(inverse_permutation)) + + return self.func(*args, canonicalize=False) + + @classmethod + def _flatten(cls, args): + args = [i for arg in args for i in (arg.args if isinstance(arg, cls) else [arg])] + return args + + def as_explicit(self): + return tensorproduct(*[arg.as_explicit() if hasattr(arg, "as_explicit") else arg for arg in self.args]) + + +class ArrayAdd(_CodegenArrayAbstract): + r""" + Class for elementwise array additions. + """ + + def __new__(cls, *args, **kwargs): + args = [_sympify(arg) for arg in args] + ranks = [get_rank(arg) for arg in args] + ranks = list(set(ranks)) + if len(ranks) != 1: + raise ValueError("summing arrays of different ranks") + shapes = [arg.shape for arg in args] + if len({i for i in shapes if i is not None}) > 1: + raise ValueError("mismatching shapes in addition") + + canonicalize = kwargs.pop("canonicalize", False) + + obj = Basic.__new__(cls, *args) + obj._subranks = ranks + if any(i is None for i in shapes): + obj._shape = None + else: + obj._shape = shapes[0] + if canonicalize: + return obj._canonicalize() + return obj + + def _canonicalize(self): + args = self.args + + # Flatten: + args = self._flatten_args(args) + + shapes = [get_shape(arg) for arg in args] + args = [arg for arg in args if not isinstance(arg, (ZeroArray, ZeroMatrix))] + if len(args) == 0: + if any(i for i in shapes if i is None): + raise NotImplementedError("cannot handle addition of ZeroMatrix/ZeroArray and undefined shape object") + return ZeroArray(*shapes[0]) + elif len(args) == 1: + return args[0] + return self.func(*args, canonicalize=False) + + @classmethod + def _flatten_args(cls, args): + new_args = [] + for arg in args: + if isinstance(arg, ArrayAdd): + new_args.extend(arg.args) + else: + new_args.append(arg) + return new_args + + def as_explicit(self): + return reduce( + operator.add, + [arg.as_explicit() if hasattr(arg, "as_explicit") else arg for arg in self.args]) + + +class PermuteDims(_CodegenArrayAbstract): + r""" + Class to represent permutation of axes of arrays. + + Examples + ======== + + >>> from sympy.tensor.array import permutedims + >>> from sympy import MatrixSymbol + >>> M = MatrixSymbol("M", 3, 3) + >>> cg = permutedims(M, [1, 0]) + + The object ``cg`` represents the transposition of ``M``, as the permutation + ``[1, 0]`` will act on its indices by switching them: + + `M_{ij} \Rightarrow M_{ji}` + + This is evident when transforming back to matrix form: + + >>> from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix + >>> convert_array_to_matrix(cg) + M.T + + >>> N = MatrixSymbol("N", 3, 2) + >>> cg = permutedims(N, [1, 0]) + >>> cg.shape + (2, 3) + + There are optional parameters that can be used as alternative to the permutation: + + >>> from sympy.tensor.array.expressions import ArraySymbol, PermuteDims + >>> M = ArraySymbol("M", (1, 2, 3, 4, 5)) + >>> expr = PermuteDims(M, index_order_old="ijklm", index_order_new="kijml") + >>> expr + PermuteDims(M, (0 2 1)(3 4)) + >>> expr.shape + (3, 1, 2, 5, 4) + + Permutations of tensor products are simplified in order to achieve a + standard form: + + >>> from sympy.tensor.array import tensorproduct + >>> M = MatrixSymbol("M", 4, 5) + >>> tp = tensorproduct(M, N) + >>> tp.shape + (4, 5, 3, 2) + >>> perm1 = permutedims(tp, [2, 3, 1, 0]) + + The args ``(M, N)`` have been sorted and the permutation has been + simplified, the expression is equivalent: + + >>> perm1.expr.args + (N, M) + >>> perm1.shape + (3, 2, 5, 4) + >>> perm1.permutation + (2 3) + + The permutation in its array form has been simplified from + ``[2, 3, 1, 0]`` to ``[0, 1, 3, 2]``, as the arguments of the tensor + product `M` and `N` have been switched: + + >>> perm1.permutation.array_form + [0, 1, 3, 2] + + We can nest a second permutation: + + >>> perm2 = permutedims(perm1, [1, 0, 2, 3]) + >>> perm2.shape + (2, 3, 5, 4) + >>> perm2.permutation.array_form + [1, 0, 3, 2] + """ + + def __new__(cls, expr, permutation=None, index_order_old=None, index_order_new=None, **kwargs): + from sympy.combinatorics import Permutation + expr = _sympify(expr) + expr_rank = get_rank(expr) + permutation = cls._get_permutation_from_arguments(permutation, index_order_old, index_order_new, expr_rank) + permutation = Permutation(permutation) + permutation_size = permutation.size + if permutation_size != expr_rank: + raise ValueError("Permutation size must be the length of the shape of expr") + + canonicalize = kwargs.pop("canonicalize", False) + + obj = Basic.__new__(cls, expr, permutation) + obj._subranks = [get_rank(expr)] + shape = get_shape(expr) + if shape is None: + obj._shape = None + else: + obj._shape = tuple(shape[permutation(i)] for i in range(len(shape))) + if canonicalize: + return obj._canonicalize() + return obj + + def _canonicalize(self): + expr = self.expr + permutation = self.permutation + if isinstance(expr, PermuteDims): + subexpr = expr.expr + subperm = expr.permutation + permutation = permutation * subperm + expr = subexpr + if isinstance(expr, ArrayContraction): + expr, permutation = self._PermuteDims_denestarg_ArrayContraction(expr, permutation) + if isinstance(expr, ArrayTensorProduct): + expr, permutation = self._PermuteDims_denestarg_ArrayTensorProduct(expr, permutation) + if isinstance(expr, (ZeroArray, ZeroMatrix)): + return ZeroArray(*[expr.shape[i] for i in permutation.array_form]) + plist = permutation.array_form + if plist == sorted(plist): + return expr + return self.func(expr, permutation, canonicalize=False) + + @property + def expr(self): + return self.args[0] + + @property + def permutation(self): + return self.args[1] + + @classmethod + def _PermuteDims_denestarg_ArrayTensorProduct(cls, expr, permutation): + # Get the permutation in its image-form: + perm_image_form = _af_invert(permutation.array_form) + args = list(expr.args) + # Starting index global position for every arg: + cumul = list(accumulate([0] + expr.subranks)) + # Split `perm_image_form` into a list of list corresponding to the indices + # of every argument: + perm_image_form_in_components = [perm_image_form[cumul[i]:cumul[i+1]] for i in range(len(args))] + # Create an index, target-position-key array: + ps = [(i, sorted(comp)) for i, comp in enumerate(perm_image_form_in_components)] + # Sort the array according to the target-position-key: + # In this way, we define a canonical way to sort the arguments according + # to the permutation. + ps.sort(key=lambda x: x[1]) + # Read the inverse-permutation (i.e. image-form) of the args: + perm_args_image_form = [i[0] for i in ps] + # Apply the args-permutation to the `args`: + args_sorted = [args[i] for i in perm_args_image_form] + # Apply the args-permutation to the array-form of the permutation of the axes (of `expr`): + perm_image_form_sorted_args = [perm_image_form_in_components[i] for i in perm_args_image_form] + new_permutation = Permutation(_af_invert([j for i in perm_image_form_sorted_args for j in i])) + return _array_tensor_product(*args_sorted), new_permutation + + @classmethod + def _PermuteDims_denestarg_ArrayContraction(cls, expr, permutation): + if not isinstance(expr, ArrayContraction): + return expr, permutation + if not isinstance(expr.expr, ArrayTensorProduct): + return expr, permutation + args = expr.expr.args + subranks = [get_rank(arg) for arg in expr.expr.args] + + contraction_indices = expr.contraction_indices + contraction_indices_flat = [j for i in contraction_indices for j in i] + cumul = list(accumulate([0] + subranks)) + + # Spread the permutation in its array form across the args in the corresponding + # tensor-product arguments with free indices: + permutation_array_blocks_up = [] + image_form = _af_invert(permutation.array_form) + counter = 0 + for i in range(len(subranks)): + current = [] + for j in range(cumul[i], cumul[i+1]): + if j in contraction_indices_flat: + continue + current.append(image_form[counter]) + counter += 1 + permutation_array_blocks_up.append(current) + + # Get the map of axis repositioning for every argument of tensor-product: + index_blocks = [list(range(cumul[i], cumul[i+1])) for i, e in enumerate(expr.subranks)] + index_blocks_up = expr._push_indices_up(expr.contraction_indices, index_blocks) + inverse_permutation = permutation**(-1) + index_blocks_up_permuted = [[inverse_permutation(j) for j in i if j is not None] for i in index_blocks_up] + + # Sorting key is a list of tuple, first element is the index of `args`, second element of + # the tuple is the sorting key to sort `args` of the tensor product: + sorting_keys = list(enumerate(index_blocks_up_permuted)) + sorting_keys.sort(key=lambda x: x[1]) + + # Now we can get the permutation acting on the args in its image-form: + new_perm_image_form = [i[0] for i in sorting_keys] + # Apply the args-level permutation to various elements: + new_index_blocks = [index_blocks[i] for i in new_perm_image_form] + new_index_perm_array_form = _af_invert([j for i in new_index_blocks for j in i]) + new_args = [args[i] for i in new_perm_image_form] + new_contraction_indices = [tuple(new_index_perm_array_form[j] for j in i) for i in contraction_indices] + new_expr = _array_contraction(_array_tensor_product(*new_args), *new_contraction_indices) + new_permutation = Permutation(_af_invert([j for i in [permutation_array_blocks_up[k] for k in new_perm_image_form] for j in i])) + return new_expr, new_permutation + + @classmethod + def _check_permutation_mapping(cls, expr, permutation): + subranks = expr.subranks + index2arg = [i for i, arg in enumerate(expr.args) for j in range(expr.subranks[i])] + permuted_indices = [permutation(i) for i in range(expr.subrank())] + new_args = list(expr.args) + arg_candidate_index = index2arg[permuted_indices[0]] + current_indices = [] + new_permutation = [] + inserted_arg_cand_indices = set() + for i, idx in enumerate(permuted_indices): + if index2arg[idx] != arg_candidate_index: + new_permutation.extend(current_indices) + current_indices = [] + arg_candidate_index = index2arg[idx] + current_indices.append(idx) + arg_candidate_rank = subranks[arg_candidate_index] + if len(current_indices) == arg_candidate_rank: + new_permutation.extend(sorted(current_indices)) + local_current_indices = [j - min(current_indices) for j in current_indices] + i1 = index2arg[i] + new_args[i1] = _permute_dims(new_args[i1], Permutation(local_current_indices)) + inserted_arg_cand_indices.add(arg_candidate_index) + current_indices = [] + new_permutation.extend(current_indices) + + # TODO: swap args positions in order to simplify the expression: + # TODO: this should be in a function + args_positions = list(range(len(new_args))) + # Get possible shifts: + maps = {} + cumulative_subranks = [0] + list(accumulate(subranks)) + for i in range(len(subranks)): + s = {index2arg[new_permutation[j]] for j in range(cumulative_subranks[i], cumulative_subranks[i+1])} + if len(s) != 1: + continue + elem = next(iter(s)) + if i != elem: + maps[i] = elem + + # Find cycles in the map: + lines = [] + current_line = [] + while maps: + if len(current_line) == 0: + k, v = maps.popitem() + current_line.append(k) + else: + k = current_line[-1] + if k not in maps: + current_line = [] + continue + v = maps.pop(k) + if v in current_line: + lines.append(current_line) + current_line = [] + continue + current_line.append(v) + for line in lines: + for i, e in enumerate(line): + args_positions[line[(i + 1) % len(line)]] = e + + # TODO: function in order to permute the args: + permutation_blocks = [[new_permutation[cumulative_subranks[i] + j] for j in range(e)] for i, e in enumerate(subranks)] + new_args = [new_args[i] for i in args_positions] + new_permutation_blocks = [permutation_blocks[i] for i in args_positions] + new_permutation2 = [j for i in new_permutation_blocks for j in i] + return _array_tensor_product(*new_args), Permutation(new_permutation2) # **(-1) + + @classmethod + def _check_if_there_are_closed_cycles(cls, expr, permutation): + args = list(expr.args) + subranks = expr.subranks + cyclic_form = permutation.cyclic_form + cumulative_subranks = [0] + list(accumulate(subranks)) + cyclic_min = [min(i) for i in cyclic_form] + cyclic_max = [max(i) for i in cyclic_form] + cyclic_keep = [] + for i, cycle in enumerate(cyclic_form): + flag = True + for j in range(len(cumulative_subranks) - 1): + if cyclic_min[i] >= cumulative_subranks[j] and cyclic_max[i] < cumulative_subranks[j+1]: + # Found a sinkable cycle. + args[j] = _permute_dims(args[j], Permutation([[k - cumulative_subranks[j] for k in cycle]])) + flag = False + break + if flag: + cyclic_keep.append(cycle) + return _array_tensor_product(*args), Permutation(cyclic_keep, size=permutation.size) + + def nest_permutation(self): + r""" + DEPRECATED. + """ + ret = self._nest_permutation(self.expr, self.permutation) + if ret is None: + return self + return ret + + @classmethod + def _nest_permutation(cls, expr, permutation): + if isinstance(expr, ArrayTensorProduct): + return _permute_dims(*cls._check_if_there_are_closed_cycles(expr, permutation)) + elif isinstance(expr, ArrayContraction): + # Invert tree hierarchy: put the contraction above. + cycles = permutation.cyclic_form + newcycles = ArrayContraction._convert_outer_indices_to_inner_indices(expr, *cycles) + newpermutation = Permutation(newcycles) + new_contr_indices = [tuple(newpermutation(j) for j in i) for i in expr.contraction_indices] + return _array_contraction(PermuteDims(expr.expr, newpermutation), *new_contr_indices) + elif isinstance(expr, ArrayAdd): + return _array_add(*[PermuteDims(arg, permutation) for arg in expr.args]) + return None + + def as_explicit(self): + expr = self.expr + if hasattr(expr, "as_explicit"): + expr = expr.as_explicit() + return permutedims(expr, self.permutation) + + @classmethod + def _get_permutation_from_arguments(cls, permutation, index_order_old, index_order_new, dim): + if permutation is None: + if index_order_new is None or index_order_old is None: + raise ValueError("Permutation not defined") + return PermuteDims._get_permutation_from_index_orders(index_order_old, index_order_new, dim) + else: + if index_order_new is not None: + raise ValueError("index_order_new cannot be defined with permutation") + if index_order_old is not None: + raise ValueError("index_order_old cannot be defined with permutation") + return permutation + + @classmethod + def _get_permutation_from_index_orders(cls, index_order_old, index_order_new, dim): + if len(set(index_order_new)) != dim: + raise ValueError("wrong number of indices in index_order_new") + if len(set(index_order_old)) != dim: + raise ValueError("wrong number of indices in index_order_old") + if len(set.symmetric_difference(set(index_order_new), set(index_order_old))) > 0: + raise ValueError("index_order_new and index_order_old must have the same indices") + permutation = [index_order_old.index(i) for i in index_order_new] + return permutation + + +class ArrayDiagonal(_CodegenArrayAbstract): + r""" + Class to represent the diagonal operator. + + Explanation + =========== + + In a 2-dimensional array it returns the diagonal, this looks like the + operation: + + `A_{ij} \rightarrow A_{ii}` + + The diagonal over axes 1 and 2 (the second and third) of the tensor product + of two 2-dimensional arrays `A \otimes B` is + + `\Big[ A_{ab} B_{cd} \Big]_{abcd} \rightarrow \Big[ A_{ai} B_{id} \Big]_{adi}` + + In this last example the array expression has been reduced from + 4-dimensional to 3-dimensional. Notice that no contraction has occurred, + rather there is a new index `i` for the diagonal, contraction would have + reduced the array to 2 dimensions. + + Notice that the diagonalized out dimensions are added as new dimensions at + the end of the indices. + """ + + def __new__(cls, expr, *diagonal_indices, **kwargs): + expr = _sympify(expr) + diagonal_indices = [Tuple(*sorted(i)) for i in diagonal_indices] + canonicalize = kwargs.get("canonicalize", False) + + shape = get_shape(expr) + if shape is not None: + cls._validate(expr, *diagonal_indices, **kwargs) + # Get new shape: + positions, shape = cls._get_positions_shape(shape, diagonal_indices) + else: + positions = None + if len(diagonal_indices) == 0: + return expr + obj = Basic.__new__(cls, expr, *diagonal_indices) + obj._positions = positions + obj._subranks = _get_subranks(expr) + obj._shape = shape + if canonicalize: + return obj._canonicalize() + return obj + + def _canonicalize(self): + expr = self.expr + diagonal_indices = self.diagonal_indices + trivial_diags = [i for i in diagonal_indices if len(i) == 1] + if len(trivial_diags) > 0: + trivial_pos = {e[0]: i for i, e in enumerate(diagonal_indices) if len(e) == 1} + diag_pos = {e: i for i, e in enumerate(diagonal_indices) if len(e) > 1} + diagonal_indices_short = [i for i in diagonal_indices if len(i) > 1] + rank1 = get_rank(self) + rank2 = len(diagonal_indices) + rank3 = rank1 - rank2 + inv_permutation = [] + counter1 = 0 + indices_down = ArrayDiagonal._push_indices_down(diagonal_indices_short, list(range(rank1)), get_rank(expr)) + for i in indices_down: + if i in trivial_pos: + inv_permutation.append(rank3 + trivial_pos[i]) + elif isinstance(i, (Integer, int)): + inv_permutation.append(counter1) + counter1 += 1 + else: + inv_permutation.append(rank3 + diag_pos[i]) + permutation = _af_invert(inv_permutation) + if len(diagonal_indices_short) > 0: + return _permute_dims(_array_diagonal(expr, *diagonal_indices_short), permutation) + else: + return _permute_dims(expr, permutation) + if isinstance(expr, ArrayAdd): + return self._ArrayDiagonal_denest_ArrayAdd(expr, *diagonal_indices) + if isinstance(expr, ArrayDiagonal): + return self._ArrayDiagonal_denest_ArrayDiagonal(expr, *diagonal_indices) + if isinstance(expr, PermuteDims): + return self._ArrayDiagonal_denest_PermuteDims(expr, *diagonal_indices) + if isinstance(expr, (ZeroArray, ZeroMatrix)): + positions, shape = self._get_positions_shape(expr.shape, diagonal_indices) + return ZeroArray(*shape) + return self.func(expr, *diagonal_indices, canonicalize=False) + + @staticmethod + def _validate(expr, *diagonal_indices, **kwargs): + # Check that no diagonalization happens on indices with mismatched + # dimensions: + shape = get_shape(expr) + for i in diagonal_indices: + if any(j >= len(shape) for j in i): + raise ValueError("index is larger than expression shape") + if len({shape[j] for j in i}) != 1: + raise ValueError("diagonalizing indices of different dimensions") + if not kwargs.get("allow_trivial_diags", False) and len(i) <= 1: + raise ValueError("need at least two axes to diagonalize") + if len(set(i)) != len(i): + raise ValueError("axis index cannot be repeated") + + @staticmethod + def _remove_trivial_dimensions(shape, *diagonal_indices): + return [tuple(j for j in i) for i in diagonal_indices if shape[i[0]] != 1] + + @property + def expr(self): + return self.args[0] + + @property + def diagonal_indices(self): + return self.args[1:] + + @staticmethod + def _flatten(expr, *outer_diagonal_indices): + inner_diagonal_indices = expr.diagonal_indices + all_inner = [j for i in inner_diagonal_indices for j in i] + all_inner.sort() + # TODO: add API for total rank and cumulative rank: + total_rank = _get_subrank(expr) + inner_rank = len(all_inner) + outer_rank = total_rank - inner_rank + shifts = [0 for i in range(outer_rank)] + counter = 0 + pointer = 0 + for i in range(outer_rank): + while pointer < inner_rank and counter >= all_inner[pointer]: + counter += 1 + pointer += 1 + shifts[i] += pointer + counter += 1 + outer_diagonal_indices = tuple(tuple(shifts[j] + j for j in i) for i in outer_diagonal_indices) + diagonal_indices = inner_diagonal_indices + outer_diagonal_indices + return _array_diagonal(expr.expr, *diagonal_indices) + + @classmethod + def _ArrayDiagonal_denest_ArrayAdd(cls, expr, *diagonal_indices): + return _array_add(*[_array_diagonal(arg, *diagonal_indices) for arg in expr.args]) + + @classmethod + def _ArrayDiagonal_denest_ArrayDiagonal(cls, expr, *diagonal_indices): + return cls._flatten(expr, *diagonal_indices) + + @classmethod + def _ArrayDiagonal_denest_PermuteDims(cls, expr: PermuteDims, *diagonal_indices): + back_diagonal_indices = [[expr.permutation(j) for j in i] for i in diagonal_indices] + nondiag = [i for i in range(get_rank(expr)) if not any(i in j for j in diagonal_indices)] + back_nondiag = [expr.permutation(i) for i in nondiag] + remap = {e: i for i, e in enumerate(sorted(back_nondiag))} + new_permutation1 = [remap[i] for i in back_nondiag] + shift = len(new_permutation1) + diag_block_perm = [i + shift for i in range(len(back_diagonal_indices))] + new_permutation = new_permutation1 + diag_block_perm + return _permute_dims( + _array_diagonal( + expr.expr, + *back_diagonal_indices + ), + new_permutation + ) + + def _push_indices_down_nonstatic(self, indices): + transform = lambda x: self._positions[x] if x < len(self._positions) else None + return _apply_recursively_over_nested_lists(transform, indices) + + def _push_indices_up_nonstatic(self, indices): + + def transform(x): + for i, e in enumerate(self._positions): + if (isinstance(e, int) and x == e) or (isinstance(e, tuple) and x in e): + return i + + return _apply_recursively_over_nested_lists(transform, indices) + + @classmethod + def _push_indices_down(cls, diagonal_indices, indices, rank): + positions, shape = cls._get_positions_shape(range(rank), diagonal_indices) + transform = lambda x: positions[x] if x < len(positions) else None + return _apply_recursively_over_nested_lists(transform, indices) + + @classmethod + def _push_indices_up(cls, diagonal_indices, indices, rank): + positions, shape = cls._get_positions_shape(range(rank), diagonal_indices) + + def transform(x): + for i, e in enumerate(positions): + if (isinstance(e, int) and x == e) or (isinstance(e, (tuple, Tuple)) and (x in e)): + return i + + return _apply_recursively_over_nested_lists(transform, indices) + + @classmethod + def _get_positions_shape(cls, shape, diagonal_indices): + data1 = tuple((i, shp) for i, shp in enumerate(shape) if not any(i in j for j in diagonal_indices)) + pos1, shp1 = zip(*data1) if data1 else ((), ()) + data2 = tuple((i, shape[i[0]]) for i in diagonal_indices) + pos2, shp2 = zip(*data2) if data2 else ((), ()) + positions = pos1 + pos2 + shape = shp1 + shp2 + return positions, shape + + def as_explicit(self): + expr = self.expr + if hasattr(expr, "as_explicit"): + expr = expr.as_explicit() + return tensordiagonal(expr, *self.diagonal_indices) + + +class ArrayElementwiseApplyFunc(_CodegenArrayAbstract): + + def __new__(cls, function, element): + + if not isinstance(function, Lambda): + d = Dummy('d') + function = Lambda(d, function(d)) + + obj = _CodegenArrayAbstract.__new__(cls, function, element) + obj._subranks = _get_subranks(element) + return obj + + @property + def function(self): + return self.args[0] + + @property + def expr(self): + return self.args[1] + + @property + def shape(self): + return self.expr.shape + + def _get_function_fdiff(self): + d = Dummy("d") + function = self.function(d) + fdiff = function.diff(d) + if isinstance(fdiff, Function): + fdiff = type(fdiff) + else: + fdiff = Lambda(d, fdiff) + return fdiff + + def as_explicit(self): + expr = self.expr + if hasattr(expr, "as_explicit"): + expr = expr.as_explicit() + return expr.applyfunc(self.function) + + +class ArrayContraction(_CodegenArrayAbstract): + r""" + This class is meant to represent contractions of arrays in a form easily + processable by the code printers. + """ + + def __new__(cls, expr, *contraction_indices, **kwargs): + contraction_indices = _sort_contraction_indices(contraction_indices) + expr = _sympify(expr) + + canonicalize = kwargs.get("canonicalize", False) + + obj = Basic.__new__(cls, expr, *contraction_indices) + obj._subranks = _get_subranks(expr) + obj._mapping = _get_mapping_from_subranks(obj._subranks) + + free_indices_to_position = {i: i for i in range(sum(obj._subranks)) if all(i not in cind for cind in contraction_indices)} + obj._free_indices_to_position = free_indices_to_position + + shape = get_shape(expr) + cls._validate(expr, *contraction_indices) + if shape: + shape = tuple(shp for i, shp in enumerate(shape) if not any(i in j for j in contraction_indices)) + obj._shape = shape + if canonicalize: + return obj._canonicalize() + return obj + + def _canonicalize(self): + expr = self.expr + contraction_indices = self.contraction_indices + + if len(contraction_indices) == 0: + return expr + + if isinstance(expr, ArrayContraction): + return self._ArrayContraction_denest_ArrayContraction(expr, *contraction_indices) + + if isinstance(expr, (ZeroArray, ZeroMatrix)): + return self._ArrayContraction_denest_ZeroArray(expr, *contraction_indices) + + if isinstance(expr, PermuteDims): + return self._ArrayContraction_denest_PermuteDims(expr, *contraction_indices) + + if isinstance(expr, ArrayTensorProduct): + expr, contraction_indices = self._sort_fully_contracted_args(expr, contraction_indices) + expr, contraction_indices = self._lower_contraction_to_addends(expr, contraction_indices) + if len(contraction_indices) == 0: + return expr + + if isinstance(expr, ArrayDiagonal): + return self._ArrayContraction_denest_ArrayDiagonal(expr, *contraction_indices) + + if isinstance(expr, ArrayAdd): + return self._ArrayContraction_denest_ArrayAdd(expr, *contraction_indices) + + # Check single index contractions on 1-dimensional axes: + contraction_indices = [i for i in contraction_indices if len(i) > 1 or get_shape(expr)[i[0]] != 1] + if len(contraction_indices) == 0: + return expr + + return self.func(expr, *contraction_indices, canonicalize=False) + + def __mul__(self, other): + if other == 1: + return self + else: + raise NotImplementedError("Product of N-dim arrays is not uniquely defined. Use another method.") + + def __rmul__(self, other): + if other == 1: + return self + else: + raise NotImplementedError("Product of N-dim arrays is not uniquely defined. Use another method.") + + @staticmethod + def _validate(expr, *contraction_indices): + shape = get_shape(expr) + if shape is None: + return + + # Check that no contraction happens when the shape is mismatched: + for i in contraction_indices: + if len({shape[j] for j in i if shape[j] != -1}) != 1: + raise ValueError("contracting indices of different dimensions") + + @classmethod + def _push_indices_down(cls, contraction_indices, indices): + flattened_contraction_indices = [j for i in contraction_indices for j in i] + flattened_contraction_indices.sort() + transform = _build_push_indices_down_func_transformation(flattened_contraction_indices) + return _apply_recursively_over_nested_lists(transform, indices) + + @classmethod + def _push_indices_up(cls, contraction_indices, indices): + flattened_contraction_indices = [j for i in contraction_indices for j in i] + flattened_contraction_indices.sort() + transform = _build_push_indices_up_func_transformation(flattened_contraction_indices) + return _apply_recursively_over_nested_lists(transform, indices) + + @classmethod + def _lower_contraction_to_addends(cls, expr, contraction_indices): + if isinstance(expr, ArrayAdd): + raise NotImplementedError() + if not isinstance(expr, ArrayTensorProduct): + return expr, contraction_indices + subranks = expr.subranks + cumranks = list(accumulate([0] + subranks)) + contraction_indices_remaining = [] + contraction_indices_args = [[] for i in expr.args] + backshift = set() + for contraction_group in contraction_indices: + for j in range(len(expr.args)): + if not isinstance(expr.args[j], ArrayAdd): + continue + if all(cumranks[j] <= k < cumranks[j+1] for k in contraction_group): + contraction_indices_args[j].append([k - cumranks[j] for k in contraction_group]) + backshift.update(contraction_group) + break + else: + contraction_indices_remaining.append(contraction_group) + if len(contraction_indices_remaining) == len(contraction_indices): + return expr, contraction_indices + total_rank = get_rank(expr) + shifts = list(accumulate([1 if i in backshift else 0 for i in range(total_rank)])) + contraction_indices_remaining = [Tuple.fromiter(j - shifts[j] for j in i) for i in contraction_indices_remaining] + ret = _array_tensor_product(*[ + _array_contraction(arg, *contr) for arg, contr in zip(expr.args, contraction_indices_args) + ]) + return ret, contraction_indices_remaining + + def split_multiple_contractions(self): + """ + Recognize multiple contractions and attempt at rewriting them as paired-contractions. + + This allows some contractions involving more than two indices to be + rewritten as multiple contractions involving two indices, thus allowing + the expression to be rewritten as a matrix multiplication line. + + Examples: + + * `A_ij b_j0 C_jk` ===> `A*DiagMatrix(b)*C` + + Care for: + - matrix being diagonalized (i.e. `A_ii`) + - vectors being diagonalized (i.e. `a_i0`) + + Multiple contractions can be split into matrix multiplications if + not more than two arguments are non-diagonals or non-vectors. + Vectors get diagonalized while diagonal matrices remain diagonal. + The non-diagonal matrices can be at the beginning or at the end + of the final matrix multiplication line. + """ + + editor = _EditArrayContraction(self) + + contraction_indices = self.contraction_indices + + onearray_insert = [] + + for indl, links in enumerate(contraction_indices): + if len(links) <= 2: + continue + + # Check multiple contractions: + # + # Examples: + # + # * `A_ij b_j0 C_jk` ===> `A*DiagMatrix(b)*C \otimes OneArray(1)` with permutation (1 2) + # + # Care for: + # - matrix being diagonalized (i.e. `A_ii`) + # - vectors being diagonalized (i.e. `a_i0`) + + # Multiple contractions can be split into matrix multiplications if + # not more than three arguments are non-diagonals or non-vectors. + # + # Vectors get diagonalized while diagonal matrices remain diagonal. + # The non-diagonal matrices can be at the beginning or at the end + # of the final matrix multiplication line. + + positions = editor.get_mapping_for_index(indl) + + # Also consider the case of diagonal matrices being contracted: + current_dimension = self.expr.shape[links[0]] + + not_vectors = [] + vectors = [] + for arg_ind, rel_ind in positions: + arg = editor.args_with_ind[arg_ind] + mat = arg.element + abs_arg_start, abs_arg_end = editor.get_absolute_range(arg) + other_arg_pos = 1-rel_ind + other_arg_abs = abs_arg_start + other_arg_pos + if ((1 not in mat.shape) or + ((current_dimension == 1) is True and mat.shape != (1, 1)) or + any(other_arg_abs in l for li, l in enumerate(contraction_indices) if li != indl) + ): + not_vectors.append((arg, rel_ind)) + else: + vectors.append((arg, rel_ind)) + if len(not_vectors) > 2: + # If more than two arguments in the multiple contraction are + # non-vectors and non-diagonal matrices, we cannot find a way + # to split this contraction into a matrix multiplication line: + continue + # Three cases to handle: + # - zero non-vectors + # - one non-vector + # - two non-vectors + for v, rel_ind in vectors: + v.element = diagonalize_vector(v.element) + vectors_to_loop = not_vectors[:1] + vectors + not_vectors[1:] + first_not_vector, rel_ind = vectors_to_loop[0] + new_index = first_not_vector.indices[rel_ind] + + for v, rel_ind in vectors_to_loop[1:-1]: + v.indices[rel_ind] = new_index + new_index = editor.get_new_contraction_index() + assert v.indices.index(None) == 1 - rel_ind + v.indices[v.indices.index(None)] = new_index + onearray_insert.append(v) + + last_vec, rel_ind = vectors_to_loop[-1] + last_vec.indices[rel_ind] = new_index + + for v in onearray_insert: + editor.insert_after(v, _ArgE(OneArray(1), [None])) + + return editor.to_array_contraction() + + def flatten_contraction_of_diagonal(self): + if not isinstance(self.expr, ArrayDiagonal): + return self + contraction_down = self.expr._push_indices_down(self.expr.diagonal_indices, self.contraction_indices) + new_contraction_indices = [] + diagonal_indices = self.expr.diagonal_indices[:] + for i in contraction_down: + contraction_group = list(i) + for j in i: + diagonal_with = [k for k in diagonal_indices if j in k] + contraction_group.extend([l for k in diagonal_with for l in k]) + diagonal_indices = [k for k in diagonal_indices if k not in diagonal_with] + new_contraction_indices.append(sorted(set(contraction_group))) + + new_contraction_indices = ArrayDiagonal._push_indices_up(diagonal_indices, new_contraction_indices) + return _array_contraction( + _array_diagonal( + self.expr.expr, + *diagonal_indices + ), + *new_contraction_indices + ) + + @staticmethod + def _get_free_indices_to_position_map(free_indices, contraction_indices): + free_indices_to_position = {} + flattened_contraction_indices = [j for i in contraction_indices for j in i] + counter = 0 + for ind in free_indices: + while counter in flattened_contraction_indices: + counter += 1 + free_indices_to_position[ind] = counter + counter += 1 + return free_indices_to_position + + @staticmethod + def _get_index_shifts(expr): + """ + Get the mapping of indices at the positions before the contraction + occurs. + + Examples + ======== + + >>> from sympy.tensor.array import tensorproduct, tensorcontraction + >>> from sympy import MatrixSymbol + >>> M = MatrixSymbol("M", 3, 3) + >>> N = MatrixSymbol("N", 3, 3) + >>> cg = tensorcontraction(tensorproduct(M, N), [1, 2]) + >>> cg._get_index_shifts(cg) + [0, 2] + + Indeed, ``cg`` after the contraction has two dimensions, 0 and 1. They + need to be shifted by 0 and 2 to get the corresponding positions before + the contraction (that is, 0 and 3). + """ + inner_contraction_indices = expr.contraction_indices + all_inner = [j for i in inner_contraction_indices for j in i] + all_inner.sort() + # TODO: add API for total rank and cumulative rank: + total_rank = _get_subrank(expr) + inner_rank = len(all_inner) + outer_rank = total_rank - inner_rank + shifts = [0 for i in range(outer_rank)] + counter = 0 + pointer = 0 + for i in range(outer_rank): + while pointer < inner_rank and counter >= all_inner[pointer]: + counter += 1 + pointer += 1 + shifts[i] += pointer + counter += 1 + return shifts + + @staticmethod + def _convert_outer_indices_to_inner_indices(expr, *outer_contraction_indices): + shifts = ArrayContraction._get_index_shifts(expr) + outer_contraction_indices = tuple(tuple(shifts[j] + j for j in i) for i in outer_contraction_indices) + return outer_contraction_indices + + @staticmethod + def _flatten(expr, *outer_contraction_indices): + inner_contraction_indices = expr.contraction_indices + outer_contraction_indices = ArrayContraction._convert_outer_indices_to_inner_indices(expr, *outer_contraction_indices) + contraction_indices = inner_contraction_indices + outer_contraction_indices + return _array_contraction(expr.expr, *contraction_indices) + + @classmethod + def _ArrayContraction_denest_ArrayContraction(cls, expr, *contraction_indices): + return cls._flatten(expr, *contraction_indices) + + @classmethod + def _ArrayContraction_denest_ZeroArray(cls, expr, *contraction_indices): + contraction_indices_flat = [j for i in contraction_indices for j in i] + shape = [e for i, e in enumerate(expr.shape) if i not in contraction_indices_flat] + return ZeroArray(*shape) + + @classmethod + def _ArrayContraction_denest_ArrayAdd(cls, expr, *contraction_indices): + return _array_add(*[_array_contraction(i, *contraction_indices) for i in expr.args]) + + @classmethod + def _ArrayContraction_denest_PermuteDims(cls, expr, *contraction_indices): + permutation = expr.permutation + plist = permutation.array_form + new_contraction_indices = [tuple(permutation(j) for j in i) for i in contraction_indices] + new_plist = [i for i in plist if not any(i in j for j in new_contraction_indices)] + new_plist = cls._push_indices_up(new_contraction_indices, new_plist) + return _permute_dims( + _array_contraction(expr.expr, *new_contraction_indices), + Permutation(new_plist) + ) + + @classmethod + def _ArrayContraction_denest_ArrayDiagonal(cls, expr: 'ArrayDiagonal', *contraction_indices): + diagonal_indices = list(expr.diagonal_indices) + down_contraction_indices = expr._push_indices_down(expr.diagonal_indices, contraction_indices, get_rank(expr.expr)) + # Flatten diagonally contracted indices: + down_contraction_indices = [[k for j in i for k in (j if isinstance(j, (tuple, Tuple)) else [j])] for i in down_contraction_indices] + new_contraction_indices = [] + for contr_indgrp in down_contraction_indices: + ind = contr_indgrp[:] + for j, diag_indgrp in enumerate(diagonal_indices): + if diag_indgrp is None: + continue + if any(i in diag_indgrp for i in contr_indgrp): + ind.extend(diag_indgrp) + diagonal_indices[j] = None + new_contraction_indices.append(sorted(set(ind))) + + new_diagonal_indices_down = [i for i in diagonal_indices if i is not None] + new_diagonal_indices = ArrayContraction._push_indices_up(new_contraction_indices, new_diagonal_indices_down) + return _array_diagonal( + _array_contraction(expr.expr, *new_contraction_indices), + *new_diagonal_indices + ) + + @classmethod + def _sort_fully_contracted_args(cls, expr, contraction_indices): + if expr.shape is None: + return expr, contraction_indices + cumul = list(accumulate([0] + expr.subranks)) + index_blocks = [list(range(cumul[i], cumul[i+1])) for i in range(len(expr.args))] + contraction_indices_flat = {j for i in contraction_indices for j in i} + fully_contracted = [all(j in contraction_indices_flat for j in range(cumul[i], cumul[i+1])) for i, arg in enumerate(expr.args)] + new_pos = sorted(range(len(expr.args)), key=lambda x: (0, default_sort_key(expr.args[x])) if fully_contracted[x] else (1,)) + new_args = [expr.args[i] for i in new_pos] + new_index_blocks_flat = [j for i in new_pos for j in index_blocks[i]] + index_permutation_array_form = _af_invert(new_index_blocks_flat) + new_contraction_indices = [tuple(index_permutation_array_form[j] for j in i) for i in contraction_indices] + new_contraction_indices = _sort_contraction_indices(new_contraction_indices) + return _array_tensor_product(*new_args), new_contraction_indices + + def _get_contraction_tuples(self): + r""" + Return tuples containing the argument index and position within the + argument of the index position. + + Examples + ======== + + >>> from sympy import MatrixSymbol + >>> from sympy.abc import N + >>> from sympy.tensor.array import tensorproduct, tensorcontraction + >>> A = MatrixSymbol("A", N, N) + >>> B = MatrixSymbol("B", N, N) + + >>> cg = tensorcontraction(tensorproduct(A, B), (1, 2)) + >>> cg._get_contraction_tuples() + [[(0, 1), (1, 0)]] + + Notes + ===== + + Here the contraction pair `(1, 2)` meaning that the 2nd and 3rd indices + of the tensor product `A\otimes B` are contracted, has been transformed + into `(0, 1)` and `(1, 0)`, identifying the same indices in a different + notation. `(0, 1)` is the second index (1) of the first argument (i.e. + 0 or `A`). `(1, 0)` is the first index (i.e. 0) of the second + argument (i.e. 1 or `B`). + """ + mapping = self._mapping + return [[mapping[j] for j in i] for i in self.contraction_indices] + + @staticmethod + def _contraction_tuples_to_contraction_indices(expr, contraction_tuples): + # TODO: check that `expr` has `.subranks`: + ranks = expr.subranks + cumulative_ranks = [0] + list(accumulate(ranks)) + return [tuple(cumulative_ranks[j]+k for j, k in i) for i in contraction_tuples] + + @property + def free_indices(self): + return self._free_indices[:] + + @property + def free_indices_to_position(self): + return dict(self._free_indices_to_position) + + @property + def expr(self): + return self.args[0] + + @property + def contraction_indices(self): + return self.args[1:] + + def _contraction_indices_to_components(self): + expr = self.expr + if not isinstance(expr, ArrayTensorProduct): + raise NotImplementedError("only for contractions of tensor products") + ranks = expr.subranks + mapping = {} + counter = 0 + for i, rank in enumerate(ranks): + for j in range(rank): + mapping[counter] = (i, j) + counter += 1 + return mapping + + def sort_args_by_name(self): + """ + Sort arguments in the tensor product so that their order is lexicographical. + + Examples + ======== + + >>> from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array + >>> from sympy import MatrixSymbol + >>> from sympy.abc import N + >>> A = MatrixSymbol("A", N, N) + >>> B = MatrixSymbol("B", N, N) + >>> C = MatrixSymbol("C", N, N) + >>> D = MatrixSymbol("D", N, N) + + >>> cg = convert_matrix_to_array(C*D*A*B) + >>> cg + ArrayContraction(ArrayTensorProduct(A, D, C, B), (0, 3), (1, 6), (2, 5)) + >>> cg.sort_args_by_name() + ArrayContraction(ArrayTensorProduct(A, D, B, C), (0, 3), (1, 4), (2, 7)) + """ + expr = self.expr + if not isinstance(expr, ArrayTensorProduct): + return self + args = expr.args + sorted_data = sorted(enumerate(args), key=lambda x: default_sort_key(x[1])) + pos_sorted, args_sorted = zip(*sorted_data) + reordering_map = {i: pos_sorted.index(i) for i, arg in enumerate(args)} + contraction_tuples = self._get_contraction_tuples() + contraction_tuples = [[(reordering_map[j], k) for j, k in i] for i in contraction_tuples] + c_tp = _array_tensor_product(*args_sorted) + new_contr_indices = self._contraction_tuples_to_contraction_indices( + c_tp, + contraction_tuples + ) + return _array_contraction(c_tp, *new_contr_indices) + + def _get_contraction_links(self): + r""" + Returns a dictionary of links between arguments in the tensor product + being contracted. + + See the example for an explanation of the values. + + Examples + ======== + + >>> from sympy import MatrixSymbol + >>> from sympy.abc import N + >>> from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array + >>> A = MatrixSymbol("A", N, N) + >>> B = MatrixSymbol("B", N, N) + >>> C = MatrixSymbol("C", N, N) + >>> D = MatrixSymbol("D", N, N) + + Matrix multiplications are pairwise contractions between neighboring + matrices: + + `A_{ij} B_{jk} C_{kl} D_{lm}` + + >>> cg = convert_matrix_to_array(A*B*C*D) + >>> cg + ArrayContraction(ArrayTensorProduct(B, C, A, D), (0, 5), (1, 2), (3, 6)) + + >>> cg._get_contraction_links() + {0: {0: (2, 1), 1: (1, 0)}, 1: {0: (0, 1), 1: (3, 0)}, 2: {1: (0, 0)}, 3: {0: (1, 1)}} + + This dictionary is interpreted as follows: argument in position 0 (i.e. + matrix `A`) has its second index (i.e. 1) contracted to `(1, 0)`, that + is argument in position 1 (matrix `B`) on the first index slot of `B`, + this is the contraction provided by the index `j` from `A`. + + The argument in position 1 (that is, matrix `B`) has two contractions, + the ones provided by the indices `j` and `k`, respectively the first + and second indices (0 and 1 in the sub-dict). The link `(0, 1)` and + `(2, 0)` respectively. `(0, 1)` is the index slot 1 (the 2nd) of + argument in position 0 (that is, `A_{\ldot j}`), and so on. + """ + args, dlinks = _get_contraction_links([self], self.subranks, *self.contraction_indices) + return dlinks + + def as_explicit(self): + expr = self.expr + if hasattr(expr, "as_explicit"): + expr = expr.as_explicit() + return tensorcontraction(expr, *self.contraction_indices) + + +class Reshape(_CodegenArrayAbstract): + """ + Reshape the dimensions of an array expression. + + Examples + ======== + + >>> from sympy.tensor.array.expressions import ArraySymbol, Reshape + >>> A = ArraySymbol("A", (6,)) + >>> A.shape + (6,) + >>> Reshape(A, (3, 2)).shape + (3, 2) + + Check the component-explicit forms: + + >>> A.as_explicit() + [A[0], A[1], A[2], A[3], A[4], A[5]] + >>> Reshape(A, (3, 2)).as_explicit() + [[A[0], A[1]], [A[2], A[3]], [A[4], A[5]]] + + """ + + def __new__(cls, expr, shape): + expr = _sympify(expr) + if not isinstance(shape, Tuple): + shape = Tuple(*shape) + if Equality(Mul.fromiter(expr.shape), Mul.fromiter(shape)) == False: + raise ValueError("shape mismatch") + obj = Expr.__new__(cls, expr, shape) + obj._shape = tuple(shape) + obj._expr = expr + return obj + + @property + def shape(self): + return self._shape + + @property + def expr(self): + return self._expr + + def doit(self, *args, **kwargs): + if kwargs.get("deep", True): + expr = self.expr.doit(*args, **kwargs) + else: + expr = self.expr + if isinstance(expr, (MatrixBase, NDimArray)): + return expr.reshape(*self.shape) + return Reshape(expr, self.shape) + + def as_explicit(self): + ee = self.expr + if hasattr(ee, "as_explicit"): + ee = ee.as_explicit() + if isinstance(ee, MatrixBase): + from sympy import Array + ee = Array(ee) + elif isinstance(ee, MatrixExpr): + return self + return ee.reshape(*self.shape) + + +class _ArgE: + """ + The ``_ArgE`` object contains references to the array expression + (``.element``) and a list containing the information about index + contractions (``.indices``). + + Index contractions are numbered and contracted indices show the number of + the contraction. Uncontracted indices have ``None`` value. + + For example: + ``_ArgE(M, [None, 3])`` + This object means that expression ``M`` is part of an array contraction + and has two indices, the first is not contracted (value ``None``), + the second index is contracted to the 4th (i.e. number ``3``) group of the + array contraction object. + """ + indices: list[int | None] + + def __init__(self, element, indices: list[int | None] | None = None): + self.element = element + if indices is None: + self.indices = [None for i in range(get_rank(element))] + else: + self.indices = indices + + def __str__(self): + return "_ArgE(%s, %s)" % (self.element, self.indices) + + __repr__ = __str__ + + +class _IndPos: + """ + Index position, requiring two integers in the constructor: + + - arg: the position of the argument in the tensor product, + - rel: the relative position of the index inside the argument. + """ + def __init__(self, arg: int, rel: int): + self.arg = arg + self.rel = rel + + def __str__(self): + return "_IndPos(%i, %i)" % (self.arg, self.rel) + + __repr__ = __str__ + + def __iter__(self): + yield from [self.arg, self.rel] + + +class _EditArrayContraction: + """ + Utility class to help manipulate array contraction objects. + + This class takes as input an ``ArrayContraction`` object and turns it into + an editable object. + + The field ``args_with_ind`` of this class is a list of ``_ArgE`` objects + which can be used to easily edit the contraction structure of the + expression. + + Once editing is finished, the ``ArrayContraction`` object may be recreated + by calling the ``.to_array_contraction()`` method. + """ + + def __init__(self, base_array: typing.Union[ArrayContraction, ArrayDiagonal, ArrayTensorProduct]): + + expr: Basic + diagonalized: tuple[tuple[int, ...], ...] + contraction_indices: list[tuple[int]] + if isinstance(base_array, ArrayContraction): + mapping = _get_mapping_from_subranks(base_array.subranks) + expr = base_array.expr + contraction_indices = base_array.contraction_indices + diagonalized = () + elif isinstance(base_array, ArrayDiagonal): + + if isinstance(base_array.expr, ArrayContraction): + mapping = _get_mapping_from_subranks(base_array.expr.subranks) + expr = base_array.expr.expr + diagonalized = ArrayContraction._push_indices_down(base_array.expr.contraction_indices, base_array.diagonal_indices) + contraction_indices = base_array.expr.contraction_indices + elif isinstance(base_array.expr, ArrayTensorProduct): + mapping = {} + expr = base_array.expr + diagonalized = base_array.diagonal_indices + contraction_indices = [] + else: + mapping = {} + expr = base_array.expr + diagonalized = base_array.diagonal_indices + contraction_indices = [] + + elif isinstance(base_array, ArrayTensorProduct): + expr = base_array + contraction_indices = [] + diagonalized = () + else: + raise NotImplementedError() + + if isinstance(expr, ArrayTensorProduct): + args = list(expr.args) + else: + args = [expr] + + args_with_ind: list[_ArgE] = [_ArgE(arg) for arg in args] + for i, contraction_tuple in enumerate(contraction_indices): + for j in contraction_tuple: + arg_pos, rel_pos = mapping[j] + args_with_ind[arg_pos].indices[rel_pos] = i + self.args_with_ind: list[_ArgE] = args_with_ind + self.number_of_contraction_indices: int = len(contraction_indices) + self._track_permutation: list[list[int]] | None = None + + mapping = _get_mapping_from_subranks(base_array.subranks) + + # Trick: add diagonalized indices as negative indices into the editor object: + for i, e in enumerate(diagonalized): + for j in e: + arg_pos, rel_pos = mapping[j] + self.args_with_ind[arg_pos].indices[rel_pos] = -1 - i + + def insert_after(self, arg: _ArgE, new_arg: _ArgE): + pos = self.args_with_ind.index(arg) + self.args_with_ind.insert(pos + 1, new_arg) + + def get_new_contraction_index(self): + self.number_of_contraction_indices += 1 + return self.number_of_contraction_indices - 1 + + def refresh_indices(self): + updates = {} + for arg_with_ind in self.args_with_ind: + updates.update({i: -1 for i in arg_with_ind.indices if i is not None}) + for i, e in enumerate(sorted(updates)): + updates[e] = i + self.number_of_contraction_indices = len(updates) + for arg_with_ind in self.args_with_ind: + arg_with_ind.indices = [updates.get(i, None) for i in arg_with_ind.indices] + + def merge_scalars(self): + scalars = [] + for arg_with_ind in self.args_with_ind: + if len(arg_with_ind.indices) == 0: + scalars.append(arg_with_ind) + for i in scalars: + self.args_with_ind.remove(i) + scalar = Mul.fromiter([i.element for i in scalars]) + if len(self.args_with_ind) == 0: + self.args_with_ind.append(_ArgE(scalar)) + else: + from sympy.tensor.array.expressions.from_array_to_matrix import _a2m_tensor_product + self.args_with_ind[0].element = _a2m_tensor_product(scalar, self.args_with_ind[0].element) + + def to_array_contraction(self): + + # Count the ranks of the arguments: + counter = 0 + # Create a collector for the new diagonal indices: + diag_indices = defaultdict(list) + + count_index_freq = Counter() + for arg_with_ind in self.args_with_ind: + count_index_freq.update(Counter(arg_with_ind.indices)) + + free_index_count = count_index_freq[None] + + # Construct the inverse permutation: + inv_perm1 = [] + inv_perm2 = [] + # Keep track of which diagonal indices have already been processed: + done = set() + + # Counter for the diagonal indices: + counter4 = 0 + + for arg_with_ind in self.args_with_ind: + # If some diagonalization axes have been removed, they should be + # permuted in order to keep the permutation. + # Add permutation here + counter2 = 0 # counter for the indices + for i in arg_with_ind.indices: + if i is None: + inv_perm1.append(counter4) + counter2 += 1 + counter4 += 1 + continue + if i >= 0: + continue + # Reconstruct the diagonal indices: + diag_indices[-1 - i].append(counter + counter2) + if count_index_freq[i] == 1 and i not in done: + inv_perm1.append(free_index_count - 1 - i) + done.add(i) + elif i not in done: + inv_perm2.append(free_index_count - 1 - i) + done.add(i) + counter2 += 1 + # Remove negative indices to restore a proper editor object: + arg_with_ind.indices = [i if i is not None and i >= 0 else None for i in arg_with_ind.indices] + counter += len([i for i in arg_with_ind.indices if i is None or i < 0]) + + inverse_permutation = inv_perm1 + inv_perm2 + permutation = _af_invert(inverse_permutation) + + # Get the diagonal indices after the detection of HadamardProduct in the expression: + diag_indices_filtered = [tuple(v) for v in diag_indices.values() if len(v) > 1] + + self.merge_scalars() + self.refresh_indices() + args = [arg.element for arg in self.args_with_ind] + contraction_indices = self.get_contraction_indices() + expr = _array_contraction(_array_tensor_product(*args), *contraction_indices) + expr2 = _array_diagonal(expr, *diag_indices_filtered) + if self._track_permutation is not None: + permutation2 = _af_invert([j for i in self._track_permutation for j in i]) + expr2 = _permute_dims(expr2, permutation2) + + expr3 = _permute_dims(expr2, permutation) + return expr3 + + def get_contraction_indices(self) -> list[list[int]]: + contraction_indices: list[list[int]] = [[] for i in range(self.number_of_contraction_indices)] + current_position: int = 0 + for arg_with_ind in self.args_with_ind: + for j in arg_with_ind.indices: + if j is not None: + contraction_indices[j].append(current_position) + current_position += 1 + return contraction_indices + + def get_mapping_for_index(self, ind) -> list[_IndPos]: + if ind >= self.number_of_contraction_indices: + raise ValueError("index value exceeding the index range") + positions: list[_IndPos] = [] + for i, arg_with_ind in enumerate(self.args_with_ind): + for j, arg_ind in enumerate(arg_with_ind.indices): + if ind == arg_ind: + positions.append(_IndPos(i, j)) + return positions + + def get_contraction_indices_to_ind_rel_pos(self) -> list[list[_IndPos]]: + contraction_indices: list[list[_IndPos]] = [[] for i in range(self.number_of_contraction_indices)] + for i, arg_with_ind in enumerate(self.args_with_ind): + for j, ind in enumerate(arg_with_ind.indices): + if ind is not None: + contraction_indices[ind].append(_IndPos(i, j)) + return contraction_indices + + def count_args_with_index(self, index: int) -> int: + """ + Count the number of arguments that have the given index. + """ + counter: int = 0 + for arg_with_ind in self.args_with_ind: + if index in arg_with_ind.indices: + counter += 1 + return counter + + def get_args_with_index(self, index: int) -> list[_ArgE]: + """ + Get a list of arguments having the given index. + """ + ret: list[_ArgE] = [i for i in self.args_with_ind if index in i.indices] + return ret + + @property + def number_of_diagonal_indices(self): + data = set() + for arg in self.args_with_ind: + data.update({i for i in arg.indices if i is not None and i < 0}) + return len(data) + + def track_permutation_start(self): + permutation = [] + perm_diag = [] + counter = 0 + counter2 = -1 + for arg_with_ind in self.args_with_ind: + perm = [] + for i in arg_with_ind.indices: + if i is not None: + if i < 0: + perm_diag.append(counter2) + counter2 -= 1 + continue + perm.append(counter) + counter += 1 + permutation.append(perm) + max_ind = max(max(i) if i else -1 for i in permutation) if permutation else -1 + perm_diag = [max_ind - i for i in perm_diag] + self._track_permutation = permutation + [perm_diag] + + def track_permutation_merge(self, destination: _ArgE, from_element: _ArgE): + index_destination = self.args_with_ind.index(destination) + index_element = self.args_with_ind.index(from_element) + self._track_permutation[index_destination].extend(self._track_permutation[index_element]) # type: ignore + self._track_permutation.pop(index_element) # type: ignore + + def get_absolute_free_range(self, arg: _ArgE) -> typing.Tuple[int, int]: + """ + Return the range of the free indices of the arg as absolute positions + among all free indices. + """ + counter = 0 + for arg_with_ind in self.args_with_ind: + number_free_indices = len([i for i in arg_with_ind.indices if i is None]) + if arg_with_ind == arg: + return counter, counter + number_free_indices + counter += number_free_indices + raise IndexError("argument not found") + + def get_absolute_range(self, arg: _ArgE) -> typing.Tuple[int, int]: + """ + Return the absolute range of indices for arg, disregarding dummy + indices. + """ + counter = 0 + for arg_with_ind in self.args_with_ind: + number_indices = len(arg_with_ind.indices) + if arg_with_ind == arg: + return counter, counter + number_indices + counter += number_indices + raise IndexError("argument not found") + + +def get_rank(expr): + if isinstance(expr, (MatrixExpr, MatrixElement)): + return 2 + if isinstance(expr, _CodegenArrayAbstract): + return len(expr.shape) + if isinstance(expr, NDimArray): + return expr.rank() + if isinstance(expr, Indexed): + return expr.rank + if isinstance(expr, IndexedBase): + shape = expr.shape + if shape is None: + return -1 + else: + return len(shape) + if hasattr(expr, "shape"): + return len(expr.shape) + return 0 + + +def _get_subrank(expr): + if isinstance(expr, _CodegenArrayAbstract): + return expr.subrank() + return get_rank(expr) + + +def _get_subranks(expr): + if isinstance(expr, _CodegenArrayAbstract): + return expr.subranks + else: + return [get_rank(expr)] + + +def get_shape(expr): + if hasattr(expr, "shape"): + return expr.shape + return () + + +def nest_permutation(expr): + if isinstance(expr, PermuteDims): + return expr.nest_permutation() + else: + return expr + + +def _array_tensor_product(*args, **kwargs): + return ArrayTensorProduct(*args, canonicalize=True, **kwargs) + + +def _array_contraction(expr, *contraction_indices, **kwargs): + return ArrayContraction(expr, *contraction_indices, canonicalize=True, **kwargs) + + +def _array_diagonal(expr, *diagonal_indices, **kwargs): + return ArrayDiagonal(expr, *diagonal_indices, canonicalize=True, **kwargs) + + +def _permute_dims(expr, permutation, **kwargs): + return PermuteDims(expr, permutation, canonicalize=True, **kwargs) + + +def _array_add(*args, **kwargs): + return ArrayAdd(*args, canonicalize=True, **kwargs) + + +def _get_array_element_or_slice(expr, indices): + return ArrayElement(expr, indices) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/arrayexpr_derivatives.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/arrayexpr_derivatives.py new file mode 100644 index 0000000000000000000000000000000000000000..ab44a6fbf715ac7f2b8c287dcc84a49289f2dd76 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/arrayexpr_derivatives.py @@ -0,0 +1,194 @@ +import operator +from functools import reduce, singledispatch + +from sympy.core.expr import Expr +from sympy.core.singleton import S +from sympy.matrices.expressions.hadamard import HadamardProduct +from sympy.matrices.expressions.inverse import Inverse +from sympy.matrices.expressions.matexpr import (MatrixExpr, MatrixSymbol) +from sympy.matrices.expressions.special import Identity, OneMatrix +from sympy.matrices.expressions.transpose import Transpose +from sympy.combinatorics.permutations import _af_invert +from sympy.matrices.expressions.applyfunc import ElementwiseApplyFunction +from sympy.tensor.array.expressions.array_expressions import ( + _ArrayExpr, ZeroArray, ArraySymbol, ArrayTensorProduct, ArrayAdd, + PermuteDims, ArrayDiagonal, ArrayElementwiseApplyFunc, get_rank, + get_shape, ArrayContraction, _array_tensor_product, _array_contraction, + _array_diagonal, _array_add, _permute_dims, Reshape) +from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array + + +@singledispatch +def array_derive(expr, x): + """ + Derivatives (gradients) for array expressions. + """ + raise NotImplementedError(f"not implemented for type {type(expr)}") + + +@array_derive.register(Expr) +def _(expr: Expr, x: _ArrayExpr): + return ZeroArray(*x.shape) + + +@array_derive.register(ArrayTensorProduct) +def _(expr: ArrayTensorProduct, x: Expr): + args = expr.args + addend_list = [] + for i, arg in enumerate(expr.args): + darg = array_derive(arg, x) + if darg == 0: + continue + args_prev = args[:i] + args_succ = args[i+1:] + shape_prev = reduce(operator.add, map(get_shape, args_prev), ()) + shape_succ = reduce(operator.add, map(get_shape, args_succ), ()) + addend = _array_tensor_product(*args_prev, darg, *args_succ) + tot1 = len(get_shape(x)) + tot2 = tot1 + len(shape_prev) + tot3 = tot2 + len(get_shape(arg)) + tot4 = tot3 + len(shape_succ) + perm = list(range(tot1, tot2)) + \ + list(range(tot1)) + list(range(tot2, tot3)) + \ + list(range(tot3, tot4)) + addend = _permute_dims(addend, _af_invert(perm)) + addend_list.append(addend) + if len(addend_list) == 1: + return addend_list[0] + elif len(addend_list) == 0: + return S.Zero + else: + return _array_add(*addend_list) + + +@array_derive.register(ArraySymbol) +def _(expr: ArraySymbol, x: _ArrayExpr): + if expr == x: + return _permute_dims( + ArrayTensorProduct.fromiter(Identity(i) for i in expr.shape), + [2*i for i in range(len(expr.shape))] + [2*i+1 for i in range(len(expr.shape))] + ) + return ZeroArray(*(x.shape + expr.shape)) + + +@array_derive.register(MatrixSymbol) +def _(expr: MatrixSymbol, x: _ArrayExpr): + m, n = expr.shape + if expr == x: + return _permute_dims( + _array_tensor_product(Identity(m), Identity(n)), + [0, 2, 1, 3] + ) + return ZeroArray(*(x.shape + expr.shape)) + + +@array_derive.register(Identity) +def _(expr: Identity, x: _ArrayExpr): + return ZeroArray(*(x.shape + expr.shape)) + + +@array_derive.register(OneMatrix) +def _(expr: OneMatrix, x: _ArrayExpr): + return ZeroArray(*(x.shape + expr.shape)) + + +@array_derive.register(Transpose) +def _(expr: Transpose, x: Expr): + # D(A.T, A) ==> (m,n,i,j) ==> D(A_ji, A_mn) = d_mj d_ni + # D(B.T, A) ==> (m,n,i,j) ==> D(B_ji, A_mn) + fd = array_derive(expr.arg, x) + return _permute_dims(fd, [0, 1, 3, 2]) + + +@array_derive.register(Inverse) +def _(expr: Inverse, x: Expr): + mat = expr.I + dexpr = array_derive(mat, x) + tp = _array_tensor_product(-expr, dexpr, expr) + mp = _array_contraction(tp, (1, 4), (5, 6)) + pp = _permute_dims(mp, [1, 2, 0, 3]) + return pp + + +@array_derive.register(ElementwiseApplyFunction) +def _(expr: ElementwiseApplyFunction, x: Expr): + assert get_rank(expr) == 2 + assert get_rank(x) == 2 + fdiff = expr._get_function_fdiff() + dexpr = array_derive(expr.expr, x) + tp = _array_tensor_product( + ElementwiseApplyFunction(fdiff, expr.expr), + dexpr + ) + td = _array_diagonal( + tp, (0, 4), (1, 5) + ) + return td + + +@array_derive.register(ArrayElementwiseApplyFunc) +def _(expr: ArrayElementwiseApplyFunc, x: Expr): + fdiff = expr._get_function_fdiff() + subexpr = expr.expr + dsubexpr = array_derive(subexpr, x) + tp = _array_tensor_product( + dsubexpr, + ArrayElementwiseApplyFunc(fdiff, subexpr) + ) + b = get_rank(x) + c = get_rank(expr) + diag_indices = [(b + i, b + c + i) for i in range(c)] + return _array_diagonal(tp, *diag_indices) + + +@array_derive.register(MatrixExpr) +def _(expr: MatrixExpr, x: Expr): + cg = convert_matrix_to_array(expr) + return array_derive(cg, x) + + +@array_derive.register(HadamardProduct) +def _(expr: HadamardProduct, x: Expr): + raise NotImplementedError() + + +@array_derive.register(ArrayContraction) +def _(expr: ArrayContraction, x: Expr): + fd = array_derive(expr.expr, x) + rank_x = len(get_shape(x)) + contraction_indices = expr.contraction_indices + new_contraction_indices = [tuple(j + rank_x for j in i) for i in contraction_indices] + return _array_contraction(fd, *new_contraction_indices) + + +@array_derive.register(ArrayDiagonal) +def _(expr: ArrayDiagonal, x: Expr): + dsubexpr = array_derive(expr.expr, x) + rank_x = len(get_shape(x)) + diag_indices = [[j + rank_x for j in i] for i in expr.diagonal_indices] + return _array_diagonal(dsubexpr, *diag_indices) + + +@array_derive.register(ArrayAdd) +def _(expr: ArrayAdd, x: Expr): + return _array_add(*[array_derive(arg, x) for arg in expr.args]) + + +@array_derive.register(PermuteDims) +def _(expr: PermuteDims, x: Expr): + de = array_derive(expr.expr, x) + perm = [0, 1] + [i + 2 for i in expr.permutation.array_form] + return _permute_dims(de, perm) + + +@array_derive.register(Reshape) +def _(expr: Reshape, x: Expr): + de = array_derive(expr.expr, x) + return Reshape(de, get_shape(x) + expr.shape) + + +def matrix_derive(expr, x): + from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix + ce = convert_matrix_to_array(expr) + dce = array_derive(ce, x) + return convert_array_to_matrix(dce).doit() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_indexed.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_indexed.py new file mode 100644 index 0000000000000000000000000000000000000000..1929c3401e131cca0a83080131ead9198b37bcbb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_indexed.py @@ -0,0 +1,12 @@ +from sympy.tensor.array.expressions import from_array_to_indexed +from sympy.utilities.decorator import deprecated + + +_conv_to_from_decorator = deprecated( + "module has been renamed by replacing 'conv_' with 'from_' in its name", + deprecated_since_version="1.11", + active_deprecations_target="deprecated-conv-array-expr-module-names", +) + + +convert_array_to_indexed = _conv_to_from_decorator(from_array_to_indexed.convert_array_to_indexed) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_matrix.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..2708e74aaa98d6ee38eae46d97d4483a546e0776 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_array_to_matrix.py @@ -0,0 +1,6 @@ +from sympy.tensor.array.expressions import from_array_to_matrix +from sympy.tensor.array.expressions.conv_array_to_indexed import _conv_to_from_decorator + +convert_array_to_matrix = _conv_to_from_decorator(from_array_to_matrix.convert_array_to_matrix) +_array2matrix = _conv_to_from_decorator(from_array_to_matrix._array2matrix) +_remove_trivial_dims = _conv_to_from_decorator(from_array_to_matrix._remove_trivial_dims) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_indexed_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_indexed_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..6058b31f20778834ea23a01553d594b7965eb6bb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_indexed_to_array.py @@ -0,0 +1,4 @@ +from sympy.tensor.array.expressions import from_indexed_to_array +from sympy.tensor.array.expressions.conv_array_to_indexed import _conv_to_from_decorator + +convert_indexed_to_array = _conv_to_from_decorator(from_indexed_to_array.convert_indexed_to_array) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_matrix_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_matrix_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..46469df60703c237527c0b2834235309640afe7c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/conv_matrix_to_array.py @@ -0,0 +1,4 @@ +from sympy.tensor.array.expressions import from_matrix_to_array +from sympy.tensor.array.expressions.conv_array_to_indexed import _conv_to_from_decorator + +convert_matrix_to_array = _conv_to_from_decorator(from_matrix_to_array.convert_matrix_to_array) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_indexed.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_indexed.py new file mode 100644 index 0000000000000000000000000000000000000000..9eb86e7cfbe31ebfe7c9649803d9cb5e34b98276 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_indexed.py @@ -0,0 +1,84 @@ +import collections.abc +import operator +from itertools import accumulate + +from sympy import Mul, Sum, Dummy, Add +from sympy.tensor.array.expressions import PermuteDims, ArrayAdd, ArrayElementwiseApplyFunc, Reshape +from sympy.tensor.array.expressions.array_expressions import ArrayTensorProduct, get_rank, ArrayContraction, \ + ArrayDiagonal, get_shape, _get_array_element_or_slice, _ArrayExpr +from sympy.tensor.array.expressions.utils import _apply_permutation_to_list + + +def convert_array_to_indexed(expr, indices): + return _ConvertArrayToIndexed().do_convert(expr, indices) + + +class _ConvertArrayToIndexed: + + def __init__(self): + self.count_dummies = 0 + + def do_convert(self, expr, indices): + if isinstance(expr, ArrayTensorProduct): + cumul = list(accumulate([0] + [get_rank(arg) for arg in expr.args])) + indices_grp = [indices[cumul[i]:cumul[i+1]] for i in range(len(expr.args))] + return Mul.fromiter(self.do_convert(arg, ind) for arg, ind in zip(expr.args, indices_grp)) + if isinstance(expr, ArrayContraction): + new_indices = [None for i in range(get_rank(expr.expr))] + limits = [] + bottom_shape = get_shape(expr.expr) + for contraction_index_grp in expr.contraction_indices: + d = Dummy(f"d{self.count_dummies}") + self.count_dummies += 1 + dim = bottom_shape[contraction_index_grp[0]] + limits.append((d, 0, dim-1)) + for i in contraction_index_grp: + new_indices[i] = d + j = 0 + for i in range(len(new_indices)): + if new_indices[i] is None: + new_indices[i] = indices[j] + j += 1 + newexpr = self.do_convert(expr.expr, new_indices) + return Sum(newexpr, *limits) + if isinstance(expr, ArrayDiagonal): + new_indices = [None for i in range(get_rank(expr.expr))] + ind_pos = expr._push_indices_down(expr.diagonal_indices, list(range(len(indices))), get_rank(expr)) + for i, index in zip(ind_pos, indices): + if isinstance(i, collections.abc.Iterable): + for j in i: + new_indices[j] = index + else: + new_indices[i] = index + newexpr = self.do_convert(expr.expr, new_indices) + return newexpr + if isinstance(expr, PermuteDims): + permuted_indices = _apply_permutation_to_list(expr.permutation, indices) + return self.do_convert(expr.expr, permuted_indices) + if isinstance(expr, ArrayAdd): + return Add.fromiter(self.do_convert(arg, indices) for arg in expr.args) + if isinstance(expr, _ArrayExpr): + return expr.__getitem__(tuple(indices)) + if isinstance(expr, ArrayElementwiseApplyFunc): + return expr.function(self.do_convert(expr.expr, indices)) + if isinstance(expr, Reshape): + shape_up = expr.shape + shape_down = get_shape(expr.expr) + cumul = list(accumulate([1] + list(reversed(shape_up)), operator.mul)) + one_index = Add.fromiter(i*s for i, s in zip(reversed(indices), cumul)) + dest_indices = [None for _ in shape_down] + c = 1 + for i, e in enumerate(reversed(shape_down)): + if c == 1: + if i == len(shape_down) - 1: + dest_indices[i] = one_index + else: + dest_indices[i] = one_index % e + elif i == len(shape_down) - 1: + dest_indices[i] = one_index // c + else: + dest_indices[i] = one_index // c % e + c *= e + dest_indices.reverse() + return self.do_convert(expr.expr, dest_indices) + return _get_array_element_or_slice(expr, indices) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_matrix.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..debfdd7eb5c4533996b3d72b55d679be3daf3afe --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_array_to_matrix.py @@ -0,0 +1,1004 @@ +from __future__ import annotations +import itertools +from collections import defaultdict +from typing import FrozenSet +from functools import singledispatch +from itertools import accumulate + +from sympy import MatMul, Basic, Wild, KroneckerProduct +from sympy.assumptions.ask import (Q, ask) +from sympy.core.mul import Mul +from sympy.core.singleton import S +from sympy.matrices.expressions.diagonal import DiagMatrix +from sympy.matrices.expressions.hadamard import hadamard_product, HadamardPower +from sympy.matrices.expressions.matexpr import MatrixExpr +from sympy.matrices.expressions.special import (Identity, ZeroMatrix, OneMatrix) +from sympy.matrices.expressions.trace import Trace +from sympy.matrices.expressions.transpose import Transpose +from sympy.combinatorics.permutations import _af_invert, Permutation +from sympy.matrices.matrixbase import MatrixBase +from sympy.matrices.expressions.applyfunc import ElementwiseApplyFunction +from sympy.matrices.expressions.matexpr import MatrixElement +from sympy.tensor.array.expressions.array_expressions import PermuteDims, ArrayDiagonal, \ + ArrayTensorProduct, OneArray, get_rank, _get_subrank, ZeroArray, ArrayContraction, \ + ArrayAdd, _CodegenArrayAbstract, get_shape, ArrayElementwiseApplyFunc, _ArrayExpr, _EditArrayContraction, _ArgE, \ + ArrayElement, _array_tensor_product, _array_contraction, _array_diagonal, _array_add, _permute_dims +from sympy.tensor.array.expressions.utils import _get_mapping_from_subranks + + +def _get_candidate_for_matmul_from_contraction(scan_indices: list[int | None], remaining_args: list[_ArgE]) -> tuple[_ArgE | None, bool, int]: + + scan_indices_int: list[int] = [i for i in scan_indices if i is not None] + if len(scan_indices_int) == 0: + return None, False, -1 + + transpose: bool = False + candidate: _ArgE | None = None + candidate_index: int = -1 + for arg_with_ind2 in remaining_args: + if not isinstance(arg_with_ind2.element, MatrixExpr): + continue + for index in scan_indices_int: + if candidate_index != -1 and candidate_index != index: + # A candidate index has already been selected, check + # repetitions only for that index: + continue + if index in arg_with_ind2.indices: + if set(arg_with_ind2.indices) == {index}: + # Index repeated twice in arg_with_ind2 + candidate = None + break + if candidate is None: + candidate = arg_with_ind2 + candidate_index = index + transpose = (index == arg_with_ind2.indices[1]) + else: + # Index repeated more than twice, break + candidate = None + break + return candidate, transpose, candidate_index + + +def _insert_candidate_into_editor(editor: _EditArrayContraction, arg_with_ind: _ArgE, candidate: _ArgE, transpose1: bool, transpose2: bool): + other = candidate.element + other_index: int | None + if transpose2: + other = Transpose(other) + other_index = candidate.indices[0] + else: + other_index = candidate.indices[1] + new_element = (Transpose(arg_with_ind.element) if transpose1 else arg_with_ind.element) * other + editor.args_with_ind.remove(candidate) + new_arge = _ArgE(new_element) + return new_arge, other_index + + +def _support_function_tp1_recognize(contraction_indices, args): + if len(contraction_indices) == 0: + return _a2m_tensor_product(*args) + + ac = _array_contraction(_array_tensor_product(*args), *contraction_indices) + editor = _EditArrayContraction(ac) + editor.track_permutation_start() + + while True: + flag_stop = True + for i, arg_with_ind in enumerate(editor.args_with_ind): + if not isinstance(arg_with_ind.element, MatrixExpr): + continue + + first_index = arg_with_ind.indices[0] + second_index = arg_with_ind.indices[1] + + first_frequency = editor.count_args_with_index(first_index) + second_frequency = editor.count_args_with_index(second_index) + + if first_index is not None and first_frequency == 1 and first_index == second_index: + flag_stop = False + arg_with_ind.element = Trace(arg_with_ind.element)._normalize() + arg_with_ind.indices = [] + break + + scan_indices = [] + if first_frequency == 2: + scan_indices.append(first_index) + if second_frequency == 2: + scan_indices.append(second_index) + + candidate, transpose, found_index = _get_candidate_for_matmul_from_contraction(scan_indices, editor.args_with_ind[i+1:]) + if candidate is not None: + flag_stop = False + editor.track_permutation_merge(arg_with_ind, candidate) + transpose1 = found_index == first_index + new_arge, other_index = _insert_candidate_into_editor(editor, arg_with_ind, candidate, transpose1, transpose) + if found_index == first_index: + new_arge.indices = [second_index, other_index] + else: + new_arge.indices = [first_index, other_index] + set_indices = set(new_arge.indices) + if len(set_indices) == 1 and set_indices != {None}: + # This is a trace: + new_arge.element = Trace(new_arge.element)._normalize() + new_arge.indices = [] + editor.args_with_ind[i] = new_arge + # TODO: is this break necessary? + break + + if flag_stop: + break + + editor.refresh_indices() + return editor.to_array_contraction() + + +def _find_trivial_matrices_rewrite(expr: ArrayTensorProduct): + # If there are matrices of trivial shape in the tensor product (i.e. shape + # (1, 1)), try to check if there is a suitable non-trivial MatMul where the + # expression can be inserted. + + # For example, if "a" has shape (1, 1) and "b" has shape (k, 1), the + # expressions "_array_tensor_product(a, b*b.T)" can be rewritten as + # "b*a*b.T" + + trivial_matrices = [] + pos: int | None = None # must be initialized else causes UnboundLocalError + first: MatrixExpr | None = None # may cause UnboundLocalError if not initialized + second: MatrixExpr | None = None # may cause UnboundLocalError if not initialized + removed: list[int] = [] + counter: int = 0 + args: list[Basic | None] = list(expr.args) + for i, arg in enumerate(expr.args): + if isinstance(arg, MatrixExpr): + if arg.shape == (1, 1): + trivial_matrices.append(arg) + args[i] = None + removed.extend([counter, counter+1]) + elif pos is None and isinstance(arg, MatMul): + margs = arg.args + for j, e in enumerate(margs): + if isinstance(e, MatrixExpr) and e.shape[1] == 1: + pos = i + first = MatMul.fromiter(margs[:j+1]) + second = MatMul.fromiter(margs[j+1:]) + break + counter += get_rank(arg) + if pos is None: + return expr, [] + args[pos] = (first*MatMul.fromiter(i for i in trivial_matrices)*second).doit() + return _array_tensor_product(*[i for i in args if i is not None]), removed + + +def _find_trivial_kronecker_products_broadcast(expr: ArrayTensorProduct): + newargs: list[Basic] = [] + removed = [] + count_dims = 0 + for arg in expr.args: + count_dims += get_rank(arg) + shape = get_shape(arg) + current_range = [count_dims-i for i in range(len(shape), 0, -1)] + if (shape == (1, 1) and len(newargs) > 0 and 1 not in get_shape(newargs[-1]) and + isinstance(newargs[-1], MatrixExpr) and isinstance(arg, MatrixExpr)): + # KroneckerProduct object allows the trick of broadcasting: + newargs[-1] = KroneckerProduct(newargs[-1], arg) + removed.extend(current_range) + elif 1 not in shape and len(newargs) > 0 and get_shape(newargs[-1]) == (1, 1): + # Broadcast: + newargs[-1] = KroneckerProduct(newargs[-1], arg) + prev_range = [i for i in range(min(current_range)) if i not in removed] + removed.extend(prev_range[-2:]) + else: + newargs.append(arg) + return _array_tensor_product(*newargs), removed + + +@singledispatch +def _array2matrix(expr): + return expr + + +@_array2matrix.register(ZeroArray) +def _(expr: ZeroArray): + if get_rank(expr) == 2: + return ZeroMatrix(*expr.shape) + else: + return expr + + +@_array2matrix.register(ArrayTensorProduct) +def _(expr: ArrayTensorProduct): + return _a2m_tensor_product(*[_array2matrix(arg) for arg in expr.args]) + + +@_array2matrix.register(ArrayContraction) +def _(expr: ArrayContraction): + expr = expr.flatten_contraction_of_diagonal() + expr = identify_removable_identity_matrices(expr) + expr = expr.split_multiple_contractions() + expr = identify_hadamard_products(expr) + if not isinstance(expr, ArrayContraction): + return _array2matrix(expr) + subexpr = expr.expr + contraction_indices: tuple[tuple[int]] = expr.contraction_indices + if contraction_indices == ((0,), (1,)) or ( + contraction_indices == ((0,),) and subexpr.shape[1] == 1 + ) or ( + contraction_indices == ((1,),) and subexpr.shape[0] == 1 + ): + shape = subexpr.shape + subexpr = _array2matrix(subexpr) + if isinstance(subexpr, MatrixExpr): + return OneMatrix(1, shape[0])*subexpr*OneMatrix(shape[1], 1) + if isinstance(subexpr, ArrayTensorProduct): + newexpr = _array_contraction(_array2matrix(subexpr), *contraction_indices) + contraction_indices = newexpr.contraction_indices + if any(i > 2 for i in newexpr.subranks): + addends = _array_add(*[_a2m_tensor_product(*j) for j in itertools.product(*[i.args if isinstance(i, + ArrayAdd) else [i] for i in expr.expr.args])]) + newexpr = _array_contraction(addends, *contraction_indices) + if isinstance(newexpr, ArrayAdd): + ret = _array2matrix(newexpr) + return ret + assert isinstance(newexpr, ArrayContraction) + ret = _support_function_tp1_recognize(contraction_indices, list(newexpr.expr.args)) + return ret + elif not isinstance(subexpr, _CodegenArrayAbstract): + ret = _array2matrix(subexpr) + if isinstance(ret, MatrixExpr): + assert expr.contraction_indices == ((0, 1),) + return _a2m_trace(ret) + else: + return _array_contraction(ret, *expr.contraction_indices) + + +@_array2matrix.register(ArrayDiagonal) +def _(expr: ArrayDiagonal): + pexpr = _array_diagonal(_array2matrix(expr.expr), *expr.diagonal_indices) + pexpr = identify_hadamard_products(pexpr) + if isinstance(pexpr, ArrayDiagonal): + pexpr = _array_diag2contr_diagmatrix(pexpr) + if expr == pexpr: + return expr + return _array2matrix(pexpr) + + +@_array2matrix.register(PermuteDims) +def _(expr: PermuteDims): + if expr.permutation.array_form == [1, 0]: + return _a2m_transpose(_array2matrix(expr.expr)) + elif isinstance(expr.expr, ArrayTensorProduct): + ranks = expr.expr.subranks + inv_permutation = expr.permutation**(-1) + newrange = [inv_permutation(i) for i in range(sum(ranks))] + newpos = [] + counter = 0 + for rank in ranks: + newpos.append(newrange[counter:counter+rank]) + counter += rank + newargs = [] + newperm = [] + scalars = [] + for pos, arg in zip(newpos, expr.expr.args): + if len(pos) == 0: + scalars.append(_array2matrix(arg)) + elif pos == sorted(pos): + newargs.append((_array2matrix(arg), pos[0])) + newperm.extend(pos) + elif len(pos) == 2: + newargs.append((_a2m_transpose(_array2matrix(arg)), pos[0])) + newperm.extend(reversed(pos)) + else: + raise NotImplementedError() + newargs = [i[0] for i in newargs] + return _permute_dims(_a2m_tensor_product(*scalars, *newargs), _af_invert(newperm)) + elif isinstance(expr.expr, ArrayContraction): + mat_mul_lines = _array2matrix(expr.expr) + if not isinstance(mat_mul_lines, ArrayTensorProduct): + return _permute_dims(mat_mul_lines, expr.permutation) + # TODO: this assumes that all arguments are matrices, it may not be the case: + permutation = Permutation(2*len(mat_mul_lines.args)-1)*expr.permutation + permuted = [permutation(i) for i in range(2*len(mat_mul_lines.args))] + args_array = [None for i in mat_mul_lines.args] + for i in range(len(mat_mul_lines.args)): + p1 = permuted[2*i] + p2 = permuted[2*i+1] + if p1 // 2 != p2 // 2: + return _permute_dims(mat_mul_lines, permutation) + if p1 > p2: + args_array[i] = _a2m_transpose(mat_mul_lines.args[p1 // 2]) + else: + args_array[i] = mat_mul_lines.args[p1 // 2] + return _a2m_tensor_product(*args_array) + else: + return expr + + +@_array2matrix.register(ArrayAdd) +def _(expr: ArrayAdd): + addends = [_array2matrix(arg) for arg in expr.args] + return _a2m_add(*addends) + + +@_array2matrix.register(ArrayElementwiseApplyFunc) +def _(expr: ArrayElementwiseApplyFunc): + subexpr = _array2matrix(expr.expr) + if isinstance(subexpr, MatrixExpr): + if subexpr.shape != (1, 1): + d = expr.function.bound_symbols[0] + w = Wild("w", exclude=[d]) + p = Wild("p", exclude=[d]) + m = expr.function.expr.match(w*d**p) + if m is not None: + return m[w]*HadamardPower(subexpr, m[p]) + return ElementwiseApplyFunction(expr.function, subexpr) + else: + return ArrayElementwiseApplyFunc(expr.function, subexpr) + + +@_array2matrix.register(ArrayElement) +def _(expr: ArrayElement): + ret = _array2matrix(expr.name) + if isinstance(ret, MatrixExpr): + return MatrixElement(ret, *expr.indices) + return ArrayElement(ret, expr.indices) + + +@singledispatch +def _remove_trivial_dims(expr): + return expr, [] + + +@_remove_trivial_dims.register(ArrayTensorProduct) +def _(expr: ArrayTensorProduct): + # Recognize expressions like [x, y] with shape (k, 1, k, 1) as `x*y.T`. + # The matrix expression has to be equivalent to the tensor product of the + # matrices, with trivial dimensions (i.e. dim=1) dropped. + # That is, add contractions over trivial dimensions: + + removed = [] + newargs = [] + cumul = list(accumulate([0] + [get_rank(arg) for arg in expr.args])) + pending = None + prev_i = None + for i, arg in enumerate(expr.args): + current_range = list(range(cumul[i], cumul[i+1])) + if isinstance(arg, OneArray): + removed.extend(current_range) + continue + if not isinstance(arg, (MatrixExpr, MatrixBase)): + rarg, rem = _remove_trivial_dims(arg) + removed.extend(rem) + newargs.append(rarg) + continue + elif getattr(arg, "is_Identity", False) and arg.shape == (1, 1): + if arg.shape == (1, 1): + # Ignore identity matrices of shape (1, 1) - they are equivalent to scalar 1. + removed.extend(current_range) + continue + elif arg.shape == (1, 1): + arg, _ = _remove_trivial_dims(arg) + # Matrix is equivalent to scalar: + if len(newargs) == 0: + newargs.append(arg) + elif 1 in get_shape(newargs[-1]): + if newargs[-1].shape[1] == 1: + newargs[-1] = newargs[-1]*arg + else: + newargs[-1] = arg*newargs[-1] + removed.extend(current_range) + else: + newargs.append(arg) + elif 1 in arg.shape: + k = [i for i in arg.shape if i != 1][0] + if pending is None: + pending = k + prev_i = i + newargs.append(arg) + elif pending == k: + prev = newargs[-1] + if prev.shape[0] == 1: + d1 = cumul[prev_i] # type: ignore + prev = _a2m_transpose(prev) + else: + d1 = cumul[prev_i] + 1 # type: ignore + if arg.shape[1] == 1: + d2 = cumul[i] + 1 + arg = _a2m_transpose(arg) + else: + d2 = cumul[i] + newargs[-1] = prev*arg + pending = None + removed.extend([d1, d2]) + else: + newargs.append(arg) + pending = k + prev_i = i + else: + newargs.append(arg) + pending = None + newexpr, newremoved = _a2m_tensor_product(*newargs), sorted(removed) + if isinstance(newexpr, ArrayTensorProduct): + newexpr, newremoved2 = _find_trivial_matrices_rewrite(newexpr) + newremoved = _combine_removed(-1, newremoved, newremoved2) + if isinstance(newexpr, ArrayTensorProduct): + newexpr, newremoved2 = _find_trivial_kronecker_products_broadcast(newexpr) + newremoved = _combine_removed(-1, newremoved, newremoved2) + return newexpr, newremoved + + +@_remove_trivial_dims.register(ArrayAdd) +def _(expr: ArrayAdd): + rec = [_remove_trivial_dims(arg) for arg in expr.args] + newargs, removed = zip(*rec) + if len({get_shape(i) for i in newargs}) > 1: + return expr, [] + if len(removed) == 0: + return expr, removed + removed1 = removed[0] + return _a2m_add(*newargs), removed1 + + +@_remove_trivial_dims.register(PermuteDims) +def _(expr: PermuteDims): + subexpr, subremoved = _remove_trivial_dims(expr.expr) + p = expr.permutation.array_form + pinv = _af_invert(expr.permutation.array_form) + shift = list(accumulate([1 if i in subremoved else 0 for i in range(len(p))])) + premoved = [pinv[i] for i in subremoved] + p2 = [e - shift[e] for e in p if e not in subremoved] + # TODO: check if subremoved should be permuted as well... + newexpr = _permute_dims(subexpr, p2) + premoved = sorted(premoved) + if newexpr != expr: + newexpr, removed2 = _remove_trivial_dims(_array2matrix(newexpr)) + premoved = _combine_removed(-1, premoved, removed2) + return newexpr, premoved + + +@_remove_trivial_dims.register(ArrayContraction) +def _(expr: ArrayContraction): + new_expr, removed0 = _array_contraction_to_diagonal_multiple_identity(expr) + if new_expr != expr: + new_expr2, removed1 = _remove_trivial_dims(_array2matrix(new_expr)) + removed = _combine_removed(-1, removed0, removed1) + return new_expr2, removed + rank1 = get_rank(expr) + expr, removed1 = remove_identity_matrices(expr) + if not isinstance(expr, ArrayContraction): + expr2, removed2 = _remove_trivial_dims(expr) + return expr2, _combine_removed(rank1, removed1, removed2) + newexpr, removed2 = _remove_trivial_dims(expr.expr) + shifts = list(accumulate([1 if i in removed2 else 0 for i in range(get_rank(expr.expr))])) + new_contraction_indices = [tuple(j for j in i if j not in removed2) for i in expr.contraction_indices] + # Remove possible empty tuples "()": + new_contraction_indices = [i for i in new_contraction_indices if len(i) > 0] + contraction_indices_flat = [j for i in expr.contraction_indices for j in i] + removed2 = [i for i in removed2 if i not in contraction_indices_flat] + new_contraction_indices = [tuple(j - shifts[j] for j in i) for i in new_contraction_indices] + # Shift removed2: + removed2 = ArrayContraction._push_indices_up(expr.contraction_indices, removed2) + removed = _combine_removed(rank1, removed1, removed2) + return _array_contraction(newexpr, *new_contraction_indices), list(removed) + + +def _remove_diagonalized_identity_matrices(expr: ArrayDiagonal): + assert isinstance(expr, ArrayDiagonal) + editor = _EditArrayContraction(expr) + mapping = {i: {j for j in editor.args_with_ind if i in j.indices} for i in range(-1, -1-editor.number_of_diagonal_indices, -1)} + removed = [] + counter: int = 0 + for i, arg_with_ind in enumerate(editor.args_with_ind): + counter += len(arg_with_ind.indices) + if isinstance(arg_with_ind.element, Identity): + if None in arg_with_ind.indices and any(i is not None and (i < 0) == True for i in arg_with_ind.indices): + diag_ind = [j for j in arg_with_ind.indices if j is not None][0] + other = [j for j in mapping[diag_ind] if j != arg_with_ind][0] + if not isinstance(other.element, MatrixExpr): + continue + if 1 not in other.element.shape: + continue + if None not in other.indices: + continue + editor.args_with_ind[i].element = None + none_index = other.indices.index(None) + other.element = DiagMatrix(other.element) + other_range = editor.get_absolute_range(other) + removed.extend([other_range[0] + none_index]) + editor.args_with_ind = [i for i in editor.args_with_ind if i.element is not None] + removed = ArrayDiagonal._push_indices_up(expr.diagonal_indices, removed, get_rank(expr.expr)) + return editor.to_array_contraction(), removed + + +@_remove_trivial_dims.register(ArrayDiagonal) +def _(expr: ArrayDiagonal): + newexpr, removed = _remove_trivial_dims(expr.expr) + shifts = list(accumulate([0] + [1 if i in removed else 0 for i in range(get_rank(expr.expr))])) + new_diag_indices_map = {i: tuple(j for j in i if j not in removed) for i in expr.diagonal_indices} + for old_diag_tuple, new_diag_tuple in new_diag_indices_map.items(): + if len(new_diag_tuple) == 1: + removed = [i for i in removed if i not in old_diag_tuple] + new_diag_indices = [tuple(j - shifts[j] for j in i) for i in new_diag_indices_map.values()] + rank = get_rank(expr.expr) + removed = ArrayDiagonal._push_indices_up(expr.diagonal_indices, removed, rank) + removed = sorted(set(removed)) + # If there are single axes to diagonalize remaining, it means that their + # corresponding dimension has been removed, they no longer need diagonalization: + new_diag_indices = [i for i in new_diag_indices if len(i) > 0] + if len(new_diag_indices) > 0: + newexpr2 = _array_diagonal(newexpr, *new_diag_indices, allow_trivial_diags=True) + else: + newexpr2 = newexpr + if isinstance(newexpr2, ArrayDiagonal): + newexpr3, removed2 = _remove_diagonalized_identity_matrices(newexpr2) + removed = _combine_removed(-1, removed, removed2) + return newexpr3, removed + else: + return newexpr2, removed + + +@_remove_trivial_dims.register(ElementwiseApplyFunction) +def _(expr: ElementwiseApplyFunction): + subexpr, removed = _remove_trivial_dims(expr.expr) + if subexpr.shape == (1, 1): + # TODO: move this to ElementwiseApplyFunction + return expr.function(subexpr), removed + [0, 1] + return ElementwiseApplyFunction(expr.function, subexpr), [] + + +@_remove_trivial_dims.register(ArrayElementwiseApplyFunc) +def _(expr: ArrayElementwiseApplyFunc): + subexpr, removed = _remove_trivial_dims(expr.expr) + return ArrayElementwiseApplyFunc(expr.function, subexpr), removed + + +def convert_array_to_matrix(expr): + r""" + Recognize matrix expressions in codegen objects. + + If more than one matrix multiplication line have been detected, return a + list with the matrix expressions. + + Examples + ======== + + >>> from sympy.tensor.array.expressions.from_indexed_to_array import convert_indexed_to_array + >>> from sympy.tensor.array import tensorcontraction, tensorproduct + >>> from sympy import MatrixSymbol, Sum + >>> from sympy.abc import i, j, k, l, N + >>> from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array + >>> from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix + >>> A = MatrixSymbol("A", N, N) + >>> B = MatrixSymbol("B", N, N) + >>> C = MatrixSymbol("C", N, N) + >>> D = MatrixSymbol("D", N, N) + + >>> expr = Sum(A[i, j]*B[j, k], (j, 0, N-1)) + >>> cg = convert_indexed_to_array(expr) + >>> convert_array_to_matrix(cg) + A*B + >>> cg = convert_indexed_to_array(expr, first_indices=[k]) + >>> convert_array_to_matrix(cg) + B.T*A.T + + Transposition is detected: + + >>> expr = Sum(A[j, i]*B[j, k], (j, 0, N-1)) + >>> cg = convert_indexed_to_array(expr) + >>> convert_array_to_matrix(cg) + A.T*B + >>> cg = convert_indexed_to_array(expr, first_indices=[k]) + >>> convert_array_to_matrix(cg) + B.T*A + + Detect the trace: + + >>> expr = Sum(A[i, i], (i, 0, N-1)) + >>> cg = convert_indexed_to_array(expr) + >>> convert_array_to_matrix(cg) + Trace(A) + + Recognize some more complex traces: + + >>> expr = Sum(A[i, j]*B[j, i], (i, 0, N-1), (j, 0, N-1)) + >>> cg = convert_indexed_to_array(expr) + >>> convert_array_to_matrix(cg) + Trace(A*B) + + More complicated expressions: + + >>> expr = Sum(A[i, j]*B[k, j]*A[l, k], (j, 0, N-1), (k, 0, N-1)) + >>> cg = convert_indexed_to_array(expr) + >>> convert_array_to_matrix(cg) + A*B.T*A.T + + Expressions constructed from matrix expressions do not contain literal + indices, the positions of free indices are returned instead: + + >>> expr = A*B + >>> cg = convert_matrix_to_array(expr) + >>> convert_array_to_matrix(cg) + A*B + + If more than one line of matrix multiplications is detected, return + separate matrix multiplication factors embedded in a tensor product object: + + >>> cg = tensorcontraction(tensorproduct(A, B, C, D), (1, 2), (5, 6)) + >>> convert_array_to_matrix(cg) + ArrayTensorProduct(A*B, C*D) + + The two lines have free indices at axes 0, 3 and 4, 7, respectively. + """ + rec = _array2matrix(expr) + rec, removed = _remove_trivial_dims(rec) + return rec + + +def _array_diag2contr_diagmatrix(expr: ArrayDiagonal): + if isinstance(expr.expr, ArrayTensorProduct): + args = list(expr.expr.args) + diag_indices = list(expr.diagonal_indices) + mapping = _get_mapping_from_subranks([_get_subrank(arg) for arg in args]) + tuple_links = [[mapping[j] for j in i] for i in diag_indices] + contr_indices = [] + total_rank = get_rank(expr) + replaced = [False for arg in args] + for i, (abs_pos, rel_pos) in enumerate(zip(diag_indices, tuple_links)): + if len(abs_pos) != 2: + continue + (pos1_outer, pos1_inner), (pos2_outer, pos2_inner) = rel_pos + arg1 = args[pos1_outer] + arg2 = args[pos2_outer] + if get_rank(arg1) != 2 or get_rank(arg2) != 2: + if replaced[pos1_outer]: + diag_indices[i] = None + if replaced[pos2_outer]: + diag_indices[i] = None + continue + pos1_in2 = 1 - pos1_inner + pos2_in2 = 1 - pos2_inner + if arg1.shape[pos1_in2] == 1: + if arg1.shape[pos1_inner] != 1: + darg1 = DiagMatrix(arg1) + else: + darg1 = arg1 + args.append(darg1) + contr_indices.append(((pos2_outer, pos2_inner), (len(args)-1, pos1_inner))) + total_rank += 1 + diag_indices[i] = None + args[pos1_outer] = OneArray(arg1.shape[pos1_in2]) + replaced[pos1_outer] = True + elif arg2.shape[pos2_in2] == 1: + if arg2.shape[pos2_inner] != 1: + darg2 = DiagMatrix(arg2) + else: + darg2 = arg2 + args.append(darg2) + contr_indices.append(((pos1_outer, pos1_inner), (len(args)-1, pos2_inner))) + total_rank += 1 + diag_indices[i] = None + args[pos2_outer] = OneArray(arg2.shape[pos2_in2]) + replaced[pos2_outer] = True + diag_indices_new = [i for i in diag_indices if i is not None] + cumul = list(accumulate([0] + [get_rank(arg) for arg in args])) + contr_indices2 = [tuple(cumul[a] + b for a, b in i) for i in contr_indices] + tc = _array_contraction( + _array_tensor_product(*args), *contr_indices2 + ) + td = _array_diagonal(tc, *diag_indices_new) + return td + return expr + + +def _a2m_mul(*args): + if not any(isinstance(i, _CodegenArrayAbstract) for i in args): + from sympy.matrices.expressions.matmul import MatMul + return MatMul(*args).doit() + else: + return _array_contraction( + _array_tensor_product(*args), + *[(2*i-1, 2*i) for i in range(1, len(args))] + ) + + +def _a2m_tensor_product(*args): + scalars = [] + arrays = [] + for arg in args: + if isinstance(arg, (MatrixExpr, _ArrayExpr, _CodegenArrayAbstract)): + arrays.append(arg) + else: + scalars.append(arg) + scalar = Mul.fromiter(scalars) + if len(arrays) == 0: + return scalar + if scalar != 1: + if isinstance(arrays[0], _CodegenArrayAbstract): + arrays = [scalar] + arrays + else: + arrays[0] *= scalar + return _array_tensor_product(*arrays) + + +def _a2m_add(*args): + if not any(isinstance(i, _CodegenArrayAbstract) for i in args): + from sympy.matrices.expressions.matadd import MatAdd + return MatAdd(*args).doit() + else: + return _array_add(*args) + + +def _a2m_trace(arg): + if isinstance(arg, _CodegenArrayAbstract): + return _array_contraction(arg, (0, 1)) + else: + from sympy.matrices.expressions.trace import Trace + return Trace(arg) + + +def _a2m_transpose(arg): + if isinstance(arg, _CodegenArrayAbstract): + return _permute_dims(arg, [1, 0]) + else: + from sympy.matrices.expressions.transpose import Transpose + return Transpose(arg).doit() + + +def identify_hadamard_products(expr: ArrayContraction | ArrayDiagonal): + + editor: _EditArrayContraction = _EditArrayContraction(expr) + + map_contr_to_args: dict[FrozenSet, list[_ArgE]] = defaultdict(list) + map_ind_to_inds: dict[int | None, int] = defaultdict(int) + for arg_with_ind in editor.args_with_ind: + for ind in arg_with_ind.indices: + map_ind_to_inds[ind] += 1 + if None in arg_with_ind.indices: + continue + map_contr_to_args[frozenset(arg_with_ind.indices)].append(arg_with_ind) + + k: FrozenSet[int] + v: list[_ArgE] + for k, v in map_contr_to_args.items(): + make_trace: bool = False + if len(k) == 1 and next(iter(k)) >= 0 and sum(next(iter(k)) in i for i in map_contr_to_args) == 1: + # This is a trace: the arguments are fully contracted with only one + # index, and the index isn't used anywhere else: + make_trace = True + first_element = S.One + elif len(k) != 2: + # Hadamard product only defined for matrices: + continue + if len(v) == 1: + # Hadamard product with a single argument makes no sense: + continue + for ind in k: + if map_ind_to_inds[ind] <= 2: + # There is no other contraction, skip: + continue + + def check_transpose(x): + x = [i if i >= 0 else -1-i for i in x] + return x == sorted(x) + + # Check if expression is a trace: + if all(map_ind_to_inds[j] == len(v) and j >= 0 for j in k) and all(j >= 0 for j in k): + # This is a trace + make_trace = True + first_element = v[0].element + if not check_transpose(v[0].indices): + first_element = first_element.T # type: ignore + hadamard_factors = v[1:] + else: + hadamard_factors = v + + # This is a Hadamard product: + + hp = hadamard_product(*[i.element if check_transpose(i.indices) else Transpose(i.element) for i in hadamard_factors]) + hp_indices = v[0].indices + if not check_transpose(hadamard_factors[0].indices): + hp_indices = list(reversed(hp_indices)) + if make_trace: + hp = Trace(first_element*hp.T)._normalize() + hp_indices = [] + editor.insert_after(v[0], _ArgE(hp, hp_indices)) + for i in v: + editor.args_with_ind.remove(i) + + return editor.to_array_contraction() + + +def identify_removable_identity_matrices(expr): + editor = _EditArrayContraction(expr) + + flag = True + while flag: + flag = False + for arg_with_ind in editor.args_with_ind: + if isinstance(arg_with_ind.element, Identity): + k = arg_with_ind.element.shape[0] + # Candidate for removal: + if arg_with_ind.indices == [None, None]: + # Free identity matrix, will be cleared by _remove_trivial_dims: + continue + elif None in arg_with_ind.indices: + ind = [j for j in arg_with_ind.indices if j is not None][0] + counted = editor.count_args_with_index(ind) + if counted == 1: + # Identity matrix contracted only on one index with itself, + # transform to a OneArray(k) element: + editor.insert_after(arg_with_ind, OneArray(k)) + editor.args_with_ind.remove(arg_with_ind) + flag = True + break + elif counted > 2: + # Case counted = 2 is a matrix multiplication by identity matrix, skip it. + # Case counted > 2 is a multiple contraction, + # this is a case where the contraction becomes a diagonalization if the + # identity matrix is dropped. + continue + elif arg_with_ind.indices[0] == arg_with_ind.indices[1]: + ind = arg_with_ind.indices[0] + counted = editor.count_args_with_index(ind) + if counted > 1: + editor.args_with_ind.remove(arg_with_ind) + flag = True + break + else: + # This is a trace, skip it as it will be recognized somewhere else: + pass + elif ask(Q.diagonal(arg_with_ind.element)): + if arg_with_ind.indices == [None, None]: + continue + elif None in arg_with_ind.indices: + pass + elif arg_with_ind.indices[0] == arg_with_ind.indices[1]: + ind = arg_with_ind.indices[0] + counted = editor.count_args_with_index(ind) + if counted == 3: + # A_ai B_bi D_ii ==> A_ai D_ij B_bj + ind_new = editor.get_new_contraction_index() + other_args = [j for j in editor.args_with_ind if j != arg_with_ind] + other_args[1].indices = [ind_new if j == ind else j for j in other_args[1].indices] + arg_with_ind.indices = [ind, ind_new] + flag = True + break + + return editor.to_array_contraction() + + +def remove_identity_matrices(expr: ArrayContraction): + editor = _EditArrayContraction(expr) + removed: list[int] = [] + + permutation_map = {} + + free_indices = list(accumulate([0] + [sum(i is None for i in arg.indices) for arg in editor.args_with_ind])) + free_map = dict(zip(editor.args_with_ind, free_indices[:-1])) + + update_pairs = {} + + for ind in range(editor.number_of_contraction_indices): + args = editor.get_args_with_index(ind) + identity_matrices = [i for i in args if isinstance(i.element, Identity)] + number_identity_matrices = len(identity_matrices) + # If the contraction involves a non-identity matrix and multiple identity matrices: + if number_identity_matrices != len(args) - 1 or number_identity_matrices == 0: + continue + # Get the non-identity element: + non_identity = [i for i in args if not isinstance(i.element, Identity)][0] + # Check that all identity matrices have at least one free index + # (otherwise they would be contractions to some other elements) + if any(None not in i.indices for i in identity_matrices): + continue + # Mark the identity matrices for removal: + for i in identity_matrices: + i.element = None + removed.extend(range(free_map[i], free_map[i] + len([j for j in i.indices if j is None]))) + last_removed = removed.pop(-1) + update_pairs[last_removed, ind] = non_identity.indices[:] + # Remove the indices from the non-identity matrix, as the contraction + # no longer exists: + non_identity.indices = [None if i == ind else i for i in non_identity.indices] + + removed.sort() + + shifts = list(accumulate([1 if i in removed else 0 for i in range(get_rank(expr))])) + for (last_removed, ind), non_identity_indices in update_pairs.items(): + pos = [free_map[non_identity] + i for i, e in enumerate(non_identity_indices) if e == ind] + assert len(pos) == 1 + for j in pos: + permutation_map[j] = last_removed + + editor.args_with_ind = [i for i in editor.args_with_ind if i.element is not None] + ret_expr = editor.to_array_contraction() + permutation = [] + counter = 0 + counter2 = 0 + for j in range(get_rank(expr)): + if j in removed: + continue + if counter2 in permutation_map: + target = permutation_map[counter2] + permutation.append(target - shifts[target]) + counter2 += 1 + else: + while counter in permutation_map.values(): + counter += 1 + permutation.append(counter) + counter += 1 + counter2 += 1 + ret_expr2 = _permute_dims(ret_expr, _af_invert(permutation)) + return ret_expr2, removed + + +def _combine_removed(dim: int, removed1: list[int], removed2: list[int]) -> list[int]: + # Concatenate two axis removal operations as performed by + # _remove_trivial_dims, + removed1 = sorted(removed1) + removed2 = sorted(removed2) + i = 0 + j = 0 + removed = [] + while True: + if j >= len(removed2): + while i < len(removed1): + removed.append(removed1[i]) + i += 1 + break + elif i < len(removed1) and removed1[i] <= i + removed2[j]: + removed.append(removed1[i]) + i += 1 + else: + removed.append(i + removed2[j]) + j += 1 + return removed + + +def _array_contraction_to_diagonal_multiple_identity(expr: ArrayContraction): + editor = _EditArrayContraction(expr) + editor.track_permutation_start() + removed: list[int] = [] + diag_index_counter: int = 0 + for i in range(editor.number_of_contraction_indices): + identities = [] + args = [] + for j, arg in enumerate(editor.args_with_ind): + if i not in arg.indices: + continue + if isinstance(arg.element, Identity): + identities.append(arg) + else: + args.append(arg) + if len(identities) == 0: + continue + if len(args) + len(identities) < 3: + continue + new_diag_ind = -1 - diag_index_counter + diag_index_counter += 1 + # Variable "flag" to control whether to skip this contraction set: + flag: bool = True + for i1, id1 in enumerate(identities): + if None not in id1.indices: + flag = True + break + free_pos = list(range(*editor.get_absolute_free_range(id1)))[0] + editor._track_permutation[-1].append(free_pos) # type: ignore + id1.element = None + flag = False + break + if flag: + continue + for arg in identities[:i1] + identities[i1+1:]: + arg.element = None + removed.extend(range(*editor.get_absolute_free_range(arg))) + for arg in args: + arg.indices = [new_diag_ind if j == i else j for j in arg.indices] + for j, e in enumerate(editor.args_with_ind): + if e.element is None: + editor._track_permutation[j] = None # type: ignore + editor._track_permutation = [i for i in editor._track_permutation if i is not None] # type: ignore + # Renumber permutation array form in order to deal with deleted positions: + remap = {e: i for i, e in enumerate(sorted({k for j in editor._track_permutation for k in j}))} + editor._track_permutation = [[remap[j] for j in i] for i in editor._track_permutation] + editor.args_with_ind = [i for i in editor.args_with_ind if i.element is not None] + new_expr = editor.to_array_contraction() + return new_expr, removed diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_indexed_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_indexed_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..c219a205c4305bd7070e5117978146224521c58c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_indexed_to_array.py @@ -0,0 +1,257 @@ +from collections import defaultdict + +from sympy import Function +from sympy.combinatorics.permutations import _af_invert +from sympy.concrete.summations import Sum +from sympy.core.add import Add +from sympy.core.mul import Mul +from sympy.core.numbers import Integer +from sympy.core.power import Pow +from sympy.core.sorting import default_sort_key +from sympy.functions.special.tensor_functions import KroneckerDelta +from sympy.tensor.array.expressions import ArrayElementwiseApplyFunc +from sympy.tensor.indexed import (Indexed, IndexedBase) +from sympy.combinatorics import Permutation +from sympy.matrices.expressions.matexpr import MatrixElement +from sympy.tensor.array.expressions.array_expressions import ArrayDiagonal, \ + get_shape, ArrayElement, _array_tensor_product, _array_diagonal, _array_contraction, _array_add, \ + _permute_dims, OneArray, ArrayAdd +from sympy.tensor.array.expressions.utils import _get_argindex, _get_diagonal_indices + + +def convert_indexed_to_array(expr, first_indices=None): + r""" + Parse indexed expression into a form useful for code generation. + + Examples + ======== + + >>> from sympy.tensor.array.expressions.from_indexed_to_array import convert_indexed_to_array + >>> from sympy import MatrixSymbol, Sum, symbols + + >>> i, j, k, d = symbols("i j k d") + >>> M = MatrixSymbol("M", d, d) + >>> N = MatrixSymbol("N", d, d) + + Recognize the trace in summation form: + + >>> expr = Sum(M[i, i], (i, 0, d-1)) + >>> convert_indexed_to_array(expr) + ArrayContraction(M, (0, 1)) + + Recognize the extraction of the diagonal by using the same index `i` on + both axes of the matrix: + + >>> expr = M[i, i] + >>> convert_indexed_to_array(expr) + ArrayDiagonal(M, (0, 1)) + + This function can help perform the transformation expressed in two + different mathematical notations as: + + `\sum_{j=0}^{N-1} A_{i,j} B_{j,k} \Longrightarrow \mathbf{A}\cdot \mathbf{B}` + + Recognize the matrix multiplication in summation form: + + >>> expr = Sum(M[i, j]*N[j, k], (j, 0, d-1)) + >>> convert_indexed_to_array(expr) + ArrayContraction(ArrayTensorProduct(M, N), (1, 2)) + + Specify that ``k`` has to be the starting index: + + >>> convert_indexed_to_array(expr, first_indices=[k]) + ArrayContraction(ArrayTensorProduct(N, M), (0, 3)) + """ + + result, indices = _convert_indexed_to_array(expr) + + if any(isinstance(i, (int, Integer)) for i in indices): + result = ArrayElement(result, indices) + indices = [] + + if not first_indices: + return result + + def _check_is_in(elem, indices): + if elem in indices: + return True + if any(elem in i for i in indices if isinstance(i, frozenset)): + return True + return False + + repl = {j: i for i in indices if isinstance(i, frozenset) for j in i} + first_indices = [repl.get(i, i) for i in first_indices] + for i in first_indices: + if not _check_is_in(i, indices): + first_indices.remove(i) + first_indices.extend([i for i in indices if not _check_is_in(i, first_indices)]) + + def _get_pos(elem, indices): + if elem in indices: + return indices.index(elem) + for i, e in enumerate(indices): + if not isinstance(e, frozenset): + continue + if elem in e: + return i + raise ValueError("not found") + + permutation = _af_invert([_get_pos(i, first_indices) for i in indices]) + if isinstance(result, ArrayAdd): + return _array_add(*[_permute_dims(arg, permutation) for arg in result.args]) + else: + return _permute_dims(result, permutation) + + +def _convert_indexed_to_array(expr): + if isinstance(expr, Sum): + function = expr.function + summation_indices = expr.variables + subexpr, subindices = _convert_indexed_to_array(function) + subindicessets = {j: i for i in subindices if isinstance(i, frozenset) for j in i} + summation_indices = sorted({subindicessets.get(i, i) for i in summation_indices}, key=default_sort_key) + # TODO: check that Kronecker delta is only contracted to one other element: + kronecker_indices = set() + if isinstance(function, Mul): + for arg in function.args: + if not isinstance(arg, KroneckerDelta): + continue + arg_indices = sorted(set(arg.indices), key=default_sort_key) + if len(arg_indices) == 2: + kronecker_indices.update(arg_indices) + kronecker_indices = sorted(kronecker_indices, key=default_sort_key) + # Check dimensional consistency: + shape = get_shape(subexpr) + if shape: + for ind, istart, iend in expr.limits: + i = _get_argindex(subindices, ind) + if istart != 0 or iend+1 != shape[i]: + raise ValueError("summation index and array dimension mismatch: %s" % ind) + contraction_indices = [] + subindices = list(subindices) + if isinstance(subexpr, ArrayDiagonal): + diagonal_indices = list(subexpr.diagonal_indices) + dindices = subindices[-len(diagonal_indices):] + subindices = subindices[:-len(diagonal_indices)] + for index in summation_indices: + if index in dindices: + position = dindices.index(index) + contraction_indices.append(diagonal_indices[position]) + diagonal_indices[position] = None + diagonal_indices = [i for i in diagonal_indices if i is not None] + for i, ind in enumerate(subindices): + if ind in summation_indices: + pass + if diagonal_indices: + subexpr = _array_diagonal(subexpr.expr, *diagonal_indices) + else: + subexpr = subexpr.expr + + axes_contraction = defaultdict(list) + for i, ind in enumerate(subindices): + include = all(j not in kronecker_indices for j in ind) if isinstance(ind, frozenset) else ind not in kronecker_indices + if ind in summation_indices and include: + axes_contraction[ind].append(i) + subindices[i] = None + for k, v in axes_contraction.items(): + if any(i in kronecker_indices for i in k) if isinstance(k, frozenset) else k in kronecker_indices: + continue + contraction_indices.append(tuple(v)) + free_indices = [i for i in subindices if i is not None] + indices_ret = list(free_indices) + indices_ret.sort(key=lambda x: free_indices.index(x)) + return _array_contraction( + subexpr, + *contraction_indices, + free_indices=free_indices + ), tuple(indices_ret) + if isinstance(expr, Mul): + args, indices = zip(*[_convert_indexed_to_array(arg) for arg in expr.args]) + # Check if there are KroneckerDelta objects: + kronecker_delta_repl = {} + for arg in args: + if not isinstance(arg, KroneckerDelta): + continue + # Diagonalize two indices: + i, j = arg.indices + kindices = set(arg.indices) + if i in kronecker_delta_repl: + kindices.update(kronecker_delta_repl[i]) + if j in kronecker_delta_repl: + kindices.update(kronecker_delta_repl[j]) + kindices = frozenset(kindices) + for index in kindices: + kronecker_delta_repl[index] = kindices + # Remove KroneckerDelta objects, their relations should be handled by + # ArrayDiagonal: + newargs = [] + newindices = [] + for arg, loc_indices in zip(args, indices): + if isinstance(arg, KroneckerDelta): + continue + newargs.append(arg) + newindices.append(loc_indices) + flattened_indices = [kronecker_delta_repl.get(j, j) for i in newindices for j in i] + diagonal_indices, ret_indices = _get_diagonal_indices(flattened_indices) + tp = _array_tensor_product(*newargs) + if diagonal_indices: + return _array_diagonal(tp, *diagonal_indices), ret_indices + else: + return tp, ret_indices + if isinstance(expr, MatrixElement): + indices = expr.args[1:] + diagonal_indices, ret_indices = _get_diagonal_indices(indices) + if diagonal_indices: + return _array_diagonal(expr.args[0], *diagonal_indices), ret_indices + else: + return expr.args[0], ret_indices + if isinstance(expr, ArrayElement): + indices = expr.indices + diagonal_indices, ret_indices = _get_diagonal_indices(indices) + if diagonal_indices: + return _array_diagonal(expr.name, *diagonal_indices), ret_indices + else: + return expr.name, ret_indices + if isinstance(expr, Indexed): + indices = expr.indices + diagonal_indices, ret_indices = _get_diagonal_indices(indices) + if diagonal_indices: + return _array_diagonal(expr.base, *diagonal_indices), ret_indices + else: + return expr.args[0], ret_indices + if isinstance(expr, IndexedBase): + raise NotImplementedError + if isinstance(expr, KroneckerDelta): + return expr, expr.indices + if isinstance(expr, Add): + args, indices = zip(*[_convert_indexed_to_array(arg) for arg in expr.args]) + args = list(args) + # Check if all indices are compatible. Otherwise expand the dimensions: + index0 = [] + shape0 = [] + for arg, arg_indices in zip(args, indices): + arg_indices_set = set(arg_indices) + arg_indices_missing = arg_indices_set.difference(index0) + index0.extend([i for i in arg_indices if i in arg_indices_missing]) + arg_shape = get_shape(arg) + shape0.extend([arg_shape[i] for i, e in enumerate(arg_indices) if e in arg_indices_missing]) + for i, (arg, arg_indices) in enumerate(zip(args, indices)): + if len(arg_indices) < len(index0): + missing_indices_pos = [i for i, e in enumerate(index0) if e not in arg_indices] + missing_shape = [shape0[i] for i in missing_indices_pos] + arg_indices = tuple(index0[j] for j in missing_indices_pos) + arg_indices + args[i] = _array_tensor_product(OneArray(*missing_shape), args[i]) + permutation = Permutation([arg_indices.index(j) for j in index0]) + # Perform index permutations: + args[i] = _permute_dims(args[i], permutation) + return _array_add(*args), tuple(index0) + if isinstance(expr, Pow): + subexpr, subindices = _convert_indexed_to_array(expr.base) + if isinstance(expr.exp, (int, Integer)): + diags = zip(*[(2*i, 2*i + 1) for i in range(expr.exp)]) + arr = _array_diagonal(_array_tensor_product(*[subexpr for i in range(expr.exp)]), *diags) + return arr, subindices + if isinstance(expr, Function): + subexpr, subindices = _convert_indexed_to_array(expr.args[0]) + return ArrayElementwiseApplyFunc(type(expr), subexpr), subindices + return expr, () diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_matrix_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_matrix_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..8f66961727f6338318d65876a7768802773e4f2d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/from_matrix_to_array.py @@ -0,0 +1,87 @@ +from sympy import KroneckerProduct +from sympy.core.basic import Basic +from sympy.core.function import Lambda +from sympy.core.mul import Mul +from sympy.core.numbers import Integer +from sympy.core.power import Pow +from sympy.core.singleton import S +from sympy.core.symbol import (Dummy, symbols) +from sympy.matrices.expressions.hadamard import (HadamardPower, HadamardProduct) +from sympy.matrices.expressions.matadd import MatAdd +from sympy.matrices.expressions.matmul import MatMul +from sympy.matrices.expressions.matpow import MatPow +from sympy.matrices.expressions.trace import Trace +from sympy.matrices.expressions.transpose import Transpose +from sympy.matrices.expressions.matexpr import MatrixExpr +from sympy.tensor.array.expressions.array_expressions import \ + ArrayElementwiseApplyFunc, _array_tensor_product, _array_contraction, \ + _array_diagonal, _array_add, _permute_dims, Reshape + + +def convert_matrix_to_array(expr: Basic) -> Basic: + if isinstance(expr, MatMul): + args_nonmat = [] + args = [] + for arg in expr.args: + if isinstance(arg, MatrixExpr): + args.append(arg) + else: + args_nonmat.append(convert_matrix_to_array(arg)) + contractions = [(2*i+1, 2*i+2) for i in range(len(args)-1)] + scalar = _array_tensor_product(*args_nonmat) if args_nonmat else S.One + if scalar == 1: + tprod = _array_tensor_product( + *[convert_matrix_to_array(arg) for arg in args]) + else: + tprod = _array_tensor_product( + scalar, + *[convert_matrix_to_array(arg) for arg in args]) + return _array_contraction( + tprod, + *contractions + ) + elif isinstance(expr, MatAdd): + return _array_add( + *[convert_matrix_to_array(arg) for arg in expr.args] + ) + elif isinstance(expr, Transpose): + return _permute_dims( + convert_matrix_to_array(expr.args[0]), [1, 0] + ) + elif isinstance(expr, Trace): + inner_expr: MatrixExpr = convert_matrix_to_array(expr.arg) # type: ignore + return _array_contraction(inner_expr, (0, len(inner_expr.shape) - 1)) + elif isinstance(expr, Mul): + return _array_tensor_product(*[convert_matrix_to_array(i) for i in expr.args]) + elif isinstance(expr, Pow): + base = convert_matrix_to_array(expr.base) + if (expr.exp > 0) == True: + return _array_tensor_product(*[base for i in range(expr.exp)]) + else: + return expr + elif isinstance(expr, MatPow): + base = convert_matrix_to_array(expr.base) + if expr.exp.is_Integer != True: + b = symbols("b", cls=Dummy) + return ArrayElementwiseApplyFunc(Lambda(b, b**expr.exp), convert_matrix_to_array(base)) + elif (expr.exp > 0) == True: + return convert_matrix_to_array(MatMul.fromiter(base for i in range(expr.exp))) + else: + return expr + elif isinstance(expr, HadamardProduct): + tp = _array_tensor_product(*[convert_matrix_to_array(arg) for arg in expr.args]) + diag = [[2*i for i in range(len(expr.args))], [2*i+1 for i in range(len(expr.args))]] + return _array_diagonal(tp, *diag) + elif isinstance(expr, HadamardPower): + base, exp = expr.args + if isinstance(exp, Integer) and exp > 0: + return convert_matrix_to_array(HadamardProduct.fromiter(base for i in range(exp))) + else: + d = Dummy("d") + return ArrayElementwiseApplyFunc(Lambda(d, d**exp), base) + elif isinstance(expr, KroneckerProduct): + kp_args = [convert_matrix_to_array(arg) for arg in expr.args] + permutation = [2*i for i in range(len(kp_args))] + [2*i + 1 for i in range(len(kp_args))] + return Reshape(_permute_dims(_array_tensor_product(*kp_args), permutation), expr.shape) + else: + return expr diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9fb05bbbcd172e6d1e76cb83a8b13dd6d49f5a2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_array_expressions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_array_expressions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65e0d0108f2794445f177ac85e7a2dae59d3b4aa Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_array_expressions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_arrayexpr_derivatives.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_arrayexpr_derivatives.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e432afccd8868064d53a5a150609a120638a16b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_arrayexpr_derivatives.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_as_explicit.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_as_explicit.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2828fb3f4ea028e4a449819c2eda28a64426b7a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_as_explicit.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_indexed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_indexed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f158a575e342f0696547f5defd1de39b4e552e5c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_indexed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_matrix.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_matrix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c49d9ede81eea89db5448cfc63aad93020993ac5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_array_to_matrix.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_indexed_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_indexed_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53562f113bebe9f2ff6b6a74836a98e9b7c380de Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_indexed_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_matrix_to_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_matrix_to_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de4d2fffeef3b5b787848d548295f83d1bd4c50f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_convert_matrix_to_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_deprecated_conv_modules.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_deprecated_conv_modules.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc3a849c1b420a4e71f045a6974046dd11b59f73 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/__pycache__/test_deprecated_conv_modules.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_array_expressions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_array_expressions.py new file mode 100644 index 0000000000000000000000000000000000000000..63fb79ab7ced7bff5ecb55b1764f43e29f98609d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_array_expressions.py @@ -0,0 +1,808 @@ +import random + +from sympy import tensordiagonal, eye, KroneckerDelta, Array +from sympy.core.symbol import symbols +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.matrices.expressions.diagonal import DiagMatrix +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.matrices.expressions.special import ZeroMatrix +from sympy.tensor.array.arrayop import (permutedims, tensorcontraction, tensorproduct) +from sympy.tensor.array.dense_ndim_array import ImmutableDenseNDimArray +from sympy.combinatorics import Permutation +from sympy.tensor.array.expressions.array_expressions import ZeroArray, OneArray, ArraySymbol, ArrayElement, \ + PermuteDims, ArrayContraction, ArrayTensorProduct, ArrayDiagonal, \ + ArrayAdd, nest_permutation, ArrayElementwiseApplyFunc, _EditArrayContraction, _ArgE, _array_tensor_product, \ + _array_contraction, _array_diagonal, _array_add, _permute_dims, Reshape +from sympy.testing.pytest import raises + +i, j, k, l, m, n = symbols("i j k l m n") + + +M = ArraySymbol("M", (k, k)) +N = ArraySymbol("N", (k, k)) +P = ArraySymbol("P", (k, k)) +Q = ArraySymbol("Q", (k, k)) + +A = ArraySymbol("A", (k, k)) +B = ArraySymbol("B", (k, k)) +C = ArraySymbol("C", (k, k)) +D = ArraySymbol("D", (k, k)) + +X = ArraySymbol("X", (k, k)) +Y = ArraySymbol("Y", (k, k)) + +a = ArraySymbol("a", (k, 1)) +b = ArraySymbol("b", (k, 1)) +c = ArraySymbol("c", (k, 1)) +d = ArraySymbol("d", (k, 1)) + + +def test_array_symbol_and_element(): + A = ArraySymbol("A", (2,)) + A0 = ArrayElement(A, (0,)) + A1 = ArrayElement(A, (1,)) + assert A[0] == A0 + assert A[1] != A0 + assert A.as_explicit() == ImmutableDenseNDimArray([A0, A1]) + + A2 = tensorproduct(A, A) + assert A2.shape == (2, 2) + # TODO: not yet supported: + # assert A2.as_explicit() == Array([[A[0]*A[0], A[1]*A[0]], [A[0]*A[1], A[1]*A[1]]]) + A3 = tensorcontraction(A2, (0, 1)) + assert A3.shape == () + # TODO: not yet supported: + # assert A3.as_explicit() == Array([]) + + A = ArraySymbol("A", (2, 3, 4)) + Ae = A.as_explicit() + assert Ae == ImmutableDenseNDimArray( + [[[ArrayElement(A, (i, j, k)) for k in range(4)] for j in range(3)] for i in range(2)]) + + p = _permute_dims(A, Permutation(0, 2, 1)) + assert isinstance(p, PermuteDims) + + A = ArraySymbol("A", (2,)) + raises(IndexError, lambda: A[()]) + raises(IndexError, lambda: A[0, 1]) + raises(ValueError, lambda: A[-1]) + raises(ValueError, lambda: A[2]) + + O = OneArray(3, 4) + Z = ZeroArray(m, n) + + raises(IndexError, lambda: O[()]) + raises(IndexError, lambda: O[1, 2, 3]) + raises(ValueError, lambda: O[3, 0]) + raises(ValueError, lambda: O[0, 4]) + + assert O[1, 2] == 1 + assert Z[1, 2] == 0 + + +def test_zero_array(): + assert ZeroArray() == 0 + assert ZeroArray().is_Integer + + za = ZeroArray(3, 2, 4) + assert za.shape == (3, 2, 4) + za_e = za.as_explicit() + assert za_e.shape == (3, 2, 4) + + m, n, k = symbols("m n k") + za = ZeroArray(m, n, k, 2) + assert za.shape == (m, n, k, 2) + raises(ValueError, lambda: za.as_explicit()) + + +def test_one_array(): + assert OneArray() == 1 + assert OneArray().is_Integer + + oa = OneArray(3, 2, 4) + assert oa.shape == (3, 2, 4) + oa_e = oa.as_explicit() + assert oa_e.shape == (3, 2, 4) + + m, n, k = symbols("m n k") + oa = OneArray(m, n, k, 2) + assert oa.shape == (m, n, k, 2) + raises(ValueError, lambda: oa.as_explicit()) + + +def test_arrayexpr_contraction_construction(): + + cg = _array_contraction(A) + assert cg == A + + cg = _array_contraction(_array_tensor_product(A, B), (1, 0)) + assert cg == _array_contraction(_array_tensor_product(A, B), (0, 1)) + + cg = _array_contraction(_array_tensor_product(M, N), (0, 1)) + indtup = cg._get_contraction_tuples() + assert indtup == [[(0, 0), (0, 1)]] + assert cg._contraction_tuples_to_contraction_indices(cg.expr, indtup) == [(0, 1)] + + cg = _array_contraction(_array_tensor_product(M, N), (1, 2)) + indtup = cg._get_contraction_tuples() + assert indtup == [[(0, 1), (1, 0)]] + assert cg._contraction_tuples_to_contraction_indices(cg.expr, indtup) == [(1, 2)] + + cg = _array_contraction(_array_tensor_product(M, M, N), (1, 4), (2, 5)) + indtup = cg._get_contraction_tuples() + assert indtup == [[(0, 0), (1, 1)], [(0, 1), (2, 0)]] + assert cg._contraction_tuples_to_contraction_indices(cg.expr, indtup) == [(0, 3), (1, 4)] + + # Test removal of trivial contraction: + assert _array_contraction(a, (1,)) == a + assert _array_contraction( + _array_tensor_product(a, b), (0, 2), (1,), (3,)) == _array_contraction( + _array_tensor_product(a, b), (0, 2)) + + +def test_arrayexpr_array_flatten(): + + # Flatten nested ArrayTensorProduct objects: + expr1 = _array_tensor_product(M, N) + expr2 = _array_tensor_product(P, Q) + expr = _array_tensor_product(expr1, expr2) + assert expr == _array_tensor_product(M, N, P, Q) + assert expr.args == (M, N, P, Q) + + # Flatten mixed ArrayTensorProduct and ArrayContraction objects: + cg1 = _array_contraction(expr1, (1, 2)) + cg2 = _array_contraction(expr2, (0, 3)) + + expr = _array_tensor_product(cg1, cg2) + assert expr == _array_contraction(_array_tensor_product(M, N, P, Q), (1, 2), (4, 7)) + + expr = _array_tensor_product(M, cg1) + assert expr == _array_contraction(_array_tensor_product(M, M, N), (3, 4)) + + # Flatten nested ArrayContraction objects: + cgnested = _array_contraction(cg1, (0, 1)) + assert cgnested == _array_contraction(_array_tensor_product(M, N), (0, 3), (1, 2)) + + cgnested = _array_contraction(_array_tensor_product(cg1, cg2), (0, 3)) + assert cgnested == _array_contraction(_array_tensor_product(M, N, P, Q), (0, 6), (1, 2), (4, 7)) + + cg3 = _array_contraction(_array_tensor_product(M, N, P, Q), (1, 3), (2, 4)) + cgnested = _array_contraction(cg3, (0, 1)) + assert cgnested == _array_contraction(_array_tensor_product(M, N, P, Q), (0, 5), (1, 3), (2, 4)) + + cgnested = _array_contraction(cg3, (0, 3), (1, 2)) + assert cgnested == _array_contraction(_array_tensor_product(M, N, P, Q), (0, 7), (1, 3), (2, 4), (5, 6)) + + cg4 = _array_contraction(_array_tensor_product(M, N, P, Q), (1, 5), (3, 7)) + cgnested = _array_contraction(cg4, (0, 1)) + assert cgnested == _array_contraction(_array_tensor_product(M, N, P, Q), (0, 2), (1, 5), (3, 7)) + + cgnested = _array_contraction(cg4, (0, 1), (2, 3)) + assert cgnested == _array_contraction(_array_tensor_product(M, N, P, Q), (0, 2), (1, 5), (3, 7), (4, 6)) + + cg = _array_diagonal(cg4) + assert cg == cg4 + assert isinstance(cg, type(cg4)) + + # Flatten nested ArrayDiagonal objects: + cg1 = _array_diagonal(expr1, (1, 2)) + cg2 = _array_diagonal(expr2, (0, 3)) + cg3 = _array_diagonal(_array_tensor_product(M, N, P, Q), (1, 3), (2, 4)) + cg4 = _array_diagonal(_array_tensor_product(M, N, P, Q), (1, 5), (3, 7)) + + cgnested = _array_diagonal(cg1, (0, 1)) + assert cgnested == _array_diagonal(_array_tensor_product(M, N), (1, 2), (0, 3)) + + cgnested = _array_diagonal(cg3, (1, 2)) + assert cgnested == _array_diagonal(_array_tensor_product(M, N, P, Q), (1, 3), (2, 4), (5, 6)) + + cgnested = _array_diagonal(cg4, (1, 2)) + assert cgnested == _array_diagonal(_array_tensor_product(M, N, P, Q), (1, 5), (3, 7), (2, 4)) + + cg = _array_add(M, N) + cg2 = _array_add(cg, P) + assert isinstance(cg2, ArrayAdd) + assert cg2.args == (M, N, P) + assert cg2.shape == (k, k) + + expr = _array_tensor_product(_array_diagonal(X, (0, 1)), _array_diagonal(A, (0, 1))) + assert expr == _array_diagonal(_array_tensor_product(X, A), (0, 1), (2, 3)) + + expr1 = _array_diagonal(_array_tensor_product(X, A), (1, 2)) + expr2 = _array_tensor_product(expr1, a) + assert expr2 == _permute_dims(_array_diagonal(_array_tensor_product(X, A, a), (1, 2)), [0, 1, 4, 2, 3]) + + expr1 = _array_contraction(_array_tensor_product(X, A), (1, 2)) + expr2 = _array_tensor_product(expr1, a) + assert isinstance(expr2, ArrayContraction) + assert isinstance(expr2.expr, ArrayTensorProduct) + + cg = _array_tensor_product(_array_diagonal(_array_tensor_product(A, X, Y), (0, 3), (1, 5)), a, b) + assert cg == _permute_dims(_array_diagonal(_array_tensor_product(A, X, Y, a, b), (0, 3), (1, 5)), [0, 1, 6, 7, 2, 3, 4, 5]) + + +def test_arrayexpr_array_diagonal(): + cg = _array_diagonal(M, (1, 0)) + assert cg == _array_diagonal(M, (0, 1)) + + cg = _array_diagonal(_array_tensor_product(M, N, P), (4, 1), (2, 0)) + assert cg == _array_diagonal(_array_tensor_product(M, N, P), (1, 4), (0, 2)) + + cg = _array_diagonal(_array_tensor_product(M, N), (1, 2), (3,), allow_trivial_diags=True) + assert cg == _permute_dims(_array_diagonal(_array_tensor_product(M, N), (1, 2)), [0, 2, 1]) + + Ax = ArraySymbol("Ax", shape=(1, 2, 3, 4, 3, 5, 6, 2, 7)) + cg = _array_diagonal(Ax, (1, 7), (3,), (2, 4), (6,), allow_trivial_diags=True) + assert cg == _permute_dims(_array_diagonal(Ax, (1, 7), (2, 4)), [0, 2, 4, 5, 1, 6, 3]) + + cg = _array_diagonal(M, (0,), allow_trivial_diags=True) + assert cg == _permute_dims(M, [1, 0]) + + raises(ValueError, lambda: _array_diagonal(M, (0, 0))) + + +def test_arrayexpr_array_shape(): + expr = _array_tensor_product(M, N, P, Q) + assert expr.shape == (k, k, k, k, k, k, k, k) + Z = MatrixSymbol("Z", m, n) + expr = _array_tensor_product(M, Z) + assert expr.shape == (k, k, m, n) + expr2 = _array_contraction(expr, (0, 1)) + assert expr2.shape == (m, n) + expr2 = _array_diagonal(expr, (0, 1)) + assert expr2.shape == (m, n, k) + exprp = _permute_dims(expr, [2, 1, 3, 0]) + assert exprp.shape == (m, k, n, k) + expr3 = _array_tensor_product(N, Z) + expr2 = _array_add(expr, expr3) + assert expr2.shape == (k, k, m, n) + + # Contraction along axes with discordant dimensions: + raises(ValueError, lambda: _array_contraction(expr, (1, 2))) + # Also diagonal needs the same dimensions: + raises(ValueError, lambda: _array_diagonal(expr, (1, 2))) + # Diagonal requires at least to axes to compute the diagonal: + raises(ValueError, lambda: _array_diagonal(expr, (1,))) + + +def test_arrayexpr_permutedims_sink(): + + cg = _permute_dims(_array_tensor_product(M, N), [0, 1, 3, 2], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_tensor_product(M, _permute_dims(N, [1, 0])) + + cg = _permute_dims(_array_tensor_product(M, N), [1, 0, 3, 2], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_tensor_product(_permute_dims(M, [1, 0]), _permute_dims(N, [1, 0])) + + cg = _permute_dims(_array_tensor_product(M, N), [3, 2, 1, 0], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_tensor_product(_permute_dims(N, [1, 0]), _permute_dims(M, [1, 0])) + + cg = _permute_dims(_array_contraction(_array_tensor_product(M, N), (1, 2)), [1, 0], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_contraction(_permute_dims(_array_tensor_product(M, N), [[0, 3]]), (1, 2)) + + cg = _permute_dims(_array_tensor_product(M, N), [1, 0, 3, 2], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_tensor_product(_permute_dims(M, [1, 0]), _permute_dims(N, [1, 0])) + + cg = _permute_dims(_array_contraction(_array_tensor_product(M, N, P), (1, 2), (3, 4)), [1, 0], nest_permutation=False) + sunk = nest_permutation(cg) + assert sunk == _array_contraction(_permute_dims(_array_tensor_product(M, N, P), [[0, 5]]), (1, 2), (3, 4)) + + +def test_arrayexpr_push_indices_up_and_down(): + + indices = list(range(12)) + + contr_diag_indices = [(0, 6), (2, 8)] + assert ArrayContraction._push_indices_down(contr_diag_indices, indices) == (1, 3, 4, 5, 7, 9, 10, 11, 12, 13, 14, 15) + assert ArrayContraction._push_indices_up(contr_diag_indices, indices) == (None, 0, None, 1, 2, 3, None, 4, None, 5, 6, 7) + + assert ArrayDiagonal._push_indices_down(contr_diag_indices, indices, 10) == (1, 3, 4, 5, 7, 9, (0, 6), (2, 8), None, None, None, None) + assert ArrayDiagonal._push_indices_up(contr_diag_indices, indices, 10) == (6, 0, 7, 1, 2, 3, 6, 4, 7, 5, None, None) + + contr_diag_indices = [(1, 2), (7, 8)] + assert ArrayContraction._push_indices_down(contr_diag_indices, indices) == (0, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15) + assert ArrayContraction._push_indices_up(contr_diag_indices, indices) == (0, None, None, 1, 2, 3, 4, None, None, 5, 6, 7) + + assert ArrayDiagonal._push_indices_down(contr_diag_indices, indices, 10) == (0, 3, 4, 5, 6, 9, (1, 2), (7, 8), None, None, None, None) + assert ArrayDiagonal._push_indices_up(contr_diag_indices, indices, 10) == (0, 6, 6, 1, 2, 3, 4, 7, 7, 5, None, None) + + +def test_arrayexpr_split_multiple_contractions(): + a = MatrixSymbol("a", k, 1) + b = MatrixSymbol("b", k, 1) + A = MatrixSymbol("A", k, k) + B = MatrixSymbol("B", k, k) + C = MatrixSymbol("C", k, k) + X = MatrixSymbol("X", k, k) + + cg = _array_contraction(_array_tensor_product(A.T, a, b, b.T, (A*X*b).applyfunc(cos)), (1, 2, 8), (5, 6, 9)) + expected = _array_contraction(_array_tensor_product(A.T, DiagMatrix(a), OneArray(1), b, b.T, (A*X*b).applyfunc(cos)), (1, 3), (2, 9), (6, 7, 10)) + assert cg.split_multiple_contractions().dummy_eq(expected) + + # Check no overlap of lines: + + cg = _array_contraction(_array_tensor_product(A, a, C, a, B), (1, 2, 4), (5, 6, 8), (3, 7)) + assert cg.split_multiple_contractions() == cg + + cg = _array_contraction(_array_tensor_product(a, b, A), (0, 2, 4), (1, 3)) + assert cg.split_multiple_contractions() == cg + + +def test_arrayexpr_nested_permutations(): + + cg = _permute_dims(_permute_dims(M, (1, 0)), (1, 0)) + assert cg == M + + times = 3 + plist1 = [list(range(6)) for i in range(times)] + plist2 = [list(range(6)) for i in range(times)] + + for i in range(times): + random.shuffle(plist1[i]) + random.shuffle(plist2[i]) + + plist1.append([2, 5, 4, 1, 0, 3]) + plist2.append([3, 5, 0, 4, 1, 2]) + + plist1.append([2, 5, 4, 0, 3, 1]) + plist2.append([3, 0, 5, 1, 2, 4]) + + plist1.append([5, 4, 2, 0, 3, 1]) + plist2.append([4, 5, 0, 2, 3, 1]) + + Me = M.subs(k, 3).as_explicit() + Ne = N.subs(k, 3).as_explicit() + Pe = P.subs(k, 3).as_explicit() + cge = tensorproduct(Me, Ne, Pe) + + for permutation_array1, permutation_array2 in zip(plist1, plist2): + p1 = Permutation(permutation_array1) + p2 = Permutation(permutation_array2) + + cg = _permute_dims( + _permute_dims( + _array_tensor_product(M, N, P), + p1), + p2 + ) + result = _permute_dims( + _array_tensor_product(M, N, P), + p2*p1 + ) + assert cg == result + + # Check that `permutedims` behaves the same way with explicit-component arrays: + result1 = _permute_dims(_permute_dims(cge, p1), p2) + result2 = _permute_dims(cge, p2*p1) + assert result1 == result2 + + +def test_arrayexpr_contraction_permutation_mix(): + + Me = M.subs(k, 3).as_explicit() + Ne = N.subs(k, 3).as_explicit() + + cg1 = _array_contraction(PermuteDims(_array_tensor_product(M, N), Permutation([0, 2, 1, 3])), (2, 3)) + cg2 = _array_contraction(_array_tensor_product(M, N), (1, 3)) + assert cg1 == cg2 + cge1 = tensorcontraction(permutedims(tensorproduct(Me, Ne), Permutation([0, 2, 1, 3])), (2, 3)) + cge2 = tensorcontraction(tensorproduct(Me, Ne), (1, 3)) + assert cge1 == cge2 + + cg1 = _permute_dims(_array_tensor_product(M, N), Permutation([0, 1, 3, 2])) + cg2 = _array_tensor_product(M, _permute_dims(N, Permutation([1, 0]))) + assert cg1 == cg2 + + cg1 = _array_contraction( + _permute_dims( + _array_tensor_product(M, N, P, Q), Permutation([0, 2, 3, 1, 4, 5, 7, 6])), + (1, 2), (3, 5) + ) + cg2 = _array_contraction( + _array_tensor_product(M, N, P, _permute_dims(Q, Permutation([1, 0]))), + (1, 5), (2, 3) + ) + assert cg1 == cg2 + + cg1 = _array_contraction( + _permute_dims( + _array_tensor_product(M, N, P, Q), Permutation([1, 0, 4, 6, 2, 7, 5, 3])), + (0, 1), (2, 6), (3, 7) + ) + cg2 = _permute_dims( + _array_contraction( + _array_tensor_product(M, P, Q, N), + (0, 1), (2, 3), (4, 7)), + [1, 0] + ) + assert cg1 == cg2 + + cg1 = _array_contraction( + _permute_dims( + _array_tensor_product(M, N, P, Q), Permutation([1, 0, 4, 6, 7, 2, 5, 3])), + (0, 1), (2, 6), (3, 7) + ) + cg2 = _permute_dims( + _array_contraction( + _array_tensor_product(_permute_dims(M, [1, 0]), N, P, Q), + (0, 1), (3, 6), (4, 5) + ), + Permutation([1, 0]) + ) + assert cg1 == cg2 + + +def test_arrayexpr_permute_tensor_product(): + cg1 = _permute_dims(_array_tensor_product(M, N, P, Q), Permutation([2, 3, 1, 0, 5, 4, 6, 7])) + cg2 = _array_tensor_product(N, _permute_dims(M, [1, 0]), + _permute_dims(P, [1, 0]), Q) + assert cg1 == cg2 + + # TODO: reverse operation starting with `PermuteDims` and getting down to `bb`... + cg1 = _permute_dims(_array_tensor_product(M, N, P, Q), Permutation([2, 3, 4, 5, 0, 1, 6, 7])) + cg2 = _array_tensor_product(N, P, M, Q) + assert cg1 == cg2 + + cg1 = _permute_dims(_array_tensor_product(M, N, P, Q), Permutation([2, 3, 4, 6, 5, 7, 0, 1])) + assert cg1.expr == _array_tensor_product(N, P, Q, M) + assert cg1.permutation == Permutation([0, 1, 2, 4, 3, 5, 6, 7]) + + cg1 = _array_contraction( + _permute_dims( + _array_tensor_product(N, Q, Q, M), + [2, 1, 5, 4, 0, 3, 6, 7]), + [1, 2, 6]) + cg2 = _permute_dims(_array_contraction(_array_tensor_product(Q, Q, N, M), (3, 5, 6)), [0, 2, 3, 1, 4]) + assert cg1 == cg2 + + cg1 = _array_contraction( + _array_contraction( + _array_contraction( + _array_contraction( + _permute_dims( + _array_tensor_product(N, Q, Q, M), + [2, 1, 5, 4, 0, 3, 6, 7]), + [1, 2, 6]), + [1, 3, 4]), + [1]), + [0]) + cg2 = _array_contraction(_array_tensor_product(M, N, Q, Q), (0, 3, 5), (1, 4, 7), (2,), (6,)) + assert cg1 == cg2 + + +def test_arrayexpr_canonicalize_diagonal__permute_dims(): + tp = _array_tensor_product(M, Q, N, P) + expr = _array_diagonal( + _permute_dims(tp, [0, 1, 2, 4, 7, 6, 3, 5]), (2, 4, 5), (6, 7), + (0, 3)) + result = _array_diagonal(tp, (2, 6, 7), (3, 5), (0, 4)) + assert expr == result + + tp = _array_tensor_product(M, N, P, Q) + expr = _array_diagonal(_permute_dims(tp, [0, 5, 2, 4, 1, 6, 3, 7]), (1, 2, 6), (3, 4)) + result = _array_diagonal(_array_tensor_product(M, P, N, Q), (3, 4, 5), (1, 2)) + assert expr == result + + +def test_arrayexpr_canonicalize_diagonal_contraction(): + tp = _array_tensor_product(M, N, P, Q) + expr = _array_contraction(_array_diagonal(tp, (1, 3, 4)), (0, 3)) + result = _array_diagonal(_array_contraction(_array_tensor_product(M, N, P, Q), (0, 6)), (0, 2, 3)) + assert expr == result + + expr = _array_contraction(_array_diagonal(tp, (0, 1, 2, 3, 7)), (1, 2, 3)) + result = _array_contraction(_array_tensor_product(M, N, P, Q), (0, 1, 2, 3, 5, 6, 7)) + assert expr == result + + expr = _array_contraction(_array_diagonal(tp, (0, 2, 6, 7)), (1, 2, 3)) + result = _array_diagonal(_array_contraction(tp, (3, 4, 5)), (0, 2, 3, 4)) + assert expr == result + + td = _array_diagonal(_array_tensor_product(M, N, P, Q), (0, 3)) + expr = _array_contraction(td, (2, 1), (0, 4, 6, 5, 3)) + result = _array_contraction(_array_tensor_product(M, N, P, Q), (0, 1, 3, 5, 6, 7), (2, 4)) + assert expr == result + + +def test_arrayexpr_array_wrong_permutation_size(): + cg = _array_tensor_product(M, N) + raises(ValueError, lambda: _permute_dims(cg, [1, 0])) + raises(ValueError, lambda: _permute_dims(cg, [1, 0, 2, 3, 5, 4])) + + +def test_arrayexpr_nested_array_elementwise_add(): + cg = _array_contraction(_array_add( + _array_tensor_product(M, N), + _array_tensor_product(N, M) + ), (1, 2)) + result = _array_add( + _array_contraction(_array_tensor_product(M, N), (1, 2)), + _array_contraction(_array_tensor_product(N, M), (1, 2)) + ) + assert cg == result + + cg = _array_diagonal(_array_add( + _array_tensor_product(M, N), + _array_tensor_product(N, M) + ), (1, 2)) + result = _array_add( + _array_diagonal(_array_tensor_product(M, N), (1, 2)), + _array_diagonal(_array_tensor_product(N, M), (1, 2)) + ) + assert cg == result + + +def test_arrayexpr_array_expr_zero_array(): + za1 = ZeroArray(k, l, m, n) + zm1 = ZeroMatrix(m, n) + + za2 = ZeroArray(k, m, m, n) + zm2 = ZeroMatrix(m, m) + zm3 = ZeroMatrix(k, k) + + assert _array_tensor_product(M, N, za1) == ZeroArray(k, k, k, k, k, l, m, n) + assert _array_tensor_product(M, N, zm1) == ZeroArray(k, k, k, k, m, n) + + assert _array_contraction(za1, (3,)) == ZeroArray(k, l, m) + assert _array_contraction(zm1, (1,)) == ZeroArray(m) + assert _array_contraction(za2, (1, 2)) == ZeroArray(k, n) + assert _array_contraction(zm2, (0, 1)) == 0 + + assert _array_diagonal(za2, (1, 2)) == ZeroArray(k, n, m) + assert _array_diagonal(zm2, (0, 1)) == ZeroArray(m) + + assert _permute_dims(za1, [2, 1, 3, 0]) == ZeroArray(m, l, n, k) + assert _permute_dims(zm1, [1, 0]) == ZeroArray(n, m) + + assert _array_add(za1) == za1 + assert _array_add(zm1) == ZeroArray(m, n) + tp1 = _array_tensor_product(MatrixSymbol("A", k, l), MatrixSymbol("B", m, n)) + assert _array_add(tp1, za1) == tp1 + tp2 = _array_tensor_product(MatrixSymbol("C", k, l), MatrixSymbol("D", m, n)) + assert _array_add(tp1, za1, tp2) == _array_add(tp1, tp2) + assert _array_add(M, zm3) == M + assert _array_add(M, N, zm3) == _array_add(M, N) + + +def test_arrayexpr_array_expr_applyfunc(): + + A = ArraySymbol("A", (3, k, 2)) + aaf = ArrayElementwiseApplyFunc(sin, A) + assert aaf.shape == (3, k, 2) + + +def test_edit_array_contraction(): + cg = _array_contraction(_array_tensor_product(A, B, C, D), (1, 2, 5)) + ecg = _EditArrayContraction(cg) + assert ecg.to_array_contraction() == cg + + ecg.args_with_ind[1], ecg.args_with_ind[2] = ecg.args_with_ind[2], ecg.args_with_ind[1] + assert ecg.to_array_contraction() == _array_contraction(_array_tensor_product(A, C, B, D), (1, 3, 4)) + + ci = ecg.get_new_contraction_index() + new_arg = _ArgE(X) + new_arg.indices = [ci, ci] + ecg.args_with_ind.insert(2, new_arg) + assert ecg.to_array_contraction() == _array_contraction(_array_tensor_product(A, C, X, B, D), (1, 3, 6), (4, 5)) + + assert ecg.get_contraction_indices() == [[1, 3, 6], [4, 5]] + assert [[tuple(j) for j in i] for i in ecg.get_contraction_indices_to_ind_rel_pos()] == [[(0, 1), (1, 1), (3, 0)], [(2, 0), (2, 1)]] + assert [list(i) for i in ecg.get_mapping_for_index(0)] == [[0, 1], [1, 1], [3, 0]] + assert [list(i) for i in ecg.get_mapping_for_index(1)] == [[2, 0], [2, 1]] + raises(ValueError, lambda: ecg.get_mapping_for_index(2)) + + ecg.args_with_ind.pop(1) + assert ecg.to_array_contraction() == _array_contraction(_array_tensor_product(A, X, B, D), (1, 4), (2, 3)) + + ecg.args_with_ind[0].indices[1] = ecg.args_with_ind[1].indices[0] + ecg.args_with_ind[1].indices[1] = ecg.args_with_ind[2].indices[0] + assert ecg.to_array_contraction() == _array_contraction(_array_tensor_product(A, X, B, D), (1, 2), (3, 4)) + + ecg.insert_after(ecg.args_with_ind[1], _ArgE(C)) + assert ecg.to_array_contraction() == _array_contraction(_array_tensor_product(A, X, C, B, D), (1, 2), (3, 6)) + + +def test_array_expressions_no_canonicalization(): + + tp = _array_tensor_product(M, N, P) + + # ArrayTensorProduct: + + expr = ArrayTensorProduct(tp, N) + assert str(expr) == "ArrayTensorProduct(ArrayTensorProduct(M, N, P), N)" + assert expr.doit() == ArrayTensorProduct(M, N, P, N) + + expr = ArrayTensorProduct(ArrayContraction(M, (0, 1)), N) + assert str(expr) == "ArrayTensorProduct(ArrayContraction(M, (0, 1)), N)" + assert expr.doit() == ArrayContraction(ArrayTensorProduct(M, N), (0, 1)) + + expr = ArrayTensorProduct(ArrayDiagonal(M, (0, 1)), N) + assert str(expr) == "ArrayTensorProduct(ArrayDiagonal(M, (0, 1)), N)" + assert expr.doit() == PermuteDims(ArrayDiagonal(ArrayTensorProduct(M, N), (0, 1)), [2, 0, 1]) + + expr = ArrayTensorProduct(PermuteDims(M, [1, 0]), N) + assert str(expr) == "ArrayTensorProduct(PermuteDims(M, (0 1)), N)" + assert expr.doit() == PermuteDims(ArrayTensorProduct(M, N), [1, 0, 2, 3]) + + # ArrayContraction: + + expr = ArrayContraction(_array_contraction(tp, (0, 2)), (0, 1)) + assert isinstance(expr, ArrayContraction) + assert isinstance(expr.expr, ArrayContraction) + assert str(expr) == "ArrayContraction(ArrayContraction(ArrayTensorProduct(M, N, P), (0, 2)), (0, 1))" + assert expr.doit() == ArrayContraction(tp, (0, 2), (1, 3)) + + expr = ArrayContraction(ArrayContraction(ArrayContraction(tp, (0, 1)), (0, 1)), (0, 1)) + assert expr.doit() == ArrayContraction(tp, (0, 1), (2, 3), (4, 5)) + # assert expr._canonicalize() == ArrayContraction(ArrayContraction(tp, (0, 1)), (0, 1), (2, 3)) + + expr = ArrayContraction(ArrayDiagonal(tp, (0, 1)), (0, 1)) + assert str(expr) == "ArrayContraction(ArrayDiagonal(ArrayTensorProduct(M, N, P), (0, 1)), (0, 1))" + assert expr.doit() == ArrayDiagonal(ArrayContraction(ArrayTensorProduct(N, M, P), (0, 1)), (0, 1)) + + expr = ArrayContraction(PermuteDims(M, [1, 0]), (0, 1)) + assert str(expr) == "ArrayContraction(PermuteDims(M, (0 1)), (0, 1))" + assert expr.doit() == ArrayContraction(M, (0, 1)) + + # ArrayDiagonal: + + expr = ArrayDiagonal(ArrayDiagonal(tp, (0, 2)), (0, 1)) + assert str(expr) == "ArrayDiagonal(ArrayDiagonal(ArrayTensorProduct(M, N, P), (0, 2)), (0, 1))" + assert expr.doit() == ArrayDiagonal(tp, (0, 2), (1, 3)) + + expr = ArrayDiagonal(ArrayDiagonal(ArrayDiagonal(tp, (0, 1)), (0, 1)), (0, 1)) + assert expr.doit() == ArrayDiagonal(tp, (0, 1), (2, 3), (4, 5)) + assert expr._canonicalize() == expr.doit() + + expr = ArrayDiagonal(ArrayContraction(tp, (0, 1)), (0, 1)) + assert str(expr) == "ArrayDiagonal(ArrayContraction(ArrayTensorProduct(M, N, P), (0, 1)), (0, 1))" + assert expr.doit() == expr + + expr = ArrayDiagonal(PermuteDims(M, [1, 0]), (0, 1)) + assert str(expr) == "ArrayDiagonal(PermuteDims(M, (0 1)), (0, 1))" + assert expr.doit() == ArrayDiagonal(M, (0, 1)) + + # ArrayAdd: + + expr = ArrayAdd(M) + assert isinstance(expr, ArrayAdd) + assert expr.doit() == M + + expr = ArrayAdd(ArrayAdd(M, N), P) + assert str(expr) == "ArrayAdd(ArrayAdd(M, N), P)" + assert expr.doit() == ArrayAdd(M, N, P) + + expr = ArrayAdd(M, ArrayAdd(N, ArrayAdd(P, M))) + assert expr.doit() == ArrayAdd(M, N, P, M) + assert expr._canonicalize() == ArrayAdd(M, N, ArrayAdd(P, M)) + + expr = ArrayAdd(M, ZeroArray(k, k), N) + assert str(expr) == "ArrayAdd(M, ZeroArray(k, k), N)" + assert expr.doit() == ArrayAdd(M, N) + + # PermuteDims: + + expr = PermuteDims(PermuteDims(M, [1, 0]), [1, 0]) + assert str(expr) == "PermuteDims(PermuteDims(M, (0 1)), (0 1))" + assert expr.doit() == M + + expr = PermuteDims(PermuteDims(PermuteDims(M, [1, 0]), [1, 0]), [1, 0]) + assert expr.doit() == PermuteDims(M, [1, 0]) + assert expr._canonicalize() == expr.doit() + + # Reshape + + expr = Reshape(A, (k**2,)) + assert expr.shape == (k**2,) + assert isinstance(expr, Reshape) + + +def test_array_expr_construction_with_functions(): + + tp = tensorproduct(M, N) + assert tp == ArrayTensorProduct(M, N) + + expr = tensorproduct(A, eye(2)) + assert expr == ArrayTensorProduct(A, eye(2)) + + # Contraction: + + expr = tensorcontraction(M, (0, 1)) + assert expr == ArrayContraction(M, (0, 1)) + + expr = tensorcontraction(tp, (1, 2)) + assert expr == ArrayContraction(tp, (1, 2)) + + expr = tensorcontraction(tensorcontraction(tp, (1, 2)), (0, 1)) + assert expr == ArrayContraction(tp, (0, 3), (1, 2)) + + # Diagonalization: + + expr = tensordiagonal(M, (0, 1)) + assert expr == ArrayDiagonal(M, (0, 1)) + + expr = tensordiagonal(tensordiagonal(tp, (0, 1)), (0, 1)) + assert expr == ArrayDiagonal(tp, (0, 1), (2, 3)) + + # Permutation of dimensions: + + expr = permutedims(M, [1, 0]) + assert expr == PermuteDims(M, [1, 0]) + + expr = permutedims(PermuteDims(tp, [1, 0, 2, 3]), [0, 1, 3, 2]) + assert expr == PermuteDims(tp, [1, 0, 3, 2]) + + expr = PermuteDims(tp, index_order_new=["a", "b", "c", "d"], index_order_old=["d", "c", "b", "a"]) + assert expr == PermuteDims(tp, [3, 2, 1, 0]) + + arr = Array(range(32)).reshape(2, 2, 2, 2, 2) + expr = PermuteDims(arr, index_order_new=["a", "b", "c", "d", "e"], index_order_old=['b', 'e', 'a', 'd', 'c']) + assert expr == PermuteDims(arr, [2, 0, 4, 3, 1]) + assert expr.as_explicit() == permutedims(arr, index_order_new=["a", "b", "c", "d", "e"], index_order_old=['b', 'e', 'a', 'd', 'c']) + + +def test_array_element_expressions(): + # Check commutative property: + assert M[0, 0]*N[0, 0] == N[0, 0]*M[0, 0] + + # Check derivatives: + assert M[0, 0].diff(M[0, 0]) == 1 + assert M[0, 0].diff(M[1, 0]) == 0 + assert M[0, 0].diff(N[0, 0]) == 0 + assert M[0, 1].diff(M[i, j]) == KroneckerDelta(i, 0)*KroneckerDelta(j, 1) + assert M[0, 1].diff(N[i, j]) == 0 + + K4 = ArraySymbol("K4", shape=(k, k, k, k)) + + assert K4[i, j, k, l].diff(K4[1, 2, 3, 4]) == ( + KroneckerDelta(i, 1)*KroneckerDelta(j, 2)*KroneckerDelta(k, 3)*KroneckerDelta(l, 4) + ) + + +def test_array_expr_reshape(): + + A = MatrixSymbol("A", 2, 2) + B = ArraySymbol("B", (2, 2, 2)) + C = Array([1, 2, 3, 4]) + + expr = Reshape(A, (4,)) + assert expr.expr == A + assert expr.shape == (4,) + assert expr.as_explicit() == Array([A[0, 0], A[0, 1], A[1, 0], A[1, 1]]) + + expr = Reshape(B, (2, 4)) + assert expr.expr == B + assert expr.shape == (2, 4) + ee = expr.as_explicit() + assert isinstance(ee, ImmutableDenseNDimArray) + assert ee.shape == (2, 4) + assert ee == Array([[B[0, 0, 0], B[0, 0, 1], B[0, 1, 0], B[0, 1, 1]], [B[1, 0, 0], B[1, 0, 1], B[1, 1, 0], B[1, 1, 1]]]) + + expr = Reshape(A, (k, 2)) + assert expr.shape == (k, 2) + + raises(ValueError, lambda: Reshape(A, (2, 3))) + raises(ValueError, lambda: Reshape(A, (3,))) + + expr = Reshape(C, (2, 2)) + assert expr.expr == C + assert expr.shape == (2, 2) + assert expr.doit() == Array([[1, 2], [3, 4]]) + + +def test_array_expr_as_explicit_with_explicit_component_arrays(): + # Test if .as_explicit() works with explicit-component arrays + # nested in array expressions: + from sympy.abc import x, y, z, t + A = Array([[x, y], [z, t]]) + assert ArrayTensorProduct(A, A).as_explicit() == tensorproduct(A, A) + assert ArrayDiagonal(A, (0, 1)).as_explicit() == tensordiagonal(A, (0, 1)) + assert ArrayContraction(A, (0, 1)).as_explicit() == tensorcontraction(A, (0, 1)) + assert ArrayAdd(A, A).as_explicit() == A + A + assert ArrayElementwiseApplyFunc(sin, A).as_explicit() == A.applyfunc(sin) + assert PermuteDims(A, [1, 0]).as_explicit() == permutedims(A, [1, 0]) + assert Reshape(A, [4]).as_explicit() == A.reshape(4) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_arrayexpr_derivatives.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_arrayexpr_derivatives.py new file mode 100644 index 0000000000000000000000000000000000000000..bc0fcf63f2607b23feb38758e4f0994de4f0384b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_arrayexpr_derivatives.py @@ -0,0 +1,78 @@ +from sympy.core.symbol import symbols +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.matrices.expressions.special import Identity +from sympy.matrices.expressions.applyfunc import ElementwiseApplyFunction +from sympy.tensor.array.expressions.array_expressions import ArraySymbol, ArrayTensorProduct, \ + PermuteDims, ArrayDiagonal, ArrayElementwiseApplyFunc, ArrayContraction, _permute_dims, Reshape +from sympy.tensor.array.expressions.arrayexpr_derivatives import array_derive + +k = symbols("k") + +I = Identity(k) +X = MatrixSymbol("X", k, k) +x = MatrixSymbol("x", k, 1) + +A = MatrixSymbol("A", k, k) +B = MatrixSymbol("B", k, k) +C = MatrixSymbol("C", k, k) +D = MatrixSymbol("D", k, k) + +A1 = ArraySymbol("A", (3, 2, k)) + + +def test_arrayexpr_derivatives1(): + + res = array_derive(X, X) + assert res == PermuteDims(ArrayTensorProduct(I, I), [0, 2, 1, 3]) + + cg = ArrayTensorProduct(A, X, B) + res = array_derive(cg, X) + assert res == _permute_dims( + ArrayTensorProduct(I, A, I, B), + [0, 4, 2, 3, 1, 5, 6, 7]) + + cg = ArrayContraction(X, (0, 1)) + res = array_derive(cg, X) + assert res == ArrayContraction(ArrayTensorProduct(I, I), (1, 3)) + + cg = ArrayDiagonal(X, (0, 1)) + res = array_derive(cg, X) + assert res == ArrayDiagonal(ArrayTensorProduct(I, I), (1, 3)) + + cg = ElementwiseApplyFunction(sin, X) + res = array_derive(cg, X) + assert res.dummy_eq(ArrayDiagonal( + ArrayTensorProduct( + ElementwiseApplyFunction(cos, X), + I, + I + ), (0, 3), (1, 5))) + + cg = ArrayElementwiseApplyFunc(sin, X) + res = array_derive(cg, X) + assert res.dummy_eq(ArrayDiagonal( + ArrayTensorProduct( + I, + I, + ArrayElementwiseApplyFunc(cos, X) + ), (1, 4), (3, 5))) + + res = array_derive(A1, A1) + assert res == PermuteDims( + ArrayTensorProduct(Identity(3), Identity(2), Identity(k)), + [0, 2, 4, 1, 3, 5] + ) + + cg = ArrayElementwiseApplyFunc(sin, A1) + res = array_derive(cg, A1) + assert res.dummy_eq(ArrayDiagonal( + ArrayTensorProduct( + Identity(3), Identity(2), Identity(k), + ArrayElementwiseApplyFunc(cos, A1) + ), (1, 6), (3, 7), (5, 8) + )) + + cg = Reshape(A, (k**2,)) + res = array_derive(cg, A) + assert res == Reshape(PermuteDims(ArrayTensorProduct(I, I), [0, 2, 1, 3]), (k, k, k**2)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_as_explicit.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_as_explicit.py new file mode 100644 index 0000000000000000000000000000000000000000..30cc61b1ee651ca032e165cd67926fa33c71354f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_as_explicit.py @@ -0,0 +1,63 @@ +from sympy.core.symbol import Symbol +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.tensor.array.arrayop import (permutedims, tensorcontraction, tensordiagonal, tensorproduct) +from sympy.tensor.array.dense_ndim_array import ImmutableDenseNDimArray +from sympy.tensor.array.expressions.array_expressions import ZeroArray, OneArray, ArraySymbol, \ + ArrayTensorProduct, PermuteDims, ArrayDiagonal, ArrayContraction, ArrayAdd +from sympy.testing.pytest import raises + + +def test_array_as_explicit_call(): + + assert ZeroArray(3, 2, 4).as_explicit() == ImmutableDenseNDimArray.zeros(3, 2, 4) + assert OneArray(3, 2, 4).as_explicit() == ImmutableDenseNDimArray([1 for i in range(3*2*4)]).reshape(3, 2, 4) + + k = Symbol("k") + X = ArraySymbol("X", (k, 3, 2)) + raises(ValueError, lambda: X.as_explicit()) + raises(ValueError, lambda: ZeroArray(k, 2, 3).as_explicit()) + raises(ValueError, lambda: OneArray(2, k, 2).as_explicit()) + + A = ArraySymbol("A", (3, 3)) + B = ArraySymbol("B", (3, 3)) + + texpr = tensorproduct(A, B) + assert isinstance(texpr, ArrayTensorProduct) + assert texpr.as_explicit() == tensorproduct(A.as_explicit(), B.as_explicit()) + + texpr = tensorcontraction(A, (0, 1)) + assert isinstance(texpr, ArrayContraction) + assert texpr.as_explicit() == A[0, 0] + A[1, 1] + A[2, 2] + + texpr = tensordiagonal(A, (0, 1)) + assert isinstance(texpr, ArrayDiagonal) + assert texpr.as_explicit() == ImmutableDenseNDimArray([A[0, 0], A[1, 1], A[2, 2]]) + + texpr = permutedims(A, [1, 0]) + assert isinstance(texpr, PermuteDims) + assert texpr.as_explicit() == permutedims(A.as_explicit(), [1, 0]) + + +def test_array_as_explicit_matrix_symbol(): + + A = MatrixSymbol("A", 3, 3) + B = MatrixSymbol("B", 3, 3) + + texpr = tensorproduct(A, B) + assert isinstance(texpr, ArrayTensorProduct) + assert texpr.as_explicit() == tensorproduct(A.as_explicit(), B.as_explicit()) + + texpr = tensorcontraction(A, (0, 1)) + assert isinstance(texpr, ArrayContraction) + assert texpr.as_explicit() == A[0, 0] + A[1, 1] + A[2, 2] + + texpr = tensordiagonal(A, (0, 1)) + assert isinstance(texpr, ArrayDiagonal) + assert texpr.as_explicit() == ImmutableDenseNDimArray([A[0, 0], A[1, 1], A[2, 2]]) + + texpr = permutedims(A, [1, 0]) + assert isinstance(texpr, PermuteDims) + assert texpr.as_explicit() == permutedims(A.as_explicit(), [1, 0]) + + expr = ArrayAdd(ArrayTensorProduct(A, B), ArrayTensorProduct(B, A)) + assert expr.as_explicit() == expr.args[0].as_explicit() + expr.args[1].as_explicit() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_indexed.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_indexed.py new file mode 100644 index 0000000000000000000000000000000000000000..a6b713fbec94ab7808c5a8a778b3313402d9d0c7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_indexed.py @@ -0,0 +1,61 @@ +from sympy import Sum, Dummy, sin +from sympy.tensor.array.expressions import ArraySymbol, ArrayTensorProduct, ArrayContraction, PermuteDims, \ + ArrayDiagonal, ArrayAdd, OneArray, ZeroArray, convert_indexed_to_array, ArrayElementwiseApplyFunc, Reshape +from sympy.tensor.array.expressions.from_array_to_indexed import convert_array_to_indexed + +from sympy.abc import i, j, k, l, m, n, o + + +def test_convert_array_to_indexed_main(): + A = ArraySymbol("A", (3, 3, 3)) + B = ArraySymbol("B", (3, 3)) + C = ArraySymbol("C", (3, 3)) + + d_ = Dummy("d_") + + assert convert_array_to_indexed(A, [i, j, k]) == A[i, j, k] + + expr = ArrayTensorProduct(A, B, C) + conv = convert_array_to_indexed(expr, [i,j,k,l,m,n,o]) + assert conv == A[i,j,k]*B[l,m]*C[n,o] + assert convert_indexed_to_array(conv, [i,j,k,l,m,n,o]) == expr + + expr = ArrayContraction(A, (0, 2)) + assert convert_array_to_indexed(expr, [i]).dummy_eq(Sum(A[d_, i, d_], (d_, 0, 2))) + + expr = ArrayDiagonal(A, (0, 2)) + assert convert_array_to_indexed(expr, [i, j]) == A[j, i, j] + + expr = PermuteDims(A, [1, 2, 0]) + conv = convert_array_to_indexed(expr, [i, j, k]) + assert conv == A[k, i, j] + assert convert_indexed_to_array(conv, [i, j, k]) == expr + + expr = ArrayAdd(B, C, PermuteDims(C, [1, 0])) + conv = convert_array_to_indexed(expr, [i, j]) + assert conv == B[i, j] + C[i, j] + C[j, i] + assert convert_indexed_to_array(conv, [i, j]) == expr + + expr = ArrayElementwiseApplyFunc(sin, A) + conv = convert_array_to_indexed(expr, [i, j, k]) + assert conv == sin(A[i, j, k]) + assert convert_indexed_to_array(conv, [i, j, k]).dummy_eq(expr) + + assert convert_array_to_indexed(OneArray(3, 3), [i, j]) == 1 + assert convert_array_to_indexed(ZeroArray(3, 3), [i, j]) == 0 + + expr = Reshape(A, (27,)) + assert convert_array_to_indexed(expr, [i]) == A[i // 9, i // 3 % 3, i % 3] + + X = ArraySymbol("X", (2, 3, 4, 5, 6)) + expr = Reshape(X, (2*3*4*5*6,)) + assert convert_array_to_indexed(expr, [i]) == X[i // 360, i // 120 % 3, i // 30 % 4, i // 6 % 5, i % 6] + + expr = Reshape(X, (4, 9, 2, 2, 5)) + one_index = 180*i + 20*j + 10*k + 5*l + m + expected = X[one_index // (3*4*5*6), one_index // (4*5*6) % 3, one_index // (5*6) % 4, one_index // 6 % 5, one_index % 6] + assert convert_array_to_indexed(expr, [i, j, k, l, m]) == expected + + X = ArraySymbol("X", (2*3*5,)) + expr = Reshape(X, (2, 3, 5)) + assert convert_array_to_indexed(expr, [i, j, k]) == X[15*i + 5*j + k] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_matrix.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..26839d5e7cec0554948c6b726482f9d8ca250b1c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_array_to_matrix.py @@ -0,0 +1,689 @@ +from sympy import Lambda, S, Dummy, KroneckerProduct +from sympy.core.symbol import symbols +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import cos, sin +from sympy.matrices.expressions.hadamard import HadamardProduct, HadamardPower +from sympy.matrices.expressions.special import (Identity, OneMatrix, ZeroMatrix) +from sympy.matrices.expressions.matexpr import MatrixElement +from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array +from sympy.tensor.array.expressions.from_array_to_matrix import _support_function_tp1_recognize, \ + _array_diag2contr_diagmatrix, convert_array_to_matrix, _remove_trivial_dims, _array2matrix, \ + _combine_removed, identify_removable_identity_matrices, _array_contraction_to_diagonal_multiple_identity +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.combinatorics import Permutation +from sympy.matrices.expressions.diagonal import DiagMatrix, DiagonalMatrix +from sympy.matrices import Trace, MatMul, Transpose +from sympy.tensor.array.expressions.array_expressions import ZeroArray, OneArray, \ + ArrayElement, ArraySymbol, ArrayElementwiseApplyFunc, _array_tensor_product, _array_contraction, \ + _array_diagonal, _permute_dims, PermuteDims, ArrayAdd, ArrayDiagonal, ArrayContraction, ArrayTensorProduct +from sympy.testing.pytest import raises + + +i, j, k, l, m, n = symbols("i j k l m n") + +I = Identity(k) +I1 = Identity(1) + +M = MatrixSymbol("M", k, k) +N = MatrixSymbol("N", k, k) +P = MatrixSymbol("P", k, k) +Q = MatrixSymbol("Q", k, k) + +A = MatrixSymbol("A", k, k) +B = MatrixSymbol("B", k, k) +C = MatrixSymbol("C", k, k) +D = MatrixSymbol("D", k, k) + +X = MatrixSymbol("X", k, k) +Y = MatrixSymbol("Y", k, k) + +a = MatrixSymbol("a", k, 1) +b = MatrixSymbol("b", k, 1) +c = MatrixSymbol("c", k, 1) +d = MatrixSymbol("d", k, 1) + +x = MatrixSymbol("x", k, 1) +y = MatrixSymbol("y", k, 1) + + +def test_arrayexpr_convert_array_to_matrix(): + + cg = _array_contraction(_array_tensor_product(M), (0, 1)) + assert convert_array_to_matrix(cg) == Trace(M) + + cg = _array_contraction(_array_tensor_product(M, N), (0, 1), (2, 3)) + assert convert_array_to_matrix(cg) == Trace(M) * Trace(N) + + cg = _array_contraction(_array_tensor_product(M, N), (0, 3), (1, 2)) + assert convert_array_to_matrix(cg) == Trace(M * N) + + cg = _array_contraction(_array_tensor_product(M, N), (0, 2), (1, 3)) + assert convert_array_to_matrix(cg) == Trace(M * N.T) + + cg = convert_matrix_to_array(M * N * P) + assert convert_array_to_matrix(cg) == M * N * P + + cg = convert_matrix_to_array(M * N.T * P) + assert convert_array_to_matrix(cg) == M * N.T * P + + cg = _array_contraction(_array_tensor_product(M,N,P,Q), (1, 2), (5, 6)) + assert convert_array_to_matrix(cg) == _array_tensor_product(M * N, P * Q) + + cg = _array_contraction(_array_tensor_product(-2, M, N), (1, 2)) + assert convert_array_to_matrix(cg) == -2 * M * N + + a = MatrixSymbol("a", k, 1) + b = MatrixSymbol("b", k, 1) + c = MatrixSymbol("c", k, 1) + cg = PermuteDims( + _array_contraction( + _array_tensor_product( + a, + ArrayAdd( + _array_tensor_product(b, c), + _array_tensor_product(c, b), + ) + ), (2, 4)), [0, 1, 3, 2]) + assert convert_array_to_matrix(cg) == a * (b.T * c + c.T * b) + + za = ZeroArray(m, n) + assert convert_array_to_matrix(za) == ZeroMatrix(m, n) + + cg = _array_tensor_product(3, M) + assert convert_array_to_matrix(cg) == 3 * M + + # Partial conversion to matrix multiplication: + expr = _array_contraction(_array_tensor_product(M, N, P, Q), (0, 2), (1, 4, 6)) + assert convert_array_to_matrix(expr) == _array_contraction(_array_tensor_product(M.T*N, P, Q), (0, 2, 4)) + + x = MatrixSymbol("x", k, 1) + cg = PermuteDims( + _array_contraction(_array_tensor_product(OneArray(1), x, OneArray(1), DiagMatrix(Identity(1))), + (0, 5)), Permutation(1, 2, 3)) + assert convert_array_to_matrix(cg) == x + + expr = ArrayAdd(M, PermuteDims(M, [1, 0])) + assert convert_array_to_matrix(expr) == M + Transpose(M) + + +def test_arrayexpr_convert_array_to_matrix2(): + cg = _array_contraction(_array_tensor_product(M, N), (1, 3)) + assert convert_array_to_matrix(cg) == M * N.T + + cg = PermuteDims(_array_tensor_product(M, N), Permutation([0, 1, 3, 2])) + assert convert_array_to_matrix(cg) == _array_tensor_product(M, N.T) + + cg = _array_tensor_product(M, PermuteDims(N, Permutation([1, 0]))) + assert convert_array_to_matrix(cg) == _array_tensor_product(M, N.T) + + cg = _array_contraction( + PermuteDims( + _array_tensor_product(M, N, P, Q), Permutation([0, 2, 3, 1, 4, 5, 7, 6])), + (1, 2), (3, 5) + ) + assert convert_array_to_matrix(cg) == _array_tensor_product(M * P.T * Trace(N), Q.T) + + cg = _array_contraction( + _array_tensor_product(M, N, P, PermuteDims(Q, Permutation([1, 0]))), + (1, 5), (2, 3) + ) + assert convert_array_to_matrix(cg) == _array_tensor_product(M * P.T * Trace(N), Q.T) + + cg = _array_tensor_product(M, PermuteDims(N, [1, 0])) + assert convert_array_to_matrix(cg) == _array_tensor_product(M, N.T) + + cg = _array_tensor_product(PermuteDims(M, [1, 0]), PermuteDims(N, [1, 0])) + assert convert_array_to_matrix(cg) == _array_tensor_product(M.T, N.T) + + cg = _array_tensor_product(PermuteDims(N, [1, 0]), PermuteDims(M, [1, 0])) + assert convert_array_to_matrix(cg) == _array_tensor_product(N.T, M.T) + + cg = _array_contraction(M, (0,), (1,)) + assert convert_array_to_matrix(cg) == OneMatrix(1, k)*M*OneMatrix(k, 1) + + cg = _array_contraction(x, (0,), (1,)) + assert convert_array_to_matrix(cg) == OneMatrix(1, k)*x + + Xm = MatrixSymbol("Xm", m, n) + cg = _array_contraction(Xm, (0,), (1,)) + assert convert_array_to_matrix(cg) == OneMatrix(1, m)*Xm*OneMatrix(n, 1) + + +def test_arrayexpr_convert_array_to_diagonalized_vector(): + + # Check matrix recognition over trivial dimensions: + + cg = _array_tensor_product(a, b) + assert convert_array_to_matrix(cg) == a * b.T + + cg = _array_tensor_product(I1, a, b) + assert convert_array_to_matrix(cg) == a * b.T + + # Recognize trace inside a tensor product: + + cg = _array_contraction(_array_tensor_product(A, B, C), (0, 3), (1, 2)) + assert convert_array_to_matrix(cg) == Trace(A * B) * C + + # Transform diagonal operator to contraction: + + cg = _array_diagonal(_array_tensor_product(A, a), (1, 2)) + assert _array_diag2contr_diagmatrix(cg) == _array_contraction(_array_tensor_product(A, OneArray(1), DiagMatrix(a)), (1, 3)) + assert convert_array_to_matrix(cg) == A * DiagMatrix(a) + + cg = _array_diagonal(_array_tensor_product(a, b), (0, 2)) + assert _array_diag2contr_diagmatrix(cg) == _permute_dims( + _array_contraction(_array_tensor_product(DiagMatrix(a), OneArray(1), b), (0, 3)), [1, 2, 0] + ) + assert convert_array_to_matrix(cg) == b.T * DiagMatrix(a) + + cg = _array_diagonal(_array_tensor_product(A, a), (0, 2)) + assert _array_diag2contr_diagmatrix(cg) == _array_contraction(_array_tensor_product(A, OneArray(1), DiagMatrix(a)), (0, 3)) + assert convert_array_to_matrix(cg) == A.T * DiagMatrix(a) + + cg = _array_diagonal(_array_tensor_product(I, x, I1), (0, 2), (3, 5)) + assert _array_diag2contr_diagmatrix(cg) == _array_contraction(_array_tensor_product(I, OneArray(1), I1, DiagMatrix(x)), (0, 5)) + assert convert_array_to_matrix(cg) == DiagMatrix(x) + + cg = _array_diagonal(_array_tensor_product(I, x, A, B), (1, 2), (5, 6)) + assert _array_diag2contr_diagmatrix(cg) == _array_diagonal(_array_contraction(_array_tensor_product(I, OneArray(1), A, B, DiagMatrix(x)), (1, 7)), (5, 6)) + # TODO: this is returning a wrong result: + # convert_array_to_matrix(cg) + + cg = _array_diagonal(_array_tensor_product(I1, a, b), (1, 3, 5)) + assert convert_array_to_matrix(cg) == a*b.T + + cg = _array_diagonal(_array_tensor_product(I1, a, b), (1, 3)) + assert _array_diag2contr_diagmatrix(cg) == _array_contraction(_array_tensor_product(OneArray(1), a, b, I1), (2, 6)) + assert convert_array_to_matrix(cg) == a*b.T + + cg = _array_diagonal(_array_tensor_product(x, I1), (1, 2)) + assert isinstance(cg, ArrayDiagonal) + assert cg.diagonal_indices == ((1, 2),) + assert convert_array_to_matrix(cg) == x + + cg = _array_diagonal(_array_tensor_product(x, I), (0, 2)) + assert _array_diag2contr_diagmatrix(cg) == _array_contraction(_array_tensor_product(OneArray(1), I, DiagMatrix(x)), (1, 3)) + assert convert_array_to_matrix(cg).doit() == DiagMatrix(x) + + raises(ValueError, lambda: _array_diagonal(x, (1,))) + + # Ignore identity matrices with contractions: + + cg = _array_contraction(_array_tensor_product(I, A, I, I), (0, 2), (1, 3), (5, 7)) + assert cg.split_multiple_contractions() == cg + assert convert_array_to_matrix(cg) == Trace(A) * I + + cg = _array_contraction(_array_tensor_product(Trace(A) * I, I, I), (1, 5), (3, 4)) + assert cg.split_multiple_contractions() == cg + assert convert_array_to_matrix(cg).doit() == Trace(A) * I + + # Add DiagMatrix when required: + + cg = _array_contraction(_array_tensor_product(A, a), (1, 2)) + assert cg.split_multiple_contractions() == cg + assert convert_array_to_matrix(cg) == A * a + + cg = _array_contraction(_array_tensor_product(A, a, B), (1, 2, 4)) + assert cg.split_multiple_contractions() == _array_contraction(_array_tensor_product(A, DiagMatrix(a), OneArray(1), B), (1, 2), (3, 5)) + assert convert_array_to_matrix(cg) == A * DiagMatrix(a) * B + + cg = _array_contraction(_array_tensor_product(A, a, B), (0, 2, 4)) + assert cg.split_multiple_contractions() == _array_contraction(_array_tensor_product(A, DiagMatrix(a), OneArray(1), B), (0, 2), (3, 5)) + assert convert_array_to_matrix(cg) == A.T * DiagMatrix(a) * B + + cg = _array_contraction(_array_tensor_product(A, a, b, a.T, B), (0, 2, 4, 7, 9)) + assert cg.split_multiple_contractions() == _array_contraction(_array_tensor_product(A, DiagMatrix(a), OneArray(1), + DiagMatrix(b), OneArray(1), DiagMatrix(a), OneArray(1), B), + (0, 2), (3, 5), (6, 9), (8, 12)) + assert convert_array_to_matrix(cg) == A.T * DiagMatrix(a) * DiagMatrix(b) * DiagMatrix(a) * B.T + + cg = _array_contraction(_array_tensor_product(I1, I1, I1), (1, 2, 4)) + assert cg.split_multiple_contractions() == _array_contraction(_array_tensor_product(I1, I1, OneArray(1), I1), (1, 2), (3, 5)) + assert convert_array_to_matrix(cg) == 1 + + cg = _array_contraction(_array_tensor_product(I, I, I, I, A), (1, 2, 8), (5, 6, 9)) + assert convert_array_to_matrix(cg.split_multiple_contractions()).doit() == A + + cg = _array_contraction(_array_tensor_product(A, a, C, a, B), (1, 2, 4), (5, 6, 8)) + expected = _array_contraction(_array_tensor_product(A, DiagMatrix(a), OneArray(1), C, DiagMatrix(a), OneArray(1), B), (1, 3), (2, 5), (6, 7), (8, 10)) + assert cg.split_multiple_contractions() == expected + assert convert_array_to_matrix(cg) == A * DiagMatrix(a) * C * DiagMatrix(a) * B + + cg = _array_contraction(_array_tensor_product(a, I1, b, I1, (a.T*b).applyfunc(cos)), (1, 2, 8), (5, 6, 9)) + expected = _array_contraction(_array_tensor_product(a, I1, OneArray(1), b, I1, OneArray(1), (a.T*b).applyfunc(cos)), + (1, 3), (2, 10), (6, 8), (7, 11)) + assert cg.split_multiple_contractions().dummy_eq(expected) + assert convert_array_to_matrix(cg).doit().dummy_eq(MatMul(a, (a.T * b).applyfunc(cos), b.T)) + + +def test_arrayexpr_convert_array_contraction_tp_additions(): + a = ArrayAdd( + _array_tensor_product(M, N), + _array_tensor_product(N, M) + ) + tp = _array_tensor_product(P, a, Q) + expr = _array_contraction(tp, (3, 4)) + expected = _array_tensor_product( + P, + ArrayAdd( + _array_contraction(_array_tensor_product(M, N), (1, 2)), + _array_contraction(_array_tensor_product(N, M), (1, 2)), + ), + Q + ) + assert expr == expected + assert convert_array_to_matrix(expr) == _array_tensor_product(P, M * N + N * M, Q) + + expr = _array_contraction(tp, (1, 2), (3, 4), (5, 6)) + result = _array_contraction( + _array_tensor_product( + P, + ArrayAdd( + _array_contraction(_array_tensor_product(M, N), (1, 2)), + _array_contraction(_array_tensor_product(N, M), (1, 2)), + ), + Q + ), (1, 2), (3, 4)) + assert expr == result + assert convert_array_to_matrix(expr) == P * (M * N + N * M) * Q + + +def test_arrayexpr_convert_array_to_implicit_matmul(): + # Trivial dimensions are suppressed, so the result can be expressed in matrix form: + + cg = _array_tensor_product(a, b) + assert convert_array_to_matrix(cg) == a * b.T + + cg = _array_tensor_product(a, b, I) + assert convert_array_to_matrix(cg) == _array_tensor_product(a*b.T, I) + + cg = _array_tensor_product(I, a, b) + assert convert_array_to_matrix(cg) == _array_tensor_product(I, a*b.T) + + cg = _array_tensor_product(a, I, b) + assert convert_array_to_matrix(cg) == _array_tensor_product(a, I, b) + + cg = _array_contraction(_array_tensor_product(I, I), (1, 2)) + assert convert_array_to_matrix(cg) == I + + cg = PermuteDims(_array_tensor_product(I, Identity(1)), [0, 2, 1, 3]) + assert convert_array_to_matrix(cg) == I + + +def test_arrayexpr_convert_array_to_matrix_remove_trivial_dims(): + + # Tensor Product: + assert _remove_trivial_dims(_array_tensor_product(a, b)) == (a * b.T, [1, 3]) + assert _remove_trivial_dims(_array_tensor_product(a.T, b)) == (a * b.T, [0, 3]) + assert _remove_trivial_dims(_array_tensor_product(a, b.T)) == (a * b.T, [1, 2]) + assert _remove_trivial_dims(_array_tensor_product(a.T, b.T)) == (a * b.T, [0, 2]) + + assert _remove_trivial_dims(_array_tensor_product(I, a.T, b.T)) == (_array_tensor_product(I, a * b.T), [2, 4]) + assert _remove_trivial_dims(_array_tensor_product(a.T, I, b.T)) == (_array_tensor_product(a.T, I, b.T), []) + + assert _remove_trivial_dims(_array_tensor_product(a, I)) == (_array_tensor_product(a, I), []) + assert _remove_trivial_dims(_array_tensor_product(I, a)) == (_array_tensor_product(I, a), []) + + assert _remove_trivial_dims(_array_tensor_product(a.T, b.T, c, d)) == ( + _array_tensor_product(a * b.T, c * d.T), [0, 2, 5, 7]) + assert _remove_trivial_dims(_array_tensor_product(a.T, I, b.T, c, d, I)) == ( + _array_tensor_product(a.T, I, b*c.T, d, I), [4, 7]) + + # Addition: + + cg = ArrayAdd(_array_tensor_product(a, b), _array_tensor_product(c, d)) + assert _remove_trivial_dims(cg) == (a * b.T + c * d.T, [1, 3]) + + # Permute Dims: + + cg = PermuteDims(_array_tensor_product(a, b), Permutation(3)(1, 2)) + assert _remove_trivial_dims(cg) == (a * b.T, [2, 3]) + + cg = PermuteDims(_array_tensor_product(a, I, b), Permutation(5)(1, 2, 3, 4)) + assert _remove_trivial_dims(cg) == (cg, []) + + cg = PermuteDims(_array_tensor_product(I, b, a), Permutation(5)(1, 2, 4, 5, 3)) + assert _remove_trivial_dims(cg) == (PermuteDims(_array_tensor_product(I, b * a.T), [0, 2, 3, 1]), [4, 5]) + + # Diagonal: + + cg = _array_diagonal(_array_tensor_product(M, a), (1, 2)) + assert _remove_trivial_dims(cg) == (cg, []) + + # Contraction: + + cg = _array_contraction(_array_tensor_product(M, a), (1, 2)) + assert _remove_trivial_dims(cg) == (cg, []) + + # A few more cases to test the removal and shift of nested removed axes + # with array contractions and array diagonals: + tp = _array_tensor_product( + OneMatrix(1, 1), + M, + x, + OneMatrix(1, 1), + Identity(1), + ) + + expr = _array_contraction(tp, (1, 8)) + rexpr, removed = _remove_trivial_dims(expr) + assert removed == [0, 5, 6, 7] + + expr = _array_contraction(tp, (1, 8), (3, 4)) + rexpr, removed = _remove_trivial_dims(expr) + assert removed == [0, 3, 4, 5] + + expr = _array_diagonal(tp, (1, 8)) + rexpr, removed = _remove_trivial_dims(expr) + assert removed == [0, 5, 6, 7, 8] + + expr = _array_diagonal(tp, (1, 8), (3, 4)) + rexpr, removed = _remove_trivial_dims(expr) + assert removed == [0, 3, 4, 5, 6] + + expr = _array_diagonal(_array_contraction(_array_tensor_product(A, x, I, I1), (1, 2, 5)), (1, 4)) + rexpr, removed = _remove_trivial_dims(expr) + assert removed == [2, 3] + + cg = _array_diagonal(_array_tensor_product(PermuteDims(_array_tensor_product(x, I1), Permutation(1, 2, 3)), (x.T*x).applyfunc(sqrt)), (2, 4), (3, 5)) + rexpr, removed = _remove_trivial_dims(cg) + assert removed == [1, 2] + + # Contractions with identity matrices need to be followed by a permutation + # in order + cg = _array_contraction(_array_tensor_product(A, B, C, M, I), (1, 8)) + ret, removed = _remove_trivial_dims(cg) + assert ret == PermuteDims(_array_tensor_product(A, B, C, M), [0, 2, 3, 4, 5, 6, 7, 1]) + assert removed == [] + + cg = _array_contraction(_array_tensor_product(A, B, C, M, I), (1, 8), (3, 4)) + ret, removed = _remove_trivial_dims(cg) + assert ret == PermuteDims(_array_contraction(_array_tensor_product(A, B, C, M), (3, 4)), [0, 2, 3, 4, 5, 1]) + assert removed == [] + + # Trivial matrices are sometimes inserted into MatMul expressions: + + cg = _array_tensor_product(b*b.T, a.T*a) + ret, removed = _remove_trivial_dims(cg) + assert ret == b*a.T*a*b.T + assert removed == [2, 3] + + Xs = ArraySymbol("X", (3, 2, k)) + cg = _array_tensor_product(M, Xs, b.T*c, a*a.T, b*b.T, c.T*d) + ret, removed = _remove_trivial_dims(cg) + assert ret == _array_tensor_product(M, Xs, a*b.T*c*c.T*d*a.T, b*b.T) + assert removed == [5, 6, 11, 12] + + cg = _array_diagonal(_array_tensor_product(I, I1, x), (1, 4), (3, 5)) + assert _remove_trivial_dims(cg) == (PermuteDims(_array_diagonal(_array_tensor_product(I, x), (1, 2)), Permutation(1, 2)), [1]) + + expr = _array_diagonal(_array_tensor_product(x, I, y), (0, 2)) + assert _remove_trivial_dims(expr) == (PermuteDims(_array_tensor_product(DiagMatrix(x), y), [1, 2, 3, 0]), [0]) + + expr = _array_diagonal(_array_tensor_product(x, I, y), (0, 2), (3, 4)) + assert _remove_trivial_dims(expr) == (expr, []) + + +def test_arrayexpr_convert_array_to_matrix_diag2contraction_diagmatrix(): + cg = _array_diagonal(_array_tensor_product(M, a), (1, 2)) + res = _array_diag2contr_diagmatrix(cg) + assert res.shape == cg.shape + assert res == _array_contraction(_array_tensor_product(M, OneArray(1), DiagMatrix(a)), (1, 3)) + + raises(ValueError, lambda: _array_diagonal(_array_tensor_product(a, M), (1, 2))) + + cg = _array_diagonal(_array_tensor_product(a.T, M), (1, 2)) + res = _array_diag2contr_diagmatrix(cg) + assert res.shape == cg.shape + assert res == _array_contraction(_array_tensor_product(OneArray(1), M, DiagMatrix(a.T)), (1, 4)) + + cg = _array_diagonal(_array_tensor_product(a.T, M, N, b.T), (1, 2), (4, 7)) + res = _array_diag2contr_diagmatrix(cg) + assert res.shape == cg.shape + assert res == _array_contraction( + _array_tensor_product(OneArray(1), M, N, OneArray(1), DiagMatrix(a.T), DiagMatrix(b.T)), (1, 7), (3, 9)) + + cg = _array_diagonal(_array_tensor_product(a, M, N, b.T), (0, 2), (4, 7)) + res = _array_diag2contr_diagmatrix(cg) + assert res.shape == cg.shape + assert res == _array_contraction( + _array_tensor_product(OneArray(1), M, N, OneArray(1), DiagMatrix(a), DiagMatrix(b.T)), (1, 6), (3, 9)) + + cg = _array_diagonal(_array_tensor_product(a, M, N, b.T), (0, 4), (3, 7)) + res = _array_diag2contr_diagmatrix(cg) + assert res.shape == cg.shape + assert res == _array_contraction( + _array_tensor_product(OneArray(1), M, N, OneArray(1), DiagMatrix(a), DiagMatrix(b.T)), (3, 6), (2, 9)) + + I1 = Identity(1) + x = MatrixSymbol("x", k, 1) + A = MatrixSymbol("A", k, k) + cg = _array_diagonal(_array_tensor_product(x, A.T, I1), (0, 2)) + assert _array_diag2contr_diagmatrix(cg).shape == cg.shape + assert _array2matrix(cg).shape == cg.shape + + +def test_arrayexpr_convert_array_to_matrix_support_function(): + + assert _support_function_tp1_recognize([], [2 * k]) == 2 * k + + assert _support_function_tp1_recognize([(1, 2)], [A, 2 * k, B, 3]) == 6 * k * A * B + + assert _support_function_tp1_recognize([(0, 3), (1, 2)], [A, B]) == Trace(A * B) + + assert _support_function_tp1_recognize([(1, 2)], [A, B]) == A * B + assert _support_function_tp1_recognize([(0, 2)], [A, B]) == A.T * B + assert _support_function_tp1_recognize([(1, 3)], [A, B]) == A * B.T + assert _support_function_tp1_recognize([(0, 3)], [A, B]) == A.T * B.T + + assert _support_function_tp1_recognize([(1, 2), (5, 6)], [A, B, C, D]) == _array_tensor_product(A * B, C * D) + assert _support_function_tp1_recognize([(1, 4), (3, 6)], [A, B, C, D]) == PermuteDims( + _array_tensor_product(A * C, B * D), [0, 2, 1, 3]) + + assert _support_function_tp1_recognize([(0, 3), (1, 4)], [A, B, C]) == B * A * C + + assert _support_function_tp1_recognize([(9, 10), (1, 2), (5, 6), (3, 4), (7, 8)], + [X, Y, A, B, C, D]) == X * Y * A * B * C * D + + assert _support_function_tp1_recognize([(9, 10), (1, 2), (5, 6), (3, 4)], + [X, Y, A, B, C, D]) == _array_tensor_product(X * Y * A * B, C * D) + + assert _support_function_tp1_recognize([(1, 7), (3, 8), (4, 11)], [X, Y, A, B, C, D]) == PermuteDims( + _array_tensor_product(X * B.T, Y * C, A.T * D.T), [0, 2, 4, 1, 3, 5] + ) + + assert _support_function_tp1_recognize([(0, 1), (3, 6), (5, 8)], [X, A, B, C, D]) == PermuteDims( + _array_tensor_product(Trace(X) * A * C, B * D), [0, 2, 1, 3]) + + assert _support_function_tp1_recognize([(1, 2), (3, 4), (5, 6), (7, 8)], [A, A, B, C, D]) == A ** 2 * B * C * D + assert _support_function_tp1_recognize([(1, 2), (3, 4), (5, 6), (7, 8)], [X, A, B, C, D]) == X * A * B * C * D + + assert _support_function_tp1_recognize([(1, 6), (3, 8), (5, 10)], [X, Y, A, B, C, D]) == PermuteDims( + _array_tensor_product(X * B, Y * C, A * D), [0, 2, 4, 1, 3, 5] + ) + + assert _support_function_tp1_recognize([(1, 4), (3, 6)], [A, B, C, D]) == PermuteDims( + _array_tensor_product(A * C, B * D), [0, 2, 1, 3]) + + assert _support_function_tp1_recognize([(0, 4), (1, 7), (2, 5), (3, 8)], [X, A, B, C, D]) == C*X.T*B*A*D + + assert _support_function_tp1_recognize([(0, 4), (1, 7), (2, 5), (3, 8)], [X, A, B, C, D]) == C*X.T*B*A*D + + +def test_convert_array_to_hadamard_products(): + + expr = HadamardProduct(M, N) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == expr + + expr = HadamardProduct(M, N)*P + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == expr + + expr = Q*HadamardProduct(M, N)*P + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == expr + + expr = Q*HadamardProduct(M, N.T)*P + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == expr + + expr = HadamardProduct(M, N)*HadamardProduct(Q, P) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert expr == ret + + expr = P.T*HadamardProduct(M, N)*HadamardProduct(Q, P) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert expr == ret + + # ArrayDiagonal should be converted + cg = _array_diagonal(_array_tensor_product(M, N, Q), (1, 3), (0, 2, 4)) + ret = convert_array_to_matrix(cg) + expected = PermuteDims(_array_diagonal(_array_tensor_product(HadamardProduct(M.T, N.T), Q), (1, 2)), [1, 0, 2]) + assert expected == ret + + # Special case that should return the same expression: + cg = _array_diagonal(_array_tensor_product(HadamardProduct(M, N), Q), (0, 2)) + ret = convert_array_to_matrix(cg) + assert ret == cg + + # Hadamard products with traces: + + expr = Trace(HadamardProduct(M, N)) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == Trace(HadamardProduct(M.T, N.T)) + + expr = Trace(A*HadamardProduct(M, N)) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == Trace(HadamardProduct(M, N)*A) + + expr = Trace(HadamardProduct(A, M)*N) + cg = convert_matrix_to_array(expr) + ret = convert_array_to_matrix(cg) + assert ret == Trace(HadamardProduct(M.T, N)*A) + + # These should not be converted into Hadamard products: + + cg = _array_diagonal(_array_tensor_product(M, N), (0, 1, 2, 3)) + ret = convert_array_to_matrix(cg) + assert ret == cg + + cg = _array_diagonal(_array_tensor_product(A), (0, 1)) + ret = convert_array_to_matrix(cg) + assert ret == cg + + cg = _array_diagonal(_array_tensor_product(M, N, P), (0, 2, 4), (1, 3, 5)) + assert convert_array_to_matrix(cg) == HadamardProduct(M, N, P) + + cg = _array_diagonal(_array_tensor_product(M, N, P), (0, 3, 4), (1, 2, 5)) + assert convert_array_to_matrix(cg) == HadamardProduct(M, P, N.T) + + cg = _array_diagonal(_array_tensor_product(I, I1, x), (1, 4), (3, 5)) + assert convert_array_to_matrix(cg) == DiagMatrix(x) + + +def test_identify_removable_identity_matrices(): + + D = DiagonalMatrix(MatrixSymbol("D", k, k)) + + cg = _array_contraction(_array_tensor_product(A, B, I), (1, 2, 4, 5)) + expected = _array_contraction(_array_tensor_product(A, B), (1, 2)) + assert identify_removable_identity_matrices(cg) == expected + + cg = _array_contraction(_array_tensor_product(A, B, C, I), (1, 3, 5, 6, 7)) + expected = _array_contraction(_array_tensor_product(A, B, C), (1, 3, 5)) + assert identify_removable_identity_matrices(cg) == expected + + # Tests with diagonal matrices: + + cg = _array_contraction(_array_tensor_product(A, B, D), (1, 2, 4, 5)) + ret = identify_removable_identity_matrices(cg) + expected = _array_contraction(_array_tensor_product(A, B, D), (1, 4), (2, 5)) + assert ret == expected + + cg = _array_contraction(_array_tensor_product(A, B, D, M, N), (1, 2, 4, 5, 6, 8)) + ret = identify_removable_identity_matrices(cg) + assert ret == cg + + +def test_combine_removed(): + + assert _combine_removed(6, [0, 1, 2], [0, 1, 2]) == [0, 1, 2, 3, 4, 5] + assert _combine_removed(8, [2, 5], [1, 3, 4]) == [1, 2, 4, 5, 6] + assert _combine_removed(8, [7], []) == [7] + + +def test_array_contraction_to_diagonal_multiple_identities(): + + expr = _array_contraction(_array_tensor_product(A, B, I, C), (1, 2, 4), (5, 6)) + assert _array_contraction_to_diagonal_multiple_identity(expr) == (expr, []) + assert convert_array_to_matrix(expr) == _array_contraction(_array_tensor_product(A, B, C), (1, 2, 4)) + + expr = _array_contraction(_array_tensor_product(A, I, I), (1, 2, 4)) + assert _array_contraction_to_diagonal_multiple_identity(expr) == (A, [2]) + assert convert_array_to_matrix(expr) == A + + expr = _array_contraction(_array_tensor_product(A, I, I, B), (1, 2, 4), (3, 6)) + assert _array_contraction_to_diagonal_multiple_identity(expr) == (expr, []) + + expr = _array_contraction(_array_tensor_product(A, I, I, B), (1, 2, 3, 4, 6)) + assert _array_contraction_to_diagonal_multiple_identity(expr) == (expr, []) + + +def test_convert_array_element_to_matrix(): + + expr = ArrayElement(M, (i, j)) + assert convert_array_to_matrix(expr) == MatrixElement(M, i, j) + + expr = ArrayElement(_array_contraction(_array_tensor_product(M, N), (1, 3)), (i, j)) + assert convert_array_to_matrix(expr) == MatrixElement(M*N.T, i, j) + + expr = ArrayElement(_array_tensor_product(M, N), (i, j, m, n)) + assert convert_array_to_matrix(expr) == expr + + +def test_convert_array_elementwise_function_to_matrix(): + + d = Dummy("d") + + expr = ArrayElementwiseApplyFunc(Lambda(d, sin(d)), x.T*y) + assert convert_array_to_matrix(expr) == sin(x.T*y) + + expr = ArrayElementwiseApplyFunc(Lambda(d, d**2), x.T*y) + assert convert_array_to_matrix(expr) == (x.T*y)**2 + + expr = ArrayElementwiseApplyFunc(Lambda(d, sin(d)), x) + assert convert_array_to_matrix(expr).dummy_eq(x.applyfunc(sin)) + + expr = ArrayElementwiseApplyFunc(Lambda(d, 1 / (2 * sqrt(d))), x) + assert convert_array_to_matrix(expr) == S.Half * HadamardPower(x, -S.Half) + + +def test_array2matrix(): + # See issue https://github.com/sympy/sympy/pull/22877 + expr = PermuteDims(ArrayContraction(ArrayTensorProduct(x, I, I1, x), (0, 3), (1, 7)), Permutation(2, 3)) + expected = PermuteDims(ArrayTensorProduct(x*x.T, I1), Permutation(3)(1, 2)) + assert _array2matrix(expr) == expected + + +def test_recognize_broadcasting(): + expr = ArrayTensorProduct(x.T*x, A) + assert _remove_trivial_dims(expr) == (KroneckerProduct(x.T*x, A), [0, 1]) + + expr = ArrayTensorProduct(A, x.T*x) + assert _remove_trivial_dims(expr) == (KroneckerProduct(A, x.T*x), [2, 3]) + + expr = ArrayTensorProduct(A, B, x.T*x, C) + assert _remove_trivial_dims(expr) == (ArrayTensorProduct(A, KroneckerProduct(B, x.T*x), C), [4, 5]) + + # Always prefer matrix multiplication to Kronecker product, if possible: + expr = ArrayTensorProduct(a, b, x.T*x) + assert _remove_trivial_dims(expr) == (a*x.T*x*b.T, [1, 3, 4, 5]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_indexed_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_indexed_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..258062eadeca041ae3c864dabeefd5165f1cef11 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_indexed_to_array.py @@ -0,0 +1,205 @@ +from sympy import tanh +from sympy.concrete.summations import Sum +from sympy.core.symbol import symbols +from sympy.functions.special.tensor_functions import KroneckerDelta +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.matrices.expressions.special import Identity +from sympy.tensor.array.expressions import ArrayElementwiseApplyFunc +from sympy.tensor.indexed import IndexedBase +from sympy.combinatorics import Permutation +from sympy.tensor.array.expressions.array_expressions import ArrayContraction, ArrayTensorProduct, \ + ArrayDiagonal, ArrayAdd, PermuteDims, ArrayElement, _array_tensor_product, _array_contraction, _array_diagonal, \ + _array_add, _permute_dims, ArraySymbol, OneArray +from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix +from sympy.tensor.array.expressions.from_indexed_to_array import convert_indexed_to_array, _convert_indexed_to_array +from sympy.testing.pytest import raises + + +A, B = symbols("A B", cls=IndexedBase) +i, j, k, l, m, n = symbols("i j k l m n") +d0, d1, d2, d3 = symbols("d0:4") + +I = Identity(k) + +M = MatrixSymbol("M", k, k) +N = MatrixSymbol("N", k, k) +P = MatrixSymbol("P", k, k) +Q = MatrixSymbol("Q", k, k) + +a = MatrixSymbol("a", k, 1) +b = MatrixSymbol("b", k, 1) +c = MatrixSymbol("c", k, 1) +d = MatrixSymbol("d", k, 1) + + +def test_arrayexpr_convert_index_to_array_support_function(): + expr = M[i, j] + assert _convert_indexed_to_array(expr) == (M, (i, j)) + expr = M[i, j]*N[k, l] + assert _convert_indexed_to_array(expr) == (ArrayTensorProduct(M, N), (i, j, k, l)) + expr = M[i, j]*N[j, k] + assert _convert_indexed_to_array(expr) == (ArrayDiagonal(ArrayTensorProduct(M, N), (1, 2)), (i, k, j)) + expr = Sum(M[i, j]*N[j, k], (j, 0, k-1)) + assert _convert_indexed_to_array(expr) == (ArrayContraction(ArrayTensorProduct(M, N), (1, 2)), (i, k)) + expr = M[i, j] + N[i, j] + assert _convert_indexed_to_array(expr) == (ArrayAdd(M, N), (i, j)) + expr = M[i, j] + N[j, i] + assert _convert_indexed_to_array(expr) == (ArrayAdd(M, PermuteDims(N, Permutation([1, 0]))), (i, j)) + expr = M[i, j] + M[j, i] + assert _convert_indexed_to_array(expr) == (ArrayAdd(M, PermuteDims(M, Permutation([1, 0]))), (i, j)) + expr = (M*N*P)[i, j] + assert _convert_indexed_to_array(expr) == (_array_contraction(ArrayTensorProduct(M, N, P), (1, 2), (3, 4)), (i, j)) + expr = expr.function # Disregard summation in previous expression + ret1, ret2 = _convert_indexed_to_array(expr) + assert ret1 == ArrayDiagonal(ArrayTensorProduct(M, N, P), (1, 2), (3, 4)) + assert str(ret2) == "(i, j, _i_1, _i_2)" + expr = KroneckerDelta(i, j)*M[i, k] + assert _convert_indexed_to_array(expr) == (M, ({i, j}, k)) + expr = KroneckerDelta(i, j)*KroneckerDelta(j, k)*M[i, l] + assert _convert_indexed_to_array(expr) == (M, ({i, j, k}, l)) + expr = KroneckerDelta(j, k)*(M[i, j]*N[k, l] + N[i, j]*M[k, l]) + assert _convert_indexed_to_array(expr) == (_array_diagonal(_array_add( + ArrayTensorProduct(M, N), + _permute_dims(ArrayTensorProduct(M, N), Permutation(0, 2)(1, 3)) + ), (1, 2)), (i, l, frozenset({j, k}))) + expr = KroneckerDelta(j, m)*KroneckerDelta(m, k)*(M[i, j]*N[k, l] + N[i, j]*M[k, l]) + assert _convert_indexed_to_array(expr) == (_array_diagonal(_array_add( + ArrayTensorProduct(M, N), + _permute_dims(ArrayTensorProduct(M, N), Permutation(0, 2)(1, 3)) + ), (1, 2)), (i, l, frozenset({j, m, k}))) + expr = KroneckerDelta(i, j)*KroneckerDelta(j, k)*KroneckerDelta(k,m)*M[i, 0]*KroneckerDelta(m, n) + assert _convert_indexed_to_array(expr) == (M, ({i, j, k, m, n}, 0)) + expr = M[i, i] + assert _convert_indexed_to_array(expr) == (ArrayDiagonal(M, (0, 1)), (i,)) + + +def test_arrayexpr_convert_indexed_to_array_expression(): + + s = Sum(A[i]*B[i], (i, 0, 3)) + cg = convert_indexed_to_array(s) + assert cg == ArrayContraction(ArrayTensorProduct(A, B), (0, 1)) + + expr = M*N + result = ArrayContraction(ArrayTensorProduct(M, N), (1, 2)) + elem = expr[i, j] + assert convert_indexed_to_array(elem) == result + + expr = M*N*M + elem = expr[i, j] + result = _array_contraction(_array_tensor_product(M, M, N), (1, 4), (2, 5)) + cg = convert_indexed_to_array(elem) + assert cg == result + + cg = convert_indexed_to_array((M * N * P)[i, j]) + assert cg == _array_contraction(ArrayTensorProduct(M, N, P), (1, 2), (3, 4)) + + cg = convert_indexed_to_array((M * N.T * P)[i, j]) + assert cg == _array_contraction(ArrayTensorProduct(M, N, P), (1, 3), (2, 4)) + + expr = -2*M*N + elem = expr[i, j] + cg = convert_indexed_to_array(elem) + assert cg == ArrayContraction(ArrayTensorProduct(-2, M, N), (1, 2)) + + +def test_arrayexpr_convert_array_element_to_array_expression(): + A = ArraySymbol("A", (k,)) + B = ArraySymbol("B", (k,)) + + s = Sum(A[i]*B[i], (i, 0, k-1)) + cg = convert_indexed_to_array(s) + assert cg == ArrayContraction(ArrayTensorProduct(A, B), (0, 1)) + + s = A[i]*B[i] + cg = convert_indexed_to_array(s) + assert cg == ArrayDiagonal(ArrayTensorProduct(A, B), (0, 1)) + + s = A[i]*B[j] + cg = convert_indexed_to_array(s, [i, j]) + assert cg == ArrayTensorProduct(A, B) + cg = convert_indexed_to_array(s, [j, i]) + assert cg == ArrayTensorProduct(B, A) + + s = tanh(A[i]*B[j]) + cg = convert_indexed_to_array(s, [i, j]) + assert cg.dummy_eq(ArrayElementwiseApplyFunc(tanh, ArrayTensorProduct(A, B))) + + +def test_arrayexpr_convert_indexed_to_array_and_back_to_matrix(): + + expr = a.T*b + elem = expr[0, 0] + cg = convert_indexed_to_array(elem) + assert cg == ArrayElement(ArrayContraction(ArrayTensorProduct(a, b), (0, 2)), [0, 0]) + + expr = M[i,j] + N[i,j] + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == M + N + + expr = M[i,j] + N[j,i] + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == M + N.T + + expr = M[i,j]*N[k,l] + N[i,j]*M[k,l] + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == ArrayAdd( + ArrayTensorProduct(M, N), + ArrayTensorProduct(N, M)) + + expr = (M*N*P)[i, j] + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == M * N * P + + expr = Sum(M[i,j]*(N*P)[j,m], (j, 0, k-1)) + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == M * N * P + + expr = Sum((P[j, m] + P[m, j])*(M[i,j]*N[m,n] + N[i,j]*M[m,n]), (j, 0, k-1), (m, 0, k-1)) + p1, p2 = _convert_indexed_to_array(expr) + assert convert_array_to_matrix(p1) == M * P * N + M * P.T * N + N * P * M + N * P.T * M + + +def test_arrayexpr_convert_indexed_to_array_out_of_bounds(): + + expr = Sum(M[i, i], (i, 0, 4)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + expr = Sum(M[i, i], (i, 0, k)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + expr = Sum(M[i, i], (i, 1, k-1)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + + expr = Sum(M[i, j]*N[j,m], (j, 0, 4)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + expr = Sum(M[i, j]*N[j,m], (j, 0, k)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + expr = Sum(M[i, j]*N[j,m], (j, 1, k-1)) + raises(ValueError, lambda: convert_indexed_to_array(expr)) + + +def test_arrayexpr_convert_indexed_to_array_broadcast(): + A = ArraySymbol("A", (3, 3)) + B = ArraySymbol("B", (3, 3)) + + expr = A[i, j] + B[k, l] + O2 = OneArray(3, 3) + expected = ArrayAdd(ArrayTensorProduct(A, O2), ArrayTensorProduct(O2, B)) + assert convert_indexed_to_array(expr) == expected + assert convert_indexed_to_array(expr, [i, j, k, l]) == expected + assert convert_indexed_to_array(expr, [l, k, i, j]) == ArrayAdd(PermuteDims(ArrayTensorProduct(O2, A), [1, 0, 2, 3]), PermuteDims(ArrayTensorProduct(B, O2), [1, 0, 2, 3])) + + expr = A[i, j] + B[j, k] + O1 = OneArray(3) + assert convert_indexed_to_array(expr, [i, j, k]) == ArrayAdd(ArrayTensorProduct(A, O1), ArrayTensorProduct(O1, B)) + + C = ArraySymbol("C", (d0, d1)) + D = ArraySymbol("D", (d3, d1)) + + expr = C[i, j] + D[k, j] + assert convert_indexed_to_array(expr, [i, j, k]) == ArrayAdd(ArrayTensorProduct(C, OneArray(d3)), PermuteDims(ArrayTensorProduct(OneArray(d0), D), [0, 2, 1])) + + X = ArraySymbol("X", (5, 3)) + + expr = X[i, n] - X[j, n] + assert convert_indexed_to_array(expr, [i, j, n]) == ArrayAdd(ArrayTensorProduct(-1, OneArray(5), X), PermuteDims(ArrayTensorProduct(X, OneArray(5)), [0, 2, 1])) + + raises(ValueError, lambda: convert_indexed_to_array(C[i, j] + D[i, j])) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_matrix_to_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_matrix_to_array.py new file mode 100644 index 0000000000000000000000000000000000000000..142585882588df6aa0e4648d9d8881ea755f42a0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_convert_matrix_to_array.py @@ -0,0 +1,128 @@ +from sympy import Lambda, KroneckerProduct +from sympy.core.symbol import symbols, Dummy +from sympy.matrices.expressions.hadamard import (HadamardPower, HadamardProduct) +from sympy.matrices.expressions.inverse import Inverse +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.matrices.expressions.matpow import MatPow +from sympy.matrices.expressions.special import Identity +from sympy.matrices.expressions.trace import Trace +from sympy.matrices.expressions.transpose import Transpose +from sympy.tensor.array.expressions.array_expressions import ArrayTensorProduct, ArrayContraction, \ + PermuteDims, ArrayDiagonal, ArrayElementwiseApplyFunc, _array_contraction, _array_tensor_product, Reshape +from sympy.tensor.array.expressions.from_array_to_matrix import convert_array_to_matrix +from sympy.tensor.array.expressions.from_matrix_to_array import convert_matrix_to_array + +i, j, k, l, m, n = symbols("i j k l m n") + +I = Identity(k) + +M = MatrixSymbol("M", k, k) +N = MatrixSymbol("N", k, k) +P = MatrixSymbol("P", k, k) +Q = MatrixSymbol("Q", k, k) + +A = MatrixSymbol("A", k, k) +B = MatrixSymbol("B", k, k) +C = MatrixSymbol("C", k, k) +D = MatrixSymbol("D", k, k) + +X = MatrixSymbol("X", k, k) +Y = MatrixSymbol("Y", k, k) + +a = MatrixSymbol("a", k, 1) +b = MatrixSymbol("b", k, 1) +c = MatrixSymbol("c", k, 1) +d = MatrixSymbol("d", k, 1) + + +def test_arrayexpr_convert_matrix_to_array(): + + expr = M*N + result = ArrayContraction(ArrayTensorProduct(M, N), (1, 2)) + assert convert_matrix_to_array(expr) == result + + expr = M*N*M + result = _array_contraction(ArrayTensorProduct(M, N, M), (1, 2), (3, 4)) + assert convert_matrix_to_array(expr) == result + + expr = Transpose(M) + assert convert_matrix_to_array(expr) == PermuteDims(M, [1, 0]) + + expr = M*Transpose(N) + assert convert_matrix_to_array(expr) == _array_contraction(_array_tensor_product(M, PermuteDims(N, [1, 0])), (1, 2)) + + expr = 3*M*N + res = convert_matrix_to_array(expr) + rexpr = convert_array_to_matrix(res) + assert expr == rexpr + + expr = 3*M + N*M.T*M + 4*k*N + res = convert_matrix_to_array(expr) + rexpr = convert_array_to_matrix(res) + assert expr == rexpr + + expr = Inverse(M)*N + rexpr = convert_array_to_matrix(convert_matrix_to_array(expr)) + assert expr == rexpr + + expr = M**2 + rexpr = convert_array_to_matrix(convert_matrix_to_array(expr)) + assert expr == rexpr + + expr = M*(2*N + 3*M) + res = convert_matrix_to_array(expr) + rexpr = convert_array_to_matrix(res) + assert expr == rexpr + + expr = Trace(M) + result = ArrayContraction(M, (0, 1)) + assert convert_matrix_to_array(expr) == result + + expr = 3*Trace(M) + result = ArrayContraction(ArrayTensorProduct(3, M), (0, 1)) + assert convert_matrix_to_array(expr) == result + + expr = 3*Trace(Trace(M) * M) + result = ArrayContraction(ArrayTensorProduct(3, M, M), (0, 1), (2, 3)) + assert convert_matrix_to_array(expr) == result + + expr = 3*Trace(M)**2 + result = ArrayContraction(ArrayTensorProduct(3, M, M), (0, 1), (2, 3)) + assert convert_matrix_to_array(expr) == result + + expr = HadamardProduct(M, N) + result = ArrayDiagonal(ArrayTensorProduct(M, N), (0, 2), (1, 3)) + assert convert_matrix_to_array(expr) == result + + expr = HadamardProduct(M*N, N*M) + result = ArrayDiagonal(ArrayContraction(ArrayTensorProduct(M, N, N, M), (1, 2), (5, 6)), (0, 2), (1, 3)) + assert convert_matrix_to_array(expr) == result + + expr = HadamardPower(M, 2) + result = ArrayDiagonal(ArrayTensorProduct(M, M), (0, 2), (1, 3)) + assert convert_matrix_to_array(expr) == result + + expr = HadamardPower(M*N, 2) + result = ArrayDiagonal(ArrayContraction(ArrayTensorProduct(M, N, M, N), (1, 2), (5, 6)), (0, 2), (1, 3)) + assert convert_matrix_to_array(expr) == result + + expr = HadamardPower(M, n) + d0 = Dummy("d0") + result = ArrayElementwiseApplyFunc(Lambda(d0, d0**n), M) + assert convert_matrix_to_array(expr).dummy_eq(result) + + expr = M**2 + assert isinstance(expr, MatPow) + assert convert_matrix_to_array(expr) == ArrayContraction(ArrayTensorProduct(M, M), (1, 2)) + + expr = a.T*b + cg = convert_matrix_to_array(expr) + assert cg == ArrayContraction(ArrayTensorProduct(a, b), (0, 2)) + + expr = KroneckerProduct(A, B) + cg = convert_matrix_to_array(expr) + assert cg == Reshape(PermuteDims(ArrayTensorProduct(A, B), [0, 2, 1, 3]), (k**2, k**2)) + + expr = KroneckerProduct(A, B, C, D) + cg = convert_matrix_to_array(expr) + assert cg == Reshape(PermuteDims(ArrayTensorProduct(A, B, C, D), [0, 2, 4, 6, 1, 3, 5, 7]), (k**4, k**4)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_deprecated_conv_modules.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_deprecated_conv_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..b41b6105410a308e7774fce760b235497d0303bb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/tests/test_deprecated_conv_modules.py @@ -0,0 +1,22 @@ +from sympy import MatrixSymbol, symbols, Sum +from sympy.tensor.array.expressions import conv_array_to_indexed, from_array_to_indexed, ArrayTensorProduct, \ + ArrayContraction, conv_array_to_matrix, from_array_to_matrix, conv_matrix_to_array, from_matrix_to_array, \ + conv_indexed_to_array, from_indexed_to_array +from sympy.testing.pytest import warns +from sympy.utilities.exceptions import SymPyDeprecationWarning + + +def test_deprecated_conv_module_results(): + + M = MatrixSymbol("M", 3, 3) + N = MatrixSymbol("N", 3, 3) + i, j, d = symbols("i j d") + + x = ArrayContraction(ArrayTensorProduct(M, N), (1, 2)) + y = Sum(M[i, d]*N[d, j], (d, 0, 2)) + + with warns(SymPyDeprecationWarning, test_stacklevel=False): + assert conv_array_to_indexed.convert_array_to_indexed(x, [i, j]).dummy_eq(from_array_to_indexed.convert_array_to_indexed(x, [i, j])) + assert conv_array_to_matrix.convert_array_to_matrix(x) == from_array_to_matrix.convert_array_to_matrix(x) + assert conv_matrix_to_array.convert_matrix_to_array(M*N) == from_matrix_to_array.convert_matrix_to_array(M*N) + assert conv_indexed_to_array.convert_indexed_to_array(y) == from_indexed_to_array.convert_indexed_to_array(y) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/utils.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e55c0e6ed47cdc9ff1c24cc92f006998aeb86822 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/expressions/utils.py @@ -0,0 +1,123 @@ +import bisect +from collections import defaultdict + +from sympy.combinatorics import Permutation +from sympy.core.containers import Tuple +from sympy.core.numbers import Integer + + +def _get_mapping_from_subranks(subranks): + mapping = {} + counter = 0 + for i, rank in enumerate(subranks): + for j in range(rank): + mapping[counter] = (i, j) + counter += 1 + return mapping + + +def _get_contraction_links(args, subranks, *contraction_indices): + mapping = _get_mapping_from_subranks(subranks) + contraction_tuples = [[mapping[j] for j in i] for i in contraction_indices] + dlinks = defaultdict(dict) + for links in contraction_tuples: + if len(links) == 2: + (arg1, pos1), (arg2, pos2) = links + dlinks[arg1][pos1] = (arg2, pos2) + dlinks[arg2][pos2] = (arg1, pos1) + continue + + return args, dict(dlinks) + + +def _sort_contraction_indices(pairing_indices): + pairing_indices = [Tuple(*sorted(i)) for i in pairing_indices] + pairing_indices.sort(key=lambda x: min(x)) + return pairing_indices + + +def _get_diagonal_indices(flattened_indices): + axes_contraction = defaultdict(list) + for i, ind in enumerate(flattened_indices): + if isinstance(ind, (int, Integer)): + # If the indices is a number, there can be no diagonal operation: + continue + axes_contraction[ind].append(i) + axes_contraction = {k: v for k, v in axes_contraction.items() if len(v) > 1} + # Put the diagonalized indices at the end: + ret_indices = [i for i in flattened_indices if i not in axes_contraction] + diag_indices = list(axes_contraction) + diag_indices.sort(key=lambda x: flattened_indices.index(x)) + diagonal_indices = [tuple(axes_contraction[i]) for i in diag_indices] + ret_indices += diag_indices + ret_indices = tuple(ret_indices) + return diagonal_indices, ret_indices + + +def _get_argindex(subindices, ind): + for i, sind in enumerate(subindices): + if ind == sind: + return i + if isinstance(sind, (set, frozenset)) and ind in sind: + return i + raise IndexError("%s not found in %s" % (ind, subindices)) + + +def _apply_recursively_over_nested_lists(func, arr): + if isinstance(arr, (tuple, list, Tuple)): + return tuple(_apply_recursively_over_nested_lists(func, i) for i in arr) + elif isinstance(arr, Tuple): + return Tuple.fromiter(_apply_recursively_over_nested_lists(func, i) for i in arr) + else: + return func(arr) + + +def _build_push_indices_up_func_transformation(flattened_contraction_indices): + shifts = {0: 0} + i = 0 + cumulative = 0 + while i < len(flattened_contraction_indices): + j = 1 + while i+j < len(flattened_contraction_indices): + if flattened_contraction_indices[i] + j != flattened_contraction_indices[i+j]: + break + j += 1 + cumulative += j + shifts[flattened_contraction_indices[i]] = cumulative + i += j + shift_keys = sorted(shifts.keys()) + + def func(idx): + return shifts[shift_keys[bisect.bisect_right(shift_keys, idx)-1]] + + def transform(j): + if j in flattened_contraction_indices: + return None + else: + return j - func(j) + + return transform + + +def _build_push_indices_down_func_transformation(flattened_contraction_indices): + N = flattened_contraction_indices[-1]+2 + + shifts = [i for i in range(N) if i not in flattened_contraction_indices] + + def transform(j): + if j < len(shifts): + return shifts[j] + else: + return j + shifts[-1] - len(shifts) + 1 + + return transform + + +def _apply_permutation_to_list(perm: Permutation, target_list: list): + """ + Permute a list according to the given permutation. + """ + new_list = [None for i in range(perm.size)] + for i, e in enumerate(target_list): + new_list[perm(i)] = e + return new_list diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/mutable_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/mutable_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..e1eaaf7241bc3b4a48234178d18da3aa5736e189 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/mutable_ndim_array.py @@ -0,0 +1,13 @@ +from sympy.tensor.array.ndim_array import NDimArray + + +class MutableNDimArray(NDimArray): + + def as_immutable(self): + raise NotImplementedError("abstract method") + + def as_mutable(self): + return self + + def _sympy_(self): + return self.as_immutable() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..2b9a857b8cfd9ee46646c46f274636d6b9962b6e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/ndim_array.py @@ -0,0 +1,601 @@ +from sympy.core.basic import Basic +from sympy.core.containers import (Dict, Tuple) +from sympy.core.expr import Expr +from sympy.core.kind import Kind, NumberKind, UndefinedKind +from sympy.core.numbers import Integer +from sympy.core.singleton import S +from sympy.core.sympify import sympify +from sympy.external.gmpy import SYMPY_INTS +from sympy.printing.defaults import Printable + +import itertools +from collections.abc import Iterable + + +class ArrayKind(Kind): + """ + Kind for N-dimensional array in SymPy. + + This kind represents the multidimensional array that algebraic + operations are defined. Basic class for this kind is ``NDimArray``, + but any expression representing the array can have this. + + Parameters + ========== + + element_kind : Kind + Kind of the element. Default is :obj:NumberKind ``, + which means that the array contains only numbers. + + Examples + ======== + + Any instance of array class has ``ArrayKind``. + + >>> from sympy import NDimArray + >>> NDimArray([1,2,3]).kind + ArrayKind(NumberKind) + + Although expressions representing an array may be not instance of + array class, it will have ``ArrayKind`` as well. + + >>> from sympy import Integral + >>> from sympy.tensor.array import NDimArray + >>> from sympy.abc import x + >>> intA = Integral(NDimArray([1,2,3]), x) + >>> isinstance(intA, NDimArray) + False + >>> intA.kind + ArrayKind(NumberKind) + + Use ``isinstance()`` to check for ``ArrayKind` without specifying + the element kind. Use ``is`` with specifying the element kind. + + >>> from sympy.tensor.array import ArrayKind + >>> from sympy.core import NumberKind + >>> boolA = NDimArray([True, False]) + >>> isinstance(boolA.kind, ArrayKind) + True + >>> boolA.kind is ArrayKind(NumberKind) + False + + See Also + ======== + + shape : Function to return the shape of objects with ``MatrixKind``. + + """ + def __new__(cls, element_kind=NumberKind): + obj = super().__new__(cls, element_kind) + obj.element_kind = element_kind + return obj + + def __repr__(self): + return "ArrayKind(%s)" % self.element_kind + + @classmethod + def _union(cls, kinds) -> 'ArrayKind': + elem_kinds = {e.kind for e in kinds} + if len(elem_kinds) == 1: + elemkind, = elem_kinds + else: + elemkind = UndefinedKind + return ArrayKind(elemkind) + + +class NDimArray(Printable): + """N-dimensional array. + + Examples + ======== + + Create an N-dim array of zeros: + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(2, 3, 4) + >>> a + [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]] + + Create an N-dim array from a list; + + >>> a = MutableDenseNDimArray([[2, 3], [4, 5]]) + >>> a + [[2, 3], [4, 5]] + + >>> b = MutableDenseNDimArray([[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]]) + >>> b + [[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]] + + Create an N-dim array from a flat list with dimension shape: + + >>> a = MutableDenseNDimArray([1, 2, 3, 4, 5, 6], (2, 3)) + >>> a + [[1, 2, 3], [4, 5, 6]] + + Create an N-dim array from a matrix: + + >>> from sympy import Matrix + >>> a = Matrix([[1,2],[3,4]]) + >>> a + Matrix([ + [1, 2], + [3, 4]]) + >>> b = MutableDenseNDimArray(a) + >>> b + [[1, 2], [3, 4]] + + Arithmetic operations on N-dim arrays + + >>> a = MutableDenseNDimArray([1, 1, 1, 1], (2, 2)) + >>> b = MutableDenseNDimArray([4, 4, 4, 4], (2, 2)) + >>> c = a + b + >>> c + [[5, 5], [5, 5]] + >>> a - b + [[-3, -3], [-3, -3]] + + """ + + _diff_wrt = True + is_scalar = False + + def __new__(cls, iterable, shape=None, **kwargs): + from sympy.tensor.array import ImmutableDenseNDimArray + return ImmutableDenseNDimArray(iterable, shape, **kwargs) + + def __getitem__(self, index): + raise NotImplementedError("A subclass of NDimArray should implement __getitem__") + + def _parse_index(self, index): + if isinstance(index, (SYMPY_INTS, Integer)): + if index >= self._loop_size: + raise ValueError("Only a tuple index is accepted") + return index + + if self._loop_size == 0: + raise ValueError("Index not valid with an empty array") + + if len(index) != self._rank: + raise ValueError('Wrong number of array axes') + + real_index = 0 + # check if input index can exist in current indexing + for i in range(self._rank): + if (index[i] >= self.shape[i]) or (index[i] < -self.shape[i]): + raise ValueError('Index ' + str(index) + ' out of border') + if index[i] < 0: + real_index += 1 + real_index = real_index*self.shape[i] + index[i] + + return real_index + + def _get_tuple_index(self, integer_index): + index = [] + for sh in reversed(self.shape): + index.append(integer_index % sh) + integer_index //= sh + index.reverse() + return tuple(index) + + def _check_symbolic_index(self, index): + # Check if any index is symbolic: + tuple_index = (index if isinstance(index, tuple) else (index,)) + if any((isinstance(i, Expr) and (not i.is_number)) for i in tuple_index): + for i, nth_dim in zip(tuple_index, self.shape): + if ((i < 0) == True) or ((i >= nth_dim) == True): + raise ValueError("index out of range") + from sympy.tensor import Indexed + return Indexed(self, *tuple_index) + return None + + def _setter_iterable_check(self, value): + from sympy.matrices.matrixbase import MatrixBase + if isinstance(value, (Iterable, MatrixBase, NDimArray)): + raise NotImplementedError + + @classmethod + def _scan_iterable_shape(cls, iterable): + def f(pointer): + if not isinstance(pointer, Iterable): + return [pointer], () + + if len(pointer) == 0: + return [], (0,) + + result = [] + elems, shapes = zip(*[f(i) for i in pointer]) + if len(set(shapes)) != 1: + raise ValueError("could not determine shape unambiguously") + for i in elems: + result.extend(i) + return result, (len(shapes),)+shapes[0] + + return f(iterable) + + @classmethod + def _handle_ndarray_creation_inputs(cls, iterable=None, shape=None, **kwargs): + from sympy.matrices.matrixbase import MatrixBase + from sympy.tensor.array import SparseNDimArray + + if shape is None: + if iterable is None: + shape = () + iterable = () + # Construction of a sparse array from a sparse array + elif isinstance(iterable, SparseNDimArray): + return iterable._shape, iterable._sparse_array + + # Construct N-dim array from another N-dim array: + elif isinstance(iterable, NDimArray): + shape = iterable.shape + + # Construct N-dim array from an iterable (numpy arrays included): + elif isinstance(iterable, Iterable): + iterable, shape = cls._scan_iterable_shape(iterable) + + # Construct N-dim array from a Matrix: + elif isinstance(iterable, MatrixBase): + shape = iterable.shape + + else: + shape = () + iterable = (iterable,) + + if isinstance(iterable, (Dict, dict)) and shape is not None: + new_dict = iterable.copy() + for k in new_dict: + if isinstance(k, (tuple, Tuple)): + new_key = 0 + for i, idx in enumerate(k): + new_key = new_key * shape[i] + idx + iterable[new_key] = iterable[k] + del iterable[k] + + if isinstance(shape, (SYMPY_INTS, Integer)): + shape = (shape,) + + if not all(isinstance(dim, (SYMPY_INTS, Integer)) for dim in shape): + raise TypeError("Shape should contain integers only.") + + return tuple(shape), iterable + + def __len__(self): + """Overload common function len(). Returns number of elements in array. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(3, 3) + >>> a + [[0, 0, 0], [0, 0, 0], [0, 0, 0]] + >>> len(a) + 9 + + """ + return self._loop_size + + @property + def shape(self): + """ + Returns array shape (dimension). + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(3, 3) + >>> a.shape + (3, 3) + + """ + return self._shape + + def rank(self): + """ + Returns rank of array. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(3,4,5,6,3) + >>> a.rank() + 5 + + """ + return self._rank + + def diff(self, *args, **kwargs): + """ + Calculate the derivative of each element in the array. + + Examples + ======== + + >>> from sympy import ImmutableDenseNDimArray + >>> from sympy.abc import x, y + >>> M = ImmutableDenseNDimArray([[x, y], [1, x*y]]) + >>> M.diff(x) + [[1, 0], [0, y]] + + """ + from sympy.tensor.array.array_derivatives import ArrayDerivative + kwargs.setdefault('evaluate', True) + return ArrayDerivative(self.as_immutable(), *args, **kwargs) + + def _eval_derivative(self, base): + # Types are (base: scalar, self: array) + return self.applyfunc(lambda x: base.diff(x)) + + def _eval_derivative_n_times(self, s, n): + return Basic._eval_derivative_n_times(self, s, n) + + def applyfunc(self, f): + """Apply a function to each element of the N-dim array. + + Examples + ======== + + >>> from sympy import ImmutableDenseNDimArray + >>> m = ImmutableDenseNDimArray([i*2+j for i in range(2) for j in range(2)], (2, 2)) + >>> m + [[0, 1], [2, 3]] + >>> m.applyfunc(lambda i: 2*i) + [[0, 2], [4, 6]] + """ + from sympy.tensor.array import SparseNDimArray + from sympy.tensor.array.arrayop import Flatten + + if isinstance(self, SparseNDimArray) and f(S.Zero) == 0: + return type(self)({k: f(v) for k, v in self._sparse_array.items() if f(v) != 0}, self.shape) + + return type(self)(map(f, Flatten(self)), self.shape) + + def _sympystr(self, printer): + def f(sh, shape_left, i, j): + if len(shape_left) == 1: + return "["+", ".join([printer._print(self[self._get_tuple_index(e)]) for e in range(i, j)])+"]" + + sh //= shape_left[0] + return "[" + ", ".join([f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh) for e in range(shape_left[0])]) + "]" # + "\n"*len(shape_left) + + if self.rank() == 0: + return printer._print(self[()]) + if 0 in self.shape: + return f"{self.__class__.__name__}([], {self.shape})" + return f(self._loop_size, self.shape, 0, self._loop_size) + + def tolist(self): + """ + Converting MutableDenseNDimArray to one-dim list + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray([1, 2, 3, 4], (2, 2)) + >>> a + [[1, 2], [3, 4]] + >>> b = a.tolist() + >>> b + [[1, 2], [3, 4]] + """ + + def f(sh, shape_left, i, j): + if len(shape_left) == 1: + return [self[self._get_tuple_index(e)] for e in range(i, j)] + result = [] + sh //= shape_left[0] + for e in range(shape_left[0]): + result.append(f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh)) + return result + + return f(self._loop_size, self.shape, 0, self._loop_size) + + def __add__(self, other): + from sympy.tensor.array.arrayop import Flatten + + if not isinstance(other, NDimArray): + return NotImplemented + + if self.shape != other.shape: + raise ValueError("array shape mismatch") + result_list = [i+j for i,j in zip(Flatten(self), Flatten(other))] + + return type(self)(result_list, self.shape) + + def __sub__(self, other): + from sympy.tensor.array.arrayop import Flatten + + if not isinstance(other, NDimArray): + return NotImplemented + + if self.shape != other.shape: + raise ValueError("array shape mismatch") + result_list = [i-j for i,j in zip(Flatten(self), Flatten(other))] + + return type(self)(result_list, self.shape) + + def __mul__(self, other): + from sympy.matrices.matrixbase import MatrixBase + from sympy.tensor.array import SparseNDimArray + from sympy.tensor.array.arrayop import Flatten + + if isinstance(other, (Iterable, NDimArray, MatrixBase)): + raise ValueError("scalar expected, use tensorproduct(...) for tensorial product") + + other = sympify(other) + if isinstance(self, SparseNDimArray): + if other.is_zero: + return type(self)({}, self.shape) + return type(self)({k: other*v for (k, v) in self._sparse_array.items()}, self.shape) + + result_list = [i*other for i in Flatten(self)] + return type(self)(result_list, self.shape) + + def __rmul__(self, other): + from sympy.matrices.matrixbase import MatrixBase + from sympy.tensor.array import SparseNDimArray + from sympy.tensor.array.arrayop import Flatten + + if isinstance(other, (Iterable, NDimArray, MatrixBase)): + raise ValueError("scalar expected, use tensorproduct(...) for tensorial product") + + other = sympify(other) + if isinstance(self, SparseNDimArray): + if other.is_zero: + return type(self)({}, self.shape) + return type(self)({k: other*v for (k, v) in self._sparse_array.items()}, self.shape) + + result_list = [other*i for i in Flatten(self)] + return type(self)(result_list, self.shape) + + def __truediv__(self, other): + from sympy.matrices.matrixbase import MatrixBase + from sympy.tensor.array import SparseNDimArray + from sympy.tensor.array.arrayop import Flatten + + if isinstance(other, (Iterable, NDimArray, MatrixBase)): + raise ValueError("scalar expected") + + other = sympify(other) + if isinstance(self, SparseNDimArray) and other != S.Zero: + return type(self)({k: v/other for (k, v) in self._sparse_array.items()}, self.shape) + + result_list = [i/other for i in Flatten(self)] + return type(self)(result_list, self.shape) + + def __rtruediv__(self, other): + raise NotImplementedError('unsupported operation on NDimArray') + + def __neg__(self): + from sympy.tensor.array import SparseNDimArray + from sympy.tensor.array.arrayop import Flatten + + if isinstance(self, SparseNDimArray): + return type(self)({k: -v for (k, v) in self._sparse_array.items()}, self.shape) + + result_list = [-i for i in Flatten(self)] + return type(self)(result_list, self.shape) + + def __iter__(self): + def iterator(): + if self._shape: + for i in range(self._shape[0]): + yield self[i] + else: + yield self[()] + + return iterator() + + def __eq__(self, other): + """ + NDimArray instances can be compared to each other. + Instances equal if they have same shape and data. + + Examples + ======== + + >>> from sympy import MutableDenseNDimArray + >>> a = MutableDenseNDimArray.zeros(2, 3) + >>> b = MutableDenseNDimArray.zeros(2, 3) + >>> a == b + True + >>> c = a.reshape(3, 2) + >>> c == b + False + >>> a[0,0] = 1 + >>> b[0,0] = 2 + >>> a == b + False + """ + from sympy.tensor.array import SparseNDimArray + if not isinstance(other, NDimArray): + return False + + if not self.shape == other.shape: + return False + + if isinstance(self, SparseNDimArray) and isinstance(other, SparseNDimArray): + return dict(self._sparse_array) == dict(other._sparse_array) + + return list(self) == list(other) + + def __ne__(self, other): + return not self == other + + def _eval_transpose(self): + if self.rank() != 2: + raise ValueError("array rank not 2") + from .arrayop import permutedims + return permutedims(self, (1, 0)) + + def transpose(self): + return self._eval_transpose() + + def _eval_conjugate(self): + from sympy.tensor.array.arrayop import Flatten + + return self.func([i.conjugate() for i in Flatten(self)], self.shape) + + def conjugate(self): + return self._eval_conjugate() + + def _eval_adjoint(self): + return self.transpose().conjugate() + + def adjoint(self): + return self._eval_adjoint() + + def _slice_expand(self, s, dim): + if not isinstance(s, slice): + return (s,) + start, stop, step = s.indices(dim) + return [start + i*step for i in range((stop-start)//step)] + + def _get_slice_data_for_array_access(self, index): + sl_factors = [self._slice_expand(i, dim) for (i, dim) in zip(index, self.shape)] + eindices = itertools.product(*sl_factors) + return sl_factors, eindices + + def _get_slice_data_for_array_assignment(self, index, value): + if not isinstance(value, NDimArray): + value = type(self)(value) + sl_factors, eindices = self._get_slice_data_for_array_access(index) + slice_offsets = [min(i) if isinstance(i, list) else None for i in sl_factors] + # TODO: add checks for dimensions for `value`? + return value, eindices, slice_offsets + + @classmethod + def _check_special_bounds(cls, flat_list, shape): + if shape == () and len(flat_list) != 1: + raise ValueError("arrays without shape need one scalar value") + if shape == (0,) and len(flat_list) > 0: + raise ValueError("if array shape is (0,) there cannot be elements") + + def _check_index_for_getitem(self, index): + if isinstance(index, (SYMPY_INTS, Integer, slice)): + index = (index,) + + if len(index) < self.rank(): + index = tuple(index) + \ + tuple(slice(None) for i in range(len(index), self.rank())) + + if len(index) > self.rank(): + raise ValueError('Dimension of index greater than rank of array') + + return index + + +class ImmutableNDimArray(NDimArray, Basic): + _op_priority = 11.0 + + def __hash__(self): + return Basic.__hash__(self) + + def as_immutable(self): + return self + + def as_mutable(self): + raise NotImplementedError("abstract method") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/sparse_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/sparse_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..f11aa95be8ec9d10a9104d48fb28f406fe43845e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/sparse_ndim_array.py @@ -0,0 +1,196 @@ +from sympy.core.basic import Basic +from sympy.core.containers import (Dict, Tuple) +from sympy.core.singleton import S +from sympy.core.sympify import _sympify +from sympy.tensor.array.mutable_ndim_array import MutableNDimArray +from sympy.tensor.array.ndim_array import NDimArray, ImmutableNDimArray +from sympy.utilities.iterables import flatten + +import functools + +class SparseNDimArray(NDimArray): + + def __new__(self, *args, **kwargs): + return ImmutableSparseNDimArray(*args, **kwargs) + + def __getitem__(self, index): + """ + Get an element from a sparse N-dim array. + + Examples + ======== + + >>> from sympy import MutableSparseNDimArray + >>> a = MutableSparseNDimArray(range(4), (2, 2)) + >>> a + [[0, 1], [2, 3]] + >>> a[0, 0] + 0 + >>> a[1, 1] + 3 + >>> a[0] + [0, 1] + >>> a[1] + [2, 3] + + Symbolic indexing: + + >>> from sympy.abc import i, j + >>> a[i, j] + [[0, 1], [2, 3]][i, j] + + Replace `i` and `j` to get element `(0, 0)`: + + >>> a[i, j].subs({i: 0, j: 0}) + 0 + + """ + syindex = self._check_symbolic_index(index) + if syindex is not None: + return syindex + + index = self._check_index_for_getitem(index) + + # `index` is a tuple with one or more slices: + if isinstance(index, tuple) and any(isinstance(i, slice) for i in index): + sl_factors, eindices = self._get_slice_data_for_array_access(index) + array = [self._sparse_array.get(self._parse_index(i), S.Zero) for i in eindices] + nshape = [len(el) for i, el in enumerate(sl_factors) if isinstance(index[i], slice)] + return type(self)(array, nshape) + else: + index = self._parse_index(index) + return self._sparse_array.get(index, S.Zero) + + @classmethod + def zeros(cls, *shape): + """ + Return a sparse N-dim array of zeros. + """ + return cls({}, shape) + + def tomatrix(self): + """ + Converts MutableDenseNDimArray to Matrix. Can convert only 2-dim array, else will raise error. + + Examples + ======== + + >>> from sympy import MutableSparseNDimArray + >>> a = MutableSparseNDimArray([1 for i in range(9)], (3, 3)) + >>> b = a.tomatrix() + >>> b + Matrix([ + [1, 1, 1], + [1, 1, 1], + [1, 1, 1]]) + """ + from sympy.matrices import SparseMatrix + if self.rank() != 2: + raise ValueError('Dimensions must be of size of 2') + + mat_sparse = {} + for key, value in self._sparse_array.items(): + mat_sparse[self._get_tuple_index(key)] = value + + return SparseMatrix(self.shape[0], self.shape[1], mat_sparse) + + def reshape(self, *newshape): + new_total_size = functools.reduce(lambda x,y: x*y, newshape) + if new_total_size != self._loop_size: + raise ValueError("Invalid reshape parameters " + newshape) + + return type(self)(self._sparse_array, newshape) + +class ImmutableSparseNDimArray(SparseNDimArray, ImmutableNDimArray): # type: ignore + + def __new__(cls, iterable=None, shape=None, **kwargs): + shape, flat_list = cls._handle_ndarray_creation_inputs(iterable, shape, **kwargs) + shape = Tuple(*map(_sympify, shape)) + cls._check_special_bounds(flat_list, shape) + loop_size = functools.reduce(lambda x,y: x*y, shape) if shape else len(flat_list) + + # Sparse array: + if isinstance(flat_list, (dict, Dict)): + sparse_array = Dict(flat_list) + else: + sparse_array = {} + for i, el in enumerate(flatten(flat_list)): + if el != 0: + sparse_array[i] = _sympify(el) + + sparse_array = Dict(sparse_array) + + self = Basic.__new__(cls, sparse_array, shape, **kwargs) + self._shape = shape + self._rank = len(shape) + self._loop_size = loop_size + self._sparse_array = sparse_array + + return self + + def __setitem__(self, index, value): + raise TypeError("immutable N-dim array") + + def as_mutable(self): + return MutableSparseNDimArray(self) + + +class MutableSparseNDimArray(MutableNDimArray, SparseNDimArray): + + def __new__(cls, iterable=None, shape=None, **kwargs): + shape, flat_list = cls._handle_ndarray_creation_inputs(iterable, shape, **kwargs) + self = object.__new__(cls) + self._shape = shape + self._rank = len(shape) + self._loop_size = functools.reduce(lambda x,y: x*y, shape) if shape else len(flat_list) + + # Sparse array: + if isinstance(flat_list, (dict, Dict)): + self._sparse_array = dict(flat_list) + return self + + self._sparse_array = {} + + for i, el in enumerate(flatten(flat_list)): + if el != 0: + self._sparse_array[i] = _sympify(el) + + return self + + def __setitem__(self, index, value): + """Allows to set items to MutableDenseNDimArray. + + Examples + ======== + + >>> from sympy import MutableSparseNDimArray + >>> a = MutableSparseNDimArray.zeros(2, 2) + >>> a[0, 0] = 1 + >>> a[1, 1] = 1 + >>> a + [[1, 0], [0, 1]] + """ + if isinstance(index, tuple) and any(isinstance(i, slice) for i in index): + value, eindices, slice_offsets = self._get_slice_data_for_array_assignment(index, value) + for i in eindices: + other_i = [ind - j for ind, j in zip(i, slice_offsets) if j is not None] + other_value = value[other_i] + complete_index = self._parse_index(i) + if other_value != 0: + self._sparse_array[complete_index] = other_value + elif complete_index in self._sparse_array: + self._sparse_array.pop(complete_index) + else: + index = self._parse_index(index) + value = _sympify(value) + if value == 0 and index in self._sparse_array: + self._sparse_array.pop(index) + else: + self._sparse_array[index] = value + + def as_immutable(self): + return ImmutableSparseNDimArray(self) + + @property + def free_symbols(self): + return {i for j in self._sparse_array.values() for i in j.free_symbols} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e0b53cf1b0b8226985d246813b98cc5b8fc7165 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_comprehension.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_comprehension.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81628c0205caf843873e344dca22fb4d0707584d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_comprehension.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_derivatives.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_derivatives.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a157f872c1c80a97bd6c5bd5f9f67ab2147f1dc6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_array_derivatives.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_arrayop.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_arrayop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b2d780565d48c7d79d169ad111a316e7d86fc0f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_arrayop.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_immutable_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_immutable_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52ffee100ce3c04f04b0415feab74696b4b91f3d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_immutable_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_mutable_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_mutable_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7455b9f87ad582ed556c93e834dff10ae4108357 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_mutable_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..092123db20af931e08e5f3acbb116439e7abec8f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array_conversions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array_conversions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d85a4942774b2ab950c2953c47c2397c68b801c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/__pycache__/test_ndim_array_conversions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_comprehension.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_comprehension.py new file mode 100644 index 0000000000000000000000000000000000000000..510e068f287fa04419712e5e9a16a314e522a62d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_comprehension.py @@ -0,0 +1,78 @@ +from sympy.tensor.array.array_comprehension import ArrayComprehension, ArrayComprehensionMap +from sympy.tensor.array import ImmutableDenseNDimArray +from sympy.abc import i, j, k, l +from sympy.testing.pytest import raises +from sympy.matrices import Matrix + + +def test_array_comprehension(): + a = ArrayComprehension(i*j, (i, 1, 3), (j, 2, 4)) + b = ArrayComprehension(i, (i, 1, j+1)) + c = ArrayComprehension(i+j+k+l, (i, 1, 2), (j, 1, 3), (k, 1, 4), (l, 1, 5)) + d = ArrayComprehension(k, (i, 1, 5)) + e = ArrayComprehension(i, (j, k+1, k+5)) + assert a.doit().tolist() == [[2, 3, 4], [4, 6, 8], [6, 9, 12]] + assert a.shape == (3, 3) + assert a.is_shape_numeric == True + assert a.tolist() == [[2, 3, 4], [4, 6, 8], [6, 9, 12]] + assert a.tomatrix() == Matrix([ + [2, 3, 4], + [4, 6, 8], + [6, 9, 12]]) + assert len(a) == 9 + assert isinstance(b.doit(), ArrayComprehension) + assert isinstance(a.doit(), ImmutableDenseNDimArray) + assert b.subs(j, 3) == ArrayComprehension(i, (i, 1, 4)) + assert b.free_symbols == {j} + assert b.shape == (j + 1,) + assert b.rank() == 1 + assert b.is_shape_numeric == False + assert c.free_symbols == set() + assert c.function == i + j + k + l + assert c.limits == ((i, 1, 2), (j, 1, 3), (k, 1, 4), (l, 1, 5)) + assert c.doit().tolist() == [[[[4, 5, 6, 7, 8], [5, 6, 7, 8, 9], [6, 7, 8, 9, 10], [7, 8, 9, 10, 11]], + [[5, 6, 7, 8, 9], [6, 7, 8, 9, 10], [7, 8, 9, 10, 11], [8, 9, 10, 11, 12]], + [[6, 7, 8, 9, 10], [7, 8, 9, 10, 11], [8, 9, 10, 11, 12], [9, 10, 11, 12, 13]]], + [[[5, 6, 7, 8, 9], [6, 7, 8, 9, 10], [7, 8, 9, 10, 11], [8, 9, 10, 11, 12]], + [[6, 7, 8, 9, 10], [7, 8, 9, 10, 11], [8, 9, 10, 11, 12], [9, 10, 11, 12, 13]], + [[7, 8, 9, 10, 11], [8, 9, 10, 11, 12], [9, 10, 11, 12, 13], [10, 11, 12, 13, 14]]]] + assert c.free_symbols == set() + assert c.variables == [i, j, k, l] + assert c.bound_symbols == [i, j, k, l] + assert d.doit().tolist() == [k, k, k, k, k] + assert len(e) == 5 + raises(TypeError, lambda: ArrayComprehension(i*j, (i, 1, 3), (j, 2, [1, 3, 2]))) + raises(ValueError, lambda: ArrayComprehension(i*j, (i, 1, 3), (j, 2, 1))) + raises(ValueError, lambda: ArrayComprehension(i*j, (i, 1, 3), (j, 2, j+1))) + raises(ValueError, lambda: len(ArrayComprehension(i*j, (i, 1, 3), (j, 2, j+4)))) + raises(TypeError, lambda: ArrayComprehension(i*j, (i, 0, i + 1.5), (j, 0, 2))) + raises(ValueError, lambda: b.tolist()) + raises(ValueError, lambda: b.tomatrix()) + raises(ValueError, lambda: c.tomatrix()) + +def test_arraycomprehensionmap(): + a = ArrayComprehensionMap(lambda i: i+1, (i, 1, 5)) + assert a.doit().tolist() == [2, 3, 4, 5, 6] + assert a.shape == (5,) + assert a.is_shape_numeric + assert a.tolist() == [2, 3, 4, 5, 6] + assert len(a) == 5 + assert isinstance(a.doit(), ImmutableDenseNDimArray) + expr = ArrayComprehensionMap(lambda i: i+1, (i, 1, k)) + assert expr.doit() == expr + assert expr.subs(k, 4) == ArrayComprehensionMap(lambda i: i+1, (i, 1, 4)) + assert expr.subs(k, 4).doit() == ImmutableDenseNDimArray([2, 3, 4, 5]) + b = ArrayComprehensionMap(lambda i: i+1, (i, 1, 2), (i, 1, 3), (i, 1, 4), (i, 1, 5)) + assert b.doit().tolist() == [[[[2, 3, 4, 5, 6], [3, 5, 7, 9, 11], [4, 7, 10, 13, 16], [5, 9, 13, 17, 21]], + [[3, 5, 7, 9, 11], [5, 9, 13, 17, 21], [7, 13, 19, 25, 31], [9, 17, 25, 33, 41]], + [[4, 7, 10, 13, 16], [7, 13, 19, 25, 31], [10, 19, 28, 37, 46], [13, 25, 37, 49, 61]]], + [[[3, 5, 7, 9, 11], [5, 9, 13, 17, 21], [7, 13, 19, 25, 31], [9, 17, 25, 33, 41]], + [[5, 9, 13, 17, 21], [9, 17, 25, 33, 41], [13, 25, 37, 49, 61], [17, 33, 49, 65, 81]], + [[7, 13, 19, 25, 31], [13, 25, 37, 49, 61], [19, 37, 55, 73, 91], [25, 49, 73, 97, 121]]]] + + # tests about lambda expression + assert ArrayComprehensionMap(lambda: 3, (i, 1, 5)).doit().tolist() == [3, 3, 3, 3, 3] + assert ArrayComprehensionMap(lambda i: i+1, (i, 1, 5)).doit().tolist() == [2, 3, 4, 5, 6] + raises(ValueError, lambda: ArrayComprehensionMap(i*j, (i, 1, 3), (j, 2, 4))) + a = ArrayComprehensionMap(lambda i, j: i+j, (i, 1, 5)) + raises(ValueError, lambda: a.doit()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_derivatives.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_derivatives.py new file mode 100644 index 0000000000000000000000000000000000000000..7f6c777c55a9170704f309bf74387d140bf2ec32 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_array_derivatives.py @@ -0,0 +1,52 @@ +from sympy.core.symbol import symbols +from sympy.matrices.dense import Matrix +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.tensor.array.ndim_array import NDimArray +from sympy.matrices.matrixbase import MatrixBase +from sympy.tensor.array.array_derivatives import ArrayDerivative + +x, y, z, t = symbols("x y z t") + +m = Matrix([[x, y], [z, t]]) + +M = MatrixSymbol("M", 3, 2) +N = MatrixSymbol("N", 4, 3) + + +def test_array_derivative_construction(): + + d = ArrayDerivative(x, m, evaluate=False) + assert d.shape == (2, 2) + expr = d.doit() + assert isinstance(expr, MatrixBase) + assert expr.shape == (2, 2) + + d = ArrayDerivative(m, m, evaluate=False) + assert d.shape == (2, 2, 2, 2) + expr = d.doit() + assert isinstance(expr, NDimArray) + assert expr.shape == (2, 2, 2, 2) + + d = ArrayDerivative(m, x, evaluate=False) + assert d.shape == (2, 2) + expr = d.doit() + assert isinstance(expr, MatrixBase) + assert expr.shape == (2, 2) + + d = ArrayDerivative(M, N, evaluate=False) + assert d.shape == (4, 3, 3, 2) + expr = d.doit() + assert isinstance(expr, ArrayDerivative) + assert expr.shape == (4, 3, 3, 2) + + d = ArrayDerivative(M, (N, 2), evaluate=False) + assert d.shape == (4, 3, 4, 3, 3, 2) + expr = d.doit() + assert isinstance(expr, ArrayDerivative) + assert expr.shape == (4, 3, 4, 3, 3, 2) + + d = ArrayDerivative(M.as_explicit(), (N.as_explicit(), 2), evaluate=False) + assert d.doit().shape == (4, 3, 4, 3, 3, 2) + expr = d.doit() + assert isinstance(expr, NDimArray) + assert expr.shape == (4, 3, 4, 3, 3, 2) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_arrayop.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_arrayop.py new file mode 100644 index 0000000000000000000000000000000000000000..de56e81e0064f1e303a7a58e41932d15f2d0b41e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_arrayop.py @@ -0,0 +1,361 @@ +import itertools +import random + +from sympy.combinatorics import Permutation +from sympy.combinatorics.permutations import _af_invert +from sympy.testing.pytest import raises + +from sympy.core.function import diff +from sympy.core.symbol import symbols +from sympy.functions.elementary.complexes import (adjoint, conjugate, transpose) +from sympy.functions.elementary.exponential import (exp, log) +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.tensor.array import Array, ImmutableDenseNDimArray, ImmutableSparseNDimArray, MutableSparseNDimArray + +from sympy.tensor.array.arrayop import tensorproduct, tensorcontraction, derive_by_array, permutedims, Flatten, \ + tensordiagonal + + +def test_import_NDimArray(): + from sympy.tensor.array import NDimArray + del NDimArray + + +def test_tensorproduct(): + x,y,z,t = symbols('x y z t') + from sympy.abc import a,b,c,d + assert tensorproduct() == 1 + assert tensorproduct([x]) == Array([x]) + assert tensorproduct([x], [y]) == Array([[x*y]]) + assert tensorproduct([x], [y], [z]) == Array([[[x*y*z]]]) + assert tensorproduct([x], [y], [z], [t]) == Array([[[[x*y*z*t]]]]) + + assert tensorproduct(x) == x + assert tensorproduct(x, y) == x*y + assert tensorproduct(x, y, z) == x*y*z + assert tensorproduct(x, y, z, t) == x*y*z*t + + for ArrayType in [ImmutableDenseNDimArray, ImmutableSparseNDimArray]: + A = ArrayType([x, y]) + B = ArrayType([1, 2, 3]) + C = ArrayType([a, b, c, d]) + + assert tensorproduct(A, B, C) == ArrayType([[[a*x, b*x, c*x, d*x], [2*a*x, 2*b*x, 2*c*x, 2*d*x], [3*a*x, 3*b*x, 3*c*x, 3*d*x]], + [[a*y, b*y, c*y, d*y], [2*a*y, 2*b*y, 2*c*y, 2*d*y], [3*a*y, 3*b*y, 3*c*y, 3*d*y]]]) + + assert tensorproduct([x, y], [1, 2, 3]) == tensorproduct(A, B) + + assert tensorproduct(A, 2) == ArrayType([2*x, 2*y]) + assert tensorproduct(A, [2]) == ArrayType([[2*x], [2*y]]) + assert tensorproduct([2], A) == ArrayType([[2*x, 2*y]]) + assert tensorproduct(a, A) == ArrayType([a*x, a*y]) + assert tensorproduct(a, A, B) == ArrayType([[a*x, 2*a*x, 3*a*x], [a*y, 2*a*y, 3*a*y]]) + assert tensorproduct(A, B, a) == ArrayType([[a*x, 2*a*x, 3*a*x], [a*y, 2*a*y, 3*a*y]]) + assert tensorproduct(B, a, A) == ArrayType([[a*x, a*y], [2*a*x, 2*a*y], [3*a*x, 3*a*y]]) + + # tests for large scale sparse array + for SparseArrayType in [ImmutableSparseNDimArray, MutableSparseNDimArray]: + a = SparseArrayType({1:2, 3:4},(1000, 2000)) + b = SparseArrayType({1:2, 3:4},(1000, 2000)) + assert tensorproduct(a, b) == ImmutableSparseNDimArray({2000001: 4, 2000003: 8, 6000001: 8, 6000003: 16}, (1000, 2000, 1000, 2000)) + + +def test_tensorcontraction(): + from sympy.abc import a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x + B = Array(range(18), (2, 3, 3)) + assert tensorcontraction(B, (1, 2)) == Array([12, 39]) + C1 = Array([a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x], (2, 3, 2, 2)) + + assert tensorcontraction(C1, (0, 2)) == Array([[a + o, b + p], [e + s, f + t], [i + w, j + x]]) + assert tensorcontraction(C1, (0, 2, 3)) == Array([a + p, e + t, i + x]) + assert tensorcontraction(C1, (2, 3)) == Array([[a + d, e + h, i + l], [m + p, q + t, u + x]]) + + +def test_derivative_by_array(): + from sympy.abc import i, j, t, x, y, z + + bexpr = x*y**2*exp(z)*log(t) + sexpr = sin(bexpr) + cexpr = cos(bexpr) + + a = Array([sexpr]) + + assert derive_by_array(sexpr, t) == x*y**2*exp(z)*cos(x*y**2*exp(z)*log(t))/t + assert derive_by_array(sexpr, [x, y, z]) == Array([bexpr/x*cexpr, 2*y*bexpr/y**2*cexpr, bexpr*cexpr]) + assert derive_by_array(a, [x, y, z]) == Array([[bexpr/x*cexpr], [2*y*bexpr/y**2*cexpr], [bexpr*cexpr]]) + + assert derive_by_array(sexpr, [[x, y], [z, t]]) == Array([[bexpr/x*cexpr, 2*y*bexpr/y**2*cexpr], [bexpr*cexpr, bexpr/log(t)/t*cexpr]]) + assert derive_by_array(a, [[x, y], [z, t]]) == Array([[[bexpr/x*cexpr], [2*y*bexpr/y**2*cexpr]], [[bexpr*cexpr], [bexpr/log(t)/t*cexpr]]]) + assert derive_by_array([[x, y], [z, t]], [x, y]) == Array([[[1, 0], [0, 0]], [[0, 1], [0, 0]]]) + assert derive_by_array([[x, y], [z, t]], [[x, y], [z, t]]) == Array([[[[1, 0], [0, 0]], [[0, 1], [0, 0]]], + [[[0, 0], [1, 0]], [[0, 0], [0, 1]]]]) + + assert diff(sexpr, t) == x*y**2*exp(z)*cos(x*y**2*exp(z)*log(t))/t + assert diff(sexpr, Array([x, y, z])) == Array([bexpr/x*cexpr, 2*y*bexpr/y**2*cexpr, bexpr*cexpr]) + assert diff(a, Array([x, y, z])) == Array([[bexpr/x*cexpr], [2*y*bexpr/y**2*cexpr], [bexpr*cexpr]]) + + assert diff(sexpr, Array([[x, y], [z, t]])) == Array([[bexpr/x*cexpr, 2*y*bexpr/y**2*cexpr], [bexpr*cexpr, bexpr/log(t)/t*cexpr]]) + assert diff(a, Array([[x, y], [z, t]])) == Array([[[bexpr/x*cexpr], [2*y*bexpr/y**2*cexpr]], [[bexpr*cexpr], [bexpr/log(t)/t*cexpr]]]) + assert diff(Array([[x, y], [z, t]]), Array([x, y])) == Array([[[1, 0], [0, 0]], [[0, 1], [0, 0]]]) + assert diff(Array([[x, y], [z, t]]), Array([[x, y], [z, t]])) == Array([[[[1, 0], [0, 0]], [[0, 1], [0, 0]]], + [[[0, 0], [1, 0]], [[0, 0], [0, 1]]]]) + + # test for large scale sparse array + for SparseArrayType in [ImmutableSparseNDimArray, MutableSparseNDimArray]: + b = MutableSparseNDimArray({0:i, 1:j}, (10000, 20000)) + assert derive_by_array(b, i) == ImmutableSparseNDimArray({0: 1}, (10000, 20000)) + assert derive_by_array(b, (i, j)) == ImmutableSparseNDimArray({0: 1, 200000001: 1}, (2, 10000, 20000)) + + #https://github.com/sympy/sympy/issues/20655 + U = Array([x, y, z]) + E = 2 + assert derive_by_array(E, U) == ImmutableDenseNDimArray([0, 0, 0]) + + +def test_issue_emerged_while_discussing_10972(): + ua = Array([-1,0]) + Fa = Array([[0, 1], [-1, 0]]) + po = tensorproduct(Fa, ua, Fa, ua) + assert tensorcontraction(po, (1, 2), (4, 5)) == Array([[0, 0], [0, 1]]) + + sa = symbols('a0:144') + po = Array(sa, [2, 2, 3, 3, 2, 2]) + assert tensorcontraction(po, (0, 1), (2, 3), (4, 5)) == sa[0] + sa[108] + sa[111] + sa[124] + sa[127] + sa[140] + sa[143] + sa[16] + sa[19] + sa[3] + sa[32] + sa[35] + assert tensorcontraction(po, (0, 1, 4, 5), (2, 3)) == sa[0] + sa[111] + sa[127] + sa[143] + sa[16] + sa[32] + assert tensorcontraction(po, (0, 1), (4, 5)) == Array([[sa[0] + sa[108] + sa[111] + sa[3], sa[112] + sa[115] + sa[4] + sa[7], + sa[11] + sa[116] + sa[119] + sa[8]], [sa[12] + sa[120] + sa[123] + sa[15], + sa[124] + sa[127] + sa[16] + sa[19], sa[128] + sa[131] + sa[20] + sa[23]], + [sa[132] + sa[135] + sa[24] + sa[27], sa[136] + sa[139] + sa[28] + sa[31], + sa[140] + sa[143] + sa[32] + sa[35]]]) + assert tensorcontraction(po, (0, 1), (2, 3)) == Array([[sa[0] + sa[108] + sa[124] + sa[140] + sa[16] + sa[32], sa[1] + sa[109] + sa[125] + sa[141] + sa[17] + sa[33]], + [sa[110] + sa[126] + sa[142] + sa[18] + sa[2] + sa[34], sa[111] + sa[127] + sa[143] + sa[19] + sa[3] + sa[35]]]) + + +def test_array_permutedims(): + sa = symbols('a0:144') + + for ArrayType in [ImmutableDenseNDimArray, ImmutableSparseNDimArray]: + m1 = ArrayType(sa[:6], (2, 3)) + assert permutedims(m1, (1, 0)) == transpose(m1) + assert m1.tomatrix().T == permutedims(m1, (1, 0)).tomatrix() + + assert m1.tomatrix().T == transpose(m1).tomatrix() + assert m1.tomatrix().C == conjugate(m1).tomatrix() + assert m1.tomatrix().H == adjoint(m1).tomatrix() + + assert m1.tomatrix().T == m1.transpose().tomatrix() + assert m1.tomatrix().C == m1.conjugate().tomatrix() + assert m1.tomatrix().H == m1.adjoint().tomatrix() + + raises(ValueError, lambda: permutedims(m1, (0,))) + raises(ValueError, lambda: permutedims(m1, (0, 0))) + raises(ValueError, lambda: permutedims(m1, (1, 2, 0))) + + # Some tests with random arrays: + dims = 6 + shape = [random.randint(1,5) for i in range(dims)] + elems = [random.random() for i in range(tensorproduct(*shape))] + ra = ArrayType(elems, shape) + perm = list(range(dims)) + # Randomize the permutation: + random.shuffle(perm) + # Test inverse permutation: + assert permutedims(permutedims(ra, perm), _af_invert(perm)) == ra + # Test that permuted shape corresponds to action by `Permutation`: + assert permutedims(ra, perm).shape == tuple(Permutation(perm)(shape)) + + z = ArrayType.zeros(4,5,6,7) + + assert permutedims(z, (2, 3, 1, 0)).shape == (6, 7, 5, 4) + assert permutedims(z, [2, 3, 1, 0]).shape == (6, 7, 5, 4) + assert permutedims(z, Permutation([2, 3, 1, 0])).shape == (6, 7, 5, 4) + + po = ArrayType(sa, [2, 2, 3, 3, 2, 2]) + + raises(ValueError, lambda: permutedims(po, (1, 1))) + raises(ValueError, lambda: po.transpose()) + raises(ValueError, lambda: po.adjoint()) + + assert permutedims(po, reversed(range(po.rank()))) == ArrayType( + [[[[[[sa[0], sa[72]], [sa[36], sa[108]]], [[sa[12], sa[84]], [sa[48], sa[120]]], [[sa[24], + sa[96]], [sa[60], sa[132]]]], + [[[sa[4], sa[76]], [sa[40], sa[112]]], [[sa[16], + sa[88]], [sa[52], sa[124]]], + [[sa[28], sa[100]], [sa[64], sa[136]]]], + [[[sa[8], + sa[80]], [sa[44], sa[116]]], [[sa[20], sa[92]], [sa[56], sa[128]]], [[sa[32], + sa[104]], [sa[68], sa[140]]]]], + [[[[sa[2], sa[74]], [sa[38], sa[110]]], [[sa[14], + sa[86]], [sa[50], sa[122]]], [[sa[26], sa[98]], [sa[62], sa[134]]]], + [[[sa[6], + sa[78]], [sa[42], sa[114]]], [[sa[18], sa[90]], [sa[54], sa[126]]], [[sa[30], + sa[102]], [sa[66], sa[138]]]], + [[[sa[10], sa[82]], [sa[46], sa[118]]], [[sa[22], + sa[94]], [sa[58], sa[130]]], + [[sa[34], sa[106]], [sa[70], sa[142]]]]]], + [[[[[sa[1], + sa[73]], [sa[37], sa[109]]], [[sa[13], sa[85]], [sa[49], sa[121]]], [[sa[25], + sa[97]], [sa[61], sa[133]]]], + [[[sa[5], sa[77]], [sa[41], sa[113]]], [[sa[17], + sa[89]], [sa[53], sa[125]]], + [[sa[29], sa[101]], [sa[65], sa[137]]]], + [[[sa[9], + sa[81]], [sa[45], sa[117]]], [[sa[21], sa[93]], [sa[57], sa[129]]], [[sa[33], + sa[105]], [sa[69], sa[141]]]]], + [[[[sa[3], sa[75]], [sa[39], sa[111]]], [[sa[15], + sa[87]], [sa[51], sa[123]]], [[sa[27], sa[99]], [sa[63], sa[135]]]], + [[[sa[7], + sa[79]], [sa[43], sa[115]]], [[sa[19], sa[91]], [sa[55], sa[127]]], [[sa[31], + sa[103]], [sa[67], sa[139]]]], + [[[sa[11], sa[83]], [sa[47], sa[119]]], [[sa[23], + sa[95]], [sa[59], sa[131]]], + [[sa[35], sa[107]], [sa[71], sa[143]]]]]]]) + + assert permutedims(po, (1, 0, 2, 3, 4, 5)) == ArrayType( + [[[[[[sa[0], sa[1]], [sa[2], sa[3]]], [[sa[4], sa[5]], [sa[6], sa[7]]], [[sa[8], sa[9]], [sa[10], + sa[11]]]], + [[[sa[12], sa[13]], [sa[14], sa[15]]], [[sa[16], sa[17]], [sa[18], + sa[19]]], [[sa[20], sa[21]], [sa[22], sa[23]]]], + [[[sa[24], sa[25]], [sa[26], + sa[27]]], [[sa[28], sa[29]], [sa[30], sa[31]]], [[sa[32], sa[33]], [sa[34], + sa[35]]]]], + [[[[sa[72], sa[73]], [sa[74], sa[75]]], [[sa[76], sa[77]], [sa[78], + sa[79]]], [[sa[80], sa[81]], [sa[82], sa[83]]]], + [[[sa[84], sa[85]], [sa[86], + sa[87]]], [[sa[88], sa[89]], [sa[90], sa[91]]], [[sa[92], sa[93]], [sa[94], + sa[95]]]], + [[[sa[96], sa[97]], [sa[98], sa[99]]], [[sa[100], sa[101]], [sa[102], + sa[103]]], + [[sa[104], sa[105]], [sa[106], sa[107]]]]]], [[[[[sa[36], sa[37]], [sa[38], + sa[39]]], + [[sa[40], sa[41]], [sa[42], sa[43]]], + [[sa[44], sa[45]], [sa[46], + sa[47]]]], + [[[sa[48], sa[49]], [sa[50], sa[51]]], + [[sa[52], sa[53]], [sa[54], + sa[55]]], + [[sa[56], sa[57]], [sa[58], sa[59]]]], + [[[sa[60], sa[61]], [sa[62], + sa[63]]], + [[sa[64], sa[65]], [sa[66], sa[67]]], + [[sa[68], sa[69]], [sa[70], + sa[71]]]]], [ + [[[sa[108], sa[109]], [sa[110], sa[111]]], + [[sa[112], sa[113]], [sa[114], + sa[115]]], + [[sa[116], sa[117]], [sa[118], sa[119]]]], + [[[sa[120], sa[121]], [sa[122], + sa[123]]], + [[sa[124], sa[125]], [sa[126], sa[127]]], + [[sa[128], sa[129]], [sa[130], + sa[131]]]], + [[[sa[132], sa[133]], [sa[134], sa[135]]], + [[sa[136], sa[137]], [sa[138], + sa[139]]], + [[sa[140], sa[141]], [sa[142], sa[143]]]]]]]) + + assert permutedims(po, (0, 2, 1, 4, 3, 5)) == ArrayType( + [[[[[[sa[0], sa[1]], [sa[4], sa[5]], [sa[8], sa[9]]], [[sa[2], sa[3]], [sa[6], sa[7]], [sa[10], + sa[11]]]], + [[[sa[36], sa[37]], [sa[40], sa[41]], [sa[44], sa[45]]], [[sa[38], + sa[39]], [sa[42], sa[43]], [sa[46], sa[47]]]]], + [[[[sa[12], sa[13]], [sa[16], + sa[17]], [sa[20], sa[21]]], [[sa[14], sa[15]], [sa[18], sa[19]], [sa[22], + sa[23]]]], + [[[sa[48], sa[49]], [sa[52], sa[53]], [sa[56], sa[57]]], [[sa[50], + sa[51]], [sa[54], sa[55]], [sa[58], sa[59]]]]], + [[[[sa[24], sa[25]], [sa[28], + sa[29]], [sa[32], sa[33]]], [[sa[26], sa[27]], [sa[30], sa[31]], [sa[34], + sa[35]]]], + [[[sa[60], sa[61]], [sa[64], sa[65]], [sa[68], sa[69]]], [[sa[62], + sa[63]], [sa[66], sa[67]], [sa[70], sa[71]]]]]], + [[[[[sa[72], sa[73]], [sa[76], + sa[77]], [sa[80], sa[81]]], [[sa[74], sa[75]], [sa[78], sa[79]], [sa[82], + sa[83]]]], + [[[sa[108], sa[109]], [sa[112], sa[113]], [sa[116], sa[117]]], [[sa[110], + sa[111]], [sa[114], sa[115]], + [sa[118], sa[119]]]]], + [[[[sa[84], sa[85]], [sa[88], + sa[89]], [sa[92], sa[93]]], [[sa[86], sa[87]], [sa[90], sa[91]], [sa[94], + sa[95]]]], + [[[sa[120], sa[121]], [sa[124], sa[125]], [sa[128], sa[129]]], [[sa[122], + sa[123]], [sa[126], sa[127]], + [sa[130], sa[131]]]]], + [[[[sa[96], sa[97]], [sa[100], + sa[101]], [sa[104], sa[105]]], [[sa[98], sa[99]], [sa[102], sa[103]], [sa[106], + sa[107]]]], + [[[sa[132], sa[133]], [sa[136], sa[137]], [sa[140], sa[141]]], [[sa[134], + sa[135]], [sa[138], sa[139]], + [sa[142], sa[143]]]]]]]) + + po2 = po.reshape(4, 9, 2, 2) + assert po2 == ArrayType([[[[sa[0], sa[1]], [sa[2], sa[3]]], [[sa[4], sa[5]], [sa[6], sa[7]]], [[sa[8], sa[9]], [sa[10], sa[11]]], [[sa[12], sa[13]], [sa[14], sa[15]]], [[sa[16], sa[17]], [sa[18], sa[19]]], [[sa[20], sa[21]], [sa[22], sa[23]]], [[sa[24], sa[25]], [sa[26], sa[27]]], [[sa[28], sa[29]], [sa[30], sa[31]]], [[sa[32], sa[33]], [sa[34], sa[35]]]], [[[sa[36], sa[37]], [sa[38], sa[39]]], [[sa[40], sa[41]], [sa[42], sa[43]]], [[sa[44], sa[45]], [sa[46], sa[47]]], [[sa[48], sa[49]], [sa[50], sa[51]]], [[sa[52], sa[53]], [sa[54], sa[55]]], [[sa[56], sa[57]], [sa[58], sa[59]]], [[sa[60], sa[61]], [sa[62], sa[63]]], [[sa[64], sa[65]], [sa[66], sa[67]]], [[sa[68], sa[69]], [sa[70], sa[71]]]], [[[sa[72], sa[73]], [sa[74], sa[75]]], [[sa[76], sa[77]], [sa[78], sa[79]]], [[sa[80], sa[81]], [sa[82], sa[83]]], [[sa[84], sa[85]], [sa[86], sa[87]]], [[sa[88], sa[89]], [sa[90], sa[91]]], [[sa[92], sa[93]], [sa[94], sa[95]]], [[sa[96], sa[97]], [sa[98], sa[99]]], [[sa[100], sa[101]], [sa[102], sa[103]]], [[sa[104], sa[105]], [sa[106], sa[107]]]], [[[sa[108], sa[109]], [sa[110], sa[111]]], [[sa[112], sa[113]], [sa[114], sa[115]]], [[sa[116], sa[117]], [sa[118], sa[119]]], [[sa[120], sa[121]], [sa[122], sa[123]]], [[sa[124], sa[125]], [sa[126], sa[127]]], [[sa[128], sa[129]], [sa[130], sa[131]]], [[sa[132], sa[133]], [sa[134], sa[135]]], [[sa[136], sa[137]], [sa[138], sa[139]]], [[sa[140], sa[141]], [sa[142], sa[143]]]]]) + + assert permutedims(po2, (3, 2, 0, 1)) == ArrayType([[[[sa[0], sa[4], sa[8], sa[12], sa[16], sa[20], sa[24], sa[28], sa[32]], [sa[36], sa[40], sa[44], sa[48], sa[52], sa[56], sa[60], sa[64], sa[68]], [sa[72], sa[76], sa[80], sa[84], sa[88], sa[92], sa[96], sa[100], sa[104]], [sa[108], sa[112], sa[116], sa[120], sa[124], sa[128], sa[132], sa[136], sa[140]]], [[sa[2], sa[6], sa[10], sa[14], sa[18], sa[22], sa[26], sa[30], sa[34]], [sa[38], sa[42], sa[46], sa[50], sa[54], sa[58], sa[62], sa[66], sa[70]], [sa[74], sa[78], sa[82], sa[86], sa[90], sa[94], sa[98], sa[102], sa[106]], [sa[110], sa[114], sa[118], sa[122], sa[126], sa[130], sa[134], sa[138], sa[142]]]], [[[sa[1], sa[5], sa[9], sa[13], sa[17], sa[21], sa[25], sa[29], sa[33]], [sa[37], sa[41], sa[45], sa[49], sa[53], sa[57], sa[61], sa[65], sa[69]], [sa[73], sa[77], sa[81], sa[85], sa[89], sa[93], sa[97], sa[101], sa[105]], [sa[109], sa[113], sa[117], sa[121], sa[125], sa[129], sa[133], sa[137], sa[141]]], [[sa[3], sa[7], sa[11], sa[15], sa[19], sa[23], sa[27], sa[31], sa[35]], [sa[39], sa[43], sa[47], sa[51], sa[55], sa[59], sa[63], sa[67], sa[71]], [sa[75], sa[79], sa[83], sa[87], sa[91], sa[95], sa[99], sa[103], sa[107]], [sa[111], sa[115], sa[119], sa[123], sa[127], sa[131], sa[135], sa[139], sa[143]]]]]) + + # test for large scale sparse array + for SparseArrayType in [ImmutableSparseNDimArray, MutableSparseNDimArray]: + A = SparseArrayType({1:1, 10000:2}, (10000, 20000, 10000)) + assert permutedims(A, (0, 1, 2)) == A + assert permutedims(A, (1, 0, 2)) == SparseArrayType({1: 1, 100000000: 2}, (20000, 10000, 10000)) + B = SparseArrayType({1:1, 20000:2}, (10000, 20000)) + assert B.transpose() == SparseArrayType({10000: 1, 1: 2}, (20000, 10000)) + + +def test_permutedims_with_indices(): + A = Array(range(32)).reshape(2, 2, 2, 2, 2) + indices_new = list("abcde") + indices_old = list("ebdac") + new_A = permutedims(A, index_order_new=indices_new, index_order_old=indices_old) + for a, b, c, d, e in itertools.product(range(2), range(2), range(2), range(2), range(2)): + assert new_A[a, b, c, d, e] == A[e, b, d, a, c] + indices_old = list("cabed") + new_A = permutedims(A, index_order_new=indices_new, index_order_old=indices_old) + for a, b, c, d, e in itertools.product(range(2), range(2), range(2), range(2), range(2)): + assert new_A[a, b, c, d, e] == A[c, a, b, e, d] + raises(ValueError, lambda: permutedims(A, index_order_old=list("aacde"), index_order_new=list("abcde"))) + raises(ValueError, lambda: permutedims(A, index_order_old=list("abcde"), index_order_new=list("abcce"))) + raises(ValueError, lambda: permutedims(A, index_order_old=list("abcde"), index_order_new=list("abce"))) + raises(ValueError, lambda: permutedims(A, index_order_old=list("abce"), index_order_new=list("abce"))) + raises(ValueError, lambda: permutedims(A, [2, 1, 0, 3, 4], index_order_old=list("abcde"))) + raises(ValueError, lambda: permutedims(A, [2, 1, 0, 3, 4], index_order_new=list("abcde"))) + + +def test_flatten(): + from sympy.matrices.dense import Matrix + for ArrayType in [ImmutableDenseNDimArray, ImmutableSparseNDimArray, Matrix]: + A = ArrayType(range(24)).reshape(4, 6) + assert list(Flatten(A)) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] + + for i, v in enumerate(Flatten(A)): + assert i == v + + +def test_tensordiagonal(): + from sympy.matrices.dense import eye + expr = Array(range(9)).reshape(3, 3) + raises(ValueError, lambda: tensordiagonal(expr, [0], [1])) + raises(ValueError, lambda: tensordiagonal(expr, [0, 0])) + assert tensordiagonal(eye(3), [0, 1]) == Array([1, 1, 1]) + assert tensordiagonal(expr, [0, 1]) == Array([0, 4, 8]) + x, y, z = symbols("x y z") + expr2 = tensorproduct([x, y, z], expr) + assert tensordiagonal(expr2, [1, 2]) == Array([[0, 4*x, 8*x], [0, 4*y, 8*y], [0, 4*z, 8*z]]) + assert tensordiagonal(expr2, [0, 1]) == Array([[0, 3*y, 6*z], [x, 4*y, 7*z], [2*x, 5*y, 8*z]]) + assert tensordiagonal(expr2, [0, 1, 2]) == Array([0, 4*y, 8*z]) + # assert tensordiagonal(expr2, [0]) == permutedims(expr2, [1, 2, 0]) + # assert tensordiagonal(expr2, [1]) == permutedims(expr2, [0, 2, 1]) + # assert tensordiagonal(expr2, [2]) == expr2 + # assert tensordiagonal(expr2, [1], [2]) == expr2 + # assert tensordiagonal(expr2, [0], [1]) == permutedims(expr2, [2, 0, 1]) + + a, b, c, X, Y, Z = symbols("a b c X Y Z") + expr3 = tensorproduct([x, y, z], [1, 2, 3], [a, b, c], [X, Y, Z]) + assert tensordiagonal(expr3, [0, 1, 2, 3]) == Array([x*a*X, 2*y*b*Y, 3*z*c*Z]) + assert tensordiagonal(expr3, [0, 1], [2, 3]) == tensorproduct([x, 2*y, 3*z], [a*X, b*Y, c*Z]) + + # assert tensordiagonal(expr3, [0], [1, 2], [3]) == tensorproduct([x, y, z], [a, 2*b, 3*c], [X, Y, Z]) + assert tensordiagonal(tensordiagonal(expr3, [2, 3]), [0, 1]) == tensorproduct([a*X, b*Y, c*Z], [x, 2*y, 3*z]) + + raises(ValueError, lambda: tensordiagonal([[1, 2, 3], [4, 5, 6]], [0, 1])) + raises(ValueError, lambda: tensordiagonal(expr3.reshape(3, 3, 9), [1, 2])) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_immutable_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_immutable_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..c6bed4b605c424284b4752592b03b13a9178aac8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_immutable_ndim_array.py @@ -0,0 +1,452 @@ +from copy import copy + +from sympy.tensor.array.dense_ndim_array import ImmutableDenseNDimArray +from sympy.core.containers import Dict +from sympy.core.function import diff +from sympy.core.numbers import Rational +from sympy.core.singleton import S +from sympy.core.symbol import (Symbol, symbols) +from sympy.matrices import SparseMatrix +from sympy.tensor.indexed import (Indexed, IndexedBase) +from sympy.matrices import Matrix +from sympy.tensor.array.sparse_ndim_array import ImmutableSparseNDimArray +from sympy.testing.pytest import raises + + +def test_ndim_array_initiation(): + arr_with_no_elements = ImmutableDenseNDimArray([], shape=(0,)) + assert len(arr_with_no_elements) == 0 + assert arr_with_no_elements.rank() == 1 + + raises(ValueError, lambda: ImmutableDenseNDimArray([0], shape=(0,))) + raises(ValueError, lambda: ImmutableDenseNDimArray([1, 2, 3], shape=(0,))) + raises(ValueError, lambda: ImmutableDenseNDimArray([], shape=())) + + raises(ValueError, lambda: ImmutableSparseNDimArray([0], shape=(0,))) + raises(ValueError, lambda: ImmutableSparseNDimArray([1, 2, 3], shape=(0,))) + raises(ValueError, lambda: ImmutableSparseNDimArray([], shape=())) + + arr_with_one_element = ImmutableDenseNDimArray([23]) + assert len(arr_with_one_element) == 1 + assert arr_with_one_element[0] == 23 + assert arr_with_one_element[:] == ImmutableDenseNDimArray([23]) + assert arr_with_one_element.rank() == 1 + + arr_with_symbol_element = ImmutableDenseNDimArray([Symbol('x')]) + assert len(arr_with_symbol_element) == 1 + assert arr_with_symbol_element[0] == Symbol('x') + assert arr_with_symbol_element[:] == ImmutableDenseNDimArray([Symbol('x')]) + assert arr_with_symbol_element.rank() == 1 + + number5 = 5 + vector = ImmutableDenseNDimArray.zeros(number5) + assert len(vector) == number5 + assert vector.shape == (number5,) + assert vector.rank() == 1 + + vector = ImmutableSparseNDimArray.zeros(number5) + assert len(vector) == number5 + assert vector.shape == (number5,) + assert vector._sparse_array == Dict() + assert vector.rank() == 1 + + n_dim_array = ImmutableDenseNDimArray(range(3**4), (3, 3, 3, 3,)) + assert len(n_dim_array) == 3 * 3 * 3 * 3 + assert n_dim_array.shape == (3, 3, 3, 3) + assert n_dim_array.rank() == 4 + + array_shape = (3, 3, 3, 3) + sparse_array = ImmutableSparseNDimArray.zeros(*array_shape) + assert len(sparse_array._sparse_array) == 0 + assert len(sparse_array) == 3 * 3 * 3 * 3 + assert n_dim_array.shape == array_shape + assert n_dim_array.rank() == 4 + + one_dim_array = ImmutableDenseNDimArray([2, 3, 1]) + assert len(one_dim_array) == 3 + assert one_dim_array.shape == (3,) + assert one_dim_array.rank() == 1 + assert one_dim_array.tolist() == [2, 3, 1] + + shape = (3, 3) + array_with_many_args = ImmutableSparseNDimArray.zeros(*shape) + assert len(array_with_many_args) == 3 * 3 + assert array_with_many_args.shape == shape + assert array_with_many_args[0, 0] == 0 + assert array_with_many_args.rank() == 2 + + shape = (int(3), int(3)) + array_with_long_shape = ImmutableSparseNDimArray.zeros(*shape) + assert len(array_with_long_shape) == 3 * 3 + assert array_with_long_shape.shape == shape + assert array_with_long_shape[int(0), int(0)] == 0 + assert array_with_long_shape.rank() == 2 + + vector_with_long_shape = ImmutableDenseNDimArray(range(5), int(5)) + assert len(vector_with_long_shape) == 5 + assert vector_with_long_shape.shape == (int(5),) + assert vector_with_long_shape.rank() == 1 + raises(ValueError, lambda: vector_with_long_shape[int(5)]) + + from sympy.abc import x + for ArrayType in [ImmutableDenseNDimArray, ImmutableSparseNDimArray]: + rank_zero_array = ArrayType(x) + assert len(rank_zero_array) == 1 + assert rank_zero_array.shape == () + assert rank_zero_array.rank() == 0 + assert rank_zero_array[()] == x + raises(ValueError, lambda: rank_zero_array[0]) + + +def test_reshape(): + array = ImmutableDenseNDimArray(range(50), 50) + assert array.shape == (50,) + assert array.rank() == 1 + + array = array.reshape(5, 5, 2) + assert array.shape == (5, 5, 2) + assert array.rank() == 3 + assert len(array) == 50 + + +def test_getitem(): + for ArrayType in [ImmutableDenseNDimArray, ImmutableSparseNDimArray]: + array = ArrayType(range(24)).reshape(2, 3, 4) + assert array.tolist() == [[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]] + assert array[0] == ArrayType([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]) + assert array[0, 0] == ArrayType([0, 1, 2, 3]) + value = 0 + for i in range(2): + for j in range(3): + for k in range(4): + assert array[i, j, k] == value + value += 1 + + raises(ValueError, lambda: array[3, 4, 5]) + raises(ValueError, lambda: array[3, 4, 5, 6]) + raises(ValueError, lambda: array[3, 4, 5, 3:4]) + + +def test_iterator(): + array = ImmutableDenseNDimArray(range(4), (2, 2)) + assert array[0] == ImmutableDenseNDimArray([0, 1]) + assert array[1] == ImmutableDenseNDimArray([2, 3]) + + array = array.reshape(4) + j = 0 + for i in array: + assert i == j + j += 1 + + +def test_sparse(): + sparse_array = ImmutableSparseNDimArray([0, 0, 0, 1], (2, 2)) + assert len(sparse_array) == 2 * 2 + # dictionary where all data is, only non-zero entries are actually stored: + assert len(sparse_array._sparse_array) == 1 + + assert sparse_array.tolist() == [[0, 0], [0, 1]] + + for i, j in zip(sparse_array, [[0, 0], [0, 1]]): + assert i == ImmutableSparseNDimArray(j) + + def sparse_assignment(): + sparse_array[0, 0] = 123 + + assert len(sparse_array._sparse_array) == 1 + raises(TypeError, sparse_assignment) + assert len(sparse_array._sparse_array) == 1 + assert sparse_array[0, 0] == 0 + assert sparse_array/0 == ImmutableSparseNDimArray([[S.NaN, S.NaN], [S.NaN, S.ComplexInfinity]], (2, 2)) + + # test for large scale sparse array + # equality test + assert ImmutableSparseNDimArray.zeros(100000, 200000) == ImmutableSparseNDimArray.zeros(100000, 200000) + + # __mul__ and __rmul__ + a = ImmutableSparseNDimArray({200001: 1}, (100000, 200000)) + assert a * 3 == ImmutableSparseNDimArray({200001: 3}, (100000, 200000)) + assert 3 * a == ImmutableSparseNDimArray({200001: 3}, (100000, 200000)) + assert a * 0 == ImmutableSparseNDimArray({}, (100000, 200000)) + assert 0 * a == ImmutableSparseNDimArray({}, (100000, 200000)) + + # __truediv__ + assert a/3 == ImmutableSparseNDimArray({200001: Rational(1, 3)}, (100000, 200000)) + + # __neg__ + assert -a == ImmutableSparseNDimArray({200001: -1}, (100000, 200000)) + + +def test_calculation(): + + a = ImmutableDenseNDimArray([1]*9, (3, 3)) + b = ImmutableDenseNDimArray([9]*9, (3, 3)) + + c = a + b + for i in c: + assert i == ImmutableDenseNDimArray([10, 10, 10]) + + assert c == ImmutableDenseNDimArray([10]*9, (3, 3)) + assert c == ImmutableSparseNDimArray([10]*9, (3, 3)) + + c = b - a + for i in c: + assert i == ImmutableDenseNDimArray([8, 8, 8]) + + assert c == ImmutableDenseNDimArray([8]*9, (3, 3)) + assert c == ImmutableSparseNDimArray([8]*9, (3, 3)) + + +def test_ndim_array_converting(): + dense_array = ImmutableDenseNDimArray([1, 2, 3, 4], (2, 2)) + alist = dense_array.tolist() + + assert alist == [[1, 2], [3, 4]] + + matrix = dense_array.tomatrix() + assert (isinstance(matrix, Matrix)) + + for i in range(len(dense_array)): + assert dense_array[dense_array._get_tuple_index(i)] == matrix[i] + assert matrix.shape == dense_array.shape + + assert ImmutableDenseNDimArray(matrix) == dense_array + assert ImmutableDenseNDimArray(matrix.as_immutable()) == dense_array + assert ImmutableDenseNDimArray(matrix.as_mutable()) == dense_array + + sparse_array = ImmutableSparseNDimArray([1, 2, 3, 4], (2, 2)) + alist = sparse_array.tolist() + + assert alist == [[1, 2], [3, 4]] + + matrix = sparse_array.tomatrix() + assert(isinstance(matrix, SparseMatrix)) + + for i in range(len(sparse_array)): + assert sparse_array[sparse_array._get_tuple_index(i)] == matrix[i] + assert matrix.shape == sparse_array.shape + + assert ImmutableSparseNDimArray(matrix) == sparse_array + assert ImmutableSparseNDimArray(matrix.as_immutable()) == sparse_array + assert ImmutableSparseNDimArray(matrix.as_mutable()) == sparse_array + + +def test_converting_functions(): + arr_list = [1, 2, 3, 4] + arr_matrix = Matrix(((1, 2), (3, 4))) + + # list + arr_ndim_array = ImmutableDenseNDimArray(arr_list, (2, 2)) + assert (isinstance(arr_ndim_array, ImmutableDenseNDimArray)) + assert arr_matrix.tolist() == arr_ndim_array.tolist() + + # Matrix + arr_ndim_array = ImmutableDenseNDimArray(arr_matrix) + assert (isinstance(arr_ndim_array, ImmutableDenseNDimArray)) + assert arr_matrix.tolist() == arr_ndim_array.tolist() + assert arr_matrix.shape == arr_ndim_array.shape + + +def test_equality(): + first_list = [1, 2, 3, 4] + second_list = [1, 2, 3, 4] + third_list = [4, 3, 2, 1] + assert first_list == second_list + assert first_list != third_list + + first_ndim_array = ImmutableDenseNDimArray(first_list, (2, 2)) + second_ndim_array = ImmutableDenseNDimArray(second_list, (2, 2)) + fourth_ndim_array = ImmutableDenseNDimArray(first_list, (2, 2)) + + assert first_ndim_array == second_ndim_array + + def assignment_attempt(a): + a[0, 0] = 0 + + raises(TypeError, lambda: assignment_attempt(second_ndim_array)) + assert first_ndim_array == second_ndim_array + assert first_ndim_array == fourth_ndim_array + + +def test_arithmetic(): + a = ImmutableDenseNDimArray([3 for i in range(9)], (3, 3)) + b = ImmutableDenseNDimArray([7 for i in range(9)], (3, 3)) + + c1 = a + b + c2 = b + a + assert c1 == c2 + + d1 = a - b + d2 = b - a + assert d1 == d2 * (-1) + + e1 = a * 5 + e2 = 5 * a + e3 = copy(a) + e3 *= 5 + assert e1 == e2 == e3 + + f1 = a / 5 + f2 = copy(a) + f2 /= 5 + assert f1 == f2 + assert f1[0, 0] == f1[0, 1] == f1[0, 2] == f1[1, 0] == f1[1, 1] == \ + f1[1, 2] == f1[2, 0] == f1[2, 1] == f1[2, 2] == Rational(3, 5) + + assert type(a) == type(b) == type(c1) == type(c2) == type(d1) == type(d2) \ + == type(e1) == type(e2) == type(e3) == type(f1) + + z0 = -a + assert z0 == ImmutableDenseNDimArray([-3 for i in range(9)], (3, 3)) + + +def test_higher_dimenions(): + m3 = ImmutableDenseNDimArray(range(10, 34), (2, 3, 4)) + + assert m3.tolist() == [[[10, 11, 12, 13], + [14, 15, 16, 17], + [18, 19, 20, 21]], + + [[22, 23, 24, 25], + [26, 27, 28, 29], + [30, 31, 32, 33]]] + + assert m3._get_tuple_index(0) == (0, 0, 0) + assert m3._get_tuple_index(1) == (0, 0, 1) + assert m3._get_tuple_index(4) == (0, 1, 0) + assert m3._get_tuple_index(12) == (1, 0, 0) + + assert str(m3) == '[[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]]' + + m3_rebuilt = ImmutableDenseNDimArray([[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]]) + assert m3 == m3_rebuilt + + m3_other = ImmutableDenseNDimArray([[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]], (2, 3, 4)) + + assert m3 == m3_other + + +def test_rebuild_immutable_arrays(): + sparr = ImmutableSparseNDimArray(range(10, 34), (2, 3, 4)) + densarr = ImmutableDenseNDimArray(range(10, 34), (2, 3, 4)) + + assert sparr == sparr.func(*sparr.args) + assert densarr == densarr.func(*densarr.args) + + +def test_slices(): + md = ImmutableDenseNDimArray(range(10, 34), (2, 3, 4)) + + assert md[:] == ImmutableDenseNDimArray(range(10, 34), (2, 3, 4)) + assert md[:, :, 0].tomatrix() == Matrix([[10, 14, 18], [22, 26, 30]]) + assert md[0, 1:2, :].tomatrix() == Matrix([[14, 15, 16, 17]]) + assert md[0, 1:3, :].tomatrix() == Matrix([[14, 15, 16, 17], [18, 19, 20, 21]]) + assert md[:, :, :] == md + + sd = ImmutableSparseNDimArray(range(10, 34), (2, 3, 4)) + assert sd == ImmutableSparseNDimArray(md) + + assert sd[:] == ImmutableSparseNDimArray(range(10, 34), (2, 3, 4)) + assert sd[:, :, 0].tomatrix() == Matrix([[10, 14, 18], [22, 26, 30]]) + assert sd[0, 1:2, :].tomatrix() == Matrix([[14, 15, 16, 17]]) + assert sd[0, 1:3, :].tomatrix() == Matrix([[14, 15, 16, 17], [18, 19, 20, 21]]) + assert sd[:, :, :] == sd + + +def test_diff_and_applyfunc(): + from sympy.abc import x, y, z + md = ImmutableDenseNDimArray([[x, y], [x*z, x*y*z]]) + assert md.diff(x) == ImmutableDenseNDimArray([[1, 0], [z, y*z]]) + assert diff(md, x) == ImmutableDenseNDimArray([[1, 0], [z, y*z]]) + + sd = ImmutableSparseNDimArray(md) + assert sd == ImmutableSparseNDimArray([x, y, x*z, x*y*z], (2, 2)) + assert sd.diff(x) == ImmutableSparseNDimArray([[1, 0], [z, y*z]]) + assert diff(sd, x) == ImmutableSparseNDimArray([[1, 0], [z, y*z]]) + + mdn = md.applyfunc(lambda x: x*3) + assert mdn == ImmutableDenseNDimArray([[3*x, 3*y], [3*x*z, 3*x*y*z]]) + assert md != mdn + + sdn = sd.applyfunc(lambda x: x/2) + assert sdn == ImmutableSparseNDimArray([[x/2, y/2], [x*z/2, x*y*z/2]]) + assert sd != sdn + + sdp = sd.applyfunc(lambda x: x+1) + assert sdp == ImmutableSparseNDimArray([[x + 1, y + 1], [x*z + 1, x*y*z + 1]]) + assert sd != sdp + + +def test_op_priority(): + from sympy.abc import x + md = ImmutableDenseNDimArray([1, 2, 3]) + e1 = (1+x)*md + e2 = md*(1+x) + assert e1 == ImmutableDenseNDimArray([1+x, 2+2*x, 3+3*x]) + assert e1 == e2 + + sd = ImmutableSparseNDimArray([1, 2, 3]) + e3 = (1+x)*sd + e4 = sd*(1+x) + assert e3 == ImmutableDenseNDimArray([1+x, 2+2*x, 3+3*x]) + assert e3 == e4 + + +def test_symbolic_indexing(): + x, y, z, w = symbols("x y z w") + M = ImmutableDenseNDimArray([[x, y], [z, w]]) + i, j = symbols("i, j") + Mij = M[i, j] + assert isinstance(Mij, Indexed) + Ms = ImmutableSparseNDimArray([[2, 3*x], [4, 5]]) + msij = Ms[i, j] + assert isinstance(msij, Indexed) + for oi, oj in [(0, 0), (0, 1), (1, 0), (1, 1)]: + assert Mij.subs({i: oi, j: oj}) == M[oi, oj] + assert msij.subs({i: oi, j: oj}) == Ms[oi, oj] + A = IndexedBase("A", (0, 2)) + assert A[0, 0].subs(A, M) == x + assert A[i, j].subs(A, M) == M[i, j] + assert M[i, j].subs(M, A) == A[i, j] + + assert isinstance(M[3 * i - 2, j], Indexed) + assert M[3 * i - 2, j].subs({i: 1, j: 0}) == M[1, 0] + assert isinstance(M[i, 0], Indexed) + assert M[i, 0].subs(i, 0) == M[0, 0] + assert M[0, i].subs(i, 1) == M[0, 1] + + assert M[i, j].diff(x) == ImmutableDenseNDimArray([[1, 0], [0, 0]])[i, j] + assert Ms[i, j].diff(x) == ImmutableSparseNDimArray([[0, 3], [0, 0]])[i, j] + + Mo = ImmutableDenseNDimArray([1, 2, 3]) + assert Mo[i].subs(i, 1) == 2 + Mos = ImmutableSparseNDimArray([1, 2, 3]) + assert Mos[i].subs(i, 1) == 2 + + raises(ValueError, lambda: M[i, 2]) + raises(ValueError, lambda: M[i, -1]) + raises(ValueError, lambda: M[2, i]) + raises(ValueError, lambda: M[-1, i]) + + raises(ValueError, lambda: Ms[i, 2]) + raises(ValueError, lambda: Ms[i, -1]) + raises(ValueError, lambda: Ms[2, i]) + raises(ValueError, lambda: Ms[-1, i]) + + +def test_issue_12665(): + # Testing Python 3 hash of immutable arrays: + arr = ImmutableDenseNDimArray([1, 2, 3]) + # This should NOT raise an exception: + hash(arr) + + +def test_zeros_without_shape(): + arr = ImmutableDenseNDimArray.zeros() + assert arr == ImmutableDenseNDimArray(0) + +def test_issue_21870(): + a0 = ImmutableDenseNDimArray(0) + assert a0.rank() == 0 + a1 = ImmutableDenseNDimArray(a0) + assert a1.rank() == 0 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_mutable_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_mutable_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..9a232f399bbc0639d326217975fb0a12e645a984 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_mutable_ndim_array.py @@ -0,0 +1,374 @@ +from copy import copy + +from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray +from sympy.core.function import diff +from sympy.core.numbers import Rational +from sympy.core.singleton import S +from sympy.core.symbol import Symbol +from sympy.core.sympify import sympify +from sympy.matrices import SparseMatrix +from sympy.matrices import Matrix +from sympy.tensor.array.sparse_ndim_array import MutableSparseNDimArray +from sympy.testing.pytest import raises + + +def test_ndim_array_initiation(): + arr_with_one_element = MutableDenseNDimArray([23]) + assert len(arr_with_one_element) == 1 + assert arr_with_one_element[0] == 23 + assert arr_with_one_element.rank() == 1 + raises(ValueError, lambda: arr_with_one_element[1]) + + arr_with_symbol_element = MutableDenseNDimArray([Symbol('x')]) + assert len(arr_with_symbol_element) == 1 + assert arr_with_symbol_element[0] == Symbol('x') + assert arr_with_symbol_element.rank() == 1 + + number5 = 5 + vector = MutableDenseNDimArray.zeros(number5) + assert len(vector) == number5 + assert vector.shape == (number5,) + assert vector.rank() == 1 + raises(ValueError, lambda: arr_with_one_element[5]) + + vector = MutableSparseNDimArray.zeros(number5) + assert len(vector) == number5 + assert vector.shape == (number5,) + assert vector._sparse_array == {} + assert vector.rank() == 1 + + n_dim_array = MutableDenseNDimArray(range(3**4), (3, 3, 3, 3,)) + assert len(n_dim_array) == 3 * 3 * 3 * 3 + assert n_dim_array.shape == (3, 3, 3, 3) + assert n_dim_array.rank() == 4 + raises(ValueError, lambda: n_dim_array[0, 0, 0, 3]) + raises(ValueError, lambda: n_dim_array[3, 0, 0, 0]) + raises(ValueError, lambda: n_dim_array[3**4]) + + array_shape = (3, 3, 3, 3) + sparse_array = MutableSparseNDimArray.zeros(*array_shape) + assert len(sparse_array._sparse_array) == 0 + assert len(sparse_array) == 3 * 3 * 3 * 3 + assert n_dim_array.shape == array_shape + assert n_dim_array.rank() == 4 + + one_dim_array = MutableDenseNDimArray([2, 3, 1]) + assert len(one_dim_array) == 3 + assert one_dim_array.shape == (3,) + assert one_dim_array.rank() == 1 + assert one_dim_array.tolist() == [2, 3, 1] + + shape = (3, 3) + array_with_many_args = MutableSparseNDimArray.zeros(*shape) + assert len(array_with_many_args) == 3 * 3 + assert array_with_many_args.shape == shape + assert array_with_many_args[0, 0] == 0 + assert array_with_many_args.rank() == 2 + + shape = (int(3), int(3)) + array_with_long_shape = MutableSparseNDimArray.zeros(*shape) + assert len(array_with_long_shape) == 3 * 3 + assert array_with_long_shape.shape == shape + assert array_with_long_shape[int(0), int(0)] == 0 + assert array_with_long_shape.rank() == 2 + + vector_with_long_shape = MutableDenseNDimArray(range(5), int(5)) + assert len(vector_with_long_shape) == 5 + assert vector_with_long_shape.shape == (int(5),) + assert vector_with_long_shape.rank() == 1 + raises(ValueError, lambda: vector_with_long_shape[int(5)]) + + from sympy.abc import x + for ArrayType in [MutableDenseNDimArray, MutableSparseNDimArray]: + rank_zero_array = ArrayType(x) + assert len(rank_zero_array) == 1 + assert rank_zero_array.shape == () + assert rank_zero_array.rank() == 0 + assert rank_zero_array[()] == x + raises(ValueError, lambda: rank_zero_array[0]) + +def test_sympify(): + from sympy.abc import x, y, z, t + arr = MutableDenseNDimArray([[x, y], [1, z*t]]) + arr_other = sympify(arr) + assert arr_other.shape == (2, 2) + assert arr_other == arr + + +def test_reshape(): + array = MutableDenseNDimArray(range(50), 50) + assert array.shape == (50,) + assert array.rank() == 1 + + array = array.reshape(5, 5, 2) + assert array.shape == (5, 5, 2) + assert array.rank() == 3 + assert len(array) == 50 + + +def test_iterator(): + array = MutableDenseNDimArray(range(4), (2, 2)) + assert array[0] == MutableDenseNDimArray([0, 1]) + assert array[1] == MutableDenseNDimArray([2, 3]) + + array = array.reshape(4) + j = 0 + for i in array: + assert i == j + j += 1 + + +def test_getitem(): + for ArrayType in [MutableDenseNDimArray, MutableSparseNDimArray]: + array = ArrayType(range(24)).reshape(2, 3, 4) + assert array.tolist() == [[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]] + assert array[0] == ArrayType([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]) + assert array[0, 0] == ArrayType([0, 1, 2, 3]) + value = 0 + for i in range(2): + for j in range(3): + for k in range(4): + assert array[i, j, k] == value + value += 1 + + raises(ValueError, lambda: array[3, 4, 5]) + raises(ValueError, lambda: array[3, 4, 5, 6]) + raises(ValueError, lambda: array[3, 4, 5, 3:4]) + + +def test_sparse(): + sparse_array = MutableSparseNDimArray([0, 0, 0, 1], (2, 2)) + assert len(sparse_array) == 2 * 2 + # dictionary where all data is, only non-zero entries are actually stored: + assert len(sparse_array._sparse_array) == 1 + + assert sparse_array.tolist() == [[0, 0], [0, 1]] + + for i, j in zip(sparse_array, [[0, 0], [0, 1]]): + assert i == MutableSparseNDimArray(j) + + sparse_array[0, 0] = 123 + assert len(sparse_array._sparse_array) == 2 + assert sparse_array[0, 0] == 123 + assert sparse_array/0 == MutableSparseNDimArray([[S.ComplexInfinity, S.NaN], [S.NaN, S.ComplexInfinity]], (2, 2)) + + # when element in sparse array become zero it will disappear from + # dictionary + sparse_array[0, 0] = 0 + assert len(sparse_array._sparse_array) == 1 + sparse_array[1, 1] = 0 + assert len(sparse_array._sparse_array) == 0 + assert sparse_array[0, 0] == 0 + + # test for large scale sparse array + # equality test + a = MutableSparseNDimArray.zeros(100000, 200000) + b = MutableSparseNDimArray.zeros(100000, 200000) + assert a == b + a[1, 1] = 1 + b[1, 1] = 2 + assert a != b + + # __mul__ and __rmul__ + assert a * 3 == MutableSparseNDimArray({200001: 3}, (100000, 200000)) + assert 3 * a == MutableSparseNDimArray({200001: 3}, (100000, 200000)) + assert a * 0 == MutableSparseNDimArray({}, (100000, 200000)) + assert 0 * a == MutableSparseNDimArray({}, (100000, 200000)) + + # __truediv__ + assert a/3 == MutableSparseNDimArray({200001: Rational(1, 3)}, (100000, 200000)) + + # __neg__ + assert -a == MutableSparseNDimArray({200001: -1}, (100000, 200000)) + + +def test_calculation(): + + a = MutableDenseNDimArray([1]*9, (3, 3)) + b = MutableDenseNDimArray([9]*9, (3, 3)) + + c = a + b + for i in c: + assert i == MutableDenseNDimArray([10, 10, 10]) + + assert c == MutableDenseNDimArray([10]*9, (3, 3)) + assert c == MutableSparseNDimArray([10]*9, (3, 3)) + + c = b - a + for i in c: + assert i == MutableSparseNDimArray([8, 8, 8]) + + assert c == MutableDenseNDimArray([8]*9, (3, 3)) + assert c == MutableSparseNDimArray([8]*9, (3, 3)) + + +def test_ndim_array_converting(): + dense_array = MutableDenseNDimArray([1, 2, 3, 4], (2, 2)) + alist = dense_array.tolist() + + assert alist == [[1, 2], [3, 4]] + + matrix = dense_array.tomatrix() + assert (isinstance(matrix, Matrix)) + + for i in range(len(dense_array)): + assert dense_array[dense_array._get_tuple_index(i)] == matrix[i] + assert matrix.shape == dense_array.shape + + assert MutableDenseNDimArray(matrix) == dense_array + assert MutableDenseNDimArray(matrix.as_immutable()) == dense_array + assert MutableDenseNDimArray(matrix.as_mutable()) == dense_array + + sparse_array = MutableSparseNDimArray([1, 2, 3, 4], (2, 2)) + alist = sparse_array.tolist() + + assert alist == [[1, 2], [3, 4]] + + matrix = sparse_array.tomatrix() + assert(isinstance(matrix, SparseMatrix)) + + for i in range(len(sparse_array)): + assert sparse_array[sparse_array._get_tuple_index(i)] == matrix[i] + assert matrix.shape == sparse_array.shape + + assert MutableSparseNDimArray(matrix) == sparse_array + assert MutableSparseNDimArray(matrix.as_immutable()) == sparse_array + assert MutableSparseNDimArray(matrix.as_mutable()) == sparse_array + + +def test_converting_functions(): + arr_list = [1, 2, 3, 4] + arr_matrix = Matrix(((1, 2), (3, 4))) + + # list + arr_ndim_array = MutableDenseNDimArray(arr_list, (2, 2)) + assert (isinstance(arr_ndim_array, MutableDenseNDimArray)) + assert arr_matrix.tolist() == arr_ndim_array.tolist() + + # Matrix + arr_ndim_array = MutableDenseNDimArray(arr_matrix) + assert (isinstance(arr_ndim_array, MutableDenseNDimArray)) + assert arr_matrix.tolist() == arr_ndim_array.tolist() + assert arr_matrix.shape == arr_ndim_array.shape + + +def test_equality(): + first_list = [1, 2, 3, 4] + second_list = [1, 2, 3, 4] + third_list = [4, 3, 2, 1] + assert first_list == second_list + assert first_list != third_list + + first_ndim_array = MutableDenseNDimArray(first_list, (2, 2)) + second_ndim_array = MutableDenseNDimArray(second_list, (2, 2)) + third_ndim_array = MutableDenseNDimArray(third_list, (2, 2)) + fourth_ndim_array = MutableDenseNDimArray(first_list, (2, 2)) + + assert first_ndim_array == second_ndim_array + second_ndim_array[0, 0] = 0 + assert first_ndim_array != second_ndim_array + assert first_ndim_array != third_ndim_array + assert first_ndim_array == fourth_ndim_array + + +def test_arithmetic(): + a = MutableDenseNDimArray([3 for i in range(9)], (3, 3)) + b = MutableDenseNDimArray([7 for i in range(9)], (3, 3)) + + c1 = a + b + c2 = b + a + assert c1 == c2 + + d1 = a - b + d2 = b - a + assert d1 == d2 * (-1) + + e1 = a * 5 + e2 = 5 * a + e3 = copy(a) + e3 *= 5 + assert e1 == e2 == e3 + + f1 = a / 5 + f2 = copy(a) + f2 /= 5 + assert f1 == f2 + assert f1[0, 0] == f1[0, 1] == f1[0, 2] == f1[1, 0] == f1[1, 1] == \ + f1[1, 2] == f1[2, 0] == f1[2, 1] == f1[2, 2] == Rational(3, 5) + + assert type(a) == type(b) == type(c1) == type(c2) == type(d1) == type(d2) \ + == type(e1) == type(e2) == type(e3) == type(f1) + + z0 = -a + assert z0 == MutableDenseNDimArray([-3 for i in range(9)], (3, 3)) + + +def test_higher_dimenions(): + m3 = MutableDenseNDimArray(range(10, 34), (2, 3, 4)) + + assert m3.tolist() == [[[10, 11, 12, 13], + [14, 15, 16, 17], + [18, 19, 20, 21]], + + [[22, 23, 24, 25], + [26, 27, 28, 29], + [30, 31, 32, 33]]] + + assert m3._get_tuple_index(0) == (0, 0, 0) + assert m3._get_tuple_index(1) == (0, 0, 1) + assert m3._get_tuple_index(4) == (0, 1, 0) + assert m3._get_tuple_index(12) == (1, 0, 0) + + assert str(m3) == '[[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]]' + + m3_rebuilt = MutableDenseNDimArray([[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]]) + assert m3 == m3_rebuilt + + m3_other = MutableDenseNDimArray([[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19, 20, 21]], [[22, 23, 24, 25], [26, 27, 28, 29], [30, 31, 32, 33]]], (2, 3, 4)) + + assert m3 == m3_other + + +def test_slices(): + md = MutableDenseNDimArray(range(10, 34), (2, 3, 4)) + + assert md[:] == MutableDenseNDimArray(range(10, 34), (2, 3, 4)) + assert md[:, :, 0].tomatrix() == Matrix([[10, 14, 18], [22, 26, 30]]) + assert md[0, 1:2, :].tomatrix() == Matrix([[14, 15, 16, 17]]) + assert md[0, 1:3, :].tomatrix() == Matrix([[14, 15, 16, 17], [18, 19, 20, 21]]) + assert md[:, :, :] == md + + sd = MutableSparseNDimArray(range(10, 34), (2, 3, 4)) + assert sd == MutableSparseNDimArray(md) + + assert sd[:] == MutableSparseNDimArray(range(10, 34), (2, 3, 4)) + assert sd[:, :, 0].tomatrix() == Matrix([[10, 14, 18], [22, 26, 30]]) + assert sd[0, 1:2, :].tomatrix() == Matrix([[14, 15, 16, 17]]) + assert sd[0, 1:3, :].tomatrix() == Matrix([[14, 15, 16, 17], [18, 19, 20, 21]]) + assert sd[:, :, :] == sd + + +def test_slices_assign(): + a = MutableDenseNDimArray(range(12), shape=(4, 3)) + b = MutableSparseNDimArray(range(12), shape=(4, 3)) + + for i in [a, b]: + assert i.tolist() == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + i[0, :] = [2, 2, 2] + assert i.tolist() == [[2, 2, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + i[0, 1:] = [8, 8] + assert i.tolist() == [[2, 8, 8], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + i[1:3, 1] = [20, 44] + assert i.tolist() == [[2, 8, 8], [3, 20, 5], [6, 44, 8], [9, 10, 11]] + + +def test_diff(): + from sympy.abc import x, y, z + md = MutableDenseNDimArray([[x, y], [x*z, x*y*z]]) + assert md.diff(x) == MutableDenseNDimArray([[1, 0], [z, y*z]]) + assert diff(md, x) == MutableDenseNDimArray([[1, 0], [z, y*z]]) + + sd = MutableSparseNDimArray(md) + assert sd == MutableSparseNDimArray([x, y, x*z, x*y*z], (2, 2)) + assert sd.diff(x) == MutableSparseNDimArray([[1, 0], [z, y*z]]) + assert diff(sd, x) == MutableSparseNDimArray([[1, 0], [z, y*z]]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array.py new file mode 100644 index 0000000000000000000000000000000000000000..7ff9b032631c01272c00478e4cdf0dcbc6997990 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array.py @@ -0,0 +1,73 @@ +from sympy.testing.pytest import raises +from sympy.functions.elementary.trigonometric import sin, cos +from sympy.matrices.dense import Matrix +from sympy.simplify import simplify +from sympy.tensor.array import Array +from sympy.tensor.array.dense_ndim_array import ( + ImmutableDenseNDimArray, MutableDenseNDimArray) +from sympy.tensor.array.sparse_ndim_array import ( + ImmutableSparseNDimArray, MutableSparseNDimArray) + +from sympy.abc import x, y + +mutable_array_types = [ + MutableDenseNDimArray, + MutableSparseNDimArray +] + +array_types = [ + ImmutableDenseNDimArray, + ImmutableSparseNDimArray, + MutableDenseNDimArray, + MutableSparseNDimArray +] + + +def test_array_negative_indices(): + for ArrayType in array_types: + test_array = ArrayType([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]) + assert test_array[:, -1] == Array([5, 10]) + assert test_array[:, -2] == Array([4, 9]) + assert test_array[:, -3] == Array([3, 8]) + assert test_array[:, -4] == Array([2, 7]) + assert test_array[:, -5] == Array([1, 6]) + assert test_array[:, 0] == Array([1, 6]) + assert test_array[:, 1] == Array([2, 7]) + assert test_array[:, 2] == Array([3, 8]) + assert test_array[:, 3] == Array([4, 9]) + assert test_array[:, 4] == Array([5, 10]) + + raises(ValueError, lambda: test_array[:, -6]) + raises(ValueError, lambda: test_array[-3, :]) + + assert test_array[-1, -1] == 10 + + +def test_issue_18361(): + A = Array([sin(2 * x) - 2 * sin(x) * cos(x)]) + B = Array([sin(x)**2 + cos(x)**2, 0]) + C = Array([(x + x**2)/(x*sin(y)**2 + x*cos(y)**2), 2*sin(x)*cos(x)]) + assert simplify(A) == Array([0]) + assert simplify(B) == Array([1, 0]) + assert simplify(C) == Array([x + 1, sin(2*x)]) + + +def test_issue_20222(): + A = Array([[1, 2], [3, 4]]) + B = Matrix([[1,2],[3,4]]) + raises(TypeError, lambda: A - B) + + +def test_issue_17851(): + for array_type in array_types: + A = array_type([]) + assert isinstance(A, array_type) + assert A.shape == (0,) + assert list(A) == [] + + +def test_issue_and_18715(): + for array_type in mutable_array_types: + A = array_type([0, 1, 2]) + A[0] += 5 + assert A[0] == 5 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array_conversions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array_conversions.py new file mode 100644 index 0000000000000000000000000000000000000000..f43260ccc636ac461ba0c06dbfcf3fe3a8d5338d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/array/tests/test_ndim_array_conversions.py @@ -0,0 +1,22 @@ +from sympy.tensor.array import (ImmutableDenseNDimArray, + ImmutableSparseNDimArray, MutableDenseNDimArray, MutableSparseNDimArray) +from sympy.abc import x, y, z + + +def test_NDim_array_conv(): + MD = MutableDenseNDimArray([x, y, z]) + MS = MutableSparseNDimArray([x, y, z]) + ID = ImmutableDenseNDimArray([x, y, z]) + IS = ImmutableSparseNDimArray([x, y, z]) + + assert MD.as_immutable() == ID + assert MD.as_mutable() == MD + + assert MS.as_immutable() == IS + assert MS.as_mutable() == MS + + assert ID.as_immutable() == ID + assert ID.as_mutable() == MD + + assert IS.as_immutable() == IS + assert IS.as_mutable() == MS diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd4b15cee0104fc96de6162b39e794de6350a09a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_functions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..431a825a95392a1e079406aa449c08057e2c54b8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_functions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_index_methods.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_index_methods.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..47ef98e25e31d0b139e1e61456b977e2add893f1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_index_methods.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_indexed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_indexed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..132205dd8aa754e5d5e419e6583da990e9807022 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_indexed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_printing.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_printing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a6b02f5fc2a33abf6d638a75a4875b197639333 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_printing.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_element.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_element.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab661487d36b1ae36203829ddcd9ad70f1fddfde Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_element.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_operators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_operators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..675d4b562f11c247eac88c95ad58d4ca9e37a696 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/__pycache__/test_tensor_operators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_functions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..ae40865d1bddffaa976dc3d94ae1ef1b6c97ca35 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_functions.py @@ -0,0 +1,57 @@ +from sympy.tensor.functions import TensorProduct +from sympy.matrices.dense import Matrix +from sympy.matrices.expressions.matexpr import MatrixSymbol +from sympy.tensor.array import Array +from sympy.abc import x, y, z +from sympy.abc import i, j, k, l + + +A = MatrixSymbol("A", 3, 3) +B = MatrixSymbol("B", 3, 3) +C = MatrixSymbol("C", 3, 3) + + +def test_TensorProduct_construction(): + assert TensorProduct(3, 4) == 12 + assert isinstance(TensorProduct(A, A), TensorProduct) + + expr = TensorProduct(TensorProduct(x, y), z) + assert expr == x*y*z + + expr = TensorProduct(TensorProduct(A, B), C) + assert expr == TensorProduct(A, B, C) + + expr = TensorProduct(Matrix.eye(2), Array([[0, -1], [1, 0]])) + assert expr == Array([ + [ + [[0, -1], [1, 0]], + [[0, 0], [0, 0]] + ], + [ + [[0, 0], [0, 0]], + [[0, -1], [1, 0]] + ] + ]) + + +def test_TensorProduct_shape(): + + expr = TensorProduct(3, 4, evaluate=False) + assert expr.shape == () + assert expr.rank() == 0 + + expr = TensorProduct(Array([1, 2]), Array([x, y]), evaluate=False) + assert expr.shape == (2, 2) + assert expr.rank() == 2 + expr = TensorProduct(expr, expr, evaluate=False) + assert expr.shape == (2, 2, 2, 2) + assert expr.rank() == 4 + + expr = TensorProduct(Matrix.eye(2), Array([[0, -1], [1, 0]]), evaluate=False) + assert expr.shape == (2, 2, 2, 2) + assert expr.rank() == 4 + + +def test_TensorProduct_getitem(): + expr = TensorProduct(A, B) + assert expr[i, j, k, l] == A[i, j]*B[k, l] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_index_methods.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_index_methods.py new file mode 100644 index 0000000000000000000000000000000000000000..df20f7e7c1ab392321e8350b95dd07c5639c1865 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_index_methods.py @@ -0,0 +1,227 @@ +from sympy.core import symbols, S, Pow, Function +from sympy.functions import exp +from sympy.testing.pytest import raises +from sympy.tensor.indexed import Idx, IndexedBase +from sympy.tensor.index_methods import IndexConformanceException + +from sympy.tensor.index_methods import (get_contraction_structure, get_indices) + + +def test_trivial_indices(): + x, y = symbols('x y') + assert get_indices(x) == (set(), {}) + assert get_indices(x*y) == (set(), {}) + assert get_indices(x + y) == (set(), {}) + assert get_indices(x**y) == (set(), {}) + + +def test_get_indices_Indexed(): + x = IndexedBase('x') + i, j = Idx('i'), Idx('j') + assert get_indices(x[i, j]) == ({i, j}, {}) + assert get_indices(x[j, i]) == ({j, i}, {}) + + +def test_get_indices_Idx(): + f = Function('f') + i, j = Idx('i'), Idx('j') + assert get_indices(f(i)*j) == ({i, j}, {}) + assert get_indices(f(j, i)) == ({j, i}, {}) + assert get_indices(f(i)*i) == (set(), {}) + + +def test_get_indices_mul(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j = Idx('i'), Idx('j') + assert get_indices(x[j]*y[i]) == ({i, j}, {}) + assert get_indices(x[i]*y[j]) == ({i, j}, {}) + + +def test_get_indices_exceptions(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j = Idx('i'), Idx('j') + raises(IndexConformanceException, lambda: get_indices(x[i] + y[j])) + + +def test_scalar_broadcast(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j = Idx('i'), Idx('j') + assert get_indices(x[i] + y[i, i]) == ({i}, {}) + assert get_indices(x[i] + y[j, j]) == ({i}, {}) + + +def test_get_indices_add(): + x = IndexedBase('x') + y = IndexedBase('y') + A = IndexedBase('A') + i, j, k = Idx('i'), Idx('j'), Idx('k') + assert get_indices(x[i] + 2*y[i]) == ({i}, {}) + assert get_indices(y[i] + 2*A[i, j]*x[j]) == ({i}, {}) + assert get_indices(y[i] + 2*(x[i] + A[i, j]*x[j])) == ({i}, {}) + assert get_indices(y[i] + x[i]*(A[j, j] + 1)) == ({i}, {}) + assert get_indices( + y[i] + x[i]*x[j]*(y[j] + A[j, k]*x[k])) == ({i}, {}) + + +def test_get_indices_Pow(): + x = IndexedBase('x') + y = IndexedBase('y') + A = IndexedBase('A') + i, j, k = Idx('i'), Idx('j'), Idx('k') + assert get_indices(Pow(x[i], y[j])) == ({i, j}, {}) + assert get_indices(Pow(x[i, k], y[j, k])) == ({i, j, k}, {}) + assert get_indices(Pow(A[i, k], y[k] + A[k, j]*x[j])) == ({i, k}, {}) + assert get_indices(Pow(2, x[i])) == get_indices(exp(x[i])) + + # test of a design decision, this may change: + assert get_indices(Pow(x[i], 2)) == ({i}, {}) + + +def test_get_contraction_structure_basic(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j = Idx('i'), Idx('j') + assert get_contraction_structure(x[i]*y[j]) == {None: {x[i]*y[j]}} + assert get_contraction_structure(x[i] + y[j]) == {None: {x[i], y[j]}} + assert get_contraction_structure(x[i]*y[i]) == {(i,): {x[i]*y[i]}} + assert get_contraction_structure( + 1 + x[i]*y[i]) == {None: {S.One}, (i,): {x[i]*y[i]}} + assert get_contraction_structure(x[i]**y[i]) == {None: {x[i]**y[i]}} + + +def test_get_contraction_structure_complex(): + x = IndexedBase('x') + y = IndexedBase('y') + A = IndexedBase('A') + i, j, k = Idx('i'), Idx('j'), Idx('k') + expr1 = y[i] + A[i, j]*x[j] + d1 = {None: {y[i]}, (j,): {A[i, j]*x[j]}} + assert get_contraction_structure(expr1) == d1 + expr2 = expr1*A[k, i] + x[k] + d2 = {None: {x[k]}, (i,): {expr1*A[k, i]}, expr1*A[k, i]: [d1]} + assert get_contraction_structure(expr2) == d2 + + +def test_contraction_structure_simple_Pow(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j, k = Idx('i'), Idx('j'), Idx('k') + ii_jj = x[i, i]**y[j, j] + assert get_contraction_structure(ii_jj) == { + None: {ii_jj}, + ii_jj: [ + {(i,): {x[i, i]}}, + {(j,): {y[j, j]}} + ] + } + + ii_jk = x[i, i]**y[j, k] + assert get_contraction_structure(ii_jk) == { + None: {x[i, i]**y[j, k]}, + x[i, i]**y[j, k]: [ + {(i,): {x[i, i]}} + ] + } + + +def test_contraction_structure_Mul_and_Pow(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j, k = Idx('i'), Idx('j'), Idx('k') + + i_ji = x[i]**(y[j]*x[i]) + assert get_contraction_structure(i_ji) == {None: {i_ji}} + ij_i = (x[i]*y[j])**(y[i]) + assert get_contraction_structure(ij_i) == {None: {ij_i}} + j_ij_i = x[j]*(x[i]*y[j])**(y[i]) + assert get_contraction_structure(j_ij_i) == {(j,): {j_ij_i}} + j_i_ji = x[j]*x[i]**(y[j]*x[i]) + assert get_contraction_structure(j_i_ji) == {(j,): {j_i_ji}} + ij_exp_kki = x[i]*y[j]*exp(y[i]*y[k, k]) + result = get_contraction_structure(ij_exp_kki) + expected = { + (i,): {ij_exp_kki}, + ij_exp_kki: [{ + None: {exp(y[i]*y[k, k])}, + exp(y[i]*y[k, k]): [{ + None: {y[i]*y[k, k]}, + y[i]*y[k, k]: [{(k,): {y[k, k]}}] + }]} + ] + } + assert result == expected + + +def test_contraction_structure_Add_in_Pow(): + x = IndexedBase('x') + y = IndexedBase('y') + i, j, k = Idx('i'), Idx('j'), Idx('k') + s_ii_jj_s = (1 + x[i, i])**(1 + y[j, j]) + expected = { + None: {s_ii_jj_s}, + s_ii_jj_s: [ + {None: {S.One}, (i,): {x[i, i]}}, + {None: {S.One}, (j,): {y[j, j]}} + ] + } + result = get_contraction_structure(s_ii_jj_s) + assert result == expected + + s_ii_jk_s = (1 + x[i, i]) ** (1 + y[j, k]) + expected_2 = { + None: {(x[i, i] + 1)**(y[j, k] + 1)}, + s_ii_jk_s: [ + {None: {S.One}, (i,): {x[i, i]}} + ] + } + result_2 = get_contraction_structure(s_ii_jk_s) + assert result_2 == expected_2 + + +def test_contraction_structure_Pow_in_Pow(): + x = IndexedBase('x') + y = IndexedBase('y') + z = IndexedBase('z') + i, j, k = Idx('i'), Idx('j'), Idx('k') + ii_jj_kk = x[i, i]**y[j, j]**z[k, k] + expected = { + None: {ii_jj_kk}, + ii_jj_kk: [ + {(i,): {x[i, i]}}, + { + None: {y[j, j]**z[k, k]}, + y[j, j]**z[k, k]: [ + {(j,): {y[j, j]}}, + {(k,): {z[k, k]}} + ] + } + ] + } + assert get_contraction_structure(ii_jj_kk) == expected + + +def test_ufunc_support(): + f = Function('f') + g = Function('g') + x = IndexedBase('x') + y = IndexedBase('y') + i, j = Idx('i'), Idx('j') + a = symbols('a') + + assert get_indices(f(x[i])) == ({i}, {}) + assert get_indices(f(x[i], y[j])) == ({i, j}, {}) + assert get_indices(f(y[i])*g(x[i])) == (set(), {}) + assert get_indices(f(a, x[i])) == ({i}, {}) + assert get_indices(f(a, y[i], x[j])*g(x[i])) == ({j}, {}) + assert get_indices(g(f(x[i]))) == ({i}, {}) + + assert get_contraction_structure(f(x[i])) == {None: {f(x[i])}} + assert get_contraction_structure( + f(y[i])*g(x[i])) == {(i,): {f(y[i])*g(x[i])}} + assert get_contraction_structure( + f(y[i])*g(f(x[i]))) == {(i,): {f(y[i])*g(f(x[i]))}} + assert get_contraction_structure( + f(x[j], y[i])*g(x[i])) == {(i,): {f(x[j], y[i])*g(x[i])}} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_indexed.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_indexed.py new file mode 100644 index 0000000000000000000000000000000000000000..689ec932c8fcefe0a24de289dd2ffd6820c63f19 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_indexed.py @@ -0,0 +1,511 @@ +from sympy.core import symbols, Symbol, Tuple, oo, Dummy +from sympy.tensor.indexed import IndexException +from sympy.testing.pytest import raises +from sympy.utilities.iterables import iterable + +# import test: +from sympy.concrete.summations import Sum +from sympy.core.function import Function, Subs, Derivative +from sympy.core.relational import (StrictLessThan, GreaterThan, + StrictGreaterThan, LessThan) +from sympy.core.singleton import S +from sympy.functions.elementary.exponential import exp, log +from sympy.functions.elementary.trigonometric import cos, sin +from sympy.functions.special.tensor_functions import KroneckerDelta +from sympy.series.order import Order +from sympy.sets.fancysets import Range +from sympy.tensor.indexed import IndexedBase, Idx, Indexed + + +def test_Idx_construction(): + i, a, b = symbols('i a b', integer=True) + assert Idx(i) != Idx(i, 1) + assert Idx(i, a) == Idx(i, (0, a - 1)) + assert Idx(i, oo) == Idx(i, (0, oo)) + + x = symbols('x', integer=False) + raises(TypeError, lambda: Idx(x)) + raises(TypeError, lambda: Idx(0.5)) + raises(TypeError, lambda: Idx(i, x)) + raises(TypeError, lambda: Idx(i, 0.5)) + raises(TypeError, lambda: Idx(i, (x, 5))) + raises(TypeError, lambda: Idx(i, (2, x))) + raises(TypeError, lambda: Idx(i, (2, 3.5))) + + +def test_Idx_properties(): + i, a, b = symbols('i a b', integer=True) + assert Idx(i).is_integer + assert Idx(i).name == 'i' + assert Idx(i + 2).name == 'i + 2' + assert Idx('foo').name == 'foo' + + +def test_Idx_bounds(): + i, a, b = symbols('i a b', integer=True) + assert Idx(i).lower is None + assert Idx(i).upper is None + assert Idx(i, a).lower == 0 + assert Idx(i, a).upper == a - 1 + assert Idx(i, 5).lower == 0 + assert Idx(i, 5).upper == 4 + assert Idx(i, oo).lower == 0 + assert Idx(i, oo).upper is oo + assert Idx(i, (a, b)).lower == a + assert Idx(i, (a, b)).upper == b + assert Idx(i, (1, 5)).lower == 1 + assert Idx(i, (1, 5)).upper == 5 + assert Idx(i, (-oo, oo)).lower is -oo + assert Idx(i, (-oo, oo)).upper is oo + + +def test_Idx_fixed_bounds(): + i, a, b, x = symbols('i a b x', integer=True) + assert Idx(x).lower is None + assert Idx(x).upper is None + assert Idx(x, a).lower == 0 + assert Idx(x, a).upper == a - 1 + assert Idx(x, 5).lower == 0 + assert Idx(x, 5).upper == 4 + assert Idx(x, oo).lower == 0 + assert Idx(x, oo).upper is oo + assert Idx(x, (a, b)).lower == a + assert Idx(x, (a, b)).upper == b + assert Idx(x, (1, 5)).lower == 1 + assert Idx(x, (1, 5)).upper == 5 + assert Idx(x, (-oo, oo)).lower is -oo + assert Idx(x, (-oo, oo)).upper is oo + + +def test_Idx_inequalities(): + i14 = Idx("i14", (1, 4)) + i79 = Idx("i79", (7, 9)) + i46 = Idx("i46", (4, 6)) + i35 = Idx("i35", (3, 5)) + + assert i14 <= 5 + assert i14 < 5 + assert not (i14 >= 5) + assert not (i14 > 5) + + assert 5 >= i14 + assert 5 > i14 + assert not (5 <= i14) + assert not (5 < i14) + + assert LessThan(i14, 5) + assert StrictLessThan(i14, 5) + assert not GreaterThan(i14, 5) + assert not StrictGreaterThan(i14, 5) + + assert i14 <= 4 + assert isinstance(i14 < 4, StrictLessThan) + assert isinstance(i14 >= 4, GreaterThan) + assert not (i14 > 4) + + assert isinstance(i14 <= 1, LessThan) + assert not (i14 < 1) + assert i14 >= 1 + assert isinstance(i14 > 1, StrictGreaterThan) + + assert not (i14 <= 0) + assert not (i14 < 0) + assert i14 >= 0 + assert i14 > 0 + + from sympy.abc import x + + assert isinstance(i14 < x, StrictLessThan) + assert isinstance(i14 > x, StrictGreaterThan) + assert isinstance(i14 <= x, LessThan) + assert isinstance(i14 >= x, GreaterThan) + + assert i14 < i79 + assert i14 <= i79 + assert not (i14 > i79) + assert not (i14 >= i79) + + assert i14 <= i46 + assert isinstance(i14 < i46, StrictLessThan) + assert isinstance(i14 >= i46, GreaterThan) + assert not (i14 > i46) + + assert isinstance(i14 < i35, StrictLessThan) + assert isinstance(i14 > i35, StrictGreaterThan) + assert isinstance(i14 <= i35, LessThan) + assert isinstance(i14 >= i35, GreaterThan) + + iNone1 = Idx("iNone1") + iNone2 = Idx("iNone2") + + assert isinstance(iNone1 < iNone2, StrictLessThan) + assert isinstance(iNone1 > iNone2, StrictGreaterThan) + assert isinstance(iNone1 <= iNone2, LessThan) + assert isinstance(iNone1 >= iNone2, GreaterThan) + + +def test_Idx_inequalities_current_fails(): + i14 = Idx("i14", (1, 4)) + + assert S(5) >= i14 + assert S(5) > i14 + assert not (S(5) <= i14) + assert not (S(5) < i14) + + +def test_Idx_func_args(): + i, a, b = symbols('i a b', integer=True) + ii = Idx(i) + assert ii.func(*ii.args) == ii + ii = Idx(i, a) + assert ii.func(*ii.args) == ii + ii = Idx(i, (a, b)) + assert ii.func(*ii.args) == ii + + +def test_Idx_subs(): + i, a, b = symbols('i a b', integer=True) + assert Idx(i, a).subs(a, b) == Idx(i, b) + assert Idx(i, a).subs(i, b) == Idx(b, a) + + assert Idx(i).subs(i, 2) == Idx(2) + assert Idx(i, a).subs(a, 2) == Idx(i, 2) + assert Idx(i, (a, b)).subs(i, 2) == Idx(2, (a, b)) + + +def test_IndexedBase_sugar(): + i, j = symbols('i j', integer=True) + a = symbols('a') + A1 = Indexed(a, i, j) + A2 = IndexedBase(a) + assert A1 == A2[i, j] + assert A1 == A2[(i, j)] + assert A1 == A2[[i, j]] + assert A1 == A2[Tuple(i, j)] + assert all(a.is_Integer for a in A2[1, 0].args[1:]) + + +def test_IndexedBase_subs(): + i = symbols('i', integer=True) + a, b = symbols('a b') + A = IndexedBase(a) + B = IndexedBase(b) + assert A[i] == B[i].subs(b, a) + C = {1: 2} + assert C[1] == A[1].subs(A, C) + + +def test_IndexedBase_shape(): + i, j, m, n = symbols('i j m n', integer=True) + a = IndexedBase('a', shape=(m, m)) + b = IndexedBase('a', shape=(m, n)) + assert b.shape == Tuple(m, n) + assert a[i, j] != b[i, j] + assert a[i, j] == b[i, j].subs(n, m) + assert b.func(*b.args) == b + assert b[i, j].func(*b[i, j].args) == b[i, j] + raises(IndexException, lambda: b[i]) + raises(IndexException, lambda: b[i, i, j]) + F = IndexedBase("F", shape=m) + assert F.shape == Tuple(m) + assert F[i].subs(i, j) == F[j] + raises(IndexException, lambda: F[i, j]) + + +def test_IndexedBase_assumptions(): + i = Symbol('i', integer=True) + a = Symbol('a') + A = IndexedBase(a, positive=True) + for c in (A, A[i]): + assert c.is_real + assert c.is_complex + assert not c.is_imaginary + assert c.is_nonnegative + assert c.is_nonzero + assert c.is_commutative + assert log(exp(c)) == c + + assert A != IndexedBase(a) + assert A == IndexedBase(a, positive=True, real=True) + assert A[i] != Indexed(a, i) + + +def test_IndexedBase_assumptions_inheritance(): + I = Symbol('I', integer=True) + I_inherit = IndexedBase(I) + I_explicit = IndexedBase('I', integer=True) + + assert I_inherit.is_integer + assert I_explicit.is_integer + assert I_inherit.label.is_integer + assert I_explicit.label.is_integer + assert I_inherit == I_explicit + + +def test_issue_17652(): + """Regression test issue #17652. + + IndexedBase.label should not upcast subclasses of Symbol + """ + class SubClass(Symbol): + pass + + x = SubClass('X') + assert type(x) == SubClass + base = IndexedBase(x) + assert type(x) == SubClass + assert type(base.label) == SubClass + + +def test_Indexed_constructor(): + i, j = symbols('i j', integer=True) + A = Indexed('A', i, j) + assert A == Indexed(Symbol('A'), i, j) + assert A == Indexed(IndexedBase('A'), i, j) + raises(TypeError, lambda: Indexed(A, i, j)) + raises(IndexException, lambda: Indexed("A")) + assert A.free_symbols == {A, A.base.label, i, j} + + +def test_Indexed_func_args(): + i, j = symbols('i j', integer=True) + a = symbols('a') + A = Indexed(a, i, j) + assert A == A.func(*A.args) + + +def test_Indexed_subs(): + i, j, k = symbols('i j k', integer=True) + a, b = symbols('a b') + A = IndexedBase(a) + B = IndexedBase(b) + assert A[i, j] == B[i, j].subs(b, a) + assert A[i, j] == A[i, k].subs(k, j) + + +def test_Indexed_properties(): + i, j = symbols('i j', integer=True) + A = Indexed('A', i, j) + assert A.name == 'A[i, j]' + assert A.rank == 2 + assert A.indices == (i, j) + assert A.base == IndexedBase('A') + assert A.ranges == [None, None] + raises(IndexException, lambda: A.shape) + + n, m = symbols('n m', integer=True) + assert Indexed('A', Idx( + i, m), Idx(j, n)).ranges == [Tuple(0, m - 1), Tuple(0, n - 1)] + assert Indexed('A', Idx(i, m), Idx(j, n)).shape == Tuple(m, n) + raises(IndexException, lambda: Indexed("A", Idx(i, m), Idx(j)).shape) + + +def test_Indexed_shape_precedence(): + i, j = symbols('i j', integer=True) + o, p = symbols('o p', integer=True) + n, m = symbols('n m', integer=True) + a = IndexedBase('a', shape=(o, p)) + assert a.shape == Tuple(o, p) + assert Indexed( + a, Idx(i, m), Idx(j, n)).ranges == [Tuple(0, m - 1), Tuple(0, n - 1)] + assert Indexed(a, Idx(i, m), Idx(j, n)).shape == Tuple(o, p) + assert Indexed( + a, Idx(i, m), Idx(j)).ranges == [Tuple(0, m - 1), (None, None)] + assert Indexed(a, Idx(i, m), Idx(j)).shape == Tuple(o, p) + + +def test_complex_indices(): + i, j = symbols('i j', integer=True) + A = Indexed('A', i, i + j) + assert A.rank == 2 + assert A.indices == (i, i + j) + + +def test_not_interable(): + i, j = symbols('i j', integer=True) + A = Indexed('A', i, i + j) + assert not iterable(A) + + +def test_Indexed_coeff(): + N = Symbol('N', integer=True) + len_y = N + i = Idx('i', len_y-1) + y = IndexedBase('y', shape=(len_y,)) + a = (1/y[i+1]*y[i]).coeff(y[i]) + b = (y[i]/y[i+1]).coeff(y[i]) + assert a == b + + +def test_differentiation(): + from sympy.functions.special.tensor_functions import KroneckerDelta + i, j, k, l = symbols('i j k l', cls=Idx) + a = symbols('a') + m, n = symbols("m, n", integer=True, finite=True) + assert m.is_real + h, L = symbols('h L', cls=IndexedBase) + hi, hj = h[i], h[j] + + expr = hi + assert expr.diff(hj) == KroneckerDelta(i, j) + assert expr.diff(hi) == KroneckerDelta(i, i) + + expr = S(2) * hi + assert expr.diff(hj) == S(2) * KroneckerDelta(i, j) + assert expr.diff(hi) == S(2) * KroneckerDelta(i, i) + assert expr.diff(a) is S.Zero + + assert Sum(expr, (i, -oo, oo)).diff(hj) == Sum(2*KroneckerDelta(i, j), (i, -oo, oo)) + assert Sum(expr.diff(hj), (i, -oo, oo)) == Sum(2*KroneckerDelta(i, j), (i, -oo, oo)) + assert Sum(expr, (i, -oo, oo)).diff(hj).doit() == 2 + + assert Sum(expr.diff(hi), (i, -oo, oo)).doit() == Sum(2, (i, -oo, oo)).doit() + assert Sum(expr, (i, -oo, oo)).diff(hi).doit() is oo + + expr = a * hj * hj / S(2) + assert expr.diff(hi) == a * h[j] * KroneckerDelta(i, j) + assert expr.diff(a) == hj * hj / S(2) + assert expr.diff(a, 2) is S.Zero + + assert Sum(expr, (i, -oo, oo)).diff(hi) == Sum(a*KroneckerDelta(i, j)*h[j], (i, -oo, oo)) + assert Sum(expr.diff(hi), (i, -oo, oo)) == Sum(a*KroneckerDelta(i, j)*h[j], (i, -oo, oo)) + assert Sum(expr, (i, -oo, oo)).diff(hi).doit() == a*h[j] + + assert Sum(expr, (j, -oo, oo)).diff(hi) == Sum(a*KroneckerDelta(i, j)*h[j], (j, -oo, oo)) + assert Sum(expr.diff(hi), (j, -oo, oo)) == Sum(a*KroneckerDelta(i, j)*h[j], (j, -oo, oo)) + assert Sum(expr, (j, -oo, oo)).diff(hi).doit() == a*h[i] + + expr = a * sin(hj * hj) + assert expr.diff(hi) == 2*a*cos(hj * hj) * hj * KroneckerDelta(i, j) + assert expr.diff(hj) == 2*a*cos(hj * hj) * hj + + expr = a * L[i, j] * h[j] + assert expr.diff(hi) == a*L[i, j]*KroneckerDelta(i, j) + assert expr.diff(hj) == a*L[i, j] + assert expr.diff(L[i, j]) == a*h[j] + assert expr.diff(L[k, l]) == a*KroneckerDelta(i, k)*KroneckerDelta(j, l)*h[j] + assert expr.diff(L[i, l]) == a*KroneckerDelta(j, l)*h[j] + + assert Sum(expr, (j, -oo, oo)).diff(L[k, l]) == Sum(a * KroneckerDelta(i, k) * KroneckerDelta(j, l) * h[j], (j, -oo, oo)) + assert Sum(expr, (j, -oo, oo)).diff(L[k, l]).doit() == a * KroneckerDelta(i, k) * h[l] + + assert h[m].diff(h[m]) == 1 + assert h[m].diff(h[n]) == KroneckerDelta(m, n) + assert Sum(a*h[m], (m, -oo, oo)).diff(h[n]) == Sum(a*KroneckerDelta(m, n), (m, -oo, oo)) + assert Sum(a*h[m], (m, -oo, oo)).diff(h[n]).doit() == a + assert Sum(a*h[m], (n, -oo, oo)).diff(h[n]) == Sum(a*KroneckerDelta(m, n), (n, -oo, oo)) + assert Sum(a*h[m], (m, -oo, oo)).diff(h[m]).doit() == oo*a + + +def test_indexed_series(): + A = IndexedBase("A") + i = symbols("i", integer=True) + assert sin(A[i]).series(A[i]) == A[i] - A[i]**3/6 + A[i]**5/120 + Order(A[i]**6, A[i]) + + +def test_indexed_is_constant(): + A = IndexedBase("A") + i, j, k = symbols("i,j,k") + assert not A[i].is_constant() + assert A[i].is_constant(j) + assert not A[1+2*i, k].is_constant() + assert not A[1+2*i, k].is_constant(i) + assert A[1+2*i, k].is_constant(j) + assert not A[1+2*i, k].is_constant(k) + + +def test_issue_12533(): + d = IndexedBase('d') + assert IndexedBase(range(5)) == Range(0, 5, 1) + assert d[0].subs(Symbol("d"), range(5)) == 0 + assert d[0].subs(d, range(5)) == 0 + assert d[1].subs(d, range(5)) == 1 + assert Indexed(Range(5), 2) == 2 + + +def test_issue_12780(): + n = symbols("n") + i = Idx("i", (0, n)) + raises(TypeError, lambda: i.subs(n, 1.5)) + + +def test_issue_18604(): + m = symbols("m") + assert Idx("i", m).name == 'i' + assert Idx("i", m).lower == 0 + assert Idx("i", m).upper == m - 1 + m = symbols("m", real=False) + raises(TypeError, lambda: Idx("i", m)) + +def test_Subs_with_Indexed(): + A = IndexedBase("A") + i, j, k = symbols("i,j,k") + x, y, z = symbols("x,y,z") + f = Function("f") + + assert Subs(A[i], A[i], A[j]).diff(A[j]) == 1 + assert Subs(A[i], A[i], x).diff(A[i]) == 0 + assert Subs(A[i], A[i], x).diff(A[j]) == 0 + assert Subs(A[i], A[i], x).diff(x) == 1 + assert Subs(A[i], A[i], x).diff(y) == 0 + assert Subs(A[i], A[i], A[j]).diff(A[k]) == KroneckerDelta(j, k) + assert Subs(x, x, A[i]).diff(A[j]) == KroneckerDelta(i, j) + assert Subs(f(A[i]), A[i], x).diff(A[j]) == 0 + assert Subs(f(A[i]), A[i], A[k]).diff(A[j]) == Derivative(f(A[k]), A[k])*KroneckerDelta(j, k) + assert Subs(x, x, A[i]**2).diff(A[j]) == 2*KroneckerDelta(i, j)*A[i] + assert Subs(A[i], A[i], A[j]**2).diff(A[k]) == 2*KroneckerDelta(j, k)*A[j] + + assert Subs(A[i]*x, x, A[i]).diff(A[i]) == 2*A[i] + assert Subs(A[i]*x, x, A[i]).diff(A[j]) == 2*A[i]*KroneckerDelta(i, j) + assert Subs(A[i]*x, x, A[j]).diff(A[i]) == A[j] + A[i]*KroneckerDelta(i, j) + assert Subs(A[i]*x, x, A[j]).diff(A[j]) == A[i] + A[j]*KroneckerDelta(i, j) + assert Subs(A[i]*x, x, A[i]).diff(A[k]) == 2*A[i]*KroneckerDelta(i, k) + assert Subs(A[i]*x, x, A[j]).diff(A[k]) == KroneckerDelta(i, k)*A[j] + KroneckerDelta(j, k)*A[i] + + assert Subs(A[i]*x, A[i], x).diff(A[i]) == 0 + assert Subs(A[i]*x, A[i], x).diff(A[j]) == 0 + assert Subs(A[i]*x, A[j], x).diff(A[i]) == x + assert Subs(A[i]*x, A[j], x).diff(A[j]) == x*KroneckerDelta(i, j) + assert Subs(A[i]*x, A[i], x).diff(A[k]) == 0 + assert Subs(A[i]*x, A[j], x).diff(A[k]) == x*KroneckerDelta(i, k) + + +def test_complicated_derivative_with_Indexed(): + x, y = symbols("x,y", cls=IndexedBase) + sigma = symbols("sigma") + i, j, k = symbols("i,j,k") + m0,m1,m2,m3,m4,m5 = symbols("m0:6") + f = Function("f") + + expr = f((x[i] - y[i])**2/sigma) + _xi_1 = symbols("xi_1", cls=Dummy) + assert expr.diff(x[m0]).dummy_eq( + (x[i] - y[i])*KroneckerDelta(i, m0)*\ + 2*Subs( + Derivative(f(_xi_1), _xi_1), + (_xi_1,), + ((x[i] - y[i])**2/sigma,) + )/sigma + ) + assert expr.diff(x[m0]).diff(x[m1]).dummy_eq( + 2*KroneckerDelta(i, m0)*\ + KroneckerDelta(i, m1)*Subs( + Derivative(f(_xi_1), _xi_1), + (_xi_1,), + ((x[i] - y[i])**2/sigma,) + )/sigma + \ + 4*(x[i] - y[i])**2*KroneckerDelta(i, m0)*KroneckerDelta(i, m1)*\ + Subs( + Derivative(f(_xi_1), _xi_1, _xi_1), + (_xi_1,), + ((x[i] - y[i])**2/sigma,) + )/sigma**2 + ) + + +def test_IndexedBase_commutative(): + t = IndexedBase('t', commutative=False) + u = IndexedBase('u', commutative=False) + v = IndexedBase('v') + assert t[0]*v[0] == v[0]*t[0] + assert t[0]*u[0] != u[0]*t[0] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_printing.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_printing.py new file mode 100644 index 0000000000000000000000000000000000000000..9f3cf7f0591a7012c93354ab7b8d7e010def38bb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_printing.py @@ -0,0 +1,13 @@ +from sympy.tensor.tensor import TensorIndexType, tensor_indices, TensorHead +from sympy import I + +def test_printing_TensMul(): + R3 = TensorIndexType('R3', dim=3) + p, q = tensor_indices("p q", R3) + K = TensorHead("K", [R3]) + + assert repr(2*K(p)) == "2*K(p)" + assert repr(-K(p)) == "-K(p)" + assert repr(-2*K(p)*K(q)) == "-2*K(p)*K(q)" + assert repr(-I*K(p)) == "-I*K(p)" + assert repr(I*K(p)) == "I*K(p)" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_tensor.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..3113f5be9bcd32224f3525b5d831b6d7476c39e3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/tensor/tests/test_tensor.py @@ -0,0 +1,2218 @@ +from sympy.concrete.summations import Sum +from sympy.core.function import expand +from sympy.core.numbers import Integer +from sympy.matrices.dense import (Matrix, eye) +from sympy.tensor.indexed import Indexed +from sympy.combinatorics import Permutation +from sympy.core import S, Rational, Symbol, Basic, Add, Wild, Function +from sympy.core.containers import Tuple +from sympy.core.symbol import symbols +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.integrals import integrate +from sympy.tensor.array import Array +from sympy.tensor.tensor import TensorIndexType, tensor_indices, TensorSymmetry, \ + get_symmetric_group_sgs, TensorIndex, tensor_mul, TensAdd, \ + riemann_cyclic_replace, riemann_cyclic, TensMul, tensor_heads, \ + TensorManager, TensExpr, TensorHead, canon_bp, \ + tensorhead, tensorsymmetry, TensorType, substitute_indices, \ + WildTensorIndex, WildTensorHead, _WildTensExpr +from sympy.testing.pytest import raises, XFAIL, warns_deprecated_sympy +from sympy.matrices import diag + +def _is_equal(arg1, arg2): + if isinstance(arg1, TensExpr): + return arg1.equals(arg2) + elif isinstance(arg2, TensExpr): + return arg2.equals(arg1) + return arg1 == arg2 + + +#################### Tests from tensor_can.py ####################### +def test_canonicalize_no_slot_sym(): + # A_d0 * B^d0; T_c = A^d0*B_d0 + Lorentz = TensorIndexType('Lorentz', dummy_name='L') + a, b, d0, d1 = tensor_indices('a,b,d0,d1', Lorentz) + A, B = tensor_heads('A,B', [Lorentz], TensorSymmetry.no_symmetry(1)) + t = A(-d0)*B(d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0)*B(-L_0)' + + # A^a * B^b; T_c = T + t = A(a)*B(b) + tc = t.canon_bp() + assert tc == t + # B^b * A^a + t1 = B(b)*A(a) + tc = t1.canon_bp() + assert str(tc) == 'A(a)*B(b)' + + # A symmetric + # A^{b}_{d0}*A^{d0, a}; T_c = A^{a d0}*A{b}_{d0} + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(b, -d0)*A(d0, a) + tc = t.canon_bp() + assert str(tc) == 'A(a, L_0)*A(b, -L_0)' + + # A^{d1}_{d0}*B^d0*C_d1 + # T_c = A^{d0 d1}*B_d0*C_d1 + B, C = tensor_heads('B,C', [Lorentz], TensorSymmetry.no_symmetry(1)) + t = A(d1, -d0)*B(d0)*C(-d1) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-L_0)*C(-L_1)' + + # A without symmetry + # A^{d1}_{d0}*B^d0*C_d1 ord=[d0,-d0,d1,-d1]; g = [2,1,0,3,4,5] + # T_c = A^{d0 d1}*B_d1*C_d0; can = [0,2,3,1,4,5] + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.no_symmetry(2)) + t = A(d1, -d0)*B(d0)*C(-d1) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-L_1)*C(-L_0)' + + # A, B without symmetry + # A^{d1}_{d0}*B_{d1}^{d0} + # T_c = A^{d0 d1}*B_{d0 d1} + B = TensorHead('B', [Lorentz]*2, TensorSymmetry.no_symmetry(2)) + t = A(d1, -d0)*B(-d1, d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-L_0, -L_1)' + # A_{d0}^{d1}*B_{d1}^{d0} + # T_c = A^{d0 d1}*B_{d1 d0} + t = A(-d0, d1)*B(-d1, d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-L_1, -L_0)' + + # A, B, C without symmetry + # A^{d1 d0}*B_{a d0}*C_{d1 b} + # T_c=A^{d0 d1}*B_{a d1}*C_{d0 b} + C = TensorHead('C', [Lorentz]*2, TensorSymmetry.no_symmetry(2)) + t = A(d1, d0)*B(-a, -d0)*C(-d1, -b) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-a, -L_1)*C(-L_0, -b)' + + # A symmetric, B and C without symmetry + # A^{d1 d0}*B_{a d0}*C_{d1 b} + # T_c = A^{d0 d1}*B_{a d0}*C_{d1 b} + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(d1, d0)*B(-a, -d0)*C(-d1, -b) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-a, -L_0)*C(-L_1, -b)' + + # A and C symmetric, B without symmetry + # A^{d1 d0}*B_{a d0}*C_{d1 b} ord=[a,b,d0,-d0,d1,-d1] + # T_c = A^{d0 d1}*B_{a d0}*C_{b d1} + C = TensorHead('C', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(d1, d0)*B(-a, -d0)*C(-d1, -b) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1)*B(-a, -L_0)*C(-b, -L_1)' + +def test_canonicalize_no_dummies(): + Lorentz = TensorIndexType('Lorentz', dummy_name='L') + a, b, c, d = tensor_indices('a, b, c, d', Lorentz) + + # A commuting + # A^c A^b A^a + # T_c = A^a A^b A^c + A = TensorHead('A', [Lorentz], TensorSymmetry.no_symmetry(1)) + t = A(c)*A(b)*A(a) + tc = t.canon_bp() + assert str(tc) == 'A(a)*A(b)*A(c)' + + # A anticommuting + # A^c A^b A^a + # T_c = -A^a A^b A^c + A = TensorHead('A', [Lorentz], TensorSymmetry.no_symmetry(1), 1) + t = A(c)*A(b)*A(a) + tc = t.canon_bp() + assert str(tc) == '-A(a)*A(b)*A(c)' + + # A commuting and symmetric + # A^{b,d}*A^{c,a} + # T_c = A^{a c}*A^{b d} + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(b, d)*A(c, a) + tc = t.canon_bp() + assert str(tc) == 'A(a, c)*A(b, d)' + + # A anticommuting and symmetric + # A^{b,d}*A^{c,a} + # T_c = -A^{a c}*A^{b d} + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.fully_symmetric(2), 1) + t = A(b, d)*A(c, a) + tc = t.canon_bp() + assert str(tc) == '-A(a, c)*A(b, d)' + + # A^{c,a}*A^{b,d} + # T_c = A^{a c}*A^{b d} + t = A(c, a)*A(b, d) + tc = t.canon_bp() + assert str(tc) == 'A(a, c)*A(b, d)' + +def test_tensorhead_construction_without_symmetry(): + L = TensorIndexType('Lorentz') + A1 = TensorHead('A', [L, L]) + A2 = TensorHead('A', [L, L], TensorSymmetry.no_symmetry(2)) + assert A1 == A2 + A3 = TensorHead('A', [L, L], TensorSymmetry.fully_symmetric(2)) # Symmetric + assert A1 != A3 + +def test_no_metric_symmetry(): + # no metric symmetry; A no symmetry + # A^d1_d0 * A^d0_d1 + # T_c = A^d0_d1 * A^d1_d0 + Lorentz = TensorIndexType('Lorentz', dummy_name='L', metric_symmetry=0) + d0, d1, d2, d3 = tensor_indices('d:4', Lorentz) + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.no_symmetry(2)) + t = A(d1, -d0)*A(d0, -d1) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, -L_1)*A(L_1, -L_0)' + + # A^d1_d2 * A^d0_d3 * A^d2_d1 * A^d3_d0 + # T_c = A^d0_d1 * A^d1_d0 * A^d2_d3 * A^d3_d2 + t = A(d1, -d2)*A(d0, -d3)*A(d2, -d1)*A(d3, -d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, -L_1)*A(L_1, -L_0)*A(L_2, -L_3)*A(L_3, -L_2)' + + # A^d0_d2 * A^d1_d3 * A^d3_d0 * A^d2_d1 + # T_c = A^d0_d1 * A^d1_d2 * A^d2_d3 * A^d3_d0 + t = A(d0, -d1)*A(d1, -d2)*A(d2, -d3)*A(d3, -d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, -L_1)*A(L_1, -L_2)*A(L_2, -L_3)*A(L_3, -L_0)' + +def test_canonicalize1(): + Lorentz = TensorIndexType('Lorentz', dummy_name='L') + a, a0, a1, a2, a3, b, d0, d1, d2, d3 = \ + tensor_indices('a,a0,a1,a2,a3,b,d0,d1,d2,d3', Lorentz) + + # A_d0*A^d0; ord = [d0,-d0] + # T_c = A^d0*A_d0 + A = TensorHead('A', [Lorentz], TensorSymmetry.no_symmetry(1)) + t = A(-d0)*A(d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0)*A(-L_0)' + + # A commuting + # A_d0*A_d1*A_d2*A^d2*A^d1*A^d0 + # T_c = A^d0*A_d0*A^d1*A_d1*A^d2*A_d2 + t = A(-d0)*A(-d1)*A(-d2)*A(d2)*A(d1)*A(d0) + tc = t.canon_bp() + assert str(tc) == 'A(L_0)*A(-L_0)*A(L_1)*A(-L_1)*A(L_2)*A(-L_2)' + + # A anticommuting + # A_d0*A_d1*A_d2*A^d2*A^d1*A^d0 + # T_c 0 + A = TensorHead('A', [Lorentz], TensorSymmetry.no_symmetry(1), 1) + t = A(-d0)*A(-d1)*A(-d2)*A(d2)*A(d1)*A(d0) + tc = t.canon_bp() + assert tc == 0 + + # A commuting symmetric + # A^{d0 b}*A^a_d1*A^d1_d0 + # T_c = A^{a d0}*A^{b d1}*A_{d0 d1} + A = TensorHead('A', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(d0, b)*A(a, -d1)*A(d1, -d0) + tc = t.canon_bp() + assert str(tc) == 'A(a, L_0)*A(b, L_1)*A(-L_0, -L_1)' + + # A, B commuting symmetric + # A^{d0 b}*A^d1_d0*B^a_d1 + # T_c = A^{b d0}*A_d0^d1*B^a_d1 + B = TensorHead('B', [Lorentz]*2, TensorSymmetry.fully_symmetric(2)) + t = A(d0, b)*A(d1, -d0)*B(a, -d1) + tc = t.canon_bp() + assert str(tc) == 'A(b, L_0)*A(-L_0, L_1)*B(a, -L_1)' + + # A commuting symmetric + # A^{d1 d0 b}*A^{a}_{d1 d0}; ord=[a,b, d0,-d0,d1,-d1] + # T_c = A^{a d0 d1}*A^{b}_{d0 d1} + A = TensorHead('A', [Lorentz]*3, TensorSymmetry.fully_symmetric(3)) + t = A(d1, d0, b)*A(a, -d1, -d0) + tc = t.canon_bp() + assert str(tc) == 'A(a, L_0, L_1)*A(b, -L_0, -L_1)' + + # A^{d3 d0 d2}*A^a0_{d1 d2}*A^d1_d3^a1*A^{a2 a3}_d0 + # T_c = A^{a0 d0 d1}*A^a1_d0^d2*A^{a2 a3 d3}*A_{d1 d2 d3} + t = A(d3, d0, d2)*A(a0, -d1, -d2)*A(d1, -d3, a1)*A(a2, a3, -d0) + tc = t.canon_bp() + assert str(tc) == 'A(a0, L_0, L_1)*A(a1, -L_0, L_2)*A(a2, a3, L_3)*A(-L_1, -L_2, -L_3)' + + # A commuting symmetric, B antisymmetric + # A^{d0 d1 d2} * A_{d2 d3 d1} * B_d0^d3 + # in this esxample and in the next three, + # renaming dummy indices and using symmetry of A, + # T = A^{d0 d1 d2} * A_{d0 d1 d3} * B_d2^d3 + # can = 0 + A = TensorHead('A', [Lorentz]*3, TensorSymmetry.fully_symmetric(3)) + B = TensorHead('B', [Lorentz]*2, TensorSymmetry.fully_symmetric(-2)) + t = A(d0, d1, d2)*A(-d2, -d3, -d1)*B(-d0, d3) + tc = t.canon_bp() + assert tc == 0 + + # A anticommuting symmetric, B antisymmetric + # A^{d0 d1 d2} * A_{d2 d3 d1} * B_d0^d3 + # T_c = A^{d0 d1 d2} * A_{d0 d1}^d3 * B_{d2 d3} + A = TensorHead('A', [Lorentz]*3, TensorSymmetry.fully_symmetric(3), 1) + B = TensorHead('B', [Lorentz]*2, TensorSymmetry.fully_symmetric(-2)) + t = A(d0, d1, d2)*A(-d2, -d3, -d1)*B(-d0, d3) + tc = t.canon_bp() + assert str(tc) == 'A(L_0, L_1, L_2)*A(-L_0, -L_1, L_3)*B(-L_2, -L_3)' + + # A anticommuting symmetric, B antisymmetric commuting, antisymmetric metric + # A^{d0 d1 d2} * A_{d2 d3 d1} * B_d0^d3 + # T_c = -A^{d0 d1 d2} * A_{d0 d1}^d3 * B_{d2 d3} + Spinor = TensorIndexType('Spinor', dummy_name='S', metric_symmetry=-1) + a, a0, a1, a2, a3, b, d0, d1, d2, d3 = \ + tensor_indices('a,a0,a1,a2,a3,b,d0,d1,d2,d3', Spinor) + A = TensorHead('A', [Spinor]*3, TensorSymmetry.fully_symmetric(3), 1) + B = TensorHead('B', [Spinor]*2, TensorSymmetry.fully_symmetric(-2)) + t = A(d0, d1, d2)*A(-d2, -d3, -d1)*B(-d0, d3) + tc = t.canon_bp() + assert str(tc) == '-A(S_0, S_1, S_2)*A(-S_0, -S_1, S_3)*B(-S_2, -S_3)' + + # A anticommuting symmetric, B antisymmetric anticommuting, + # no metric symmetry + # A^{d0 d1 d2} * A_{d2 d3 d1} * B_d0^d3 + # T_c = A^{d0 d1 d2} * A_{d0 d1 d3} * B_d2^d3 + Mat = TensorIndexType('Mat', metric_symmetry=0, dummy_name='M') + a, a0, a1, a2, a3, b, d0, d1, d2, d3 = \ + tensor_indices('a,a0,a1,a2,a3,b,d0,d1,d2,d3', Mat) + A = TensorHead('A', [Mat]*3, TensorSymmetry.fully_symmetric(3), 1) + B = TensorHead('B', [Mat]*2, TensorSymmetry.fully_symmetric(-2)) + t = A(d0, d1, d2)*A(-d2, -d3, -d1)*B(-d0, d3) + tc = t.canon_bp() + assert str(tc) == 'A(M_0, M_1, M_2)*A(-M_0, -M_1, -M_3)*B(-M_2, M_3)' + + # Gamma anticommuting + # Gamma_{mu nu} * gamma^rho * Gamma^{nu mu alpha} + # T_c = -Gamma^{mu nu} * gamma^rho * Gamma_{alpha mu nu} + alpha, beta, gamma, mu, nu, rho = \ + tensor_indices('alpha,beta,gamma,mu,nu,rho', Lorentz) + Gamma = TensorHead('Gamma', [Lorentz], + TensorSymmetry.fully_symmetric(1), 2) + Gamma2 = TensorHead('Gamma', [Lorentz]*2, + TensorSymmetry.fully_symmetric(-2), 2) + Gamma3 = TensorHead('Gamma', [Lorentz]*3, + TensorSymmetry.fully_symmetric(-3), 2) + t = Gamma2(-mu, -nu)*Gamma(rho)*Gamma3(nu, mu, alpha) + tc = t.canon_bp() + assert str(tc) == '-Gamma(L_0, L_1)*Gamma(rho)*Gamma(alpha, -L_0, -L_1)' + + # Gamma_{mu nu} * Gamma^{gamma beta} * gamma_rho * Gamma^{nu mu alpha} + # T_c = Gamma^{mu nu} * Gamma^{beta gamma} * gamma_rho * Gamma^alpha_{mu nu} + t = Gamma2(mu, nu)*Gamma2(beta, gamma)*Gamma(-rho)*Gamma3(alpha, -mu, -nu) + tc = t.canon_bp() + assert str(tc) == 'Gamma(L_0, L_1)*Gamma(beta, gamma)*Gamma(-rho)*Gamma(alpha, -L_0, -L_1)' + + # f^a_{b,c} antisymmetric in b,c; A_mu^a no symmetry + # f^c_{d a} * f_{c e b} * A_mu^d * A_nu^a * A^{nu e} * A^{mu b} + # g = [8,11,5, 9,13,7, 1,10, 3,4, 2,12, 0,6, 14,15] + # T_c = -f^{a b c} * f_a^{d e} * A^mu_b * A_{mu d} * A^nu_c * A_{nu e} + Flavor = TensorIndexType('Flavor', dummy_name='F') + a, b, c, d, e, ff = tensor_indices('a,b,c,d,e,f', Flavor) + mu, nu = tensor_indices('mu,nu', Lorentz) + f = TensorHead('f', [Flavor]*3, TensorSymmetry.direct_product(1, -2)) + A = TensorHead('A', [Lorentz, Flavor], TensorSymmetry.no_symmetry(2)) + t = f(c, -d, -a)*f(-c, -e, -b)*A(-mu, d)*A(-nu, a)*A(nu, e)*A(mu, b) + tc = t.canon_bp() + assert str(tc) == '-f(F_0, F_1, F_2)*f(-F_0, F_3, F_4)*A(L_0, -F_1)*A(-L_0, -F_3)*A(L_1, -F_2)*A(-L_1, -F_4)' + + +def test_bug_correction_tensor_indices(): + # to make sure that tensor_indices does not return a list if creating + # only one index: + A = TensorIndexType("A") + i = tensor_indices('i', A) + assert not isinstance(i, (tuple, list)) + assert isinstance(i, TensorIndex) + + +def test_riemann_invariants(): + Lorentz = TensorIndexType('Lorentz', dummy_name='L') + d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11 = \ + tensor_indices('d0:12', Lorentz) + # R^{d0 d1}_{d1 d0}; ord = [d0,-d0,d1,-d1] + # T_c = -R^{d0 d1}_{d0 d1} + R = TensorHead('R', [Lorentz]*4, TensorSymmetry.riemann()) + t = R(d0, d1, -d1, -d0) + tc = t.canon_bp() + assert str(tc) == '-R(L_0, L_1, -L_0, -L_1)' + + # R_d11^d1_d0^d5 * R^{d6 d4 d0}_d5 * R_{d7 d2 d8 d9} * + # R_{d10 d3 d6 d4} * R^{d2 d7 d11}_d1 * R^{d8 d9 d3 d10} + # can = [0,2,4,6, 1,3,8,10, 5,7,12,14, 9,11,16,18, 13,15,20,22, + # 17,19,21 1: + option_parser.error('--problems must not be given more than once.') + if options.origins > 1: + option_parser.error('--origins must not be given more than once.') + if options.by_importer > 1: + option_parser.error('--by-importer must not be given more than once.') + if options.by_origin > 1: + option_parser.error('--by-origin must not be given more than once.') + options.problems = options.problems == 1 + options.origins = options.origins == 1 + options.by_importer = options.by_importer == 1 + options.by_origin = options.by_origin == 1 + if not options.problems and not options.origins: + option_parser.error( + 'At least one of --problems and --origins is required') + if options.problems and options.origins: + option_parser.error( + 'At most one of --problems and --origins is allowed') + if options.by_importer and options.by_origin: + option_parser.error( + 'At most one of --by-importer and --by-origin is allowed') + options.by_process = not options.by_importer and not options.by_origin + + builtin_import = builtins.__import__ + + class Definition: + """Information about a symbol's definition.""" + def __init__(self, name, value, definer): + self.name = name + self.value = value + self.definer = definer + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + return self.name == other.name and self.value == other.value + def __ne__(self, other): + return not (self == other) + def __repr__(self): + return 'Definition(%s, ..., %s)' % ( + repr(self.name), repr(self.definer)) + + # Maps each function/variable to name of module to define it + symbol_definers: dict[Definition, str] = {} + + def in_module(a, b): + """Is a the same module as or a submodule of b?""" + return a == b or a != None and b != None and a.startswith(b + '.') + + def relevant(module): + """Is module relevant for import checking? + + Only imports between relevant modules will be checked.""" + return in_module(module, 'sympy') + + sorted_messages = [] + + def msg(msg, *args): + if options.by_process: + print(msg % args) + else: + sorted_messages.append(msg % args) + + def tracking_import(module, globals=globals(), locals=[], fromlist=None, level=-1): + """__import__ wrapper - does not change imports at all, but tracks them. + + Default order is implemented by doing output directly. + All other orders are implemented by collecting output information into + a sorted list that will be emitted after all imports are processed. + + Indirect imports can only occur after the requested symbol has been + imported directly (because the indirect import would not have a module + to pick the symbol up from). + So this code detects indirect imports by checking whether the symbol in + question was already imported. + + Keeps the semantics of __import__ unchanged.""" + caller_frame = inspect.getframeinfo(sys._getframe(1)) + importer_filename = caller_frame.filename + importer_module = globals['__name__'] + if importer_filename == caller_frame.filename: + importer_reference = '%s line %s' % ( + importer_filename, str(caller_frame.lineno)) + else: + importer_reference = importer_filename + result = builtin_import(module, globals, locals, fromlist, level) + importee_module = result.__name__ + # We're only interested if importer and importee are in SymPy + if relevant(importer_module) and relevant(importee_module): + for symbol in result.__dict__.iterkeys(): + definition = Definition( + symbol, result.__dict__[symbol], importer_module) + if definition not in symbol_definers: + symbol_definers[definition] = importee_module + if hasattr(result, '__path__'): + ##PACKAGE## + # The existence of __path__ is documented in the tutorial on modules. + # Python 3.3 documents this in http://docs.python.org/3.3/reference/import.html + if options.by_origin: + msg('Error: %s (a package) is imported by %s', + module, importer_reference) + else: + msg('Error: %s contains package import %s', + importer_reference, module) + if fromlist != None: + symbol_list = fromlist + if '*' in symbol_list: + if (importer_filename.endswith(("__init__.py", "__init__.pyc", "__init__.pyo"))): + # We do not check starred imports inside __init__ + # That's the normal "please copy over its imports to my namespace" + symbol_list = [] + else: + symbol_list = result.__dict__.iterkeys() + for symbol in symbol_list: + if symbol not in result.__dict__: + if options.by_origin: + msg('Error: %s.%s is not defined (yet), but %s tries to import it', + importee_module, symbol, importer_reference) + else: + msg('Error: %s tries to import %s.%s, which did not define it (yet)', + importer_reference, importee_module, symbol) + else: + definition = Definition( + symbol, result.__dict__[symbol], importer_module) + symbol_definer = symbol_definers[definition] + if symbol_definer == importee_module: + ##DUPLICATE## + if options.by_origin: + msg('Error: %s.%s is imported again into %s', + importee_module, symbol, importer_reference) + else: + msg('Error: %s imports %s.%s again', + importer_reference, importee_module, symbol) + else: + ##ORIGIN## + if options.by_origin: + msg('Error: %s.%s is imported by %s, which should import %s.%s instead', + importee_module, symbol, importer_reference, symbol_definer, symbol) + else: + msg('Error: %s imports %s.%s but should import %s.%s instead', + importer_reference, importee_module, symbol, symbol_definer, symbol) + return result + + builtins.__import__ = tracking_import + __import__('sympy') + + sorted_messages.sort() + for message in sorted_messages: + print(message) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_code_quality.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_code_quality.py new file mode 100644 index 0000000000000000000000000000000000000000..9a9f363f0b9a802553f8643186b7d858d1ad0694 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_code_quality.py @@ -0,0 +1,510 @@ +# coding=utf-8 +from os import walk, sep, pardir +from os.path import split, join, abspath, exists, isfile +from glob import glob +import re +import random +import ast + +from sympy.testing.pytest import raises +from sympy.testing.quality_unicode import _test_this_file_encoding + +# System path separator (usually slash or backslash) to be +# used with excluded files, e.g. +# exclude = set([ +# "%(sep)smpmath%(sep)s" % sepd, +# ]) +sepd = {"sep": sep} + +# path and sympy_path +SYMPY_PATH = abspath(join(split(__file__)[0], pardir, pardir)) # go to sympy/ +assert exists(SYMPY_PATH) + +TOP_PATH = abspath(join(SYMPY_PATH, pardir)) +BIN_PATH = join(TOP_PATH, "bin") +EXAMPLES_PATH = join(TOP_PATH, "examples") + +# Error messages +message_space = "File contains trailing whitespace: %s, line %s." +message_implicit = "File contains an implicit import: %s, line %s." +message_tabs = "File contains tabs instead of spaces: %s, line %s." +message_carriage = "File contains carriage returns at end of line: %s, line %s" +message_str_raise = "File contains string exception: %s, line %s" +message_gen_raise = "File contains generic exception: %s, line %s" +message_old_raise = "File contains old-style raise statement: %s, line %s, \"%s\"" +message_eof = "File does not end with a newline: %s, line %s" +message_multi_eof = "File ends with more than 1 newline: %s, line %s" +message_test_suite_def = "Function should start with 'test_' or '_': %s, line %s" +message_duplicate_test = "This is a duplicate test function: %s, line %s" +message_self_assignments = "File contains assignments to self/cls: %s, line %s." +message_func_is = "File contains '.func is': %s, line %s." +message_bare_expr = "File contains bare expression: %s, line %s." + +implicit_test_re = re.compile(r'^\s*(>>> )?(\.\.\. )?from .* import .*\*') +str_raise_re = re.compile( + r'^\s*(>>> )?(\.\.\. )?raise(\s+(\'|\")|\s*(\(\s*)+(\'|\"))') +gen_raise_re = re.compile( + r'^\s*(>>> )?(\.\.\. )?raise(\s+Exception|\s*(\(\s*)+Exception)') +old_raise_re = re.compile(r'^\s*(>>> )?(\.\.\. )?raise((\s*\(\s*)|\s+)\w+\s*,') +test_suite_def_re = re.compile(r'^def\s+(?!(_|test))[^(]*\(\s*\)\s*:$') +test_ok_def_re = re.compile(r'^def\s+test_.*:$') +test_file_re = re.compile(r'.*[/\\]test_.*\.py$') +func_is_re = re.compile(r'\.\s*func\s+is') + + +def tab_in_leading(s): + """Returns True if there are tabs in the leading whitespace of a line, + including the whitespace of docstring code samples.""" + n = len(s) - len(s.lstrip()) + if not s[n:n + 3] in ['...', '>>>']: + check = s[:n] + else: + smore = s[n + 3:] + check = s[:n] + smore[:len(smore) - len(smore.lstrip())] + return not (check.expandtabs() == check) + + +def find_self_assignments(s): + """Returns a list of "bad" assignments: if there are instances + of assigning to the first argument of the class method (except + for staticmethod's). + """ + t = [n for n in ast.parse(s).body if isinstance(n, ast.ClassDef)] + + bad = [] + for c in t: + for n in c.body: + if not isinstance(n, ast.FunctionDef): + continue + if any(d.id == 'staticmethod' + for d in n.decorator_list if isinstance(d, ast.Name)): + continue + if n.name == '__new__': + continue + if not n.args.args: + continue + first_arg = n.args.args[0].arg + + for m in ast.walk(n): + if isinstance(m, ast.Assign): + for a in m.targets: + if isinstance(a, ast.Name) and a.id == first_arg: + bad.append(m) + elif (isinstance(a, ast.Tuple) and + any(q.id == first_arg for q in a.elts + if isinstance(q, ast.Name))): + bad.append(m) + + return bad + + +def check_directory_tree(base_path, file_check, exclusions=set(), pattern="*.py"): + """ + Checks all files in the directory tree (with base_path as starting point) + with the file_check function provided, skipping files that contain + any of the strings in the set provided by exclusions. + """ + if not base_path: + return + for root, dirs, files in walk(base_path): + check_files(glob(join(root, pattern)), file_check, exclusions) + + +def check_files(files, file_check, exclusions=set(), pattern=None): + """ + Checks all files with the file_check function provided, skipping files + that contain any of the strings in the set provided by exclusions. + """ + if not files: + return + for fname in files: + if not exists(fname) or not isfile(fname): + continue + if any(ex in fname for ex in exclusions): + continue + if pattern is None or re.match(pattern, fname): + file_check(fname) + + +class _Visit(ast.NodeVisitor): + """return the line number corresponding to the + line on which a bare expression appears if it is a binary op + or a comparison that is not in a with block. + + EXAMPLES + ======== + + >>> import ast + >>> class _Visit(ast.NodeVisitor): + ... def visit_Expr(self, node): + ... if isinstance(node.value, (ast.BinOp, ast.Compare)): + ... print(node.lineno) + ... def visit_With(self, node): + ... pass # no checking there + ... + >>> code='''x = 1 # line 1 + ... for i in range(3): + ... x == 2 # <-- 3 + ... if x == 2: + ... x == 3 # <-- 5 + ... x + 1 # <-- 6 + ... x = 1 + ... if x == 1: + ... print(1) + ... while x != 1: + ... x == 1 # <-- 11 + ... with raises(TypeError): + ... c == 1 + ... raise TypeError + ... assert x == 1 + ... ''' + >>> _Visit().visit(ast.parse(code)) + 3 + 5 + 6 + 11 + """ + def visit_Expr(self, node): + if isinstance(node.value, (ast.BinOp, ast.Compare)): + assert None, message_bare_expr % ('', node.lineno) + def visit_With(self, node): + pass + + +BareExpr = _Visit() + + +def line_with_bare_expr(code): + """return None or else 0-based line number of code on which + a bare expression appeared. + """ + tree = ast.parse(code) + try: + BareExpr.visit(tree) + except AssertionError as msg: + assert msg.args + msg = msg.args[0] + assert msg.startswith(message_bare_expr.split(':', 1)[0]) + return int(msg.rsplit(' ', 1)[1].rstrip('.')) # the line number + + +def test_files(): + """ + This test tests all files in SymPy and checks that: + o no lines contains a trailing whitespace + o no lines end with \r\n + o no line uses tabs instead of spaces + o that the file ends with a single newline + o there are no general or string exceptions + o there are no old style raise statements + o name of arg-less test suite functions start with _ or test_ + o no duplicate function names that start with test_ + o no assignments to self variable in class methods + o no lines contain ".func is" except in the test suite + o there is no do-nothing expression like `a == b` or `x + 1` + """ + + def test(fname): + with open(fname, encoding="utf8") as test_file: + test_this_file(fname, test_file) + with open(fname, encoding='utf8') as test_file: + _test_this_file_encoding(fname, test_file) + + def test_this_file(fname, test_file): + idx = None + code = test_file.read() + test_file.seek(0) # restore reader to head + py = fname if sep not in fname else fname.rsplit(sep, 1)[-1] + if py.startswith('test_'): + idx = line_with_bare_expr(code) + if idx is not None: + assert False, message_bare_expr % (fname, idx + 1) + + line = None # to flag the case where there were no lines in file + tests = 0 + test_set = set() + for idx, line in enumerate(test_file): + if test_file_re.match(fname): + if test_suite_def_re.match(line): + assert False, message_test_suite_def % (fname, idx + 1) + if test_ok_def_re.match(line): + tests += 1 + test_set.add(line[3:].split('(')[0].strip()) + if len(test_set) != tests: + assert False, message_duplicate_test % (fname, idx + 1) + if line.endswith((" \n", "\t\n")): + assert False, message_space % (fname, idx + 1) + if line.endswith("\r\n"): + assert False, message_carriage % (fname, idx + 1) + if tab_in_leading(line): + assert False, message_tabs % (fname, idx + 1) + if str_raise_re.search(line): + assert False, message_str_raise % (fname, idx + 1) + if gen_raise_re.search(line): + assert False, message_gen_raise % (fname, idx + 1) + if (implicit_test_re.search(line) and + not list(filter(lambda ex: ex in fname, import_exclude))): + assert False, message_implicit % (fname, idx + 1) + if func_is_re.search(line) and not test_file_re.search(fname): + assert False, message_func_is % (fname, idx + 1) + + result = old_raise_re.search(line) + + if result is not None: + assert False, message_old_raise % ( + fname, idx + 1, result.group(2)) + + if line is not None: + if line == '\n' and idx > 0: + assert False, message_multi_eof % (fname, idx + 1) + elif not line.endswith('\n'): + # eof newline check + assert False, message_eof % (fname, idx + 1) + + + # Files to test at top level + top_level_files = [join(TOP_PATH, file) for file in [ + "isympy.py", + "build.py", + "setup.py", + ]] + # Files to exclude from all tests + exclude = { + "%(sep)ssympy%(sep)sparsing%(sep)sautolev%(sep)s_antlr%(sep)sautolevparser.py" % sepd, + "%(sep)ssympy%(sep)sparsing%(sep)sautolev%(sep)s_antlr%(sep)sautolevlexer.py" % sepd, + "%(sep)ssympy%(sep)sparsing%(sep)sautolev%(sep)s_antlr%(sep)sautolevlistener.py" % sepd, + "%(sep)ssympy%(sep)sparsing%(sep)slatex%(sep)s_antlr%(sep)slatexparser.py" % sepd, + "%(sep)ssympy%(sep)sparsing%(sep)slatex%(sep)s_antlr%(sep)slatexlexer.py" % sepd, + } + # Files to exclude from the implicit import test + import_exclude = { + # glob imports are allowed in top-level __init__.py: + "%(sep)ssympy%(sep)s__init__.py" % sepd, + # these __init__.py should be fixed: + # XXX: not really, they use useful import pattern (DRY) + "%(sep)svector%(sep)s__init__.py" % sepd, + "%(sep)smechanics%(sep)s__init__.py" % sepd, + "%(sep)squantum%(sep)s__init__.py" % sepd, + "%(sep)spolys%(sep)s__init__.py" % sepd, + "%(sep)spolys%(sep)sdomains%(sep)s__init__.py" % sepd, + # interactive SymPy executes ``from sympy import *``: + "%(sep)sinteractive%(sep)ssession.py" % sepd, + # isympy.py executes ``from sympy import *``: + "%(sep)sisympy.py" % sepd, + # these two are import timing tests: + "%(sep)sbin%(sep)ssympy_time.py" % sepd, + "%(sep)sbin%(sep)ssympy_time_cache.py" % sepd, + # Taken from Python stdlib: + "%(sep)sparsing%(sep)ssympy_tokenize.py" % sepd, + # this one should be fixed: + "%(sep)splotting%(sep)spygletplot%(sep)s" % sepd, + # False positive in the docstring + "%(sep)sbin%(sep)stest_external_imports.py" % sepd, + "%(sep)sbin%(sep)stest_submodule_imports.py" % sepd, + # These are deprecated stubs that can be removed at some point: + "%(sep)sutilities%(sep)sruntests.py" % sepd, + "%(sep)sutilities%(sep)spytest.py" % sepd, + "%(sep)sutilities%(sep)srandtest.py" % sepd, + "%(sep)sutilities%(sep)stmpfiles.py" % sepd, + "%(sep)sutilities%(sep)squality_unicode.py" % sepd, + } + check_files(top_level_files, test) + check_directory_tree(BIN_PATH, test, {"~", ".pyc", ".sh"}, "*") + check_directory_tree(SYMPY_PATH, test, exclude) + check_directory_tree(EXAMPLES_PATH, test, exclude) + + +def _with_space(c): + # return c with a random amount of leading space + return random.randint(0, 10)*' ' + c + + +def test_raise_statement_regular_expression(): + candidates_ok = [ + "some text # raise Exception, 'text'", + "raise ValueError('text') # raise Exception, 'text'", + "raise ValueError('text')", + "raise ValueError", + "raise ValueError('text')", + "raise ValueError('text') #,", + # Talking about an exception in a docstring + ''''"""This function will raise ValueError, except when it doesn't"""''', + "raise (ValueError('text')", + ] + str_candidates_fail = [ + "raise 'exception'", + "raise 'Exception'", + 'raise "exception"', + 'raise "Exception"', + "raise 'ValueError'", + ] + gen_candidates_fail = [ + "raise Exception('text') # raise Exception, 'text'", + "raise Exception('text')", + "raise Exception", + "raise Exception('text')", + "raise Exception('text') #,", + "raise Exception, 'text'", + "raise Exception, 'text' # raise Exception('text')", + "raise Exception, 'text' # raise Exception, 'text'", + ">>> raise Exception, 'text'", + ">>> raise Exception, 'text' # raise Exception('text')", + ">>> raise Exception, 'text' # raise Exception, 'text'", + ] + old_candidates_fail = [ + "raise Exception, 'text'", + "raise Exception, 'text' # raise Exception('text')", + "raise Exception, 'text' # raise Exception, 'text'", + ">>> raise Exception, 'text'", + ">>> raise Exception, 'text' # raise Exception('text')", + ">>> raise Exception, 'text' # raise Exception, 'text'", + "raise ValueError, 'text'", + "raise ValueError, 'text' # raise Exception('text')", + "raise ValueError, 'text' # raise Exception, 'text'", + ">>> raise ValueError, 'text'", + ">>> raise ValueError, 'text' # raise Exception('text')", + ">>> raise ValueError, 'text' # raise Exception, 'text'", + "raise(ValueError,", + "raise (ValueError,", + "raise( ValueError,", + "raise ( ValueError,", + "raise(ValueError ,", + "raise (ValueError ,", + "raise( ValueError ,", + "raise ( ValueError ,", + ] + + for c in candidates_ok: + assert str_raise_re.search(_with_space(c)) is None, c + assert gen_raise_re.search(_with_space(c)) is None, c + assert old_raise_re.search(_with_space(c)) is None, c + for c in str_candidates_fail: + assert str_raise_re.search(_with_space(c)) is not None, c + for c in gen_candidates_fail: + assert gen_raise_re.search(_with_space(c)) is not None, c + for c in old_candidates_fail: + assert old_raise_re.search(_with_space(c)) is not None, c + + +def test_implicit_imports_regular_expression(): + candidates_ok = [ + "from sympy import something", + ">>> from sympy import something", + "from sympy.somewhere import something", + ">>> from sympy.somewhere import something", + "import sympy", + ">>> import sympy", + "import sympy.something.something", + "... import sympy", + "... import sympy.something.something", + "... from sympy import something", + "... from sympy.somewhere import something", + ">> from sympy import *", # To allow 'fake' docstrings + "# from sympy import *", + "some text # from sympy import *", + ] + candidates_fail = [ + "from sympy import *", + ">>> from sympy import *", + "from sympy.somewhere import *", + ">>> from sympy.somewhere import *", + "... from sympy import *", + "... from sympy.somewhere import *", + ] + for c in candidates_ok: + assert implicit_test_re.search(_with_space(c)) is None, c + for c in candidates_fail: + assert implicit_test_re.search(_with_space(c)) is not None, c + + +def test_test_suite_defs(): + candidates_ok = [ + " def foo():\n", + "def foo(arg):\n", + "def _foo():\n", + "def test_foo():\n", + ] + candidates_fail = [ + "def foo():\n", + "def foo() :\n", + "def foo( ):\n", + "def foo():\n", + ] + for c in candidates_ok: + assert test_suite_def_re.search(c) is None, c + for c in candidates_fail: + assert test_suite_def_re.search(c) is not None, c + + +def test_test_duplicate_defs(): + candidates_ok = [ + "def foo():\ndef foo():\n", + "def test():\ndef test_():\n", + "def test_():\ndef test__():\n", + ] + candidates_fail = [ + "def test_():\ndef test_ ():\n", + "def test_1():\ndef test_1():\n", + ] + ok = (None, 'check') + def check(file): + tests = 0 + test_set = set() + for idx, line in enumerate(file.splitlines()): + if test_ok_def_re.match(line): + tests += 1 + test_set.add(line[3:].split('(')[0].strip()) + if len(test_set) != tests: + return False, message_duplicate_test % ('check', idx + 1) + return None, 'check' + for c in candidates_ok: + assert check(c) == ok + for c in candidates_fail: + assert check(c) != ok + + +def test_find_self_assignments(): + candidates_ok = [ + "class A(object):\n def foo(self, arg): arg = self\n", + "class A(object):\n def foo(self, arg): self.prop = arg\n", + "class A(object):\n def foo(self, arg): obj, obj2 = arg, self\n", + "class A(object):\n @classmethod\n def bar(cls, arg): arg = cls\n", + "class A(object):\n def foo(var, arg): arg = var\n", + ] + candidates_fail = [ + "class A(object):\n def foo(self, arg): self = arg\n", + "class A(object):\n def foo(self, arg): obj, self = arg, arg\n", + "class A(object):\n def foo(self, arg):\n if arg: self = arg", + "class A(object):\n @classmethod\n def foo(cls, arg): cls = arg\n", + "class A(object):\n def foo(var, arg): var = arg\n", + ] + + for c in candidates_ok: + assert find_self_assignments(c) == [] + for c in candidates_fail: + assert find_self_assignments(c) != [] + + +def test_test_unicode_encoding(): + unicode_whitelist = ['foo'] + unicode_strict_whitelist = ['bar'] + + fname = 'abc' + test_file = ['α'] + raises(AssertionError, lambda: _test_this_file_encoding( + fname, test_file, unicode_whitelist, unicode_strict_whitelist)) + + fname = 'abc' + test_file = ['abc'] + _test_this_file_encoding( + fname, test_file, unicode_whitelist, unicode_strict_whitelist) + + fname = 'foo' + test_file = ['abc'] + raises(AssertionError, lambda: _test_this_file_encoding( + fname, test_file, unicode_whitelist, unicode_strict_whitelist)) + + fname = 'bar' + test_file = ['abc'] + _test_this_file_encoding( + fname, test_file, unicode_whitelist, unicode_strict_whitelist) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_deprecated.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_deprecated.py new file mode 100644 index 0000000000000000000000000000000000000000..696933d96d6232ea869da1002ec9ebee5309724d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_deprecated.py @@ -0,0 +1,5 @@ +from sympy.testing.pytest import warns_deprecated_sympy + +def test_deprecated_testing_randtest(): + with warns_deprecated_sympy(): + import sympy.testing.randtest # noqa:F401 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_module_imports.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_module_imports.py new file mode 100644 index 0000000000000000000000000000000000000000..d16dbaa98156c287c18b46ff07c0ede5d26e069a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_module_imports.py @@ -0,0 +1,42 @@ +""" +Checks that SymPy does not contain indirect imports. + +An indirect import is importing a symbol from a module that itself imported the +symbol from elsewhere. Such a constellation makes it harder to diagnose +inter-module dependencies and import order problems, and is therefore strongly +discouraged. + +(Indirect imports from end-user code is fine and in fact a best practice.) + +Implementation note: Forcing Python into actually unloading already-imported +submodules is a tricky and partly undocumented process. To avoid these issues, +the actual diagnostic code is in bin/diagnose_imports, which is run as a +separate, pristine Python process. +""" + +import subprocess +import sys +from os.path import abspath, dirname, join, normpath +import inspect + +from sympy.testing.pytest import XFAIL + +@XFAIL +def test_module_imports_are_direct(): + my_filename = abspath(inspect.getfile(inspect.currentframe())) + my_dirname = dirname(my_filename) + diagnose_imports_filename = join(my_dirname, 'diagnose_imports.py') + diagnose_imports_filename = normpath(diagnose_imports_filename) + + process = subprocess.Popen( + [ + sys.executable, + normpath(diagnose_imports_filename), + '--problems', + '--by-importer' + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=-1) + output, _ = process.communicate() + assert output == '', "There are import problems:\n" + output.decode() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_pytest.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_pytest.py new file mode 100644 index 0000000000000000000000000000000000000000..7080a4ed4602707a3efb4d9025e8e9cbe23a5ef8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_pytest.py @@ -0,0 +1,211 @@ +import warnings + +from sympy.testing.pytest import (raises, warns, ignore_warnings, + warns_deprecated_sympy, Failed) +from sympy.utilities.exceptions import sympy_deprecation_warning + + + +# Test callables + + +def test_expected_exception_is_silent_callable(): + def f(): + raise ValueError() + raises(ValueError, f) + + +# Under pytest raises will raise Failed rather than AssertionError +def test_lack_of_exception_triggers_AssertionError_callable(): + try: + raises(Exception, lambda: 1 + 1) + assert False + except Failed as e: + assert "DID NOT RAISE" in str(e) + + +def test_unexpected_exception_is_passed_through_callable(): + def f(): + raise ValueError("some error message") + try: + raises(TypeError, f) + assert False + except ValueError as e: + assert str(e) == "some error message" + +# Test with statement + +def test_expected_exception_is_silent_with(): + with raises(ValueError): + raise ValueError() + + +def test_lack_of_exception_triggers_AssertionError_with(): + try: + with raises(Exception): + 1 + 1 + assert False + except Failed as e: + assert "DID NOT RAISE" in str(e) + + +def test_unexpected_exception_is_passed_through_with(): + try: + with raises(TypeError): + raise ValueError("some error message") + assert False + except ValueError as e: + assert str(e) == "some error message" + +# Now we can use raises() instead of try/catch +# to test that a specific exception class is raised + + +def test_second_argument_should_be_callable_or_string(): + raises(TypeError, lambda: raises("irrelevant", 42)) + + +def test_warns_catches_warning(): + with warnings.catch_warnings(record=True) as w: + with warns(UserWarning): + warnings.warn('this is the warning message') + assert len(w) == 0 + + +def test_warns_raises_without_warning(): + with raises(Failed): + with warns(UserWarning): + pass + + +def test_warns_hides_other_warnings(): + with raises(RuntimeWarning): + with warns(UserWarning): + warnings.warn('this is the warning message', UserWarning) + warnings.warn('this is the other message', RuntimeWarning) + + +def test_warns_continues_after_warning(): + with warnings.catch_warnings(record=True) as w: + finished = False + with warns(UserWarning): + warnings.warn('this is the warning message') + finished = True + assert finished + assert len(w) == 0 + + +def test_warns_many_warnings(): + with warns(UserWarning): + warnings.warn('this is the warning message', UserWarning) + warnings.warn('this is the other warning message', UserWarning) + + +def test_warns_match_matching(): + with warnings.catch_warnings(record=True) as w: + with warns(UserWarning, match='this is the warning message'): + warnings.warn('this is the warning message', UserWarning) + assert len(w) == 0 + + +def test_warns_match_non_matching(): + with warnings.catch_warnings(record=True) as w: + with raises(Failed): + with warns(UserWarning, match='this is the warning message'): + warnings.warn('this is not the expected warning message', UserWarning) + assert len(w) == 0 + +def _warn_sympy_deprecation(stacklevel=3): + sympy_deprecation_warning( + "feature", + active_deprecations_target="active-deprecations", + deprecated_since_version="0.0.0", + stacklevel=stacklevel, + ) + +def test_warns_deprecated_sympy_catches_warning(): + with warnings.catch_warnings(record=True) as w: + with warns_deprecated_sympy(): + _warn_sympy_deprecation() + assert len(w) == 0 + + +def test_warns_deprecated_sympy_raises_without_warning(): + with raises(Failed): + with warns_deprecated_sympy(): + pass + +def test_warns_deprecated_sympy_wrong_stacklevel(): + with raises(Failed): + with warns_deprecated_sympy(): + _warn_sympy_deprecation(stacklevel=1) + +def test_warns_deprecated_sympy_doesnt_hide_other_warnings(): + # Unlike pytest's deprecated_call, we should not hide other warnings. + with raises(RuntimeWarning): + with warns_deprecated_sympy(): + _warn_sympy_deprecation() + warnings.warn('this is the other message', RuntimeWarning) + + +def test_warns_deprecated_sympy_continues_after_warning(): + with warnings.catch_warnings(record=True) as w: + finished = False + with warns_deprecated_sympy(): + _warn_sympy_deprecation() + finished = True + assert finished + assert len(w) == 0 + +def test_ignore_ignores_warning(): + with warnings.catch_warnings(record=True) as w: + with ignore_warnings(UserWarning): + warnings.warn('this is the warning message') + assert len(w) == 0 + + +def test_ignore_does_not_raise_without_warning(): + with warnings.catch_warnings(record=True) as w: + with ignore_warnings(UserWarning): + pass + assert len(w) == 0 + + +def test_ignore_allows_other_warnings(): + with warnings.catch_warnings(record=True) as w: + # This is needed when pytest is run as -Werror + # the setting is reverted at the end of the catch_Warnings block. + warnings.simplefilter("always") + with ignore_warnings(UserWarning): + warnings.warn('this is the warning message', UserWarning) + warnings.warn('this is the other message', RuntimeWarning) + assert len(w) == 1 + assert isinstance(w[0].message, RuntimeWarning) + assert str(w[0].message) == 'this is the other message' + + +def test_ignore_continues_after_warning(): + with warnings.catch_warnings(record=True) as w: + finished = False + with ignore_warnings(UserWarning): + warnings.warn('this is the warning message') + finished = True + assert finished + assert len(w) == 0 + + +def test_ignore_many_warnings(): + with warnings.catch_warnings(record=True) as w: + # This is needed when pytest is run as -Werror + # the setting is reverted at the end of the catch_Warnings block. + warnings.simplefilter("always") + with ignore_warnings(UserWarning): + warnings.warn('this is the warning message', UserWarning) + warnings.warn('this is the other message', RuntimeWarning) + warnings.warn('this is the warning message', UserWarning) + warnings.warn('this is the other message', RuntimeWarning) + warnings.warn('this is the other message', RuntimeWarning) + assert len(w) == 3 + for wi in w: + assert isinstance(wi.message, RuntimeWarning) + assert str(wi.message) == 'this is the other message' diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_runtests_pytest.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_runtests_pytest.py new file mode 100644 index 0000000000000000000000000000000000000000..cd56d831f3618c9dbb8a1dffe63d95a0befc6adf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/testing/tests/test_runtests_pytest.py @@ -0,0 +1,171 @@ +import pathlib +from typing import List + +import pytest + +from sympy.testing.runtests_pytest import ( + make_absolute_path, + sympy_dir, + update_args_with_paths, +) + + +class TestMakeAbsolutePath: + + @staticmethod + @pytest.mark.parametrize( + 'partial_path', ['sympy', 'sympy/core', 'sympy/nonexistant_directory'], + ) + def test_valid_partial_path(partial_path: str): + """Paths that start with `sympy` are valid.""" + _ = make_absolute_path(partial_path) + + @staticmethod + @pytest.mark.parametrize( + 'partial_path', ['not_sympy', 'also/not/sympy'], + ) + def test_invalid_partial_path_raises_value_error(partial_path: str): + """A `ValueError` is raises on paths that don't start with `sympy`.""" + with pytest.raises(ValueError): + _ = make_absolute_path(partial_path) + + +class TestUpdateArgsWithPaths: + + @staticmethod + def test_no_paths(): + """If no paths are passed, only `sympy` and `doc/src` are appended. + + `sympy` and `doc/src` are the `testpaths` stated in `pytest.ini`. They + need to be manually added as if any path-related arguments are passed + to `pytest.main` then the settings in `pytest.ini` may be ignored. + + """ + paths = [] + args = update_args_with_paths(paths=paths, keywords=None, args=[]) + expected = [ + str(pathlib.Path(sympy_dir(), 'sympy')), + str(pathlib.Path(sympy_dir(), 'doc/src')), + ] + assert args == expected + + @staticmethod + @pytest.mark.parametrize( + 'path', + ['sympy/core/tests/test_basic.py', '_basic'] + ) + def test_one_file(path: str): + """Single files/paths, full or partial, are matched correctly.""" + args = update_args_with_paths(paths=[path], keywords=None, args=[]) + expected = [ + str(pathlib.Path(sympy_dir(), 'sympy/core/tests/test_basic.py')), + ] + assert args == expected + + @staticmethod + def test_partial_path_from_root(): + """Partial paths from the root directly are matched correctly.""" + args = update_args_with_paths(paths=['sympy/functions'], keywords=None, args=[]) + expected = [str(pathlib.Path(sympy_dir(), 'sympy/functions'))] + assert args == expected + + @staticmethod + def test_multiple_paths_from_root(): + """Multiple paths, partial or full, are matched correctly.""" + paths = ['sympy/core/tests/test_basic.py', 'sympy/functions'] + args = update_args_with_paths(paths=paths, keywords=None, args=[]) + expected = [ + str(pathlib.Path(sympy_dir(), 'sympy/core/tests/test_basic.py')), + str(pathlib.Path(sympy_dir(), 'sympy/functions')), + ] + assert args == expected + + @staticmethod + @pytest.mark.parametrize( + 'paths, expected_paths', + [ + ( + ['/core', '/util'], + [ + 'doc/src/modules/utilities', + 'doc/src/reference/public/utilities', + 'sympy/core', + 'sympy/logic/utilities', + 'sympy/utilities', + ] + ), + ] + ) + def test_multiple_paths_from_non_root(paths: List[str], expected_paths: List[str]): + """Multiple partial paths are matched correctly.""" + args = update_args_with_paths(paths=paths, keywords=None, args=[]) + assert len(args) == len(expected_paths) + for arg, expected in zip(sorted(args), expected_paths): + assert expected in arg + + @staticmethod + @pytest.mark.parametrize( + 'paths', + [ + + [], + ['sympy/physics'], + ['sympy/physics/mechanics'], + ['sympy/physics/mechanics/tests'], + ['sympy/physics/mechanics/tests/test_kane3.py'], + ] + ) + def test_string_as_keyword(paths: List[str]): + """String keywords are matched correctly.""" + keywords = ('bicycle', ) + args = update_args_with_paths(paths=paths, keywords=keywords, args=[]) + expected_args = ['sympy/physics/mechanics/tests/test_kane3.py::test_bicycle'] + assert len(args) == len(expected_args) + for arg, expected in zip(sorted(args), expected_args): + assert expected in arg + + @staticmethod + @pytest.mark.parametrize( + 'paths', + [ + + [], + ['sympy/core'], + ['sympy/core/tests'], + ['sympy/core/tests/test_sympify.py'], + ] + ) + def test_integer_as_keyword(paths: List[str]): + """Integer keywords are matched correctly.""" + keywords = ('3538', ) + args = update_args_with_paths(paths=paths, keywords=keywords, args=[]) + expected_args = ['sympy/core/tests/test_sympify.py::test_issue_3538'] + assert len(args) == len(expected_args) + for arg, expected in zip(sorted(args), expected_args): + assert expected in arg + + @staticmethod + def test_multiple_keywords(): + """Multiple keywords are matched correctly.""" + keywords = ('bicycle', '3538') + args = update_args_with_paths(paths=[], keywords=keywords, args=[]) + expected_args = [ + 'sympy/core/tests/test_sympify.py::test_issue_3538', + 'sympy/physics/mechanics/tests/test_kane3.py::test_bicycle', + ] + assert len(args) == len(expected_args) + for arg, expected in zip(sorted(args), expected_args): + assert expected in arg + + @staticmethod + def test_keyword_match_in_multiple_files(): + """Keywords are matched across multiple files.""" + keywords = ('1130', ) + args = update_args_with_paths(paths=[], keywords=keywords, args=[]) + expected_args = [ + 'sympy/integrals/tests/test_heurisch.py::test_heurisch_symbolic_coeffs_1130', + 'sympy/utilities/tests/test_lambdify.py::test_python_div_zero_issue_11306', + ] + assert len(args) == len(expected_args) + for arg, expected in zip(sorted(args), expected_args): + assert expected in arg diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6fa29abc37d2541666b20378e9320cfb1e1b176a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/core.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f78f3b62d114ff01d3067c1154d4376814d692b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/core.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/rewrite.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/rewrite.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a846c63bb14a0afb572f31ad341f9f6dd5aee3b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/rewrite.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/usympy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/usympy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7034e80e24efdd7c6825df0fdac0165473673c9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/__pycache__/usympy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da73633785e5b61b90233f44db5d8041d7b50da0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_rewrite.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_rewrite.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d254f31f46e4e915af22c67342b405c408af1245 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_rewrite.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_sympy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_sympy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fcffd54e1751e7c83cb706e57abca3cb1611250 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_sympy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_unify.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_unify.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8d3991058edc8be2618951819243e46ca51a5d0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/__pycache__/test_unify.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_rewrite.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_rewrite.py new file mode 100644 index 0000000000000000000000000000000000000000..7b73e2856d5f6380c576220fa2780324df98091a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_rewrite.py @@ -0,0 +1,74 @@ +from sympy.unify.rewrite import rewriterule +from sympy.core.basic import Basic +from sympy.core.singleton import S +from sympy.core.symbol import Symbol +from sympy.functions.elementary.trigonometric import sin +from sympy.abc import x, y +from sympy.strategies.rl import rebuild +from sympy.assumptions import Q + +p, q = Symbol('p'), Symbol('q') + +def test_simple(): + rl = rewriterule(Basic(p, S(1)), Basic(p, S(2)), variables=(p,)) + assert list(rl(Basic(S(3), S(1)))) == [Basic(S(3), S(2))] + + p1 = p**2 + p2 = p**3 + rl = rewriterule(p1, p2, variables=(p,)) + + expr = x**2 + assert list(rl(expr)) == [x**3] + +def test_simple_variables(): + rl = rewriterule(Basic(x, S(1)), Basic(x, S(2)), variables=(x,)) + assert list(rl(Basic(S(3), S(1)))) == [Basic(S(3), S(2))] + + rl = rewriterule(x**2, x**3, variables=(x,)) + assert list(rl(y**2)) == [y**3] + +def test_moderate(): + p1 = p**2 + q**3 + p2 = (p*q)**4 + rl = rewriterule(p1, p2, (p, q)) + + expr = x**2 + y**3 + assert list(rl(expr)) == [(x*y)**4] + +def test_sincos(): + p1 = sin(p)**2 + sin(p)**2 + p2 = 1 + rl = rewriterule(p1, p2, (p, q)) + + assert list(rl(sin(x)**2 + sin(x)**2)) == [1] + assert list(rl(sin(y)**2 + sin(y)**2)) == [1] + +def test_Exprs_ok(): + rl = rewriterule(p+q, q+p, (p, q)) + next(rl(x+y)).is_commutative + str(next(rl(x+y))) + +def test_condition_simple(): + rl = rewriterule(x, x+1, [x], lambda x: x < 10) + assert not list(rl(S(15))) + assert rebuild(next(rl(S(5)))) == 6 + + +def test_condition_multiple(): + rl = rewriterule(x + y, x**y, [x,y], lambda x, y: x.is_integer) + + a = Symbol('a') + b = Symbol('b', integer=True) + expr = a + b + assert list(rl(expr)) == [b**a] + + c = Symbol('c', integer=True) + d = Symbol('d', integer=True) + assert set(rl(c + d)) == {c**d, d**c} + +def test_assumptions(): + rl = rewriterule(x + y, x**y, [x, y], assume=Q.integer(x)) + + a, b = map(Symbol, 'ab') + expr = a + b + assert list(rl(expr, Q.integer(b))) == [b**a] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_sympy.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_sympy.py new file mode 100644 index 0000000000000000000000000000000000000000..eca3933a91abfabdbad96f626e4da761a41b3fd2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_sympy.py @@ -0,0 +1,162 @@ +from sympy.core.add import Add +from sympy.core.basic import Basic +from sympy.core.containers import Tuple +from sympy.core.singleton import S +from sympy.core.symbol import (Symbol, symbols) +from sympy.logic.boolalg import And +from sympy.core.symbol import Str +from sympy.unify.core import Compound, Variable +from sympy.unify.usympy import (deconstruct, construct, unify, is_associative, + is_commutative) +from sympy.abc import x, y, z, n + +def test_deconstruct(): + expr = Basic(S(1), S(2), S(3)) + expected = Compound(Basic, (1, 2, 3)) + assert deconstruct(expr) == expected + + assert deconstruct(1) == 1 + assert deconstruct(x) == x + assert deconstruct(x, variables=(x,)) == Variable(x) + assert deconstruct(Add(1, x, evaluate=False)) == Compound(Add, (1, x)) + assert deconstruct(Add(1, x, evaluate=False), variables=(x,)) == \ + Compound(Add, (1, Variable(x))) + +def test_construct(): + expr = Compound(Basic, (S(1), S(2), S(3))) + expected = Basic(S(1), S(2), S(3)) + assert construct(expr) == expected + +def test_nested(): + expr = Basic(S(1), Basic(S(2)), S(3)) + cmpd = Compound(Basic, (S(1), Compound(Basic, Tuple(2)), S(3))) + assert deconstruct(expr) == cmpd + assert construct(cmpd) == expr + +def test_unify(): + expr = Basic(S(1), S(2), S(3)) + a, b, c = map(Symbol, 'abc') + pattern = Basic(a, b, c) + assert list(unify(expr, pattern, {}, (a, b, c))) == [{a: 1, b: 2, c: 3}] + assert list(unify(expr, pattern, variables=(a, b, c))) == \ + [{a: 1, b: 2, c: 3}] + +def test_unify_variables(): + assert list(unify(Basic(S(1), S(2)), Basic(S(1), x), {}, variables=(x,))) == [{x: 2}] + +def test_s_input(): + expr = Basic(S(1), S(2)) + a, b = map(Symbol, 'ab') + pattern = Basic(a, b) + assert list(unify(expr, pattern, {}, (a, b))) == [{a: 1, b: 2}] + assert list(unify(expr, pattern, {a: 5}, (a, b))) == [] + +def iterdicteq(a, b): + a = tuple(a) + b = tuple(b) + return len(a) == len(b) and all(x in b for x in a) + +def test_unify_commutative(): + expr = Add(1, 2, 3, evaluate=False) + a, b, c = map(Symbol, 'abc') + pattern = Add(a, b, c, evaluate=False) + + result = tuple(unify(expr, pattern, {}, (a, b, c))) + expected = ({a: 1, b: 2, c: 3}, + {a: 1, b: 3, c: 2}, + {a: 2, b: 1, c: 3}, + {a: 2, b: 3, c: 1}, + {a: 3, b: 1, c: 2}, + {a: 3, b: 2, c: 1}) + + assert iterdicteq(result, expected) + +def test_unify_iter(): + expr = Add(1, 2, 3, evaluate=False) + a, b, c = map(Symbol, 'abc') + pattern = Add(a, c, evaluate=False) + assert is_associative(deconstruct(pattern)) + assert is_commutative(deconstruct(pattern)) + + result = list(unify(expr, pattern, {}, (a, c))) + expected = [{a: 1, c: Add(2, 3, evaluate=False)}, + {a: 1, c: Add(3, 2, evaluate=False)}, + {a: 2, c: Add(1, 3, evaluate=False)}, + {a: 2, c: Add(3, 1, evaluate=False)}, + {a: 3, c: Add(1, 2, evaluate=False)}, + {a: 3, c: Add(2, 1, evaluate=False)}, + {a: Add(1, 2, evaluate=False), c: 3}, + {a: Add(2, 1, evaluate=False), c: 3}, + {a: Add(1, 3, evaluate=False), c: 2}, + {a: Add(3, 1, evaluate=False), c: 2}, + {a: Add(2, 3, evaluate=False), c: 1}, + {a: Add(3, 2, evaluate=False), c: 1}] + + assert iterdicteq(result, expected) + +def test_hard_match(): + from sympy.functions.elementary.trigonometric import (cos, sin) + expr = sin(x) + cos(x)**2 + p, q = map(Symbol, 'pq') + pattern = sin(p) + cos(p)**2 + assert list(unify(expr, pattern, {}, (p, q))) == [{p: x}] + +def test_matrix(): + from sympy.matrices.expressions.matexpr import MatrixSymbol + X = MatrixSymbol('X', n, n) + Y = MatrixSymbol('Y', 2, 2) + Z = MatrixSymbol('Z', 2, 3) + assert list(unify(X, Y, {}, variables=[n, Str('X')])) == [{Str('X'): Str('Y'), n: 2}] + assert list(unify(X, Z, {}, variables=[n, Str('X')])) == [] + +def test_non_frankenAdds(): + # the is_commutative property used to fail because of Basic.__new__ + # This caused is_commutative and str calls to fail + expr = x+y*2 + rebuilt = construct(deconstruct(expr)) + # Ensure that we can run these commands without causing an error + str(rebuilt) + rebuilt.is_commutative + +def test_FiniteSet_commutivity(): + from sympy.sets.sets import FiniteSet + a, b, c, x, y = symbols('a,b,c,x,y') + s = FiniteSet(a, b, c) + t = FiniteSet(x, y) + variables = (x, y) + assert {x: FiniteSet(a, c), y: b} in tuple(unify(s, t, variables=variables)) + +def test_FiniteSet_complex(): + from sympy.sets.sets import FiniteSet + a, b, c, x, y, z = symbols('a,b,c,x,y,z') + expr = FiniteSet(Basic(S(1), x), y, Basic(x, z)) + pattern = FiniteSet(a, Basic(x, b)) + variables = a, b + expected = ({b: 1, a: FiniteSet(y, Basic(x, z))}, + {b: z, a: FiniteSet(y, Basic(S(1), x))}) + assert iterdicteq(unify(expr, pattern, variables=variables), expected) + + +def test_and(): + variables = x, y + expected = ({x: z > 0, y: n < 3},) + assert iterdicteq(unify((z>0) & (n<3), And(x, y), variables=variables), + expected) + +def test_Union(): + from sympy.sets.sets import Interval + assert list(unify(Interval(0, 1) + Interval(10, 11), + Interval(0, 1) + Interval(12, 13), + variables=(Interval(12, 13),))) + +def test_is_commutative(): + assert is_commutative(deconstruct(x+y)) + assert is_commutative(deconstruct(x*y)) + assert not is_commutative(deconstruct(x**y)) + +def test_commutative_in_commutative(): + from sympy.abc import a,b,c,d + from sympy.functions.elementary.trigonometric import (cos, sin) + eq = sin(3)*sin(4)*sin(5) + 4*cos(3)*cos(4) + pat = a*cos(b)*cos(c) + d*sin(b)*sin(c) + assert next(unify(eq, pat, variables=(a,b,c,d))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_unify.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_unify.py new file mode 100644 index 0000000000000000000000000000000000000000..31153242576e1ff55dd3097efbc985aced5d574a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/unify/tests/test_unify.py @@ -0,0 +1,88 @@ +from sympy.unify.core import Compound, Variable, CondVariable, allcombinations +from sympy.unify import core + +a,b,c = 'a', 'b', 'c' +w,x,y,z = map(Variable, 'wxyz') + +C = Compound + +def is_associative(x): + return isinstance(x, Compound) and (x.op in ('Add', 'Mul', 'CAdd', 'CMul')) +def is_commutative(x): + return isinstance(x, Compound) and (x.op in ('CAdd', 'CMul')) + + +def unify(a, b, s={}): + return core.unify(a, b, s=s, is_associative=is_associative, + is_commutative=is_commutative) + +def test_basic(): + assert list(unify(a, x, {})) == [{x: a}] + assert list(unify(a, x, {x: 10})) == [] + assert list(unify(1, x, {})) == [{x: 1}] + assert list(unify(a, a, {})) == [{}] + assert list(unify((w, x), (y, z), {})) == [{w: y, x: z}] + assert list(unify(x, (a, b), {})) == [{x: (a, b)}] + + assert list(unify((a, b), (x, x), {})) == [] + assert list(unify((y, z), (x, x), {}))!= [] + assert list(unify((a, (b, c)), (a, (x, y)), {})) == [{x: b, y: c}] + +def test_ops(): + assert list(unify(C('Add', (a,b,c)), C('Add', (a,x,y)), {})) == \ + [{x:b, y:c}] + assert list(unify(C('Add', (C('Mul', (1,2)), b,c)), C('Add', (x,y,c)), {})) == \ + [{x: C('Mul', (1,2)), y:b}] + +def test_associative(): + c1 = C('Add', (1,2,3)) + c2 = C('Add', (x,y)) + assert tuple(unify(c1, c2, {})) == ({x: 1, y: C('Add', (2, 3))}, + {x: C('Add', (1, 2)), y: 3}) + +def test_commutative(): + c1 = C('CAdd', (1,2,3)) + c2 = C('CAdd', (x,y)) + result = list(unify(c1, c2, {})) + assert {x: 1, y: C('CAdd', (2, 3))} in result + assert ({x: 2, y: C('CAdd', (1, 3))} in result or + {x: 2, y: C('CAdd', (3, 1))} in result) + +def _test_combinations_assoc(): + assert set(allcombinations((1,2,3), (a,b), True)) == \ + {(((1, 2), (3,)), (a, b)), (((1,), (2, 3)), (a, b))} + +def _test_combinations_comm(): + assert set(allcombinations((1,2,3), (a,b), None)) == \ + {(((1,), (2, 3)), ('a', 'b')), (((2,), (3, 1)), ('a', 'b')), + (((3,), (1, 2)), ('a', 'b')), (((1, 2), (3,)), ('a', 'b')), + (((2, 3), (1,)), ('a', 'b')), (((3, 1), (2,)), ('a', 'b'))} + +def test_allcombinations(): + assert set(allcombinations((1,2), (1,2), 'commutative')) ==\ + {(((1,),(2,)), ((1,),(2,))), (((1,),(2,)), ((2,),(1,)))} + + +def test_commutativity(): + c1 = Compound('CAdd', (a, b)) + c2 = Compound('CAdd', (x, y)) + assert is_commutative(c1) and is_commutative(c2) + assert len(list(unify(c1, c2, {}))) == 2 + + +def test_CondVariable(): + expr = C('CAdd', (1, 2)) + x = Variable('x') + y = CondVariable('y', lambda a: a % 2 == 0) + z = CondVariable('z', lambda a: a > 3) + pattern = C('CAdd', (x, y)) + assert list(unify(expr, pattern, {})) == \ + [{x: 1, y: 2}] + + z = CondVariable('z', lambda a: a > 3) + pattern = C('CAdd', (z, y)) + + assert list(unify(expr, pattern, {})) == [] + +def test_defaultdict(): + assert next(unify(Variable('x'), 'foo')) == {Variable('x'): 'foo'} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1669704dcb2e7953c5c309b40f330a4dde264cba Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/autowrap.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/autowrap.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1e0261e6855bb82584b9db02314f37851179a21 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/autowrap.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/codegen.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/codegen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac75157b20e845840421bbac805992df5de7b2e6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/codegen.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/decorator.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/decorator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6bd1794431e161df44f5be54bc4a7940861ce61 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/decorator.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/enumerative.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/enumerative.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c4d417b1f7a459f966bcf835a132fcee2575da5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/enumerative.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/exceptions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9cb328e3386b13b7deb78dfb354aa3f14103a9e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/exceptions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/lambdify.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/lambdify.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd58bf2fc8989067bd6f14386004fd8164af5f86 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/lambdify.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/magic.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/magic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4c0f8adb1ae611ae1cde34648393ce94d6f4bde Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/magic.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/matchpy_connector.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/matchpy_connector.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be5d62d00be7e8186d56b9350ae57ba31e67f3ac Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/matchpy_connector.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/memoization.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/memoization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7db126f5a82f080cce370fd1ccde2a3704d59dc4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/memoization.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/misc.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/misc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce471bd33a77f589dac82667cd84f38d26ae918c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/misc.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pkgdata.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pkgdata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d802310bcd8469073a3ac005a455720518df026 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pkgdata.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pytest.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pytest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..258c13a67e265bea8bd2efd2f198ae68cafb0067 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/pytest.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/randtest.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/randtest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b32470830696d7f6bf09bb06fd785b17e4df308 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/randtest.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/runtests.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/runtests.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97790f66b4c38e8bc525b19ad5b1beb614e9a89b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/runtests.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/source.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/source.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41fac9843ff57a4f504a6faedb579ec0d96221d5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/source.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/timeutils.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/timeutils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8c3e694bd662f4de3376f768ae9857956aa1edf Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/timeutils.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/tmpfiles.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/tmpfiles.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0e9310addba335e5c2f0a8dda44a47f3d288186 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/__pycache__/tmpfiles.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d2c05ad48a93493bb5434256c88dfd614ac47b2d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__init__.py @@ -0,0 +1,22 @@ +""" This sub-module is private, i.e. external code should not depend on it. + +These functions are used by tests run as part of continuous integration. +Once the implementation is mature (it should support the major +platforms: Windows, OS X & Linux) it may become official API which + may be relied upon by downstream libraries. Until then API may break +without prior notice. + +TODO: +- (optionally) clean up after tempfile.mkdtemp() +- cross-platform testing +- caching of compiler choice and intermediate files + +""" + +from .compilation import compile_link_import_strings, compile_run_strings +from .availability import has_fortran, has_c, has_cxx + +__all__ = [ + 'compile_link_import_strings', 'compile_run_strings', + 'has_fortran', 'has_c', 'has_cxx', +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ac005653bc81fa3963fb1dcecf0d7f5eae19905 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/availability.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/availability.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c22f9bbbc0368a32e3f2d621ed6ffc87bac33086 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/availability.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/compilation.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/compilation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac2001ed7bd86d7afea628b1c6a582d41eee79ad Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/compilation.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/runners.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/runners.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..98aa6caa1748184a487da74815e4bd20ca62aade Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/runners.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/util.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e51bf59fb915d1b32e7ddff2008d12459b319a3b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/__pycache__/util.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/availability.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/availability.py new file mode 100644 index 0000000000000000000000000000000000000000..dc97b3e7b8c7e7307c6c21352ed4035d977aabb3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/availability.py @@ -0,0 +1,77 @@ +import os +from .compilation import compile_run_strings +from .util import CompilerNotFoundError + +def has_fortran(): + if not hasattr(has_fortran, 'result'): + try: + (stdout, stderr), info = compile_run_strings( + [('main.f90', ( + 'program foo\n' + 'print *, "hello world"\n' + 'end program' + ))], clean=True + ) + except CompilerNotFoundError: + has_fortran.result = False + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise + else: + if info['exit_status'] != os.EX_OK or 'hello world' not in stdout: + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise ValueError("Failed to compile test program:\n%s\n%s\n" % (stdout, stderr)) + has_fortran.result = False + else: + has_fortran.result = True + return has_fortran.result + + +def has_c(): + if not hasattr(has_c, 'result'): + try: + (stdout, stderr), info = compile_run_strings( + [('main.c', ( + '#include \n' + 'int main(){\n' + 'printf("hello world\\n");\n' + 'return 0;\n' + '}' + ))], clean=True + ) + except CompilerNotFoundError: + has_c.result = False + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise + else: + if info['exit_status'] != os.EX_OK or 'hello world' not in stdout: + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise ValueError("Failed to compile test program:\n%s\n%s\n" % (stdout, stderr)) + has_c.result = False + else: + has_c.result = True + return has_c.result + + +def has_cxx(): + if not hasattr(has_cxx, 'result'): + try: + (stdout, stderr), info = compile_run_strings( + [('main.cxx', ( + '#include \n' + 'int main(){\n' + 'std::cout << "hello world" << std::endl;\n' + '}' + ))], clean=True + ) + except CompilerNotFoundError: + has_cxx.result = False + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise + else: + if info['exit_status'] != os.EX_OK or 'hello world' not in stdout: + if os.environ.get('SYMPY_STRICT_COMPILER_CHECKS', '0') == '1': + raise ValueError("Failed to compile test program:\n%s\n%s\n" % (stdout, stderr)) + has_cxx.result = False + else: + has_cxx.result = True + return has_cxx.result diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/compilation.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/compilation.py new file mode 100644 index 0000000000000000000000000000000000000000..2d50a20467c08086d6cb5fb5b0d478e7a953d720 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/compilation.py @@ -0,0 +1,657 @@ +import glob +import os +import shutil +import subprocess +import sys +import tempfile +import warnings +from pathlib import Path +from sysconfig import get_config_var, get_config_vars, get_path + +from .runners import ( + CCompilerRunner, + CppCompilerRunner, + FortranCompilerRunner +) +from .util import ( + get_abspath, make_dirs, copy, Glob, ArbitraryDepthGlob, + glob_at_depth, import_module_from_file, pyx_is_cplus, + sha256_of_string, sha256_of_file, CompileError +) + +if os.name == 'posix': + objext = '.o' +elif os.name == 'nt': + objext = '.obj' +else: + warnings.warn("Unknown os.name: {}".format(os.name)) + objext = '.o' + + +def compile_sources(files, Runner=None, destdir=None, cwd=None, keep_dir_struct=False, + per_file_kwargs=None, **kwargs): + """ Compile source code files to object files. + + Parameters + ========== + + files : iterable of str + Paths to source files, if ``cwd`` is given, the paths are taken as relative. + Runner: CompilerRunner subclass (optional) + Could be e.g. ``FortranCompilerRunner``. Will be inferred from filename + extensions if missing. + destdir: str + Output directory, if cwd is given, the path is taken as relative. + cwd: str + Working directory. Specify to have compiler run in other directory. + also used as root of relative paths. + keep_dir_struct: bool + Reproduce directory structure in `destdir`. default: ``False`` + per_file_kwargs: dict + Dict mapping instances in ``files`` to keyword arguments. + \\*\\*kwargs: dict + Default keyword arguments to pass to ``Runner``. + + Returns + ======= + List of strings (paths of object files). + """ + _per_file_kwargs = {} + + if per_file_kwargs is not None: + for k, v in per_file_kwargs.items(): + if isinstance(k, Glob): + for path in glob.glob(k.pathname): + _per_file_kwargs[path] = v + elif isinstance(k, ArbitraryDepthGlob): + for path in glob_at_depth(k.filename, cwd): + _per_file_kwargs[path] = v + else: + _per_file_kwargs[k] = v + + # Set up destination directory + destdir = destdir or '.' + if not os.path.isdir(destdir): + if os.path.exists(destdir): + raise OSError("{} is not a directory".format(destdir)) + else: + make_dirs(destdir) + if cwd is None: + cwd = '.' + for f in files: + copy(f, destdir, only_update=True, dest_is_dir=True) + + # Compile files and return list of paths to the objects + dstpaths = [] + for f in files: + if keep_dir_struct: + name, ext = os.path.splitext(f) + else: + name, ext = os.path.splitext(os.path.basename(f)) + file_kwargs = kwargs.copy() + file_kwargs.update(_per_file_kwargs.get(f, {})) + dstpaths.append(src2obj(f, Runner, cwd=cwd, **file_kwargs)) + return dstpaths + + +def get_mixed_fort_c_linker(vendor=None, cplus=False, cwd=None): + vendor = vendor or os.environ.get('SYMPY_COMPILER_VENDOR', 'gnu') + + if vendor.lower() == 'intel': + if cplus: + return (FortranCompilerRunner, + {'flags': ['-nofor_main', '-cxxlib']}, vendor) + else: + return (FortranCompilerRunner, + {'flags': ['-nofor_main']}, vendor) + elif vendor.lower() == 'gnu' or 'llvm': + if cplus: + return (CppCompilerRunner, + {'lib_options': ['fortran']}, vendor) + else: + return (FortranCompilerRunner, + {}, vendor) + else: + raise ValueError("No vendor found.") + + +def link(obj_files, out_file=None, shared=False, Runner=None, + cwd=None, cplus=False, fort=False, extra_objs=None, **kwargs): + """ Link object files. + + Parameters + ========== + + obj_files: iterable of str + Paths to object files. + out_file: str (optional) + Path to executable/shared library, if ``None`` it will be + deduced from the last item in obj_files. + shared: bool + Generate a shared library? + Runner: CompilerRunner subclass (optional) + If not given the ``cplus`` and ``fort`` flags will be inspected + (fallback is the C compiler). + cwd: str + Path to the root of relative paths and working directory for compiler. + cplus: bool + C++ objects? default: ``False``. + fort: bool + Fortran objects? default: ``False``. + extra_objs: list + List of paths to extra object files / static libraries. + \\*\\*kwargs: dict + Keyword arguments passed to ``Runner``. + + Returns + ======= + + The absolute path to the generated shared object / executable. + + """ + if out_file is None: + out_file, ext = os.path.splitext(os.path.basename(obj_files[-1])) + if shared: + out_file += get_config_var('EXT_SUFFIX') + + if not Runner: + if fort: + Runner, extra_kwargs, vendor = \ + get_mixed_fort_c_linker( + vendor=kwargs.get('vendor', None), + cplus=cplus, + cwd=cwd, + ) + for k, v in extra_kwargs.items(): + if k in kwargs: + kwargs[k].expand(v) + else: + kwargs[k] = v + else: + if cplus: + Runner = CppCompilerRunner + else: + Runner = CCompilerRunner + + flags = kwargs.pop('flags', []) + if shared: + if '-shared' not in flags: + flags.append('-shared') + run_linker = kwargs.pop('run_linker', True) + if not run_linker: + raise ValueError("run_linker was set to False (nonsensical).") + + out_file = get_abspath(out_file, cwd=cwd) + runner = Runner(obj_files+(extra_objs or []), out_file, flags, cwd=cwd, **kwargs) + runner.run() + return out_file + + +def link_py_so(obj_files, so_file=None, cwd=None, libraries=None, + cplus=False, fort=False, extra_objs=None, **kwargs): + """ Link Python extension module (shared object) for importing + + Parameters + ========== + + obj_files: iterable of str + Paths to object files to be linked. + so_file: str + Name (path) of shared object file to create. If not specified it will + have the basname of the last object file in `obj_files` but with the + extension '.so' (Unix). + cwd: path string + Root of relative paths and working directory of linker. + libraries: iterable of strings + Libraries to link against, e.g. ['m']. + cplus: bool + Any C++ objects? default: ``False``. + fort: bool + Any Fortran objects? default: ``False``. + extra_objs: list + List of paths of extra object files / static libraries to link against. + kwargs**: dict + Keyword arguments passed to ``link(...)``. + + Returns + ======= + + Absolute path to the generate shared object. + """ + libraries = libraries or [] + + include_dirs = kwargs.pop('include_dirs', []) + library_dirs = kwargs.pop('library_dirs', []) + + # Add Python include and library directories + # PY_LDFLAGS does not available on all python implementations + # e.g. when with pypy, so it's LDFLAGS we need to use + if sys.platform == "win32": + warnings.warn("Windows not yet supported.") + elif sys.platform == 'darwin': + cfgDict = get_config_vars() + kwargs['linkline'] = kwargs.get('linkline', []) + [cfgDict['LDFLAGS']] + library_dirs += [cfgDict['LIBDIR']] + + # In macOS, linker needs to compile frameworks + # e.g. "-framework CoreFoundation" + is_framework = False + for opt in cfgDict['LIBS'].split(): + if is_framework: + kwargs['linkline'] = kwargs.get('linkline', []) + ['-framework', opt] + is_framework = False + elif opt.startswith('-l'): + libraries.append(opt[2:]) + elif opt.startswith('-framework'): + is_framework = True + # The python library is not included in LIBS + libfile = cfgDict['LIBRARY'] + libname = ".".join(libfile.split('.')[:-1])[3:] + libraries.append(libname) + + elif sys.platform[:3] == 'aix': + # Don't use the default code below + pass + else: + if get_config_var('Py_ENABLE_SHARED'): + cfgDict = get_config_vars() + kwargs['linkline'] = kwargs.get('linkline', []) + [cfgDict['LDFLAGS']] + library_dirs += [cfgDict['LIBDIR']] + for opt in cfgDict['BLDLIBRARY'].split(): + if opt.startswith('-l'): + libraries += [opt[2:]] + else: + pass + + flags = kwargs.pop('flags', []) + needed_flags = ('-pthread',) + for flag in needed_flags: + if flag not in flags: + flags.append(flag) + + return link(obj_files, shared=True, flags=flags, cwd=cwd, cplus=cplus, fort=fort, + include_dirs=include_dirs, libraries=libraries, + library_dirs=library_dirs, extra_objs=extra_objs, **kwargs) + + +def simple_cythonize(src, destdir=None, cwd=None, **cy_kwargs): + """ Generates a C file from a Cython source file. + + Parameters + ========== + + src: str + Path to Cython source. + destdir: str (optional) + Path to output directory (default: '.'). + cwd: path string (optional) + Root of relative paths (default: '.'). + **cy_kwargs: + Second argument passed to cy_compile. Generates a .cpp file if ``cplus=True`` in ``cy_kwargs``, + else a .c file. + """ + from Cython.Compiler.Main import ( + default_options, CompilationOptions + ) + from Cython.Compiler.Main import compile as cy_compile + + assert src.lower().endswith('.pyx') or src.lower().endswith('.py') + cwd = cwd or '.' + destdir = destdir or '.' + + ext = '.cpp' if cy_kwargs.get('cplus', False) else '.c' + c_name = os.path.splitext(os.path.basename(src))[0] + ext + + dstfile = os.path.join(destdir, c_name) + + if cwd: + ori_dir = os.getcwd() + else: + ori_dir = '.' + os.chdir(cwd) + try: + cy_options = CompilationOptions(default_options) + cy_options.__dict__.update(cy_kwargs) + # Set language_level if not set by cy_kwargs + # as not setting it is deprecated + if 'language_level' not in cy_kwargs: + cy_options.__dict__['language_level'] = 3 + cy_result = cy_compile([src], cy_options) + if cy_result.num_errors > 0: + raise ValueError("Cython compilation failed.") + + # Move generated C file to destination + # In macOS, the generated C file is in the same directory as the source + # but the /var is a symlink to /private/var, so we need to use realpath + if os.path.realpath(os.path.dirname(src)) != os.path.realpath(destdir): + if os.path.exists(dstfile): + os.unlink(dstfile) + shutil.move(os.path.join(os.path.dirname(src), c_name), destdir) + finally: + os.chdir(ori_dir) + return dstfile + + +extension_mapping = { + '.c': (CCompilerRunner, None), + '.cpp': (CppCompilerRunner, None), + '.cxx': (CppCompilerRunner, None), + '.f': (FortranCompilerRunner, None), + '.for': (FortranCompilerRunner, None), + '.ftn': (FortranCompilerRunner, None), + '.f90': (FortranCompilerRunner, None), # ifort only knows about .f90 + '.f95': (FortranCompilerRunner, 'f95'), + '.f03': (FortranCompilerRunner, 'f2003'), + '.f08': (FortranCompilerRunner, 'f2008'), +} + + +def src2obj(srcpath, Runner=None, objpath=None, cwd=None, inc_py=False, **kwargs): + """ Compiles a source code file to an object file. + + Files ending with '.pyx' assumed to be cython files and + are dispatched to pyx2obj. + + Parameters + ========== + + srcpath: str + Path to source file. + Runner: CompilerRunner subclass (optional) + If ``None``: deduced from extension of srcpath. + objpath : str (optional) + Path to generated object. If ``None``: deduced from ``srcpath``. + cwd: str (optional) + Working directory and root of relative paths. If ``None``: current dir. + inc_py: bool + Add Python include path to kwarg "include_dirs". Default: False + \\*\\*kwargs: dict + keyword arguments passed to Runner or pyx2obj + + """ + name, ext = os.path.splitext(os.path.basename(srcpath)) + if objpath is None: + if os.path.isabs(srcpath): + objpath = '.' + else: + objpath = os.path.dirname(srcpath) + objpath = objpath or '.' # avoid objpath == '' + + if os.path.isdir(objpath): + objpath = os.path.join(objpath, name + objext) + + include_dirs = kwargs.pop('include_dirs', []) + if inc_py: + py_inc_dir = get_path('include') + if py_inc_dir not in include_dirs: + include_dirs.append(py_inc_dir) + + if ext.lower() == '.pyx': + return pyx2obj(srcpath, objpath=objpath, include_dirs=include_dirs, cwd=cwd, + **kwargs) + + if Runner is None: + Runner, std = extension_mapping[ext.lower()] + if 'std' not in kwargs: + kwargs['std'] = std + + flags = kwargs.pop('flags', []) + needed_flags = ('-fPIC',) + for flag in needed_flags: + if flag not in flags: + flags.append(flag) + + # src2obj implies not running the linker... + run_linker = kwargs.pop('run_linker', False) + if run_linker: + raise CompileError("src2obj called with run_linker=True") + + runner = Runner([srcpath], objpath, include_dirs=include_dirs, + run_linker=run_linker, cwd=cwd, flags=flags, **kwargs) + runner.run() + return objpath + + +def pyx2obj(pyxpath, objpath=None, destdir=None, cwd=None, + include_dirs=None, cy_kwargs=None, cplus=None, **kwargs): + """ + Convenience function + + If cwd is specified, pyxpath and dst are taken to be relative + If only_update is set to `True` the modification time is checked + and compilation is only run if the source is newer than the + destination + + Parameters + ========== + + pyxpath: str + Path to Cython source file. + objpath: str (optional) + Path to object file to generate. + destdir: str (optional) + Directory to put generated C file. When ``None``: directory of ``objpath``. + cwd: str (optional) + Working directory and root of relative paths. + include_dirs: iterable of path strings (optional) + Passed onto src2obj and via cy_kwargs['include_path'] + to simple_cythonize. + cy_kwargs: dict (optional) + Keyword arguments passed onto `simple_cythonize` + cplus: bool (optional) + Indicate whether C++ is used. default: auto-detect using ``.util.pyx_is_cplus``. + compile_kwargs: dict + keyword arguments passed onto src2obj + + Returns + ======= + + Absolute path of generated object file. + + """ + assert pyxpath.endswith('.pyx') + cwd = cwd or '.' + objpath = objpath or '.' + destdir = destdir or os.path.dirname(objpath) + + abs_objpath = get_abspath(objpath, cwd=cwd) + + if os.path.isdir(abs_objpath): + pyx_fname = os.path.basename(pyxpath) + name, ext = os.path.splitext(pyx_fname) + objpath = os.path.join(objpath, name + objext) + + cy_kwargs = cy_kwargs or {} + cy_kwargs['output_dir'] = cwd + if cplus is None: + cplus = pyx_is_cplus(pyxpath) + cy_kwargs['cplus'] = cplus + + interm_c_file = simple_cythonize(pyxpath, destdir=destdir, cwd=cwd, **cy_kwargs) + + include_dirs = include_dirs or [] + flags = kwargs.pop('flags', []) + needed_flags = ('-fwrapv', '-pthread', '-fPIC') + for flag in needed_flags: + if flag not in flags: + flags.append(flag) + + options = kwargs.pop('options', []) + + if kwargs.pop('strict_aliasing', False): + raise CompileError("Cython requires strict aliasing to be disabled.") + + # Let's be explicit about standard + if cplus: + std = kwargs.pop('std', 'c++98') + else: + std = kwargs.pop('std', 'c99') + + return src2obj(interm_c_file, objpath=objpath, cwd=cwd, + include_dirs=include_dirs, flags=flags, std=std, + options=options, inc_py=True, strict_aliasing=False, + **kwargs) + + +def _any_X(srcs, cls): + for src in srcs: + name, ext = os.path.splitext(src) + key = ext.lower() + if key in extension_mapping: + if extension_mapping[key][0] == cls: + return True + return False + + +def any_fortran_src(srcs): + return _any_X(srcs, FortranCompilerRunner) + + +def any_cplus_src(srcs): + return _any_X(srcs, CppCompilerRunner) + + +def compile_link_import_py_ext(sources, extname=None, build_dir='.', compile_kwargs=None, + link_kwargs=None, extra_objs=None): + """ Compiles sources to a shared object (Python extension) and imports it + + Sources in ``sources`` which is imported. If shared object is newer than the sources, they + are not recompiled but instead it is imported. + + Parameters + ========== + + sources : list of strings + List of paths to sources. + extname : string + Name of extension (default: ``None``). + If ``None``: taken from the last file in ``sources`` without extension. + build_dir: str + Path to directory in which objects files etc. are generated. + compile_kwargs: dict + keyword arguments passed to ``compile_sources`` + link_kwargs: dict + keyword arguments passed to ``link_py_so`` + extra_objs: list + List of paths to (prebuilt) object files / static libraries to link against. + + Returns + ======= + + The imported module from of the Python extension. + """ + if extname is None: + extname = os.path.splitext(os.path.basename(sources[-1]))[0] + + compile_kwargs = compile_kwargs or {} + link_kwargs = link_kwargs or {} + + try: + mod = import_module_from_file(os.path.join(build_dir, extname), sources) + except ImportError: + objs = compile_sources(list(map(get_abspath, sources)), destdir=build_dir, + cwd=build_dir, **compile_kwargs) + so = link_py_so(objs, cwd=build_dir, fort=any_fortran_src(sources), + cplus=any_cplus_src(sources), extra_objs=extra_objs, **link_kwargs) + mod = import_module_from_file(so) + return mod + + +def _write_sources_to_build_dir(sources, build_dir): + build_dir = build_dir or tempfile.mkdtemp() + if not os.path.isdir(build_dir): + raise OSError("Non-existent directory: ", build_dir) + + source_files = [] + for name, src in sources: + dest = os.path.join(build_dir, name) + differs = True + sha256_in_mem = sha256_of_string(src.encode('utf-8')).hexdigest() + if os.path.exists(dest): + if os.path.exists(dest + '.sha256'): + sha256_on_disk = Path(dest + '.sha256').read_text() + else: + sha256_on_disk = sha256_of_file(dest).hexdigest() + + differs = sha256_on_disk != sha256_in_mem + if differs: + with open(dest, 'wt') as fh: + fh.write(src) + with open(dest + '.sha256', 'wt') as fh: + fh.write(sha256_in_mem) + source_files.append(dest) + return source_files, build_dir + + +def compile_link_import_strings(sources, build_dir=None, **kwargs): + """ Compiles, links and imports extension module from source. + + Parameters + ========== + + sources : iterable of name/source pair tuples + build_dir : string (default: None) + Path. ``None`` implies use a temporary directory. + **kwargs: + Keyword arguments passed onto `compile_link_import_py_ext`. + + Returns + ======= + + mod : module + The compiled and imported extension module. + info : dict + Containing ``build_dir`` as 'build_dir'. + + """ + source_files, build_dir = _write_sources_to_build_dir(sources, build_dir) + mod = compile_link_import_py_ext(source_files, build_dir=build_dir, **kwargs) + info = {"build_dir": build_dir} + return mod, info + + +def compile_run_strings(sources, build_dir=None, clean=False, compile_kwargs=None, link_kwargs=None): + """ Compiles, links and runs a program built from sources. + + Parameters + ========== + + sources : iterable of name/source pair tuples + build_dir : string (default: None) + Path. ``None`` implies use a temporary directory. + clean : bool + Whether to remove build_dir after use. This will only have an + effect if ``build_dir`` is ``None`` (which creates a temporary directory). + Passing ``clean == True`` and ``build_dir != None`` raises a ``ValueError``. + This will also set ``build_dir`` in returned info dictionary to ``None``. + compile_kwargs: dict + Keyword arguments passed onto ``compile_sources`` + link_kwargs: dict + Keyword arguments passed onto ``link`` + + Returns + ======= + + (stdout, stderr): pair of strings + info: dict + Containing exit status as 'exit_status' and ``build_dir`` as 'build_dir' + + """ + if clean and build_dir is not None: + raise ValueError("Automatic removal of build_dir is only available for temporary directory.") + try: + source_files, build_dir = _write_sources_to_build_dir(sources, build_dir) + objs = compile_sources(list(map(get_abspath, source_files)), destdir=build_dir, + cwd=build_dir, **(compile_kwargs or {})) + prog = link(objs, cwd=build_dir, + fort=any_fortran_src(source_files), + cplus=any_cplus_src(source_files), **(link_kwargs or {})) + p = subprocess.Popen([prog], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + exit_status = p.wait() + stdout, stderr = [txt.decode('utf-8') for txt in p.communicate()] + finally: + if clean and os.path.isdir(build_dir): + shutil.rmtree(build_dir) + build_dir = None + info = {"exit_status": exit_status, "build_dir": build_dir} + return (stdout, stderr), info diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/runners.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/runners.py new file mode 100644 index 0000000000000000000000000000000000000000..1f37d6cf8ac47807da7f3f00dfc5cd847c03fa8d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/runners.py @@ -0,0 +1,301 @@ +from __future__ import annotations +from typing import Callable, Optional + +from collections import OrderedDict +import os +import re +import subprocess +import warnings + +from .util import ( + find_binary_of_command, unique_list, CompileError +) + + +class CompilerRunner: + """ CompilerRunner base class. + + Parameters + ========== + + sources : list of str + Paths to sources. + out : str + flags : iterable of str + Compiler flags. + run_linker : bool + compiler_name_exe : (str, str) tuple + Tuple of compiler name & command to call. + cwd : str + Path of root of relative paths. + include_dirs : list of str + Include directories. + libraries : list of str + Libraries to link against. + library_dirs : list of str + Paths to search for shared libraries. + std : str + Standard string, e.g. ``'c++11'``, ``'c99'``, ``'f2003'``. + define: iterable of strings + macros to define + undef : iterable of strings + macros to undefine + preferred_vendor : string + name of preferred vendor e.g. 'gnu' or 'intel' + + Methods + ======= + + run(): + Invoke compilation as a subprocess. + + """ + + environ_key_compiler: str # e.g. 'CC', 'CXX', ... + environ_key_flags: str # e.g. 'CFLAGS', 'CXXFLAGS', ... + environ_key_ldflags: str = "LDFLAGS" # typically 'LDFLAGS' + + # Subclass to vendor/binary dict + compiler_dict: dict[str, str] + + # Standards should be a tuple of supported standards + # (first one will be the default) + standards: tuple[None | str, ...] + + # Subclass to dict of binary/formater-callback + std_formater: dict[str, Callable[[Optional[str]], str]] + + # subclass to be e.g. {'gcc': 'gnu', ...} + compiler_name_vendor_mapping: dict[str, str] + + def __init__(self, sources, out, flags=None, run_linker=True, compiler=None, cwd='.', + include_dirs=None, libraries=None, library_dirs=None, std=None, define=None, + undef=None, strict_aliasing=None, preferred_vendor=None, linkline=None, **kwargs): + if isinstance(sources, str): + raise ValueError("Expected argument sources to be a list of strings.") + self.sources = list(sources) + self.out = out + self.flags = flags or [] + if os.environ.get(self.environ_key_flags): + self.flags += os.environ[self.environ_key_flags].split() + self.cwd = cwd + if compiler: + self.compiler_name, self.compiler_binary = compiler + elif os.environ.get(self.environ_key_compiler): + self.compiler_binary = os.environ[self.environ_key_compiler] + for k, v in self.compiler_dict.items(): + if k in self.compiler_binary: + self.compiler_vendor = k + self.compiler_name = v + break + else: + self.compiler_vendor, self.compiler_name = list(self.compiler_dict.items())[0] + warnings.warn("failed to determine what kind of compiler %s is, assuming %s" % + (self.compiler_binary, self.compiler_name)) + else: + # Find a compiler + if preferred_vendor is None: + preferred_vendor = os.environ.get('SYMPY_COMPILER_VENDOR', None) + self.compiler_name, self.compiler_binary, self.compiler_vendor = self.find_compiler(preferred_vendor) + if self.compiler_binary is None: + raise ValueError("No compiler found (searched: {})".format(', '.join(self.compiler_dict.values()))) + self.define = define or [] + self.undef = undef or [] + self.include_dirs = include_dirs or [] + self.libraries = libraries or [] + self.library_dirs = library_dirs or [] + self.std = std or self.standards[0] + self.run_linker = run_linker + if self.run_linker: + # both gnu and intel compilers use '-c' for disabling linker + self.flags = list(filter(lambda x: x != '-c', self.flags)) + else: + if '-c' not in self.flags: + self.flags.append('-c') + + if self.std: + self.flags.append(self.std_formater[ + self.compiler_name](self.std)) + + self.linkline = (linkline or []) + [lf for lf in map( + str.strip, os.environ.get(self.environ_key_ldflags, "").split() + ) if lf != ""] + + if strict_aliasing is not None: + nsa_re = re.compile("no-strict-aliasing$") + sa_re = re.compile("strict-aliasing$") + if strict_aliasing is True: + if any(map(nsa_re.match, flags)): + raise CompileError("Strict aliasing cannot be both enforced and disabled") + elif any(map(sa_re.match, flags)): + pass # already enforced + else: + flags.append('-fstrict-aliasing') + elif strict_aliasing is False: + if any(map(nsa_re.match, flags)): + pass # already disabled + else: + if any(map(sa_re.match, flags)): + raise CompileError("Strict aliasing cannot be both enforced and disabled") + else: + flags.append('-fno-strict-aliasing') + else: + msg = "Expected argument strict_aliasing to be True/False, got {}" + raise ValueError(msg.format(strict_aliasing)) + + @classmethod + def find_compiler(cls, preferred_vendor=None): + """ Identify a suitable C/fortran/other compiler. """ + candidates = list(cls.compiler_dict.keys()) + if preferred_vendor: + if preferred_vendor in candidates: + candidates = [preferred_vendor]+candidates + else: + raise ValueError("Unknown vendor {}".format(preferred_vendor)) + name, path = find_binary_of_command([cls.compiler_dict[x] for x in candidates]) + return name, path, cls.compiler_name_vendor_mapping[name] + + def cmd(self): + """ List of arguments (str) to be passed to e.g. ``subprocess.Popen``. """ + cmd = ( + [self.compiler_binary] + + self.flags + + ['-U'+x for x in self.undef] + + ['-D'+x for x in self.define] + + ['-I'+x for x in self.include_dirs] + + self.sources + ) + if self.run_linker: + cmd += (['-L'+x for x in self.library_dirs] + + ['-l'+x for x in self.libraries] + + self.linkline) + counted = [] + for envvar in re.findall(r'\$\{(\w+)\}', ' '.join(cmd)): + if os.getenv(envvar) is None: + if envvar not in counted: + counted.append(envvar) + msg = "Environment variable '{}' undefined.".format(envvar) + raise CompileError(msg) + return cmd + + def run(self): + self.flags = unique_list(self.flags) + + # Append output flag and name to tail of flags + self.flags.extend(['-o', self.out]) + env = os.environ.copy() + env['PWD'] = self.cwd + + # NOTE: intel compilers seems to need shell=True + p = subprocess.Popen(' '.join(self.cmd()), + shell=True, + cwd=self.cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env) + comm = p.communicate() + try: + self.cmd_outerr = comm[0].decode('utf-8') + except UnicodeDecodeError: + self.cmd_outerr = comm[0].decode('iso-8859-1') # win32 + self.cmd_returncode = p.returncode + + # Error handling + if self.cmd_returncode != 0: + msg = "Error executing '{}' in {} (exited status {}):\n {}\n".format( + ' '.join(self.cmd()), self.cwd, str(self.cmd_returncode), self.cmd_outerr + ) + raise CompileError(msg) + + return self.cmd_outerr, self.cmd_returncode + + +class CCompilerRunner(CompilerRunner): + + environ_key_compiler = 'CC' + environ_key_flags = 'CFLAGS' + + compiler_dict = OrderedDict([ + ('gnu', 'gcc'), + ('intel', 'icc'), + ('llvm', 'clang'), + ]) + + standards = ('c89', 'c90', 'c99', 'c11') # First is default + + std_formater = { + 'gcc': '-std={}'.format, + 'icc': '-std={}'.format, + 'clang': '-std={}'.format, + } + + compiler_name_vendor_mapping = { + 'gcc': 'gnu', + 'icc': 'intel', + 'clang': 'llvm' + } + + +def _mk_flag_filter(cmplr_name): # helper for class initialization + not_welcome = {'g++': ("Wimplicit-interface",)} # "Wstrict-prototypes",)} + if cmplr_name in not_welcome: + def fltr(x): + for nw in not_welcome[cmplr_name]: + if nw in x: + return False + return True + else: + def fltr(x): + return True + return fltr + + +class CppCompilerRunner(CompilerRunner): + + environ_key_compiler = 'CXX' + environ_key_flags = 'CXXFLAGS' + + compiler_dict = OrderedDict([ + ('gnu', 'g++'), + ('intel', 'icpc'), + ('llvm', 'clang++'), + ]) + + # First is the default, c++0x == c++11 + standards = ('c++98', 'c++0x') + + std_formater = { + 'g++': '-std={}'.format, + 'icpc': '-std={}'.format, + 'clang++': '-std={}'.format, + } + + compiler_name_vendor_mapping = { + 'g++': 'gnu', + 'icpc': 'intel', + 'clang++': 'llvm' + } + + +class FortranCompilerRunner(CompilerRunner): + + environ_key_compiler = 'FC' + environ_key_flags = 'FFLAGS' + + standards = (None, 'f77', 'f95', 'f2003', 'f2008') + + std_formater = { + 'gfortran': lambda x: '-std=gnu' if x is None else '-std=legacy' if x == 'f77' else '-std={}'.format(x), + 'ifort': lambda x: '-stand f08' if x is None else '-stand f{}'.format(x[-2:]), # f2008 => f08 + } + + compiler_dict = OrderedDict([ + ('gnu', 'gfortran'), + ('intel', 'ifort'), + ]) + + compiler_name_vendor_mapping = { + 'gfortran': 'gnu', + 'ifort': 'intel', + } diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1ca72be9aca2ecbe55e33ca2b567cd7fdd7c3ec Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/test_compilation.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/test_compilation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56f37825fd244fa83cb9382d575ee623e53c1bbc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/__pycache__/test_compilation.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/test_compilation.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/test_compilation.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7cf86644a9645665e62b49cfc4e7ea73b2c1ab --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/_compilation/tests/test_compilation.py @@ -0,0 +1,104 @@ +import shutil +import os +import subprocess +import tempfile +from sympy.external import import_module +from sympy.testing.pytest import skip, skip_under_pyodide + +from sympy.utilities._compilation.compilation import compile_link_import_py_ext, compile_link_import_strings, compile_sources, get_abspath + +numpy = import_module('numpy') +cython = import_module('cython') + +_sources1 = [ + ('sigmoid.c', r""" +#include + +void sigmoid(int n, const double * const restrict in, + double * const restrict out, double lim){ + for (int i=0; i 0: + if not os.path.exists(parent): + make_dirs(parent) + + if not os.path.exists(path): + os.mkdir(path, 0o777) + else: + assert os.path.isdir(path) + +def missing_or_other_newer(path, other_path, cwd=None): + """ + Investigate if path is non-existent or older than provided reference + path. + + Parameters + ========== + path: string + path to path which might be missing or too old + other_path: string + reference path + cwd: string + working directory (root of relative paths) + + Returns + ======= + True if path is older or missing. + """ + cwd = cwd or '.' + path = get_abspath(path, cwd=cwd) + other_path = get_abspath(other_path, cwd=cwd) + if not os.path.exists(path): + return True + if os.path.getmtime(other_path) - 1e-6 >= os.path.getmtime(path): + # 1e-6 is needed because http://stackoverflow.com/questions/17086426/ + return True + return False + +def copy(src, dst, only_update=False, copystat=True, cwd=None, + dest_is_dir=False, create_dest_dirs=False): + """ Variation of ``shutil.copy`` with extra options. + + Parameters + ========== + + src : str + Path to source file. + dst : str + Path to destination. + only_update : bool + Only copy if source is newer than destination + (returns None if it was newer), default: ``False``. + copystat : bool + See ``shutil.copystat``. default: ``True``. + cwd : str + Path to working directory (root of relative paths). + dest_is_dir : bool + Ensures that dst is treated as a directory. default: ``False`` + create_dest_dirs : bool + Creates directories if needed. + + Returns + ======= + + Path to the copied file. + + """ + if cwd: # Handle working directory + if not os.path.isabs(src): + src = os.path.join(cwd, src) + if not os.path.isabs(dst): + dst = os.path.join(cwd, dst) + + if not os.path.exists(src): # Make sure source file exists + raise FileNotFoundError("Source: `{}` does not exist".format(src)) + + # We accept both (re)naming destination file _or_ + # passing a (possible non-existent) destination directory + if dest_is_dir: + if not dst[-1] == '/': + dst = dst+'/' + else: + if os.path.exists(dst) and os.path.isdir(dst): + dest_is_dir = True + + if dest_is_dir: + dest_dir = dst + dest_fname = os.path.basename(src) + dst = os.path.join(dest_dir, dest_fname) + else: + dest_dir = os.path.dirname(dst) + + if not os.path.exists(dest_dir): + if create_dest_dirs: + make_dirs(dest_dir) + else: + raise FileNotFoundError("You must create directory first.") + + if only_update: + if not missing_or_other_newer(dst, src): + return + + if os.path.islink(dst): + dst = os.path.abspath(os.path.realpath(dst), cwd=cwd) + + shutil.copy(src, dst) + if copystat: + shutil.copystat(src, dst) + + return dst + +Glob = namedtuple('Glob', 'pathname') +ArbitraryDepthGlob = namedtuple('ArbitraryDepthGlob', 'filename') + +def glob_at_depth(filename_glob, cwd=None): + if cwd is not None: + cwd = '.' + globbed = [] + for root, dirs, filenames in os.walk(cwd): + for fn in filenames: + # This is not tested: + if fnmatch.fnmatch(fn, filename_glob): + globbed.append(os.path.join(root, fn)) + return globbed + +def sha256_of_file(path, nblocks=128): + """ Computes the SHA256 hash of a file. + + Parameters + ========== + + path : string + Path to file to compute hash of. + nblocks : int + Number of blocks to read per iteration. + + Returns + ======= + + hashlib sha256 hash object. Use ``.digest()`` or ``.hexdigest()`` + on returned object to get binary or hex encoded string. + """ + sh = sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(nblocks*sh.block_size), b''): + sh.update(chunk) + return sh + + +def sha256_of_string(string): + """ Computes the SHA256 hash of a string. """ + sh = sha256() + sh.update(string) + return sh + + +def pyx_is_cplus(path): + """ + Inspect a Cython source file (.pyx) and look for comment line like: + + # distutils: language = c++ + + Returns True if such a file is present in the file, else False. + """ + with open(path) as fh: + for line in fh: + if line.startswith('#') and '=' in line: + splitted = line.split('=') + if len(splitted) != 2: + continue + lhs, rhs = splitted + if lhs.strip().split()[-1].lower() == 'language' and \ + rhs.strip().split()[0].lower() == 'c++': + return True + return False + +def import_module_from_file(filename, only_if_newer_than=None): + """ Imports Python extension (from shared object file) + + Provide a list of paths in `only_if_newer_than` to check + timestamps of dependencies. import_ raises an ImportError + if any is newer. + + Word of warning: The OS may cache shared objects which makes + reimporting same path of an shared object file very problematic. + + It will not detect the new time stamp, nor new checksum, but will + instead silently use old module. Use unique names for this reason. + + Parameters + ========== + + filename : str + Path to shared object. + only_if_newer_than : iterable of strings + Paths to dependencies of the shared object. + + Raises + ====== + + ``ImportError`` if any of the files specified in ``only_if_newer_than`` are newer + than the file given by filename. + """ + path, name = os.path.split(filename) + name, ext = os.path.splitext(name) + name = name.split('.')[0] + if sys.version_info[0] == 2: + from imp import find_module, load_module + fobj, filename, data = find_module(name, [path]) + if only_if_newer_than: + for dep in only_if_newer_than: + if os.path.getmtime(filename) < os.path.getmtime(dep): + raise ImportError("{} is newer than {}".format(dep, filename)) + mod = load_module(name, fobj, filename, data) + else: + import importlib.util + spec = importlib.util.spec_from_file_location(name, filename) + if spec is None: + raise ImportError("Failed to import: '%s'" % filename) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def find_binary_of_command(candidates): + """ Finds binary first matching name among candidates. + + Calls ``which`` from shutils for provided candidates and returns + first hit. + + Parameters + ========== + + candidates : iterable of str + Names of candidate commands + + Raises + ====== + + CompilerNotFoundError if no candidates match. + """ + from shutil import which + for c in candidates: + binary_path = which(c) + if c and binary_path: + return c, binary_path + + raise CompilerNotFoundError('No binary located for candidates: {}'.format(candidates)) + + +def unique_list(l): + """ Uniquify a list (skip duplicate items). """ + result = [] + for x in l: + if x not in result: + result.append(x) + return result diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eded44ee3c0f34ad1324765ba06ee9d6eb5e9899 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/__init__.py @@ -0,0 +1,122 @@ +"""Module with some functions for MathML, like transforming MathML +content in MathML presentation. + +To use this module, you will need lxml. +""" + +from pathlib import Path + +from sympy.utilities.decorator import doctest_depends_on + + +__doctest_requires__ = {('apply_xsl', 'c2p'): ['lxml']} + + +def add_mathml_headers(s): + return """""" + s + "" + + +def _read_binary(pkgname, filename): + import sys + + if sys.version_info >= (3, 10): + # files was added in Python 3.9 but only seems to work here in 3.10+ + from importlib.resources import files + return files(pkgname).joinpath(filename).read_bytes() + else: + # read_binary was deprecated in Python 3.11 + from importlib.resources import read_binary + return read_binary(pkgname, filename) + + +def _read_xsl(xsl): + # Previously these values were allowed: + if xsl == 'mathml/data/simple_mmlctop.xsl': + xsl = 'simple_mmlctop.xsl' + elif xsl == 'mathml/data/mmlctop.xsl': + xsl = 'mmlctop.xsl' + elif xsl == 'mathml/data/mmltex.xsl': + xsl = 'mmltex.xsl' + + if xsl in ['simple_mmlctop.xsl', 'mmlctop.xsl', 'mmltex.xsl']: + xslbytes = _read_binary('sympy.utilities.mathml.data', xsl) + else: + xslbytes = Path(xsl).read_bytes() + + return xslbytes + + +@doctest_depends_on(modules=('lxml',)) +def apply_xsl(mml, xsl): + """Apply a xsl to a MathML string. + + Parameters + ========== + + mml + A string with MathML code. + xsl + A string giving the name of an xsl (xml stylesheet) file which can be + found in sympy/utilities/mathml/data. The following files are supplied + with SymPy: + + - mmlctop.xsl + - mmltex.xsl + - simple_mmlctop.xsl + + Alternatively, a full path to an xsl file can be given. + + Examples + ======== + + >>> from sympy.utilities.mathml import apply_xsl + >>> xsl = 'simple_mmlctop.xsl' + >>> mml = ' a b ' + >>> res = apply_xsl(mml,xsl) + >>> print(res) + + + a + + + b + + """ + from lxml import etree + + parser = etree.XMLParser(resolve_entities=False) + ac = etree.XSLTAccessControl.DENY_ALL + + s = etree.XML(_read_xsl(xsl), parser=parser) + transform = etree.XSLT(s, access_control=ac) + doc = etree.XML(mml, parser=parser) + result = transform(doc) + s = str(result) + return s + + +@doctest_depends_on(modules=('lxml',)) +def c2p(mml, simple=False): + """Transforms a document in MathML content (like the one that sympy produces) + in one document in MathML presentation, more suitable for printing, and more + widely accepted + + Examples + ======== + + >>> from sympy.utilities.mathml import c2p + >>> mml = ' 2 ' + >>> c2p(mml,simple=True) != c2p(mml,simple=False) + True + + """ + + if not mml.startswith(' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + e + + + + + + + + + + + + + - + + + + + + + + &#x2062; + &#x2148; + + + + + + + + + + + + + - + + + + + + + + &#x2062; + &#x2148; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Polar + &#x2062; + + + + + + + + + + + + + + + Polar + &#x2062; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x03BB; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2218; + + + + + + + + + + + + + + + + + + &#x2218; + + + + + + + + + + + + + + + + id + + + id + + + + + + + + + + + + + + domain + + + codomain + + + image + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + { + + + + + + + + if + + + + + + + + + + + otherwise + + + + + + + + + + + + + + + + + + + + + + + &#x230A; + + + + + + + + + + + + + + + + + + + &#x230B; + + + + + + + + + + + + &#x2147; + + + + + + + + + + + + + + + + + ! + + + + + + + + + + + + + + + + max + + + min + + + + + + + max + + + min + + + + + + + + + + + + + | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2062; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + gcd + + + lcm + + + + + + gcd + + + lcm + + + + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2227; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2228; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x22BB; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + &#x00AC; + &#x2061; + + + + + + + + + + + + + + + &#x00AC; + &#x2061; + + + + + + + + + + + + + + + + + + + &#x2200; + + + + + + + + + + + + : + + + + , + + + + + + + + + + + + + + &#x2203; + + + + + + + + + + + + : + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00AF; + + + + + + + + + + + + + + + + + &#x211C; + + + &#x2111; + + + &#x2061; + + + + + + + + + + + + + + + + + &#x230A; + + + &#x2308; + + + + + + &#x230B; + + + &#x2309; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2260; + + + &#x2248; + + + &#x2223; + + + + + &#x2198; + + + &#x2197; + + + &#x2192; + + + + + &#x21D2; + + + &#x2208; + + + &#x2209; + + + &#x2284; + + + &#x2288; + + + + + + + + + + &#x2286; + + + &#x2282; + + + + + + + + + + + + &#x2265; + + + &#x2264; + + + &#x2261; + + + + + + + + + + + + + + + + + + + + + + ln + + + + + ln + + + + + + + + + + + + + + + + + + + + + + log + + + + + + log + + + + + + + + log + + + + log + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2146; + + &#x2146; + + + + + + + + &#x2146; + + + + &#x2146; + + + + + + + + + + + + + + + + + + &#x2032; + + + + + + + + + + + + + + + + + &#x2145; + + + + + + + + &#x2202; + + + + + &#x2202; + + + + + + + + + + + + + + + + + + + &#x2202; + + + + &#x2202; + + + + + + + + + + &#x2202; + + &#x2202; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2207; + 2 + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x222A; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2229; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00D7; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + = + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + + + + + + + + + + + + + + + + + + + + &#x222B; + + + + + + &#x222B; + + + + + + &#x222B; + + + + + + + + + + + + &#x222B; + + + + + + + + + &#x222B; + + + + + + + + + &#x2146; + + + + + + + + + + + + + + + + lim + + + + &#x2192; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x03C3; + + + + + + + + + + + + + + + + + &#x03C3; + + + + + + + 2 + + + + + + + + + + + + + median + + + + + + + + + + + + + + + + + mode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + det + + + + + + + + + + + + + + + T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00D7; + + + &#x22C5; + + + &#x2297; + + + + + + + + + + + + &#x2124; + + + + &#x211D; + + + + &#x211A; + + + + &#x2115; + + + + &#x2102; + + + + &#x2119; + + + + &#x2147; + + + + &#x2148; + + + + NaN + + + + true + + + + false + + + + &#x2205; + + + + &#x03C0; + + + + &#x213D; + + + + &#x221E; + + + diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/mmltex.xsl b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/mmltex.xsl new file mode 100644 index 0000000000000000000000000000000000000000..5e6b85e02efd5196fe76b6ce4e10def27b9a8497 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/mmltex.xsl @@ -0,0 +1,2360 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $ + + $ + + + + + + + + + + + + + + + + i + + + + + / + + + + + + _{} + + + + + e^{i + + } + + + + + E + + + + + + + + \mathrm{} + + + + + + + + + + + + + ( + + + , + + ) + + + + + () + + + + + + + \left( + + \left[ + + + , + + + + \right) + + \right] + + + + + \left\{\right\} + + + + + ^{(-1)} + + + + + + + + \mathrm{lambda}\: + + .\: + + + + + + + + + + \circ + + + + +\mathrm{id} + + + + \mathop{\mathrm{ + + }} + + + + + + + + \begin{cases} + + + \end{cases} + + + + + & \text{if $ + + $} + \\ + + + + + & \text{otherwise} + + + + + \left\lfloor\frac{ + + }{ + + }\right\rfloor + + + + + + + + ! + + + + + + + \left( + \frac{ + + + }{ + + + } + \right) + + + + + \ + + \{ + + + + , + + + + + + , + + + + \} + + + + + - + + + + + + + + + - + + + + + + + + + + ( + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + ) + + + + + + + + + ^{ + + + + } + + + + + + + \mod + + + + + + + + + + ( + + + + \times + + + + + + + + + + ) + + + + + \sqrt + + [ + + ] + + { + + } + + + +\gcd + + + + + + + + \land + + + + + + + + + + \lor + + + + + + + + + + \mathop{\mathrm{xor}} + + + + + + \neg + + + + + + + + + + \implies + + + + + + + + \ + + + + + , + + + \colon + + + + + + + \left| + + \right| + + + + + \overline{} + + + +\Re + + +\Im + + + + \left\lfloor + + \right\rfloor + + + + + \left\lceil + + \right\rceil + + + + + + + + + = + + + + + + + + + + \neq + + + + + + + + + + > + + + + + + + + + + < + + + + + + + + + + \ge + + + + + + + + + + \le + + + + + + + + + + \equiv + + + + + + + + + + \approx + + + + + + + + | + + + + + + + + \int + + _{ + + } + + + ^{ + + } + + + + \,d + + + + + + + ^\prime + + + + \frac{ + + + d^{ + + } + + }{d + + ^{ + + } + + + d + + }{d + + } + + + } + + + + + D_{ + + + , + + } + + + + + \frac{\partial^{ + + + + + + + + + + + + + + + + + + + + + } + + }{ + + \partial + + + ^{ + + } + + + } + + + + + + + + + , + + + +\mathop{\mathrm{div}} + + +\nabla^2 + + + + \{\} + + + + + \left[\right] + + + + + + + \colon + + + + + + , + + + + + + + + + + + + \cup + + + + + + + + + + \cap + + + + + + + + \in + + + + + + + + + + \notin + + + + + + + + + + + + \subseteq + + + + + + + + + + \subset + + + + + + + + + + \nsubseteq + + + + + + + + + + \not\subset + + + + + + + + + + \setminus + + + + + + | + + | + + + + + + + + + \times + + + + + + + + ^{ + + } + + + + + \sum + + + + + \prod + + + + + _{ + + + = + + + } + + + ^{ + + } + + + + + + + + \lim_{ + + } + + + + + + \to + + + + + + + + + + + + \searrow + \nearrow + \rightarrow + \to + + + + + + + + \ + + + + + + + + + \ + + + + + + \mathrm{ + + \,} + + + + + + + \mathrm{ + + } + + + + + e^{} + + + + + \lg + + + + + + + \log_{ + + } + + + + + + + + \left\langle + + + , + + \right\rangle + + + +\sigma + + + + \sigma( + + )^2 + + + + + \left\langle + + ^{ + + }\right\rangle + + _{ + + } + + + + + + + \left(\begin{array}{c} + + + \\ + + \end{array}\right) + + + + + \begin{pmatrix} + + \end{pmatrix} + + + + + + + & + + \\ + + + + + \det + + + + + + + \begin{vmatrix} + + \end{vmatrix} + + + + + + + + ^T + + + + + + + + _{ + + + , + + } + + + + + + + + + \dot + + + + + + + + + + + +\mathbb{Z} + + +\mathbb{R} + + +\mathbb{Q} + + +\mathbb{N} + + +\mathbb{C} + + +\mathbb{P} + + +e + + +i + + +NaN + + +\mbox{true} + + +\mbox{false} + + +\emptyset + + +\pi + + +\gamma + + +\infty + + + + + + + ( + + + + + + + + + ) + + + + + + + ( + + + + + + + + ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \multicolumn{ + + }{c}{ + + } + + & + + + + + + + \hfill + + + + \hfill + + + + & + + + + + + + \\ + + + + + \begin{array}{ + + | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | + + } + + \hline + + + + \\ \hline + + \end{array} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \overline{ + + + + + } + + + \overbrace{ + + + + + } + + + \underline{ + + + + + + } + + + \underbrace{ + + + + + + } + + + + + _{ + + }^{ + + } + + + \underset{ + + }{\overset{ + + }{ + + }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \overline{ + + } + + + \overbrace{ + + } + + + + + ^{ + + } + + + \stackrel{ + + }{ + + } + + + + + + + + + + + \underline{ + + } + + + \underbrace{ + + } + + + + + _{ + + } + + + \underset{ + + }{ + + } + + + + + + { + + }_{ + + }^{ + + } + + + + { + + }^{ + + } + + + + { + + }_{ + + } + + + + + + {}_{ + + } + + + {}^{ + + } + + + + + + {} + + + _{ + + } + + + ^{ + + } + + + + + + + + + + + + + + {} + + + _{ + + } + + + ^{ + + } + + + + + + + + + + + + + + + + + + \genfrac{}{}{ + + + + ex + + + .05ex + + + + .2ex + + + + + + }{}{ + + + \frac{ + + + + \hfill + + + + \hfill + + }{ + + \hfill + + + + \hfill + + } + + + + + + \sqrt[ + + ]{ + + } + + + + exception 25: + \text{exception 25:} + + + + + + \sqrt{ + + } + + + + + + + \left + + + \ + + + + \left( + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + + \right + + + \ + + + + \right) + + + + + \phantom{ + + } + + + + + + \overline{ + + \hspace{.2em}|} + + + \sqrt{ + + } + + + \overline{) + + } + + + + + + + + + + + \colorbox[rgb]{ + + + + }{$ + + + \textcolor[rgb]{ + + + + }{ + + + + } + + + $} + + + + + + + + + + + + + + + + + + + + + \mathrm{ + + } + + + + + + + + + + + + + + + + + + + + + + \text{ + + } + + + + \phantom{\rule + + [- + + ] + + { + + 0ex + + + }{ + + 0ex + + + }} + + + + + + " + + + " + + + + + + \colorbox[rgb]{ + + + + }{$ + + + \textcolor[rgb]{ + + + + }{ + + + + + \mathrm{ + + + \mathbf{ + + + \mathit{ + + + \mathbit{ + + + \mathbb{ + + + { + + + \mathcal{ + + + \mathsc{ + + + \mathfrak{ + + + \mathsf{ + + + \mathbsf{ + + + \mathsfit{ + + + \mathbsfit{ + + + \mathtt{ + + + { + + + + + + } + + + } + + + $} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + , + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + , + + + + + + + + + + + + + + 0,1,1 + 0,0,0 + 0,0,1 + 1,0,1 + .5,.5,.5 + 0,.5,0 + 0,1,0 + .5,0,0 + 0,0,.5 + .5,.5,0 + .5,0,.5 + 1,0,0 + .75,.75,.75 + 0,.5,.5 + 1,1,1 + 1,1,0 + + Exception at color template + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Exception at Hex2Decimal template + + + + + + + + + + + diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/simple_mmlctop.xsl b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/simple_mmlctop.xsl new file mode 100644 index 0000000000000000000000000000000000000000..0cd73bccc24c963ed9c6c4121cc410997b94261c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/mathml/data/simple_mmlctop.xsl @@ -0,0 +1,3166 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + e + + + + + + + + + + + + + - + + + + + + + + &#x2062; + &#x2148; + + + + + + + + + + + + + - + + + + + + + + &#x2062; + &#x2148; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Polar + &#x2062; + + + + + + + + + + + + + + + Polar + &#x2062; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [ + + + ] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x03BB; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2218; + + + + + + + + + + + + + + + + + + &#x2218; + + + + + + + + + + + + + + + + id + + + id + + + + + + + + + + + + + + domain + + + codomain + + + image + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + { + + + + + + + + if + + + + + + + + + + + otherwise + + + + + + + + + + + + + + + + + + + + + + + &#x230A; + + + + + + + + + + + + + + + + + + + &#x230B; + + + + + + + + + + + + e + + + + + + + + + + + + + + + + + ! + + + + + + + + + + + + + + + + max + + + min + + + + + + + max + + + min + + + + + + + + + + + + + | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2062; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + gcd + + + lcm + + + + + + gcd + + + lcm + + + + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2227; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2228; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x22BB; + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + &#x00AC; + &#x2061; + + + + + + + + + + + + + + + &#x00AC; + &#x2061; + + + + + + + + + + + + + + + + + + + &#x2200; + + + + + + + + + + + + : + + + + , + + + + + + + + + + + + + + &#x2203; + + + + + + + + + + + + : + + + + , + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00AF; + + + + + + + + + + + + + + + + + &#x211C; + + + &#x2111; + + + &#x2061; + + + + + + + + + + + + + + + + + &#x230A; + + + &#x2308; + + + + + + &#x230B; + + + &#x2309; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2260; + + + &#x2248; + + + &#x2223; + + + + + &#x2198; + + + &#x2197; + + + &#x2192; + + + + + &#x21D2; + + + &#x2208; + + + &#x2209; + + + &#x2284; + + + &#x2288; + + + + + + + + + + &#x2286; + + + &#x2282; + + + + + + + + + + + + &#x2265; + + + &#x2264; + + + &#x2261; + + + + + + + + + + + + + + + + + + + + + + ln + + + + + ln + + + + + + + + + + + + + + + + + + + + + + log + + + + + + log + + + + + + + + log + + + + log + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + d + + d + + + + + + + + d + + + + d + + + + + + + + + + + + + + + + + + &#x2032; + + + + + + + + + + + + + + + + + &#x2145; + + + + + + + + &#x2202; + + + + + &#x2202; + + + + + + + + + + + + + + + + + + + &#x2202; + + + + &#x2202; + + + + + + + + + + &#x2202; + + &#x2202; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2207; + 2 + + &#x2061; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x222A; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2229; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00D7; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + = + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + + + + + + + &#x2211; + + + &#x220F; + + + + + + + + + + + + + + + + + + + + + + + + &#x222B; + + + + + + &#x222B; + + + + + + &#x222B; + + + + + + + + + + + + &#x222B; + + + + + + + + + &#x222B; + + + + + + + + + d + + + + + + + + + + + + + + + + lim + + + + &#x2192; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x03C3; + + + + + + + + + + + + + + + + + &#x03C3; + + + + + + + 2 + + + + + + + + + + + + + median + + + + + + + + + + + + + + + + + mode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + det + + + + + + + + + + + + + + + T + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &#x00D7; + + + &#x22C5; + + + &#x2297; + + + + + + + + + + + + &#x2124; + + + + &#x211D; + + + + &#x211A; + + + + &#x2115; + + + + &#x2102; + + + + &#x2119; + + + + e + + + + &#x2148; + + + + NaN + + + + true + + + + false + + + + &#x2205; + + + + &#x03C0; + + + + &#x213D; + + + + &#x221E; + + + diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d9f03ce0a84cf6b9336e7378bf281326ffcc9df Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_autowrap.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_autowrap.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f4394f9d94ab48b5e48506b291c6593c5e579b0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_autowrap.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4aa9c997ef82f9521251fadc0db9ed2659b079f5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_julia.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_julia.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b75bf826db657f6e980a6f784ed968009177d170 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_julia.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_octave.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_octave.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9223810d2c5bc1db7608c554edd2920386196f0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_octave.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_rust.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_rust.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..591d777189b09eaff405a0f5408e6cfe4558ea7c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_codegen_rust.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_decorator.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_decorator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd14accd18265304df8f0342cf9cf657d41e7987 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_decorator.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_deprecated.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_deprecated.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c93977b8be9d5d59044b3810b26f828b02a92a0a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_deprecated.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_enumerative.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_enumerative.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0051f52ab0b9f1a3ab550a0890c4f64de0bfd8ad Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_enumerative.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_exceptions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78cf0883816effcc1dbcbd1b5344ba111fd1f2e7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_exceptions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_iterables.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_iterables.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78b0b17b13d20410a0f93e7595c6875fe1c755b2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_iterables.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_matchpy_connector.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_matchpy_connector.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02cbb37830e1f003b61e080f182f99db298e7945 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_matchpy_connector.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_mathml.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_mathml.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd75e856bb5c8ce204cd98d7626b898670d28141 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_mathml.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_misc.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_misc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fe10ce7e9c3d620292b34b38d52e1bd8026111f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_misc.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_pickling.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_pickling.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b743b85834240282533c134bf4564cfaeda9cde Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_pickling.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_source.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_source.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f740861abff872b15b0a8883045beaf48b3dddf6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_source.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_timeutils.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_timeutils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..281539b5699ecbf578b6728848e204c71f044fba Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_timeutils.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_xxe.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_xxe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2429e97c5b84b7978c2309b8f353419f03c4258b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/__pycache__/test_xxe.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_autowrap.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_autowrap.py new file mode 100644 index 0000000000000000000000000000000000000000..b5a33d77adb46710cfa3cfeb1ea39402e35c76cf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_autowrap.py @@ -0,0 +1,467 @@ +# Tests that require installed backends go into +# sympy/test_external/test_autowrap + +import os +import tempfile +import shutil +from io import StringIO +from pathlib import Path + +from sympy.core import symbols, Eq +from sympy.utilities.autowrap import (autowrap, binary_function, + CythonCodeWrapper, UfuncifyCodeWrapper, CodeWrapper) +from sympy.utilities.codegen import ( + CCodeGen, C99CodeGen, CodeGenArgumentListError, make_routine +) +from sympy.testing.pytest import raises +from sympy.testing.tmpfiles import TmpFileManager + + +def get_string(dump_fn, routines, prefix="file", **kwargs): + """Wrapper for dump_fn. dump_fn writes its results to a stream object and + this wrapper returns the contents of that stream as a string. This + auxiliary function is used by many tests below. + + The header and the empty lines are not generator to facilitate the + testing of the output. + """ + output = StringIO() + dump_fn(routines, output, prefix, **kwargs) + source = output.getvalue() + output.close() + return source + + +def test_cython_wrapper_scalar_function(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + routine = make_routine("test", expr) + code_gen = CythonCodeWrapper(CCodeGen()) + source = get_string(code_gen.dump_pyx, [routine]) + + expected = ( + "cdef extern from 'file.h':\n" + " double test(double x, double y, double z)\n" + "\n" + "def test_c(double x, double y, double z):\n" + "\n" + " return test(x, y, z)") + assert source == expected + + +def test_cython_wrapper_outarg(): + from sympy.core.relational import Equality + x, y, z = symbols('x,y,z') + code_gen = CythonCodeWrapper(C99CodeGen()) + + routine = make_routine("test", Equality(z, x + y)) + source = get_string(code_gen.dump_pyx, [routine]) + expected = ( + "cdef extern from 'file.h':\n" + " void test(double x, double y, double *z)\n" + "\n" + "def test_c(double x, double y):\n" + "\n" + " cdef double z = 0\n" + " test(x, y, &z)\n" + " return z") + assert source == expected + + +def test_cython_wrapper_inoutarg(): + from sympy.core.relational import Equality + x, y, z = symbols('x,y,z') + code_gen = CythonCodeWrapper(C99CodeGen()) + routine = make_routine("test", Equality(z, x + y + z)) + source = get_string(code_gen.dump_pyx, [routine]) + expected = ( + "cdef extern from 'file.h':\n" + " void test(double x, double y, double *z)\n" + "\n" + "def test_c(double x, double y, double z):\n" + "\n" + " test(x, y, &z)\n" + " return z") + assert source == expected + + +def test_cython_wrapper_compile_flags(): + from sympy.core.relational import Equality + x, y, z = symbols('x,y,z') + routine = make_routine("test", Equality(z, x + y)) + + code_gen = CythonCodeWrapper(CCodeGen()) + + expected = """\ +from setuptools import setup +from setuptools import Extension +from Cython.Build import cythonize +cy_opts = {'compiler_directives': {'language_level': '3'}} + +ext_mods = [Extension( + 'wrapper_module_%(num)s', ['wrapper_module_%(num)s.pyx', 'wrapped_code_%(num)s.c'], + include_dirs=[], + library_dirs=[], + libraries=[], + extra_compile_args=['-std=c99'], + extra_link_args=[] +)] +setup(ext_modules=cythonize(ext_mods, **cy_opts)) +""" % {'num': CodeWrapper._module_counter} + + temp_dir = tempfile.mkdtemp() + TmpFileManager.tmp_folder(temp_dir) + setup_file_path = os.path.join(temp_dir, 'setup.py') + + code_gen._prepare_files(routine, build_dir=temp_dir) + setup_text = Path(setup_file_path).read_text() + assert setup_text == expected + + code_gen = CythonCodeWrapper(CCodeGen(), + include_dirs=['/usr/local/include', '/opt/booger/include'], + library_dirs=['/user/local/lib'], + libraries=['thelib', 'nilib'], + extra_compile_args=['-slow-math'], + extra_link_args=['-lswamp', '-ltrident'], + cythonize_options={'compiler_directives': {'boundscheck': False}} + ) + expected = """\ +from setuptools import setup +from setuptools import Extension +from Cython.Build import cythonize +cy_opts = {'compiler_directives': {'boundscheck': False}} + +ext_mods = [Extension( + 'wrapper_module_%(num)s', ['wrapper_module_%(num)s.pyx', 'wrapped_code_%(num)s.c'], + include_dirs=['/usr/local/include', '/opt/booger/include'], + library_dirs=['/user/local/lib'], + libraries=['thelib', 'nilib'], + extra_compile_args=['-slow-math', '-std=c99'], + extra_link_args=['-lswamp', '-ltrident'] +)] +setup(ext_modules=cythonize(ext_mods, **cy_opts)) +""" % {'num': CodeWrapper._module_counter} + + code_gen._prepare_files(routine, build_dir=temp_dir) + setup_text = Path(setup_file_path).read_text() + assert setup_text == expected + + expected = """\ +from setuptools import setup +from setuptools import Extension +from Cython.Build import cythonize +cy_opts = {'compiler_directives': {'boundscheck': False}} +import numpy as np + +ext_mods = [Extension( + 'wrapper_module_%(num)s', ['wrapper_module_%(num)s.pyx', 'wrapped_code_%(num)s.c'], + include_dirs=['/usr/local/include', '/opt/booger/include', np.get_include()], + library_dirs=['/user/local/lib'], + libraries=['thelib', 'nilib'], + extra_compile_args=['-slow-math', '-std=c99'], + extra_link_args=['-lswamp', '-ltrident'] +)] +setup(ext_modules=cythonize(ext_mods, **cy_opts)) +""" % {'num': CodeWrapper._module_counter} + + code_gen._need_numpy = True + code_gen._prepare_files(routine, build_dir=temp_dir) + setup_text = Path(setup_file_path).read_text() + assert setup_text == expected + + TmpFileManager.cleanup() + +def test_cython_wrapper_unique_dummyvars(): + from sympy.core.relational import Equality + from sympy.core.symbol import Dummy + x, y, z = Dummy('x'), Dummy('y'), Dummy('z') + x_id, y_id, z_id = [str(d.dummy_index) for d in [x, y, z]] + expr = Equality(z, x + y) + routine = make_routine("test", expr) + code_gen = CythonCodeWrapper(CCodeGen()) + source = get_string(code_gen.dump_pyx, [routine]) + expected_template = ( + "cdef extern from 'file.h':\n" + " void test(double x_{x_id}, double y_{y_id}, double *z_{z_id})\n" + "\n" + "def test_c(double x_{x_id}, double y_{y_id}):\n" + "\n" + " cdef double z_{z_id} = 0\n" + " test(x_{x_id}, y_{y_id}, &z_{z_id})\n" + " return z_{z_id}") + expected = expected_template.format(x_id=x_id, y_id=y_id, z_id=z_id) + assert source == expected + +def test_autowrap_dummy(): + x, y, z = symbols('x y z') + + # Uses DummyWrapper to test that codegen works as expected + + f = autowrap(x + y, backend='dummy') + assert f() == str(x + y) + assert f.args == "x, y" + assert f.returns == "nameless" + f = autowrap(Eq(z, x + y), backend='dummy') + assert f() == str(x + y) + assert f.args == "x, y" + assert f.returns == "z" + f = autowrap(Eq(z, x + y + z), backend='dummy') + assert f() == str(x + y + z) + assert f.args == "x, y, z" + assert f.returns == "z" + + +def test_autowrap_args(): + x, y, z = symbols('x y z') + + raises(CodeGenArgumentListError, lambda: autowrap(Eq(z, x + y), + backend='dummy', args=[x])) + f = autowrap(Eq(z, x + y), backend='dummy', args=[y, x]) + assert f() == str(x + y) + assert f.args == "y, x" + assert f.returns == "z" + + raises(CodeGenArgumentListError, lambda: autowrap(Eq(z, x + y + z), + backend='dummy', args=[x, y])) + f = autowrap(Eq(z, x + y + z), backend='dummy', args=[y, x, z]) + assert f() == str(x + y + z) + assert f.args == "y, x, z" + assert f.returns == "z" + + f = autowrap(Eq(z, x + y + z), backend='dummy', args=(y, x, z)) + assert f() == str(x + y + z) + assert f.args == "y, x, z" + assert f.returns == "z" + +def test_autowrap_store_files(): + x, y = symbols('x y') + tmp = tempfile.mkdtemp() + TmpFileManager.tmp_folder(tmp) + + f = autowrap(x + y, backend='dummy', tempdir=tmp) + assert f() == str(x + y) + assert os.access(tmp, os.F_OK) + + TmpFileManager.cleanup() + +def test_autowrap_store_files_issue_gh12939(): + x, y = symbols('x y') + tmp = './tmp' + saved_cwd = os.getcwd() + temp_cwd = tempfile.mkdtemp() + try: + os.chdir(temp_cwd) + f = autowrap(x + y, backend='dummy', tempdir=tmp) + assert f() == str(x + y) + assert os.access(tmp, os.F_OK) + finally: + os.chdir(saved_cwd) + shutil.rmtree(temp_cwd) + + +def test_binary_function(): + x, y = symbols('x y') + f = binary_function('f', x + y, backend='dummy') + assert f._imp_() == str(x + y) + + +def test_ufuncify_source(): + x, y, z = symbols('x,y,z') + code_wrapper = UfuncifyCodeWrapper(C99CodeGen("ufuncify")) + routine = make_routine("test", x + y + z) + source = get_string(code_wrapper.dump_c, [routine]) + expected = """\ +#include "Python.h" +#include "math.h" +#include "numpy/ndarraytypes.h" +#include "numpy/ufuncobject.h" +#include "numpy/halffloat.h" +#include "file.h" + +static PyMethodDef wrapper_module_%(num)sMethods[] = { + {NULL, NULL, 0, NULL} +}; + +#ifdef NPY_1_19_API_VERSION +static void test_ufunc(char **args, const npy_intp *dimensions, const npy_intp* steps, void* data) +#else +static void test_ufunc(char **args, npy_intp *dimensions, npy_intp* steps, void* data) +#endif +{ + npy_intp i; + npy_intp n = dimensions[0]; + char *in0 = args[0]; + char *in1 = args[1]; + char *in2 = args[2]; + char *out0 = args[3]; + npy_intp in0_step = steps[0]; + npy_intp in1_step = steps[1]; + npy_intp in2_step = steps[2]; + npy_intp out0_step = steps[3]; + for (i = 0; i < n; i++) { + *((double *)out0) = test(*(double *)in0, *(double *)in1, *(double *)in2); + in0 += in0_step; + in1 += in1_step; + in2 += in2_step; + out0 += out0_step; + } +} +PyUFuncGenericFunction test_funcs[1] = {&test_ufunc}; +static char test_types[4] = {NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE}; +static void *test_data[1] = {NULL}; + +#if PY_VERSION_HEX >= 0x03000000 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "wrapper_module_%(num)s", + NULL, + -1, + wrapper_module_%(num)sMethods, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC PyInit_wrapper_module_%(num)s(void) +{ + PyObject *m, *d; + PyObject *ufunc0; + m = PyModule_Create(&moduledef); + if (!m) { + return NULL; + } + import_array(); + import_umath(); + d = PyModule_GetDict(m); + ufunc0 = PyUFunc_FromFuncAndData(test_funcs, test_data, test_types, 1, 3, 1, + PyUFunc_None, "wrapper_module_%(num)s", "Created in SymPy with Ufuncify", 0); + PyDict_SetItemString(d, "test", ufunc0); + Py_DECREF(ufunc0); + return m; +} +#else +PyMODINIT_FUNC initwrapper_module_%(num)s(void) +{ + PyObject *m, *d; + PyObject *ufunc0; + m = Py_InitModule("wrapper_module_%(num)s", wrapper_module_%(num)sMethods); + if (m == NULL) { + return; + } + import_array(); + import_umath(); + d = PyModule_GetDict(m); + ufunc0 = PyUFunc_FromFuncAndData(test_funcs, test_data, test_types, 1, 3, 1, + PyUFunc_None, "wrapper_module_%(num)s", "Created in SymPy with Ufuncify", 0); + PyDict_SetItemString(d, "test", ufunc0); + Py_DECREF(ufunc0); +} +#endif""" % {'num': CodeWrapper._module_counter} + assert source == expected + + +def test_ufuncify_source_multioutput(): + x, y, z = symbols('x,y,z') + var_symbols = (x, y, z) + expr = x + y**3 + 10*z**2 + code_wrapper = UfuncifyCodeWrapper(C99CodeGen("ufuncify")) + routines = [make_routine("func{}".format(i), expr.diff(var_symbols[i]), var_symbols) for i in range(len(var_symbols))] + source = get_string(code_wrapper.dump_c, routines, funcname='multitest') + expected = """\ +#include "Python.h" +#include "math.h" +#include "numpy/ndarraytypes.h" +#include "numpy/ufuncobject.h" +#include "numpy/halffloat.h" +#include "file.h" + +static PyMethodDef wrapper_module_%(num)sMethods[] = { + {NULL, NULL, 0, NULL} +}; + +#ifdef NPY_1_19_API_VERSION +static void multitest_ufunc(char **args, const npy_intp *dimensions, const npy_intp* steps, void* data) +#else +static void multitest_ufunc(char **args, npy_intp *dimensions, npy_intp* steps, void* data) +#endif +{ + npy_intp i; + npy_intp n = dimensions[0]; + char *in0 = args[0]; + char *in1 = args[1]; + char *in2 = args[2]; + char *out0 = args[3]; + char *out1 = args[4]; + char *out2 = args[5]; + npy_intp in0_step = steps[0]; + npy_intp in1_step = steps[1]; + npy_intp in2_step = steps[2]; + npy_intp out0_step = steps[3]; + npy_intp out1_step = steps[4]; + npy_intp out2_step = steps[5]; + for (i = 0; i < n; i++) { + *((double *)out0) = func0(*(double *)in0, *(double *)in1, *(double *)in2); + *((double *)out1) = func1(*(double *)in0, *(double *)in1, *(double *)in2); + *((double *)out2) = func2(*(double *)in0, *(double *)in1, *(double *)in2); + in0 += in0_step; + in1 += in1_step; + in2 += in2_step; + out0 += out0_step; + out1 += out1_step; + out2 += out2_step; + } +} +PyUFuncGenericFunction multitest_funcs[1] = {&multitest_ufunc}; +static char multitest_types[6] = {NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE, NPY_DOUBLE}; +static void *multitest_data[1] = {NULL}; + +#if PY_VERSION_HEX >= 0x03000000 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "wrapper_module_%(num)s", + NULL, + -1, + wrapper_module_%(num)sMethods, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC PyInit_wrapper_module_%(num)s(void) +{ + PyObject *m, *d; + PyObject *ufunc0; + m = PyModule_Create(&moduledef); + if (!m) { + return NULL; + } + import_array(); + import_umath(); + d = PyModule_GetDict(m); + ufunc0 = PyUFunc_FromFuncAndData(multitest_funcs, multitest_data, multitest_types, 1, 3, 3, + PyUFunc_None, "wrapper_module_%(num)s", "Created in SymPy with Ufuncify", 0); + PyDict_SetItemString(d, "multitest", ufunc0); + Py_DECREF(ufunc0); + return m; +} +#else +PyMODINIT_FUNC initwrapper_module_%(num)s(void) +{ + PyObject *m, *d; + PyObject *ufunc0; + m = Py_InitModule("wrapper_module_%(num)s", wrapper_module_%(num)sMethods); + if (m == NULL) { + return; + } + import_array(); + import_umath(); + d = PyModule_GetDict(m); + ufunc0 = PyUFunc_FromFuncAndData(multitest_funcs, multitest_data, multitest_types, 1, 3, 3, + PyUFunc_None, "wrapper_module_%(num)s", "Created in SymPy with Ufuncify", 0); + PyDict_SetItemString(d, "multitest", ufunc0); + Py_DECREF(ufunc0); +} +#endif""" % {'num': CodeWrapper._module_counter} + assert source == expected diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen.py new file mode 100644 index 0000000000000000000000000000000000000000..4ccc6f9a90fb0a0bec39cea22420da8091ede740 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen.py @@ -0,0 +1,1632 @@ +from io import StringIO + +from sympy.core import symbols, Eq, pi, Catalan, Lambda, Dummy +from sympy.core.relational import Equality +from sympy.core.symbol import Symbol +from sympy.functions.special.error_functions import erf +from sympy.integrals.integrals import Integral +from sympy.matrices import Matrix, MatrixSymbol +from sympy.utilities.codegen import ( + codegen, make_routine, CCodeGen, C89CodeGen, C99CodeGen, InputArgument, + CodeGenError, FCodeGen, CodeGenArgumentListError, OutputArgument, + InOutArgument) +from sympy.testing.pytest import raises +from sympy.utilities.lambdify import implemented_function + +#FIXME: Fails due to circular import in with core +# from sympy import codegen + + +def get_string(dump_fn, routines, prefix="file", header=False, empty=False): + """Wrapper for dump_fn. dump_fn writes its results to a stream object and + this wrapper returns the contents of that stream as a string. This + auxiliary function is used by many tests below. + + The header and the empty lines are not generated to facilitate the + testing of the output. + """ + output = StringIO() + dump_fn(routines, output, prefix, header, empty) + source = output.getvalue() + output.close() + return source + + +def test_Routine_argument_order(): + a, x, y, z = symbols('a x y z') + expr = (x + y)*z + raises(CodeGenArgumentListError, lambda: make_routine("test", expr, + argument_sequence=[z, x])) + raises(CodeGenArgumentListError, lambda: make_routine("test", Eq(a, + expr), argument_sequence=[z, x, y])) + r = make_routine('test', Eq(a, expr), argument_sequence=[z, x, a, y]) + assert [ arg.name for arg in r.arguments ] == [z, x, a, y] + assert [ type(arg) for arg in r.arguments ] == [ + InputArgument, InputArgument, OutputArgument, InputArgument ] + r = make_routine('test', Eq(z, expr), argument_sequence=[z, x, y]) + assert [ type(arg) for arg in r.arguments ] == [ + InOutArgument, InputArgument, InputArgument ] + + from sympy.tensor import IndexedBase, Idx + A, B = map(IndexedBase, ['A', 'B']) + m = symbols('m', integer=True) + i = Idx('i', m) + r = make_routine('test', Eq(A[i], B[i]), argument_sequence=[B, A, m]) + assert [ arg.name for arg in r.arguments ] == [B.label, A.label, m] + + expr = Integral(x*y*z, (x, 1, 2), (y, 1, 3)) + r = make_routine('test', Eq(a, expr), argument_sequence=[z, x, a, y]) + assert [ arg.name for arg in r.arguments ] == [z, x, a, y] + + +def test_empty_c_code(): + code_gen = C89CodeGen() + source = get_string(code_gen.dump_c, []) + assert source == "#include \"file.h\"\n#include \n" + + +def test_empty_c_code_with_comment(): + code_gen = C89CodeGen() + source = get_string(code_gen.dump_c, [], header=True) + assert source[:82] == ( + "/******************************************************************************\n *" + ) + # " Code generated with SymPy 0.7.2-git " + assert source[158:] == ( "*\n" + " * *\n" + " * See http://www.sympy.org/ for more information. *\n" + " * *\n" + " * This file is part of 'project' *\n" + " ******************************************************************************/\n" + "#include \"file.h\"\n" + "#include \n" + ) + + +def test_empty_c_header(): + code_gen = C99CodeGen() + source = get_string(code_gen.dump_h, []) + assert source == "#ifndef PROJECT__FILE__H\n#define PROJECT__FILE__H\n#endif\n" + + +def test_simple_c_code(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + routine = make_routine("test", expr) + code_gen = C89CodeGen() + source = get_string(code_gen.dump_c, [routine]) + expected = ( + "#include \"file.h\"\n" + "#include \n" + "double test(double x, double y, double z) {\n" + " double test_result;\n" + " test_result = z*(x + y);\n" + " return test_result;\n" + "}\n" + ) + assert source == expected + + +def test_c_code_reserved_words(): + x, y, z = symbols('if, typedef, while') + expr = (x + y) * z + routine = make_routine("test", expr) + code_gen = C99CodeGen() + source = get_string(code_gen.dump_c, [routine]) + expected = ( + "#include \"file.h\"\n" + "#include \n" + "double test(double if_, double typedef_, double while_) {\n" + " double test_result;\n" + " test_result = while_*(if_ + typedef_);\n" + " return test_result;\n" + "}\n" + ) + assert source == expected + + +def test_numbersymbol_c_code(): + routine = make_routine("test", pi**Catalan) + code_gen = C89CodeGen() + source = get_string(code_gen.dump_c, [routine]) + expected = ( + "#include \"file.h\"\n" + "#include \n" + "double test() {\n" + " double test_result;\n" + " double const Catalan = %s;\n" + " test_result = pow(M_PI, Catalan);\n" + " return test_result;\n" + "}\n" + ) % Catalan.evalf(17) + assert source == expected + + +def test_c_code_argument_order(): + x, y, z = symbols('x,y,z') + expr = x + y + routine = make_routine("test", expr, argument_sequence=[z, x, y]) + code_gen = C89CodeGen() + source = get_string(code_gen.dump_c, [routine]) + expected = ( + "#include \"file.h\"\n" + "#include \n" + "double test(double z, double x, double y) {\n" + " double test_result;\n" + " test_result = x + y;\n" + " return test_result;\n" + "}\n" + ) + assert source == expected + + +def test_simple_c_header(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + routine = make_routine("test", expr) + code_gen = C89CodeGen() + source = get_string(code_gen.dump_h, [routine]) + expected = ( + "#ifndef PROJECT__FILE__H\n" + "#define PROJECT__FILE__H\n" + "double test(double x, double y, double z);\n" + "#endif\n" + ) + assert source == expected + + +def test_simple_c_codegen(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + expected = [ + ("file.c", + "#include \"file.h\"\n" + "#include \n" + "double test(double x, double y, double z) {\n" + " double test_result;\n" + " test_result = z*(x + y);\n" + " return test_result;\n" + "}\n"), + ("file.h", + "#ifndef PROJECT__FILE__H\n" + "#define PROJECT__FILE__H\n" + "double test(double x, double y, double z);\n" + "#endif\n") + ] + result = codegen(("test", expr), "C", "file", header=False, empty=False) + assert result == expected + + +def test_multiple_results_c(): + x, y, z = symbols('x,y,z') + expr1 = (x + y)*z + expr2 = (x - y)*z + routine = make_routine( + "test", + [expr1, expr2] + ) + code_gen = C99CodeGen() + raises(CodeGenError, lambda: get_string(code_gen.dump_h, [routine])) + + +def test_no_results_c(): + raises(ValueError, lambda: make_routine("test", [])) + + +def test_ansi_math1_codegen(): + # not included: log10 + from sympy.functions.elementary.complexes import Abs + from sympy.functions.elementary.exponential import log + from sympy.functions.elementary.hyperbolic import (cosh, sinh, tanh) + from sympy.functions.elementary.integers import (ceiling, floor) + from sympy.functions.elementary.miscellaneous import sqrt + from sympy.functions.elementary.trigonometric import (acos, asin, atan, cos, sin, tan) + x = symbols('x') + name_expr = [ + ("test_fabs", Abs(x)), + ("test_acos", acos(x)), + ("test_asin", asin(x)), + ("test_atan", atan(x)), + ("test_ceil", ceiling(x)), + ("test_cos", cos(x)), + ("test_cosh", cosh(x)), + ("test_floor", floor(x)), + ("test_log", log(x)), + ("test_ln", log(x)), + ("test_sin", sin(x)), + ("test_sinh", sinh(x)), + ("test_sqrt", sqrt(x)), + ("test_tan", tan(x)), + ("test_tanh", tanh(x)), + ] + result = codegen(name_expr, "C89", "file", header=False, empty=False) + assert result[0][0] == "file.c" + assert result[0][1] == ( + '#include "file.h"\n#include \n' + 'double test_fabs(double x) {\n double test_fabs_result;\n test_fabs_result = fabs(x);\n return test_fabs_result;\n}\n' + 'double test_acos(double x) {\n double test_acos_result;\n test_acos_result = acos(x);\n return test_acos_result;\n}\n' + 'double test_asin(double x) {\n double test_asin_result;\n test_asin_result = asin(x);\n return test_asin_result;\n}\n' + 'double test_atan(double x) {\n double test_atan_result;\n test_atan_result = atan(x);\n return test_atan_result;\n}\n' + 'double test_ceil(double x) {\n double test_ceil_result;\n test_ceil_result = ceil(x);\n return test_ceil_result;\n}\n' + 'double test_cos(double x) {\n double test_cos_result;\n test_cos_result = cos(x);\n return test_cos_result;\n}\n' + 'double test_cosh(double x) {\n double test_cosh_result;\n test_cosh_result = cosh(x);\n return test_cosh_result;\n}\n' + 'double test_floor(double x) {\n double test_floor_result;\n test_floor_result = floor(x);\n return test_floor_result;\n}\n' + 'double test_log(double x) {\n double test_log_result;\n test_log_result = log(x);\n return test_log_result;\n}\n' + 'double test_ln(double x) {\n double test_ln_result;\n test_ln_result = log(x);\n return test_ln_result;\n}\n' + 'double test_sin(double x) {\n double test_sin_result;\n test_sin_result = sin(x);\n return test_sin_result;\n}\n' + 'double test_sinh(double x) {\n double test_sinh_result;\n test_sinh_result = sinh(x);\n return test_sinh_result;\n}\n' + 'double test_sqrt(double x) {\n double test_sqrt_result;\n test_sqrt_result = sqrt(x);\n return test_sqrt_result;\n}\n' + 'double test_tan(double x) {\n double test_tan_result;\n test_tan_result = tan(x);\n return test_tan_result;\n}\n' + 'double test_tanh(double x) {\n double test_tanh_result;\n test_tanh_result = tanh(x);\n return test_tanh_result;\n}\n' + ) + assert result[1][0] == "file.h" + assert result[1][1] == ( + '#ifndef PROJECT__FILE__H\n#define PROJECT__FILE__H\n' + 'double test_fabs(double x);\ndouble test_acos(double x);\n' + 'double test_asin(double x);\ndouble test_atan(double x);\n' + 'double test_ceil(double x);\ndouble test_cos(double x);\n' + 'double test_cosh(double x);\ndouble test_floor(double x);\n' + 'double test_log(double x);\ndouble test_ln(double x);\n' + 'double test_sin(double x);\ndouble test_sinh(double x);\n' + 'double test_sqrt(double x);\ndouble test_tan(double x);\n' + 'double test_tanh(double x);\n#endif\n' + ) + + +def test_ansi_math2_codegen(): + # not included: frexp, ldexp, modf, fmod + from sympy.functions.elementary.trigonometric import atan2 + x, y = symbols('x,y') + name_expr = [ + ("test_atan2", atan2(x, y)), + ("test_pow", x**y), + ] + result = codegen(name_expr, "C89", "file", header=False, empty=False) + assert result[0][0] == "file.c" + assert result[0][1] == ( + '#include "file.h"\n#include \n' + 'double test_atan2(double x, double y) {\n double test_atan2_result;\n test_atan2_result = atan2(x, y);\n return test_atan2_result;\n}\n' + 'double test_pow(double x, double y) {\n double test_pow_result;\n test_pow_result = pow(x, y);\n return test_pow_result;\n}\n' + ) + assert result[1][0] == "file.h" + assert result[1][1] == ( + '#ifndef PROJECT__FILE__H\n#define PROJECT__FILE__H\n' + 'double test_atan2(double x, double y);\n' + 'double test_pow(double x, double y);\n' + '#endif\n' + ) + + +def test_complicated_codegen(): + from sympy.functions.elementary.trigonometric import (cos, sin, tan) + x, y, z = symbols('x,y,z') + name_expr = [ + ("test1", ((sin(x) + cos(y) + tan(z))**7).expand()), + ("test2", cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))))), + ] + result = codegen(name_expr, "C89", "file", header=False, empty=False) + assert result[0][0] == "file.c" + assert result[0][1] == ( + '#include "file.h"\n#include \n' + 'double test1(double x, double y, double z) {\n' + ' double test1_result;\n' + ' test1_result = ' + 'pow(sin(x), 7) + ' + '7*pow(sin(x), 6)*cos(y) + ' + '7*pow(sin(x), 6)*tan(z) + ' + '21*pow(sin(x), 5)*pow(cos(y), 2) + ' + '42*pow(sin(x), 5)*cos(y)*tan(z) + ' + '21*pow(sin(x), 5)*pow(tan(z), 2) + ' + '35*pow(sin(x), 4)*pow(cos(y), 3) + ' + '105*pow(sin(x), 4)*pow(cos(y), 2)*tan(z) + ' + '105*pow(sin(x), 4)*cos(y)*pow(tan(z), 2) + ' + '35*pow(sin(x), 4)*pow(tan(z), 3) + ' + '35*pow(sin(x), 3)*pow(cos(y), 4) + ' + '140*pow(sin(x), 3)*pow(cos(y), 3)*tan(z) + ' + '210*pow(sin(x), 3)*pow(cos(y), 2)*pow(tan(z), 2) + ' + '140*pow(sin(x), 3)*cos(y)*pow(tan(z), 3) + ' + '35*pow(sin(x), 3)*pow(tan(z), 4) + ' + '21*pow(sin(x), 2)*pow(cos(y), 5) + ' + '105*pow(sin(x), 2)*pow(cos(y), 4)*tan(z) + ' + '210*pow(sin(x), 2)*pow(cos(y), 3)*pow(tan(z), 2) + ' + '210*pow(sin(x), 2)*pow(cos(y), 2)*pow(tan(z), 3) + ' + '105*pow(sin(x), 2)*cos(y)*pow(tan(z), 4) + ' + '21*pow(sin(x), 2)*pow(tan(z), 5) + ' + '7*sin(x)*pow(cos(y), 6) + ' + '42*sin(x)*pow(cos(y), 5)*tan(z) + ' + '105*sin(x)*pow(cos(y), 4)*pow(tan(z), 2) + ' + '140*sin(x)*pow(cos(y), 3)*pow(tan(z), 3) + ' + '105*sin(x)*pow(cos(y), 2)*pow(tan(z), 4) + ' + '42*sin(x)*cos(y)*pow(tan(z), 5) + ' + '7*sin(x)*pow(tan(z), 6) + ' + 'pow(cos(y), 7) + ' + '7*pow(cos(y), 6)*tan(z) + ' + '21*pow(cos(y), 5)*pow(tan(z), 2) + ' + '35*pow(cos(y), 4)*pow(tan(z), 3) + ' + '35*pow(cos(y), 3)*pow(tan(z), 4) + ' + '21*pow(cos(y), 2)*pow(tan(z), 5) + ' + '7*cos(y)*pow(tan(z), 6) + ' + 'pow(tan(z), 7);\n' + ' return test1_result;\n' + '}\n' + 'double test2(double x, double y, double z) {\n' + ' double test2_result;\n' + ' test2_result = cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))));\n' + ' return test2_result;\n' + '}\n' + ) + assert result[1][0] == "file.h" + assert result[1][1] == ( + '#ifndef PROJECT__FILE__H\n' + '#define PROJECT__FILE__H\n' + 'double test1(double x, double y, double z);\n' + 'double test2(double x, double y, double z);\n' + '#endif\n' + ) + + +def test_loops_c(): + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m = symbols('n m', integer=True) + A = IndexedBase('A') + x = IndexedBase('x') + y = IndexedBase('y') + i = Idx('i', m) + j = Idx('j', n) + + (f1, code), (f2, interface) = codegen( + ('matrix_vector', Eq(y[i], A[i, j]*x[j])), "C99", "file", header=False, empty=False) + + assert f1 == 'file.c' + expected = ( + '#include "file.h"\n' + '#include \n' + 'void matrix_vector(double *A, int m, int n, double *x, double *y) {\n' + ' for (int i=0; i\n' + 'void test_dummies(int m_%(mno)i, double *x, double *y) {\n' + ' for (int i_%(ino)i=0; i_%(ino)i\n' + 'void matrix_vector(double *A, int m, int n, int o, int p, double *x, double *y) {\n' + ' for (int i=o; i<%(upperi)s; i++){\n' + ' y[i] = 0;\n' + ' }\n' + ' for (int i=o; i<%(upperi)s; i++){\n' + ' for (int j=0; j\n' + 'double foo(double x, double *y) {\n' + ' (*y) = sin(x);\n' + ' double foo_result;\n' + ' foo_result = cos(x);\n' + ' return foo_result;\n' + '}\n' + ) + assert result[0][1] == expected + + +def test_output_arg_c_reserved_words(): + from sympy.core.relational import Equality + from sympy.functions.elementary.trigonometric import (cos, sin) + x, y, z = symbols("if, while, z") + r = make_routine("foo", [Equality(y, sin(x)), cos(x)]) + c = C89CodeGen() + result = c.write([r], "test", header=False, empty=False) + assert result[0][0] == "test.c" + expected = ( + '#include "test.h"\n' + '#include \n' + 'double foo(double if_, double *while_) {\n' + ' (*while_) = sin(if_);\n' + ' double foo_result;\n' + ' foo_result = cos(if_);\n' + ' return foo_result;\n' + '}\n' + ) + assert result[0][1] == expected + + +def test_multidim_c_argument_cse(): + A_sym = MatrixSymbol('A', 3, 3) + b_sym = MatrixSymbol('b', 3, 1) + A = Matrix(A_sym) + b = Matrix(b_sym) + c = A*b + cgen = CCodeGen(project="test", cse=True) + r = cgen.routine("c", c) + r.arguments[-1].result_var = "out" + r.arguments[-1]._name = "out" + code = get_string(cgen.dump_c, [r], prefix="test") + expected = ( + '#include "test.h"\n' + "#include \n" + "void c(double *A, double *b, double *out) {\n" + " out[0] = A[0]*b[0] + A[1]*b[1] + A[2]*b[2];\n" + " out[1] = A[3]*b[0] + A[4]*b[1] + A[5]*b[2];\n" + " out[2] = A[6]*b[0] + A[7]*b[1] + A[8]*b[2];\n" + "}\n" + ) + assert code == expected + + +def test_ccode_results_named_ordered(): + x, y, z = symbols('x,y,z') + B, C = symbols('B,C') + A = MatrixSymbol('A', 1, 3) + expr1 = Equality(A, Matrix([[1, 2, x]])) + expr2 = Equality(C, (x + y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + expected = ( + '#include "test.h"\n' + '#include \n' + 'void test(double x, double *C, double z, double y, double *A, double *B) {\n' + ' (*C) = z*(x + y);\n' + ' A[0] = 1;\n' + ' A[1] = 2;\n' + ' A[2] = x;\n' + ' (*B) = 2*x;\n' + '}\n' + ) + + result = codegen(name_expr, "c", "test", header=False, empty=False, + argument_sequence=(x, C, z, y, A, B)) + source = result[0][1] + assert source == expected + + +def test_ccode_matrixsymbol_slice(): + A = MatrixSymbol('A', 5, 3) + B = MatrixSymbol('B', 1, 3) + C = MatrixSymbol('C', 1, 3) + D = MatrixSymbol('D', 5, 1) + name_expr = ("test", [Equality(B, A[0, :]), + Equality(C, A[1, :]), + Equality(D, A[:, 2])]) + result = codegen(name_expr, "c99", "test", header=False, empty=False) + source = result[0][1] + expected = ( + '#include "test.h"\n' + '#include \n' + 'void test(double *A, double *B, double *C, double *D) {\n' + ' B[0] = A[0];\n' + ' B[1] = A[1];\n' + ' B[2] = A[2];\n' + ' C[0] = A[3];\n' + ' C[1] = A[4];\n' + ' C[2] = A[5];\n' + ' D[0] = A[2];\n' + ' D[1] = A[5];\n' + ' D[2] = A[8];\n' + ' D[3] = A[11];\n' + ' D[4] = A[14];\n' + '}\n' + ) + assert source == expected + +def test_ccode_cse(): + a, b, c, d = symbols('a b c d') + e = MatrixSymbol('e', 3, 1) + name_expr = ("test", [Equality(e, Matrix([[a*b], [a*b + c*d], [a*b*c*d]]))]) + generator = CCodeGen(cse=True) + result = codegen(name_expr, code_gen=generator, header=False, empty=False) + source = result[0][1] + expected = ( + '#include "test.h"\n' + '#include \n' + 'void test(double a, double b, double c, double d, double *e) {\n' + ' const double x0 = a*b;\n' + ' const double x1 = c*d;\n' + ' e[0] = x0;\n' + ' e[1] = x0 + x1;\n' + ' e[2] = x0*x1;\n' + '}\n' + ) + assert source == expected + +def test_ccode_unused_array_arg(): + x = MatrixSymbol('x', 2, 1) + # x does not appear in output + name_expr = ("test", 1.0) + generator = CCodeGen() + result = codegen(name_expr, code_gen=generator, header=False, empty=False, argument_sequence=(x,)) + source = result[0][1] + # note: x should appear as (double *) + expected = ( + '#include "test.h"\n' + '#include \n' + 'double test(double *x) {\n' + ' double test_result;\n' + ' test_result = 1.0;\n' + ' return test_result;\n' + '}\n' + ) + assert source == expected + +def test_ccode_unused_array_arg_func(): + # issue 16689 + X = MatrixSymbol('X',3,1) + Y = MatrixSymbol('Y',3,1) + z = symbols('z',integer = True) + name_expr = ('testBug', X[0] + X[1]) + result = codegen(name_expr, language='C', header=False, empty=False, argument_sequence=(X, Y, z)) + source = result[0][1] + expected = ( + '#include "testBug.h"\n' + '#include \n' + 'double testBug(double *X, double *Y, int z) {\n' + ' double testBug_result;\n' + ' testBug_result = X[0] + X[1];\n' + ' return testBug_result;\n' + '}\n' + ) + assert source == expected + +def test_empty_f_code(): + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, []) + assert source == "" + + +def test_empty_f_code_with_header(): + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [], header=True) + assert source[:82] == ( + "!******************************************************************************\n!*" + ) + # " Code generated with SymPy 0.7.2-git " + assert source[158:] == ( "*\n" + "!* *\n" + "!* See http://www.sympy.org/ for more information. *\n" + "!* *\n" + "!* This file is part of 'project' *\n" + "!******************************************************************************\n" + ) + + +def test_empty_f_header(): + code_gen = FCodeGen() + source = get_string(code_gen.dump_h, []) + assert source == "" + + +def test_simple_f_code(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + routine = make_routine("test", expr) + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [routine]) + expected = ( + "REAL*8 function test(x, y, z)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(in) :: z\n" + "test = z*(x + y)\n" + "end function\n" + ) + assert source == expected + + +def test_numbersymbol_f_code(): + routine = make_routine("test", pi**Catalan) + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [routine]) + expected = ( + "REAL*8 function test()\n" + "implicit none\n" + "REAL*8, parameter :: Catalan = %sd0\n" + "REAL*8, parameter :: pi = %sd0\n" + "test = pi**Catalan\n" + "end function\n" + ) % (Catalan.evalf(17), pi.evalf(17)) + assert source == expected + +def test_erf_f_code(): + x = symbols('x') + routine = make_routine("test", erf(x) - erf(-2 * x)) + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [routine]) + expected = ( + "REAL*8 function test(x)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "test = erf(x) + erf(2.0d0*x)\n" + "end function\n" + ) + assert source == expected, source + +def test_f_code_argument_order(): + x, y, z = symbols('x,y,z') + expr = x + y + routine = make_routine("test", expr, argument_sequence=[z, x, y]) + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [routine]) + expected = ( + "REAL*8 function test(z, x, y)\n" + "implicit none\n" + "REAL*8, intent(in) :: z\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "test = x + y\n" + "end function\n" + ) + assert source == expected + + +def test_simple_f_header(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + routine = make_routine("test", expr) + code_gen = FCodeGen() + source = get_string(code_gen.dump_h, [routine]) + expected = ( + "interface\n" + "REAL*8 function test(x, y, z)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(in) :: z\n" + "end function\n" + "end interface\n" + ) + assert source == expected + + +def test_simple_f_codegen(): + x, y, z = symbols('x,y,z') + expr = (x + y)*z + result = codegen( + ("test", expr), "F95", "file", header=False, empty=False) + expected = [ + ("file.f90", + "REAL*8 function test(x, y, z)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(in) :: z\n" + "test = z*(x + y)\n" + "end function\n"), + ("file.h", + "interface\n" + "REAL*8 function test(x, y, z)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(in) :: z\n" + "end function\n" + "end interface\n") + ] + assert result == expected + + +def test_multiple_results_f(): + x, y, z = symbols('x,y,z') + expr1 = (x + y)*z + expr2 = (x - y)*z + routine = make_routine( + "test", + [expr1, expr2] + ) + code_gen = FCodeGen() + raises(CodeGenError, lambda: get_string(code_gen.dump_h, [routine])) + + +def test_no_results_f(): + raises(ValueError, lambda: make_routine("test", [])) + + +def test_intrinsic_math_codegen(): + # not included: log10 + from sympy.functions.elementary.complexes import Abs + from sympy.functions.elementary.exponential import log + from sympy.functions.elementary.hyperbolic import (cosh, sinh, tanh) + from sympy.functions.elementary.miscellaneous import sqrt + from sympy.functions.elementary.trigonometric import (acos, asin, atan, cos, sin, tan) + x = symbols('x') + name_expr = [ + ("test_abs", Abs(x)), + ("test_acos", acos(x)), + ("test_asin", asin(x)), + ("test_atan", atan(x)), + ("test_cos", cos(x)), + ("test_cosh", cosh(x)), + ("test_log", log(x)), + ("test_ln", log(x)), + ("test_sin", sin(x)), + ("test_sinh", sinh(x)), + ("test_sqrt", sqrt(x)), + ("test_tan", tan(x)), + ("test_tanh", tanh(x)), + ] + result = codegen(name_expr, "F95", "file", header=False, empty=False) + assert result[0][0] == "file.f90" + expected = ( + 'REAL*8 function test_abs(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_abs = abs(x)\n' + 'end function\n' + 'REAL*8 function test_acos(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_acos = acos(x)\n' + 'end function\n' + 'REAL*8 function test_asin(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_asin = asin(x)\n' + 'end function\n' + 'REAL*8 function test_atan(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_atan = atan(x)\n' + 'end function\n' + 'REAL*8 function test_cos(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_cos = cos(x)\n' + 'end function\n' + 'REAL*8 function test_cosh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_cosh = cosh(x)\n' + 'end function\n' + 'REAL*8 function test_log(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_log = log(x)\n' + 'end function\n' + 'REAL*8 function test_ln(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_ln = log(x)\n' + 'end function\n' + 'REAL*8 function test_sin(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_sin = sin(x)\n' + 'end function\n' + 'REAL*8 function test_sinh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_sinh = sinh(x)\n' + 'end function\n' + 'REAL*8 function test_sqrt(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_sqrt = sqrt(x)\n' + 'end function\n' + 'REAL*8 function test_tan(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_tan = tan(x)\n' + 'end function\n' + 'REAL*8 function test_tanh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'test_tanh = tanh(x)\n' + 'end function\n' + ) + assert result[0][1] == expected + + assert result[1][0] == "file.h" + expected = ( + 'interface\n' + 'REAL*8 function test_abs(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_acos(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_asin(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_atan(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_cos(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_cosh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_log(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_ln(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_sin(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_sinh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_sqrt(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_tan(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_tanh(x)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'end function\n' + 'end interface\n' + ) + assert result[1][1] == expected + + +def test_intrinsic_math2_codegen(): + # not included: frexp, ldexp, modf, fmod + from sympy.functions.elementary.trigonometric import atan2 + x, y = symbols('x,y') + name_expr = [ + ("test_atan2", atan2(x, y)), + ("test_pow", x**y), + ] + result = codegen(name_expr, "F95", "file", header=False, empty=False) + assert result[0][0] == "file.f90" + expected = ( + 'REAL*8 function test_atan2(x, y)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'test_atan2 = atan2(x, y)\n' + 'end function\n' + 'REAL*8 function test_pow(x, y)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'test_pow = x**y\n' + 'end function\n' + ) + assert result[0][1] == expected + + assert result[1][0] == "file.h" + expected = ( + 'interface\n' + 'REAL*8 function test_atan2(x, y)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test_pow(x, y)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'end function\n' + 'end interface\n' + ) + assert result[1][1] == expected + + +def test_complicated_codegen_f95(): + from sympy.functions.elementary.trigonometric import (cos, sin, tan) + x, y, z = symbols('x,y,z') + name_expr = [ + ("test1", ((sin(x) + cos(y) + tan(z))**7).expand()), + ("test2", cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))))), + ] + result = codegen(name_expr, "F95", "file", header=False, empty=False) + assert result[0][0] == "file.f90" + expected = ( + 'REAL*8 function test1(x, y, z)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'REAL*8, intent(in) :: z\n' + 'test1 = sin(x)**7 + 7*sin(x)**6*cos(y) + 7*sin(x)**6*tan(z) + 21*sin(x) &\n' + ' **5*cos(y)**2 + 42*sin(x)**5*cos(y)*tan(z) + 21*sin(x)**5*tan(z) &\n' + ' **2 + 35*sin(x)**4*cos(y)**3 + 105*sin(x)**4*cos(y)**2*tan(z) + &\n' + ' 105*sin(x)**4*cos(y)*tan(z)**2 + 35*sin(x)**4*tan(z)**3 + 35*sin( &\n' + ' x)**3*cos(y)**4 + 140*sin(x)**3*cos(y)**3*tan(z) + 210*sin(x)**3* &\n' + ' cos(y)**2*tan(z)**2 + 140*sin(x)**3*cos(y)*tan(z)**3 + 35*sin(x) &\n' + ' **3*tan(z)**4 + 21*sin(x)**2*cos(y)**5 + 105*sin(x)**2*cos(y)**4* &\n' + ' tan(z) + 210*sin(x)**2*cos(y)**3*tan(z)**2 + 210*sin(x)**2*cos(y) &\n' + ' **2*tan(z)**3 + 105*sin(x)**2*cos(y)*tan(z)**4 + 21*sin(x)**2*tan &\n' + ' (z)**5 + 7*sin(x)*cos(y)**6 + 42*sin(x)*cos(y)**5*tan(z) + 105* &\n' + ' sin(x)*cos(y)**4*tan(z)**2 + 140*sin(x)*cos(y)**3*tan(z)**3 + 105 &\n' + ' *sin(x)*cos(y)**2*tan(z)**4 + 42*sin(x)*cos(y)*tan(z)**5 + 7*sin( &\n' + ' x)*tan(z)**6 + cos(y)**7 + 7*cos(y)**6*tan(z) + 21*cos(y)**5*tan( &\n' + ' z)**2 + 35*cos(y)**4*tan(z)**3 + 35*cos(y)**3*tan(z)**4 + 21*cos( &\n' + ' y)**2*tan(z)**5 + 7*cos(y)*tan(z)**6 + tan(z)**7\n' + 'end function\n' + 'REAL*8 function test2(x, y, z)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'REAL*8, intent(in) :: z\n' + 'test2 = cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))))\n' + 'end function\n' + ) + assert result[0][1] == expected + assert result[1][0] == "file.h" + expected = ( + 'interface\n' + 'REAL*8 function test1(x, y, z)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'REAL*8, intent(in) :: z\n' + 'end function\n' + 'end interface\n' + 'interface\n' + 'REAL*8 function test2(x, y, z)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(in) :: y\n' + 'REAL*8, intent(in) :: z\n' + 'end function\n' + 'end interface\n' + ) + assert result[1][1] == expected + + +def test_loops(): + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + + n, m = symbols('n,m', integer=True) + A, x, y = map(IndexedBase, 'Axy') + i = Idx('i', m) + j = Idx('j', n) + + (f1, code), (f2, interface) = codegen( + ('matrix_vector', Eq(y[i], A[i, j]*x[j])), "F95", "file", header=False, empty=False) + + assert f1 == 'file.f90' + expected = ( + 'subroutine matrix_vector(A, m, n, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'INTEGER*4, intent(in) :: n\n' + 'REAL*8, intent(in), dimension(1:m, 1:n) :: A\n' + 'REAL*8, intent(in), dimension(1:n) :: x\n' + 'REAL*8, intent(out), dimension(1:m) :: y\n' + 'INTEGER*4 :: i\n' + 'INTEGER*4 :: j\n' + 'do i = 1, m\n' + ' y(i) = 0\n' + 'end do\n' + 'do i = 1, m\n' + ' do j = 1, n\n' + ' y(i) = %(rhs)s + y(i)\n' + ' end do\n' + 'end do\n' + 'end subroutine\n' + ) + + assert code == expected % {'rhs': 'A(i, j)*x(j)'} or\ + code == expected % {'rhs': 'x(j)*A(i, j)'} + assert f2 == 'file.h' + assert interface == ( + 'interface\n' + 'subroutine matrix_vector(A, m, n, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'INTEGER*4, intent(in) :: n\n' + 'REAL*8, intent(in), dimension(1:m, 1:n) :: A\n' + 'REAL*8, intent(in), dimension(1:n) :: x\n' + 'REAL*8, intent(out), dimension(1:m) :: y\n' + 'end subroutine\n' + 'end interface\n' + ) + + +def test_dummy_loops_f95(): + from sympy.tensor import IndexedBase, Idx + i, m = symbols('i m', integer=True, cls=Dummy) + x = IndexedBase('x') + y = IndexedBase('y') + i = Idx(i, m) + expected = ( + 'subroutine test_dummies(m_%(mcount)i, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m_%(mcount)i\n' + 'REAL*8, intent(in), dimension(1:m_%(mcount)i) :: x\n' + 'REAL*8, intent(out), dimension(1:m_%(mcount)i) :: y\n' + 'INTEGER*4 :: i_%(icount)i\n' + 'do i_%(icount)i = 1, m_%(mcount)i\n' + ' y(i_%(icount)i) = x(i_%(icount)i)\n' + 'end do\n' + 'end subroutine\n' + ) % {'icount': i.label.dummy_index, 'mcount': m.dummy_index} + r = make_routine('test_dummies', Eq(y[i], x[i])) + c = FCodeGen() + code = get_string(c.dump_f95, [r]) + assert code == expected + + +def test_loops_InOut(): + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + + i, j, n, m = symbols('i,j,n,m', integer=True) + A, x, y = symbols('A,x,y') + A = IndexedBase(A)[Idx(i, m), Idx(j, n)] + x = IndexedBase(x)[Idx(j, n)] + y = IndexedBase(y)[Idx(i, m)] + + (f1, code), (f2, interface) = codegen( + ('matrix_vector', Eq(y, y + A*x)), "F95", "file", header=False, empty=False) + + assert f1 == 'file.f90' + expected = ( + 'subroutine matrix_vector(A, m, n, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'INTEGER*4, intent(in) :: n\n' + 'REAL*8, intent(in), dimension(1:m, 1:n) :: A\n' + 'REAL*8, intent(in), dimension(1:n) :: x\n' + 'REAL*8, intent(inout), dimension(1:m) :: y\n' + 'INTEGER*4 :: i\n' + 'INTEGER*4 :: j\n' + 'do i = 1, m\n' + ' do j = 1, n\n' + ' y(i) = %(rhs)s + y(i)\n' + ' end do\n' + 'end do\n' + 'end subroutine\n' + ) + + assert (code == expected % {'rhs': 'A(i, j)*x(j)'} or + code == expected % {'rhs': 'x(j)*A(i, j)'}) + assert f2 == 'file.h' + assert interface == ( + 'interface\n' + 'subroutine matrix_vector(A, m, n, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'INTEGER*4, intent(in) :: n\n' + 'REAL*8, intent(in), dimension(1:m, 1:n) :: A\n' + 'REAL*8, intent(in), dimension(1:n) :: x\n' + 'REAL*8, intent(inout), dimension(1:m) :: y\n' + 'end subroutine\n' + 'end interface\n' + ) + + +def test_partial_loops_f(): + # check that loop boundaries are determined by Idx, and array strides + # determined by shape of IndexedBase object. + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m, o, p = symbols('n m o p', integer=True) + A = IndexedBase('A', shape=(m, p)) + x = IndexedBase('x') + y = IndexedBase('y') + i = Idx('i', (o, m - 5)) # Note: bounds are inclusive + j = Idx('j', n) # dimension n corresponds to bounds (0, n - 1) + + (f1, code), (f2, interface) = codegen( + ('matrix_vector', Eq(y[i], A[i, j]*x[j])), "F95", "file", header=False, empty=False) + + expected = ( + 'subroutine matrix_vector(A, m, n, o, p, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'INTEGER*4, intent(in) :: n\n' + 'INTEGER*4, intent(in) :: o\n' + 'INTEGER*4, intent(in) :: p\n' + 'REAL*8, intent(in), dimension(1:m, 1:p) :: A\n' + 'REAL*8, intent(in), dimension(1:n) :: x\n' + 'REAL*8, intent(out), dimension(1:%(iup-ilow)s) :: y\n' + 'INTEGER*4 :: i\n' + 'INTEGER*4 :: j\n' + 'do i = %(ilow)s, %(iup)s\n' + ' y(i) = 0\n' + 'end do\n' + 'do i = %(ilow)s, %(iup)s\n' + ' do j = 1, n\n' + ' y(i) = %(rhs)s + y(i)\n' + ' end do\n' + 'end do\n' + 'end subroutine\n' + ) % { + 'rhs': '%(rhs)s', + 'iup': str(m - 4), + 'ilow': str(1 + o), + 'iup-ilow': str(m - 4 - o) + } + + assert code == expected % {'rhs': 'A(i, j)*x(j)'} or\ + code == expected % {'rhs': 'x(j)*A(i, j)'} + + +def test_output_arg_f(): + from sympy.core.relational import Equality + from sympy.functions.elementary.trigonometric import (cos, sin) + x, y, z = symbols("x,y,z") + r = make_routine("foo", [Equality(y, sin(x)), cos(x)]) + c = FCodeGen() + result = c.write([r], "test", header=False, empty=False) + assert result[0][0] == "test.f90" + assert result[0][1] == ( + 'REAL*8 function foo(x, y)\n' + 'implicit none\n' + 'REAL*8, intent(in) :: x\n' + 'REAL*8, intent(out) :: y\n' + 'y = sin(x)\n' + 'foo = cos(x)\n' + 'end function\n' + ) + + +def test_inline_function(): + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m = symbols('n m', integer=True) + A, x, y = map(IndexedBase, 'Axy') + i = Idx('i', m) + p = FCodeGen() + func = implemented_function('func', Lambda(n, n*(n + 1))) + routine = make_routine('test_inline', Eq(y[i], func(x[i]))) + code = get_string(p.dump_f95, [routine]) + expected = ( + 'subroutine test_inline(m, x, y)\n' + 'implicit none\n' + 'INTEGER*4, intent(in) :: m\n' + 'REAL*8, intent(in), dimension(1:m) :: x\n' + 'REAL*8, intent(out), dimension(1:m) :: y\n' + 'INTEGER*4 :: i\n' + 'do i = 1, m\n' + ' y(i) = %s*%s\n' + 'end do\n' + 'end subroutine\n' + ) + args = ('x(i)', '(x(i) + 1)') + assert code == expected % args or\ + code == expected % args[::-1] + + +def test_f_code_call_signature_wrap(): + # Issue #7934 + x = symbols('x:20') + expr = 0 + for sym in x: + expr += sym + routine = make_routine("test", expr) + code_gen = FCodeGen() + source = get_string(code_gen.dump_f95, [routine]) + expected = """\ +REAL*8 function test(x0, x1, x10, x11, x12, x13, x14, x15, x16, x17, x18, & + x19, x2, x3, x4, x5, x6, x7, x8, x9) +implicit none +REAL*8, intent(in) :: x0 +REAL*8, intent(in) :: x1 +REAL*8, intent(in) :: x10 +REAL*8, intent(in) :: x11 +REAL*8, intent(in) :: x12 +REAL*8, intent(in) :: x13 +REAL*8, intent(in) :: x14 +REAL*8, intent(in) :: x15 +REAL*8, intent(in) :: x16 +REAL*8, intent(in) :: x17 +REAL*8, intent(in) :: x18 +REAL*8, intent(in) :: x19 +REAL*8, intent(in) :: x2 +REAL*8, intent(in) :: x3 +REAL*8, intent(in) :: x4 +REAL*8, intent(in) :: x5 +REAL*8, intent(in) :: x6 +REAL*8, intent(in) :: x7 +REAL*8, intent(in) :: x8 +REAL*8, intent(in) :: x9 +test = x0 + x1 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + & + x19 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 +end function +""" + assert source == expected + + +def test_check_case(): + x, X = symbols('x,X') + raises(CodeGenError, lambda: codegen(('test', x*X), 'f95', 'prefix')) + + +def test_check_case_false_positive(): + # The upper case/lower case exception should not be triggered by SymPy + # objects that differ only because of assumptions. (It may be useful to + # have a check for that as well, but here we only want to test against + # false positives with respect to case checking.) + x1 = symbols('x') + x2 = symbols('x', my_assumption=True) + try: + codegen(('test', x1*x2), 'f95', 'prefix') + except CodeGenError as e: + if e.args[0].startswith("Fortran ignores case."): + raise AssertionError("This exception should not be raised!") + + +def test_c_fortran_omit_routine_name(): + x, y = symbols("x,y") + name_expr = [("foo", 2*x)] + result = codegen(name_expr, "F95", header=False, empty=False) + expresult = codegen(name_expr, "F95", "foo", header=False, empty=False) + assert result[0][1] == expresult[0][1] + + name_expr = ("foo", x*y) + result = codegen(name_expr, "F95", header=False, empty=False) + expresult = codegen(name_expr, "F95", "foo", header=False, empty=False) + assert result[0][1] == expresult[0][1] + + name_expr = ("foo", Matrix([[x, y], [x+y, x-y]])) + result = codegen(name_expr, "C89", header=False, empty=False) + expresult = codegen(name_expr, "C89", "foo", header=False, empty=False) + assert result[0][1] == expresult[0][1] + + +def test_fcode_matrix_output(): + x, y, z = symbols('x,y,z') + e1 = x + y + e2 = Matrix([[x, y], [z, 16]]) + name_expr = ("test", (e1, e2)) + result = codegen(name_expr, "f95", "test", header=False, empty=False) + source = result[0][1] + expected = ( + "REAL*8 function test(x, y, z, out_%(hash)s)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(in) :: z\n" + "REAL*8, intent(out), dimension(1:2, 1:2) :: out_%(hash)s\n" + "out_%(hash)s(1, 1) = x\n" + "out_%(hash)s(2, 1) = z\n" + "out_%(hash)s(1, 2) = y\n" + "out_%(hash)s(2, 2) = 16\n" + "test = x + y\n" + "end function\n" + ) + # look for the magic number + a = source.splitlines()[5] + b = a.split('_') + out = b[1] + expected = expected % {'hash': out} + assert source == expected + + +def test_fcode_results_named_ordered(): + x, y, z = symbols('x,y,z') + B, C = symbols('B,C') + A = MatrixSymbol('A', 1, 3) + expr1 = Equality(A, Matrix([[1, 2, x]])) + expr2 = Equality(C, (x + y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result = codegen(name_expr, "f95", "test", header=False, empty=False, + argument_sequence=(x, z, y, C, A, B)) + source = result[0][1] + expected = ( + "subroutine test(x, z, y, C, A, B)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: z\n" + "REAL*8, intent(in) :: y\n" + "REAL*8, intent(out) :: C\n" + "REAL*8, intent(out) :: B\n" + "REAL*8, intent(out), dimension(1:1, 1:3) :: A\n" + "C = z*(x + y)\n" + "A(1, 1) = 1\n" + "A(1, 2) = 2\n" + "A(1, 3) = x\n" + "B = 2*x\n" + "end subroutine\n" + ) + assert source == expected + + +def test_fcode_matrixsymbol_slice(): + A = MatrixSymbol('A', 2, 3) + B = MatrixSymbol('B', 1, 3) + C = MatrixSymbol('C', 1, 3) + D = MatrixSymbol('D', 2, 1) + name_expr = ("test", [Equality(B, A[0, :]), + Equality(C, A[1, :]), + Equality(D, A[:, 2])]) + result = codegen(name_expr, "f95", "test", header=False, empty=False) + source = result[0][1] + expected = ( + "subroutine test(A, B, C, D)\n" + "implicit none\n" + "REAL*8, intent(in), dimension(1:2, 1:3) :: A\n" + "REAL*8, intent(out), dimension(1:1, 1:3) :: B\n" + "REAL*8, intent(out), dimension(1:1, 1:3) :: C\n" + "REAL*8, intent(out), dimension(1:2, 1:1) :: D\n" + "B(1, 1) = A(1, 1)\n" + "B(1, 2) = A(1, 2)\n" + "B(1, 3) = A(1, 3)\n" + "C(1, 1) = A(2, 1)\n" + "C(1, 2) = A(2, 2)\n" + "C(1, 3) = A(2, 3)\n" + "D(1, 1) = A(1, 3)\n" + "D(2, 1) = A(2, 3)\n" + "end subroutine\n" + ) + assert source == expected + + +def test_fcode_matrixsymbol_slice_autoname(): + # see issue #8093 + A = MatrixSymbol('A', 2, 3) + name_expr = ("test", A[:, 1]) + result = codegen(name_expr, "f95", "test", header=False, empty=False) + source = result[0][1] + expected = ( + "subroutine test(A, out_%(hash)s)\n" + "implicit none\n" + "REAL*8, intent(in), dimension(1:2, 1:3) :: A\n" + "REAL*8, intent(out), dimension(1:2, 1:1) :: out_%(hash)s\n" + "out_%(hash)s(1, 1) = A(1, 2)\n" + "out_%(hash)s(2, 1) = A(2, 2)\n" + "end subroutine\n" + ) + # look for the magic number + a = source.splitlines()[3] + b = a.split('_') + out = b[1] + expected = expected % {'hash': out} + assert source == expected + + +def test_global_vars(): + x, y, z, t = symbols("x y z t") + result = codegen(('f', x*y), "F95", header=False, empty=False, + global_vars=(y,)) + source = result[0][1] + expected = ( + "REAL*8 function f(x)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "f = x*y\n" + "end function\n" + ) + assert source == expected + + expected = ( + '#include "f.h"\n' + '#include \n' + 'double f(double x, double y) {\n' + ' double f_result;\n' + ' f_result = x*y + z;\n' + ' return f_result;\n' + '}\n' + ) + result = codegen(('f', x*y+z), "C", header=False, empty=False, + global_vars=(z, t)) + source = result[0][1] + assert source == expected + +def test_custom_codegen(): + from sympy.printing.c import C99CodePrinter + from sympy.functions.elementary.exponential import exp + + printer = C99CodePrinter(settings={'user_functions': {'exp': 'fastexp'}}) + + x, y = symbols('x y') + expr = exp(x + y) + + # replace math.h with a different header + gen = C99CodeGen(printer=printer, + preprocessor_statements=['#include "fastexp.h"']) + + expected = ( + '#include "expr.h"\n' + '#include "fastexp.h"\n' + 'double expr(double x, double y) {\n' + ' double expr_result;\n' + ' expr_result = fastexp(x + y);\n' + ' return expr_result;\n' + '}\n' + ) + + result = codegen(('expr', expr), header=False, empty=False, code_gen=gen) + source = result[0][1] + assert source == expected + + # use both math.h and an external header + gen = C99CodeGen(printer=printer) + gen.preprocessor_statements.append('#include "fastexp.h"') + + expected = ( + '#include "expr.h"\n' + '#include \n' + '#include "fastexp.h"\n' + 'double expr(double x, double y) {\n' + ' double expr_result;\n' + ' expr_result = fastexp(x + y);\n' + ' return expr_result;\n' + '}\n' + ) + + result = codegen(('expr', expr), header=False, empty=False, code_gen=gen) + source = result[0][1] + assert source == expected + +def test_c_with_printer(): + # issue 13586 + from sympy.printing.c import C99CodePrinter + class CustomPrinter(C99CodePrinter): + def _print_Pow(self, expr): + return "fastpow({}, {})".format(self._print(expr.base), + self._print(expr.exp)) + + x = symbols('x') + expr = x**3 + expected =[ + ("file.c", + "#include \"file.h\"\n" + "#include \n" + "double test(double x) {\n" + " double test_result;\n" + " test_result = fastpow(x, 3);\n" + " return test_result;\n" + "}\n"), + ("file.h", + "#ifndef PROJECT__FILE__H\n" + "#define PROJECT__FILE__H\n" + "double test(double x);\n" + "#endif\n") + ] + result = codegen(("test", expr), "C","file", header=False, empty=False, printer = CustomPrinter()) + assert result == expected + + +def test_fcode_complex(): + import sympy.utilities.codegen + sympy.utilities.codegen.COMPLEX_ALLOWED = True + x = Symbol('x', real=True) + y = Symbol('y',real=True) + result = codegen(('test',x+y), 'f95', 'test', header=False, empty=False) + source = (result[0][1]) + expected = ( + "REAL*8 function test(x, y)\n" + "implicit none\n" + "REAL*8, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "test = x + y\n" + "end function\n") + assert source == expected + x = Symbol('x') + y = Symbol('y',real=True) + result = codegen(('test',x+y), 'f95', 'test', header=False, empty=False) + source = (result[0][1]) + expected = ( + "COMPLEX*16 function test(x, y)\n" + "implicit none\n" + "COMPLEX*16, intent(in) :: x\n" + "REAL*8, intent(in) :: y\n" + "test = x + y\n" + "end function\n" + ) + assert source==expected + sympy.utilities.codegen.COMPLEX_ALLOWED = False diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_julia.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_julia.py new file mode 100644 index 0000000000000000000000000000000000000000..12841cb7d476107e3866d91b998bed1f997f3901 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_julia.py @@ -0,0 +1,620 @@ +from io import StringIO + +from sympy.core import S, symbols, Eq, pi, Catalan, EulerGamma, Function +from sympy.core.relational import Equality +from sympy.functions.elementary.piecewise import Piecewise +from sympy.matrices import Matrix, MatrixSymbol +from sympy.utilities.codegen import JuliaCodeGen, codegen, make_routine +from sympy.testing.pytest import XFAIL +import sympy + + +x, y, z = symbols('x,y,z') + + +def test_empty_jl_code(): + code_gen = JuliaCodeGen() + output = StringIO() + code_gen.dump_jl([], output, "file", header=False, empty=False) + source = output.getvalue() + assert source == "" + + +def test_jl_simple_code(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Julia", header=False, empty=False) + assert result[0] == "test.jl" + source = result[1] + expected = ( + "function test(x, y, z)\n" + " out1 = z .* (x + y)\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +def test_jl_simple_code_with_header(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Julia", header=True, empty=False) + assert result[0] == "test.jl" + source = result[1] + expected = ( + "# Code generated with SymPy " + sympy.__version__ + "\n" + "#\n" + "# See http://www.sympy.org/ for more information.\n" + "#\n" + "# This file is part of 'project'\n" + "function test(x, y, z)\n" + " out1 = z .* (x + y)\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +def test_jl_simple_code_nameout(): + expr = Equality(z, (x + y)) + name_expr = ("test", expr) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y)\n" + " z = x + y\n" + " return z\n" + "end\n" + ) + assert source == expected + + +def test_jl_numbersymbol(): + name_expr = ("test", pi**Catalan) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test()\n" + " out1 = pi ^ catalan\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +@XFAIL +def test_jl_numbersymbol_no_inline(): + # FIXME: how to pass inline=False to the JuliaCodePrinter? + name_expr = ("test", [pi**Catalan, EulerGamma]) + result, = codegen(name_expr, "Julia", header=False, + empty=False, inline=False) + source = result[1] + expected = ( + "function test()\n" + " Catalan = 0.915965594177219\n" + " EulerGamma = 0.5772156649015329\n" + " out1 = pi ^ Catalan\n" + " out2 = EulerGamma\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_jl_code_argument_order(): + expr = x + y + routine = make_routine("test", expr, argument_sequence=[z, x, y], language="julia") + code_gen = JuliaCodeGen() + output = StringIO() + code_gen.dump_jl([routine], output, "test", header=False, empty=False) + source = output.getvalue() + expected = ( + "function test(z, x, y)\n" + " out1 = x + y\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +def test_multiple_results_m(): + # Here the output order is the input order + expr1 = (x + y)*z + expr2 = (x - y)*z + name_expr = ("test", [expr1, expr2]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y, z)\n" + " out1 = z .* (x + y)\n" + " out2 = z .* (x - y)\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_results_named_unordered(): + # Here output order is based on name_expr + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y, z)\n" + " C = z .* (x + y)\n" + " A = z .* (x - y)\n" + " B = 2 * x\n" + " return C, A, B\n" + "end\n" + ) + assert source == expected + + +def test_results_named_ordered(): + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result = codegen(name_expr, "Julia", header=False, empty=False, + argument_sequence=(x, z, y)) + assert result[0][0] == "test.jl" + source = result[0][1] + expected = ( + "function test(x, z, y)\n" + " C = z .* (x + y)\n" + " A = z .* (x - y)\n" + " B = 2 * x\n" + " return C, A, B\n" + "end\n" + ) + assert source == expected + + +def test_complicated_jl_codegen(): + from sympy.functions.elementary.trigonometric import (cos, sin, tan) + name_expr = ("testlong", + [ ((sin(x) + cos(y) + tan(z))**3).expand(), + cos(cos(cos(cos(cos(cos(cos(cos(x + y + z)))))))) + ]) + result = codegen(name_expr, "Julia", header=False, empty=False) + assert result[0][0] == "testlong.jl" + source = result[0][1] + expected = ( + "function testlong(x, y, z)\n" + " out1 = sin(x) .^ 3 + 3 * sin(x) .^ 2 .* cos(y) + 3 * sin(x) .^ 2 .* tan(z)" + " + 3 * sin(x) .* cos(y) .^ 2 + 6 * sin(x) .* cos(y) .* tan(z) + 3 * sin(x) .* tan(z) .^ 2" + " + cos(y) .^ 3 + 3 * cos(y) .^ 2 .* tan(z) + 3 * cos(y) .* tan(z) .^ 2 + tan(z) .^ 3\n" + " out2 = cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))))\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_jl_output_arg_mixed_unordered(): + # named outputs are alphabetical, unnamed output appear in the given order + from sympy.functions.elementary.trigonometric import (cos, sin) + a = symbols("a") + name_expr = ("foo", [cos(2*x), Equality(y, sin(x)), cos(x), Equality(a, sin(2*x))]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + assert result[0] == "foo.jl" + source = result[1] + expected = ( + 'function foo(x)\n' + ' out1 = cos(2 * x)\n' + ' y = sin(x)\n' + ' out3 = cos(x)\n' + ' a = sin(2 * x)\n' + ' return out1, y, out3, a\n' + 'end\n' + ) + assert source == expected + + +def test_jl_piecewise_(): + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True), evaluate=False) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function pwtest(x)\n" + " out1 = ((x < -1) ? (0) :\n" + " (x <= 1) ? (x .^ 2) :\n" + " (x > 1) ? (2 - x) : (1))\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +@XFAIL +def test_jl_piecewise_no_inline(): + # FIXME: how to pass inline=False to the JuliaCodePrinter? + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True)) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Julia", header=False, empty=False, + inline=False) + source = result[1] + expected = ( + "function pwtest(x)\n" + " if (x < -1)\n" + " out1 = 0\n" + " elseif (x <= 1)\n" + " out1 = x .^ 2\n" + " elseif (x > 1)\n" + " out1 = -x + 2\n" + " else\n" + " out1 = 1\n" + " end\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +def test_jl_multifcns_per_file(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Julia", header=False, empty=False) + assert result[0][0] == "foo.jl" + source = result[0][1] + expected = ( + "function foo(x, y)\n" + " out1 = 2 * x\n" + " out2 = 3 * y\n" + " return out1, out2\n" + "end\n" + "function bar(y)\n" + " out1 = y .^ 2\n" + " out2 = 4 * y\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_jl_multifcns_per_file_w_header(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Julia", header=True, empty=False) + assert result[0][0] == "foo.jl" + source = result[0][1] + expected = ( + "# Code generated with SymPy " + sympy.__version__ + "\n" + "#\n" + "# See http://www.sympy.org/ for more information.\n" + "#\n" + "# This file is part of 'project'\n" + "function foo(x, y)\n" + " out1 = 2 * x\n" + " out2 = 3 * y\n" + " return out1, out2\n" + "end\n" + "function bar(y)\n" + " out1 = y .^ 2\n" + " out2 = 4 * y\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_jl_filename_match_prefix(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result, = codegen(name_expr, "Julia", prefix="baz", header=False, + empty=False) + assert result[0] == "baz.jl" + + +def test_jl_matrix_named(): + e2 = Matrix([[x, 2*y, pi*z]]) + name_expr = ("test", Equality(MatrixSymbol('myout1', 1, 3), e2)) + result = codegen(name_expr, "Julia", header=False, empty=False) + assert result[0][0] == "test.jl" + source = result[0][1] + expected = ( + "function test(x, y, z)\n" + " myout1 = [x 2 * y pi * z]\n" + " return myout1\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrix_named_matsym(): + myout1 = MatrixSymbol('myout1', 1, 3) + e2 = Matrix([[x, 2*y, pi*z]]) + name_expr = ("test", Equality(myout1, e2, evaluate=False)) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y, z)\n" + " myout1 = [x 2 * y pi * z]\n" + " return myout1\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrix_output_autoname(): + expr = Matrix([[x, x+y, 3]]) + name_expr = ("test", expr) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y)\n" + " out1 = [x x + y 3]\n" + " return out1\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrix_output_autoname_2(): + e1 = (x + y) + e2 = Matrix([[2*x, 2*y, 2*z]]) + e3 = Matrix([[x], [y], [z]]) + e4 = Matrix([[x, y], [z, 16]]) + name_expr = ("test", (e1, e2, e3, e4)) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y, z)\n" + " out1 = x + y\n" + " out2 = [2 * x 2 * y 2 * z]\n" + " out3 = [x, y, z]\n" + " out4 = [x y;\n" + " z 16]\n" + " return out1, out2, out3, out4\n" + "end\n" + ) + assert source == expected + + +def test_jl_results_matrix_named_ordered(): + B, C = symbols('B,C') + A = MatrixSymbol('A', 1, 3) + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, Matrix([[1, 2, x]])) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result, = codegen(name_expr, "Julia", header=False, empty=False, + argument_sequence=(x, z, y)) + source = result[1] + expected = ( + "function test(x, z, y)\n" + " C = z .* (x + y)\n" + " A = [1 2 x]\n" + " B = 2 * x\n" + " return C, A, B\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrixsymbol_slice(): + A = MatrixSymbol('A', 2, 3) + B = MatrixSymbol('B', 1, 3) + C = MatrixSymbol('C', 1, 3) + D = MatrixSymbol('D', 2, 1) + name_expr = ("test", [Equality(B, A[0, :]), + Equality(C, A[1, :]), + Equality(D, A[:, 2])]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(A)\n" + " B = A[1,:]\n" + " C = A[2,:]\n" + " D = A[:,3]\n" + " return B, C, D\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrixsymbol_slice2(): + A = MatrixSymbol('A', 3, 4) + B = MatrixSymbol('B', 2, 2) + C = MatrixSymbol('C', 2, 2) + name_expr = ("test", [Equality(B, A[0:2, 0:2]), + Equality(C, A[0:2, 1:3])]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(A)\n" + " B = A[1:2,1:2]\n" + " C = A[1:2,2:3]\n" + " return B, C\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrixsymbol_slice3(): + A = MatrixSymbol('A', 8, 7) + B = MatrixSymbol('B', 2, 2) + C = MatrixSymbol('C', 4, 2) + name_expr = ("test", [Equality(B, A[6:, 1::3]), + Equality(C, A[::2, ::3])]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(A)\n" + " B = A[7:end,2:3:end]\n" + " C = A[1:2:end,1:3:end]\n" + " return B, C\n" + "end\n" + ) + assert source == expected + + +def test_jl_matrixsymbol_slice_autoname(): + A = MatrixSymbol('A', 2, 3) + B = MatrixSymbol('B', 1, 3) + name_expr = ("test", [Equality(B, A[0,:]), A[1,:], A[:,0], A[:,1]]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(A)\n" + " B = A[1,:]\n" + " out2 = A[2,:]\n" + " out3 = A[:,1]\n" + " out4 = A[:,2]\n" + " return B, out2, out3, out4\n" + "end\n" + ) + assert source == expected + + +def test_jl_loops(): + # Note: an Julia programmer would probably vectorize this across one or + # more dimensions. Also, size(A) would be used rather than passing in m + # and n. Perhaps users would expect us to vectorize automatically here? + # Or is it possible to represent such things using IndexedBase? + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m = symbols('n m', integer=True) + A = IndexedBase('A') + x = IndexedBase('x') + y = IndexedBase('y') + i = Idx('i', m) + j = Idx('j', n) + result, = codegen(('mat_vec_mult', Eq(y[i], A[i, j]*x[j])), "Julia", + header=False, empty=False) + source = result[1] + expected = ( + 'function mat_vec_mult(y, A, m, n, x)\n' + ' for i = 1:m\n' + ' y[i] = 0\n' + ' end\n' + ' for i = 1:m\n' + ' for j = 1:n\n' + ' y[i] = %(rhs)s + y[i]\n' + ' end\n' + ' end\n' + ' return y\n' + 'end\n' + ) + assert (source == expected % {'rhs': 'A[%s,%s] .* x[j]' % (i, j)} or + source == expected % {'rhs': 'x[j] .* A[%s,%s]' % (i, j)}) + + +def test_jl_tensor_loops_multiple_contractions(): + # see comments in previous test about vectorizing + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m, o, p = symbols('n m o p', integer=True) + A = IndexedBase('A') + B = IndexedBase('B') + y = IndexedBase('y') + i = Idx('i', m) + j = Idx('j', n) + k = Idx('k', o) + l = Idx('l', p) + result, = codegen(('tensorthing', Eq(y[i], B[j, k, l]*A[i, j, k, l])), + "Julia", header=False, empty=False) + source = result[1] + expected = ( + 'function tensorthing(y, A, B, m, n, o, p)\n' + ' for i = 1:m\n' + ' y[i] = 0\n' + ' end\n' + ' for i = 1:m\n' + ' for j = 1:n\n' + ' for k = 1:o\n' + ' for l = 1:p\n' + ' y[i] = A[i,j,k,l] .* B[j,k,l] + y[i]\n' + ' end\n' + ' end\n' + ' end\n' + ' end\n' + ' return y\n' + 'end\n' + ) + assert source == expected + + +def test_jl_InOutArgument(): + expr = Equality(x, x**2) + name_expr = ("mysqr", expr) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function mysqr(x)\n" + " x = x .^ 2\n" + " return x\n" + "end\n" + ) + assert source == expected + + +def test_jl_InOutArgument_order(): + # can specify the order as (x, y) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Julia", header=False, + empty=False, argument_sequence=(x,y)) + source = result[1] + expected = ( + "function test(x, y)\n" + " x = x .^ 2 + y\n" + " return x\n" + "end\n" + ) + assert source == expected + # make sure it gives (x, y) not (y, x) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x, y)\n" + " x = x .^ 2 + y\n" + " return x\n" + "end\n" + ) + assert source == expected + + +def test_jl_not_supported(): + f = Function('f') + name_expr = ("test", [f(x).diff(x), S.ComplexInfinity]) + result, = codegen(name_expr, "Julia", header=False, empty=False) + source = result[1] + expected = ( + "function test(x)\n" + " # unsupported: Derivative(f(x), x)\n" + " # unsupported: zoo\n" + " out1 = Derivative(f(x), x)\n" + " out2 = zoo\n" + " return out1, out2\n" + "end\n" + ) + assert source == expected + + +def test_global_vars_octave(): + x, y, z, t = symbols("x y z t") + result = codegen(('f', x*y), "Julia", header=False, empty=False, + global_vars=(y,)) + source = result[0][1] + expected = ( + "function f(x)\n" + " out1 = x .* y\n" + " return out1\n" + "end\n" + ) + assert source == expected + + result = codegen(('f', x*y+z), "Julia", header=False, empty=False, + argument_sequence=(x, y), global_vars=(z, t)) + source = result[0][1] + expected = ( + "function f(x, y)\n" + " out1 = x .* y + z\n" + " return out1\n" + "end\n" + ) + assert source == expected diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_octave.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_octave.py new file mode 100644 index 0000000000000000000000000000000000000000..77aaef7dd0d81d6855024e49fbb3d6d4c09f3384 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_octave.py @@ -0,0 +1,589 @@ +from io import StringIO + +from sympy.core import S, symbols, Eq, pi, Catalan, EulerGamma, Function +from sympy.core.relational import Equality +from sympy.functions.elementary.piecewise import Piecewise +from sympy.matrices import Matrix, MatrixSymbol +from sympy.utilities.codegen import OctaveCodeGen, codegen, make_routine +from sympy.testing.pytest import raises +from sympy.testing.pytest import XFAIL +import sympy + + +x, y, z = symbols('x,y,z') + + +def test_empty_m_code(): + code_gen = OctaveCodeGen() + output = StringIO() + code_gen.dump_m([], output, "file", header=False, empty=False) + source = output.getvalue() + assert source == "" + + +def test_m_simple_code(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Octave", header=False, empty=False) + assert result[0] == "test.m" + source = result[1] + expected = ( + "function out1 = test(x, y, z)\n" + " out1 = z.*(x + y);\n" + "end\n" + ) + assert source == expected + + +def test_m_simple_code_with_header(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Octave", header=True, empty=False) + assert result[0] == "test.m" + source = result[1] + expected = ( + "function out1 = test(x, y, z)\n" + " %TEST Autogenerated by SymPy\n" + " % Code generated with SymPy " + sympy.__version__ + "\n" + " %\n" + " % See http://www.sympy.org/ for more information.\n" + " %\n" + " % This file is part of 'project'\n" + " out1 = z.*(x + y);\n" + "end\n" + ) + assert source == expected + + +def test_m_simple_code_nameout(): + expr = Equality(z, (x + y)) + name_expr = ("test", expr) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function z = test(x, y)\n" + " z = x + y;\n" + "end\n" + ) + assert source == expected + + +def test_m_numbersymbol(): + name_expr = ("test", pi**Catalan) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function out1 = test()\n" + " out1 = pi^%s;\n" + "end\n" + ) % Catalan.evalf(17) + assert source == expected + + +@XFAIL +def test_m_numbersymbol_no_inline(): + # FIXME: how to pass inline=False to the OctaveCodePrinter? + name_expr = ("test", [pi**Catalan, EulerGamma]) + result, = codegen(name_expr, "Octave", header=False, + empty=False, inline=False) + source = result[1] + expected = ( + "function [out1, out2] = test()\n" + " Catalan = 0.915965594177219; % constant\n" + " EulerGamma = 0.5772156649015329; % constant\n" + " out1 = pi^Catalan;\n" + " out2 = EulerGamma;\n" + "end\n" + ) + assert source == expected + + +def test_m_code_argument_order(): + expr = x + y + routine = make_routine("test", expr, argument_sequence=[z, x, y], language="octave") + code_gen = OctaveCodeGen() + output = StringIO() + code_gen.dump_m([routine], output, "test", header=False, empty=False) + source = output.getvalue() + expected = ( + "function out1 = test(z, x, y)\n" + " out1 = x + y;\n" + "end\n" + ) + assert source == expected + + +def test_multiple_results_m(): + # Here the output order is the input order + expr1 = (x + y)*z + expr2 = (x - y)*z + name_expr = ("test", [expr1, expr2]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [out1, out2] = test(x, y, z)\n" + " out1 = z.*(x + y);\n" + " out2 = z.*(x - y);\n" + "end\n" + ) + assert source == expected + + +def test_results_named_unordered(): + # Here output order is based on name_expr + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [C, A, B] = test(x, y, z)\n" + " C = z.*(x + y);\n" + " A = z.*(x - y);\n" + " B = 2*x;\n" + "end\n" + ) + assert source == expected + + +def test_results_named_ordered(): + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result = codegen(name_expr, "Octave", header=False, empty=False, + argument_sequence=(x, z, y)) + assert result[0][0] == "test.m" + source = result[0][1] + expected = ( + "function [C, A, B] = test(x, z, y)\n" + " C = z.*(x + y);\n" + " A = z.*(x - y);\n" + " B = 2*x;\n" + "end\n" + ) + assert source == expected + + +def test_complicated_m_codegen(): + from sympy.functions.elementary.trigonometric import (cos, sin, tan) + name_expr = ("testlong", + [ ((sin(x) + cos(y) + tan(z))**3).expand(), + cos(cos(cos(cos(cos(cos(cos(cos(x + y + z)))))))) + ]) + result = codegen(name_expr, "Octave", header=False, empty=False) + assert result[0][0] == "testlong.m" + source = result[0][1] + expected = ( + "function [out1, out2] = testlong(x, y, z)\n" + " out1 = sin(x).^3 + 3*sin(x).^2.*cos(y) + 3*sin(x).^2.*tan(z)" + " + 3*sin(x).*cos(y).^2 + 6*sin(x).*cos(y).*tan(z) + 3*sin(x).*tan(z).^2" + " + cos(y).^3 + 3*cos(y).^2.*tan(z) + 3*cos(y).*tan(z).^2 + tan(z).^3;\n" + " out2 = cos(cos(cos(cos(cos(cos(cos(cos(x + y + z))))))));\n" + "end\n" + ) + assert source == expected + + +def test_m_output_arg_mixed_unordered(): + # named outputs are alphabetical, unnamed output appear in the given order + from sympy.functions.elementary.trigonometric import (cos, sin) + a = symbols("a") + name_expr = ("foo", [cos(2*x), Equality(y, sin(x)), cos(x), Equality(a, sin(2*x))]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + assert result[0] == "foo.m" + source = result[1] + expected = ( + 'function [out1, y, out3, a] = foo(x)\n' + ' out1 = cos(2*x);\n' + ' y = sin(x);\n' + ' out3 = cos(x);\n' + ' a = sin(2*x);\n' + 'end\n' + ) + assert source == expected + + +def test_m_piecewise_(): + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True), evaluate=False) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function out1 = pwtest(x)\n" + " out1 = ((x < -1).*(0) + (~(x < -1)).*( ...\n" + " (x <= 1).*(x.^2) + (~(x <= 1)).*( ...\n" + " (x > 1).*(2 - x) + (~(x > 1)).*(1))));\n" + "end\n" + ) + assert source == expected + + +@XFAIL +def test_m_piecewise_no_inline(): + # FIXME: how to pass inline=False to the OctaveCodePrinter? + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True)) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Octave", header=False, empty=False, + inline=False) + source = result[1] + expected = ( + "function out1 = pwtest(x)\n" + " if (x < -1)\n" + " out1 = 0;\n" + " elseif (x <= 1)\n" + " out1 = x.^2;\n" + " elseif (x > 1)\n" + " out1 = -x + 2;\n" + " else\n" + " out1 = 1;\n" + " end\n" + "end\n" + ) + assert source == expected + + +def test_m_multifcns_per_file(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Octave", header=False, empty=False) + assert result[0][0] == "foo.m" + source = result[0][1] + expected = ( + "function [out1, out2] = foo(x, y)\n" + " out1 = 2*x;\n" + " out2 = 3*y;\n" + "end\n" + "function [out1, out2] = bar(y)\n" + " out1 = y.^2;\n" + " out2 = 4*y;\n" + "end\n" + ) + assert source == expected + + +def test_m_multifcns_per_file_w_header(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Octave", header=True, empty=False) + assert result[0][0] == "foo.m" + source = result[0][1] + expected = ( + "function [out1, out2] = foo(x, y)\n" + " %FOO Autogenerated by SymPy\n" + " % Code generated with SymPy " + sympy.__version__ + "\n" + " %\n" + " % See http://www.sympy.org/ for more information.\n" + " %\n" + " % This file is part of 'project'\n" + " out1 = 2*x;\n" + " out2 = 3*y;\n" + "end\n" + "function [out1, out2] = bar(y)\n" + " out1 = y.^2;\n" + " out2 = 4*y;\n" + "end\n" + ) + assert source == expected + + +def test_m_filename_match_first_fcn(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + raises(ValueError, lambda: codegen(name_expr, + "Octave", prefix="bar", header=False, empty=False)) + + +def test_m_matrix_named(): + e2 = Matrix([[x, 2*y, pi*z]]) + name_expr = ("test", Equality(MatrixSymbol('myout1', 1, 3), e2)) + result = codegen(name_expr, "Octave", header=False, empty=False) + assert result[0][0] == "test.m" + source = result[0][1] + expected = ( + "function myout1 = test(x, y, z)\n" + " myout1 = [x 2*y pi*z];\n" + "end\n" + ) + assert source == expected + + +def test_m_matrix_named_matsym(): + myout1 = MatrixSymbol('myout1', 1, 3) + e2 = Matrix([[x, 2*y, pi*z]]) + name_expr = ("test", Equality(myout1, e2, evaluate=False)) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function myout1 = test(x, y, z)\n" + " myout1 = [x 2*y pi*z];\n" + "end\n" + ) + assert source == expected + + +def test_m_matrix_output_autoname(): + expr = Matrix([[x, x+y, 3]]) + name_expr = ("test", expr) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function out1 = test(x, y)\n" + " out1 = [x x + y 3];\n" + "end\n" + ) + assert source == expected + + +def test_m_matrix_output_autoname_2(): + e1 = (x + y) + e2 = Matrix([[2*x, 2*y, 2*z]]) + e3 = Matrix([[x], [y], [z]]) + e4 = Matrix([[x, y], [z, 16]]) + name_expr = ("test", (e1, e2, e3, e4)) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [out1, out2, out3, out4] = test(x, y, z)\n" + " out1 = x + y;\n" + " out2 = [2*x 2*y 2*z];\n" + " out3 = [x; y; z];\n" + " out4 = [x y; z 16];\n" + "end\n" + ) + assert source == expected + + +def test_m_results_matrix_named_ordered(): + B, C = symbols('B,C') + A = MatrixSymbol('A', 1, 3) + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, Matrix([[1, 2, x]])) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result, = codegen(name_expr, "Octave", header=False, empty=False, + argument_sequence=(x, z, y)) + source = result[1] + expected = ( + "function [C, A, B] = test(x, z, y)\n" + " C = z.*(x + y);\n" + " A = [1 2 x];\n" + " B = 2*x;\n" + "end\n" + ) + assert source == expected + + +def test_m_matrixsymbol_slice(): + A = MatrixSymbol('A', 2, 3) + B = MatrixSymbol('B', 1, 3) + C = MatrixSymbol('C', 1, 3) + D = MatrixSymbol('D', 2, 1) + name_expr = ("test", [Equality(B, A[0, :]), + Equality(C, A[1, :]), + Equality(D, A[:, 2])]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [B, C, D] = test(A)\n" + " B = A(1, :);\n" + " C = A(2, :);\n" + " D = A(:, 3);\n" + "end\n" + ) + assert source == expected + + +def test_m_matrixsymbol_slice2(): + A = MatrixSymbol('A', 3, 4) + B = MatrixSymbol('B', 2, 2) + C = MatrixSymbol('C', 2, 2) + name_expr = ("test", [Equality(B, A[0:2, 0:2]), + Equality(C, A[0:2, 1:3])]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [B, C] = test(A)\n" + " B = A(1:2, 1:2);\n" + " C = A(1:2, 2:3);\n" + "end\n" + ) + assert source == expected + + +def test_m_matrixsymbol_slice3(): + A = MatrixSymbol('A', 8, 7) + B = MatrixSymbol('B', 2, 2) + C = MatrixSymbol('C', 4, 2) + name_expr = ("test", [Equality(B, A[6:, 1::3]), + Equality(C, A[::2, ::3])]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [B, C] = test(A)\n" + " B = A(7:end, 2:3:end);\n" + " C = A(1:2:end, 1:3:end);\n" + "end\n" + ) + assert source == expected + + +def test_m_matrixsymbol_slice_autoname(): + A = MatrixSymbol('A', 2, 3) + B = MatrixSymbol('B', 1, 3) + name_expr = ("test", [Equality(B, A[0,:]), A[1,:], A[:,0], A[:,1]]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [B, out2, out3, out4] = test(A)\n" + " B = A(1, :);\n" + " out2 = A(2, :);\n" + " out3 = A(:, 1);\n" + " out4 = A(:, 2);\n" + "end\n" + ) + assert source == expected + + +def test_m_loops(): + # Note: an Octave programmer would probably vectorize this across one or + # more dimensions. Also, size(A) would be used rather than passing in m + # and n. Perhaps users would expect us to vectorize automatically here? + # Or is it possible to represent such things using IndexedBase? + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m = symbols('n m', integer=True) + A = IndexedBase('A') + x = IndexedBase('x') + y = IndexedBase('y') + i = Idx('i', m) + j = Idx('j', n) + result, = codegen(('mat_vec_mult', Eq(y[i], A[i, j]*x[j])), "Octave", + header=False, empty=False) + source = result[1] + expected = ( + 'function y = mat_vec_mult(A, m, n, x)\n' + ' for i = 1:m\n' + ' y(i) = 0;\n' + ' end\n' + ' for i = 1:m\n' + ' for j = 1:n\n' + ' y(i) = %(rhs)s + y(i);\n' + ' end\n' + ' end\n' + 'end\n' + ) + assert (source == expected % {'rhs': 'A(%s, %s).*x(j)' % (i, j)} or + source == expected % {'rhs': 'x(j).*A(%s, %s)' % (i, j)}) + + +def test_m_tensor_loops_multiple_contractions(): + # see comments in previous test about vectorizing + from sympy.tensor import IndexedBase, Idx + from sympy.core.symbol import symbols + n, m, o, p = symbols('n m o p', integer=True) + A = IndexedBase('A') + B = IndexedBase('B') + y = IndexedBase('y') + i = Idx('i', m) + j = Idx('j', n) + k = Idx('k', o) + l = Idx('l', p) + result, = codegen(('tensorthing', Eq(y[i], B[j, k, l]*A[i, j, k, l])), + "Octave", header=False, empty=False) + source = result[1] + expected = ( + 'function y = tensorthing(A, B, m, n, o, p)\n' + ' for i = 1:m\n' + ' y(i) = 0;\n' + ' end\n' + ' for i = 1:m\n' + ' for j = 1:n\n' + ' for k = 1:o\n' + ' for l = 1:p\n' + ' y(i) = A(i, j, k, l).*B(j, k, l) + y(i);\n' + ' end\n' + ' end\n' + ' end\n' + ' end\n' + 'end\n' + ) + assert source == expected + + +def test_m_InOutArgument(): + expr = Equality(x, x**2) + name_expr = ("mysqr", expr) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function x = mysqr(x)\n" + " x = x.^2;\n" + "end\n" + ) + assert source == expected + + +def test_m_InOutArgument_order(): + # can specify the order as (x, y) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Octave", header=False, + empty=False, argument_sequence=(x,y)) + source = result[1] + expected = ( + "function x = test(x, y)\n" + " x = x.^2 + y;\n" + "end\n" + ) + assert source == expected + # make sure it gives (x, y) not (y, x) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function x = test(x, y)\n" + " x = x.^2 + y;\n" + "end\n" + ) + assert source == expected + + +def test_m_not_supported(): + f = Function('f') + name_expr = ("test", [f(x).diff(x), S.ComplexInfinity]) + result, = codegen(name_expr, "Octave", header=False, empty=False) + source = result[1] + expected = ( + "function [out1, out2] = test(x)\n" + " % unsupported: Derivative(f(x), x)\n" + " % unsupported: zoo\n" + " out1 = Derivative(f(x), x);\n" + " out2 = zoo;\n" + "end\n" + ) + assert source == expected + + +def test_global_vars_octave(): + x, y, z, t = symbols("x y z t") + result = codegen(('f', x*y), "Octave", header=False, empty=False, + global_vars=(y,)) + source = result[0][1] + expected = ( + "function out1 = f(x)\n" + " global y\n" + " out1 = x.*y;\n" + "end\n" + ) + assert source == expected + + result = codegen(('f', x*y+z), "Octave", header=False, empty=False, + argument_sequence=(x, y), global_vars=(z, t)) + source = result[0][1] + expected = ( + "function out1 = f(x, y)\n" + " global t z\n" + " out1 = x.*y + z;\n" + "end\n" + ) + assert source == expected diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_rust.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_rust.py new file mode 100644 index 0000000000000000000000000000000000000000..bc7f82158ae8fa7dfe34bf909aa695b119fb9526 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_codegen_rust.py @@ -0,0 +1,401 @@ +from io import StringIO + +from sympy.core import S, symbols, pi, Catalan, EulerGamma, Function +from sympy.core.relational import Equality +from sympy.functions.elementary.piecewise import Piecewise +from sympy.utilities.codegen import RustCodeGen, codegen, make_routine +from sympy.testing.pytest import XFAIL +import sympy + + +x, y, z = symbols('x,y,z') + + +def test_empty_rust_code(): + code_gen = RustCodeGen() + output = StringIO() + code_gen.dump_rs([], output, "file", header=False, empty=False) + source = output.getvalue() + assert source == "" + + +def test_simple_rust_code(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Rust", header=False, empty=False) + assert result[0] == "test.rs" + source = result[1] + expected = ( + "fn test(x: f64, y: f64, z: f64) -> f64 {\n" + " let out1 = z*(x + y);\n" + " out1\n" + "}\n" + ) + assert source == expected + + +def test_simple_code_with_header(): + name_expr = ("test", (x + y)*z) + result, = codegen(name_expr, "Rust", header=True, empty=False) + assert result[0] == "test.rs" + source = result[1] + version_str = "Code generated with SymPy %s" % sympy.__version__ + version_line = version_str.center(76).rstrip() + expected = ( + "/*\n" + " *%(version_line)s\n" + " *\n" + " * See http://www.sympy.org/ for more information.\n" + " *\n" + " * This file is part of 'project'\n" + " */\n" + "fn test(x: f64, y: f64, z: f64) -> f64 {\n" + " let out1 = z*(x + y);\n" + " out1\n" + "}\n" + ) % {'version_line': version_line} + assert source == expected + + +def test_simple_code_nameout(): + expr = Equality(z, (x + y)) + name_expr = ("test", expr) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test(x: f64, y: f64) -> f64 {\n" + " let z = x + y;\n" + " z\n" + "}\n" + ) + assert source == expected + + +def test_numbersymbol(): + name_expr = ("test", pi**Catalan) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test() -> f64 {\n" + " const Catalan: f64 = %s;\n" + " let out1 = PI.powf(Catalan);\n" + " out1\n" + "}\n" + ) % Catalan.evalf(17) + assert source == expected + + +@XFAIL +def test_numbersymbol_inline(): + # FIXME: how to pass inline to the RustCodePrinter? + name_expr = ("test", [pi**Catalan, EulerGamma]) + result, = codegen(name_expr, "Rust", header=False, + empty=False, inline=True) + source = result[1] + expected = ( + "fn test() -> (f64, f64) {\n" + " const Catalan: f64 = %s;\n" + " const EulerGamma: f64 = %s;\n" + " let out1 = PI.powf(Catalan);\n" + " let out2 = EulerGamma);\n" + " (out1, out2)\n" + "}\n" + ) % (Catalan.evalf(17), EulerGamma.evalf(17)) + assert source == expected + + +def test_argument_order(): + expr = x + y + routine = make_routine("test", expr, argument_sequence=[z, x, y], language="rust") + code_gen = RustCodeGen() + output = StringIO() + code_gen.dump_rs([routine], output, "test", header=False, empty=False) + source = output.getvalue() + expected = ( + "fn test(z: f64, x: f64, y: f64) -> f64 {\n" + " let out1 = x + y;\n" + " out1\n" + "}\n" + ) + assert source == expected + + +def test_multiple_results_rust(): + # Here the output order is the input order + expr1 = (x + y)*z + expr2 = (x - y)*z + name_expr = ("test", [expr1, expr2]) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test(x: f64, y: f64, z: f64) -> (f64, f64) {\n" + " let out1 = z*(x + y);\n" + " let out2 = z*(x - y);\n" + " (out1, out2)\n" + "}\n" + ) + assert source == expected + + +def test_results_named_unordered(): + # Here output order is based on name_expr + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test(x: f64, y: f64, z: f64) -> (f64, f64, f64) {\n" + " let C = z*(x + y);\n" + " let A = z*(x - y);\n" + " let B = 2*x;\n" + " (C, A, B)\n" + "}\n" + ) + assert source == expected + + +def test_results_named_ordered(): + A, B, C = symbols('A,B,C') + expr1 = Equality(C, (x + y)*z) + expr2 = Equality(A, (x - y)*z) + expr3 = Equality(B, 2*x) + name_expr = ("test", [expr1, expr2, expr3]) + result = codegen(name_expr, "Rust", header=False, empty=False, + argument_sequence=(x, z, y)) + assert result[0][0] == "test.rs" + source = result[0][1] + expected = ( + "fn test(x: f64, z: f64, y: f64) -> (f64, f64, f64) {\n" + " let C = z*(x + y);\n" + " let A = z*(x - y);\n" + " let B = 2*x;\n" + " (C, A, B)\n" + "}\n" + ) + assert source == expected + + +def test_complicated_rs_codegen(): + from sympy.functions.elementary.trigonometric import (cos, sin, tan) + name_expr = ("testlong", + [ ((sin(x) + cos(y) + tan(z))**3).expand(), + cos(cos(cos(cos(cos(cos(cos(cos(x + y + z)))))))) + ]) + result = codegen(name_expr, "Rust", header=False, empty=False) + assert result[0][0] == "testlong.rs" + source = result[0][1] + expected = ( + "fn testlong(x: f64, y: f64, z: f64) -> (f64, f64) {\n" + " let out1 = x.sin().powi(3) + 3*x.sin().powi(2)*y.cos()" + " + 3*x.sin().powi(2)*z.tan() + 3*x.sin()*y.cos().powi(2)" + " + 6*x.sin()*y.cos()*z.tan() + 3*x.sin()*z.tan().powi(2)" + " + y.cos().powi(3) + 3*y.cos().powi(2)*z.tan()" + " + 3*y.cos()*z.tan().powi(2) + z.tan().powi(3);\n" + " let out2 = (x + y + z).cos().cos().cos().cos()" + ".cos().cos().cos().cos();\n" + " (out1, out2)\n" + "}\n" + ) + assert source == expected + + +def test_output_arg_mixed_unordered(): + # named outputs are alphabetical, unnamed output appear in the given order + from sympy.functions.elementary.trigonometric import (cos, sin) + a = symbols("a") + name_expr = ("foo", [cos(2*x), Equality(y, sin(x)), cos(x), Equality(a, sin(2*x))]) + result, = codegen(name_expr, "Rust", header=False, empty=False) + assert result[0] == "foo.rs" + source = result[1] + expected = ( + "fn foo(x: f64) -> (f64, f64, f64, f64) {\n" + " let out1 = (2*x).cos();\n" + " let y = x.sin();\n" + " let out3 = x.cos();\n" + " let a = (2*x).sin();\n" + " (out1, y, out3, a)\n" + "}\n" + ) + assert source == expected + + +def test_piecewise_(): + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True), evaluate=False) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn pwtest(x: f64) -> f64 {\n" + " let out1 = if (x < -1.0) {\n" + " 0\n" + " } else if (x <= 1.0) {\n" + " x.powi(2)\n" + " } else if (x > 1.0) {\n" + " 2 - x\n" + " } else {\n" + " 1\n" + " };\n" + " out1\n" + "}\n" + ) + assert source == expected + + +@XFAIL +def test_piecewise_inline(): + # FIXME: how to pass inline to the RustCodePrinter? + pw = Piecewise((0, x < -1), (x**2, x <= 1), (-x+2, x > 1), (1, True)) + name_expr = ("pwtest", pw) + result, = codegen(name_expr, "Rust", header=False, empty=False, + inline=True) + source = result[1] + expected = ( + "fn pwtest(x: f64) -> f64 {\n" + " let out1 = if (x < -1) { 0 } else if (x <= 1) { x.powi(2) }" + " else if (x > 1) { -x + 2 } else { 1 };\n" + " out1\n" + "}\n" + ) + assert source == expected + + +def test_multifcns_per_file(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Rust", header=False, empty=False) + assert result[0][0] == "foo.rs" + source = result[0][1] + expected = ( + "fn foo(x: f64, y: f64) -> (f64, f64) {\n" + " let out1 = 2*x;\n" + " let out2 = 3*y;\n" + " (out1, out2)\n" + "}\n" + "fn bar(y: f64) -> (f64, f64) {\n" + " let out1 = y.powi(2);\n" + " let out2 = 4*y;\n" + " (out1, out2)\n" + "}\n" + ) + assert source == expected + + +def test_multifcns_per_file_w_header(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result = codegen(name_expr, "Rust", header=True, empty=False) + assert result[0][0] == "foo.rs" + source = result[0][1] + version_str = "Code generated with SymPy %s" % sympy.__version__ + version_line = version_str.center(76).rstrip() + expected = ( + "/*\n" + " *%(version_line)s\n" + " *\n" + " * See http://www.sympy.org/ for more information.\n" + " *\n" + " * This file is part of 'project'\n" + " */\n" + "fn foo(x: f64, y: f64) -> (f64, f64) {\n" + " let out1 = 2*x;\n" + " let out2 = 3*y;\n" + " (out1, out2)\n" + "}\n" + "fn bar(y: f64) -> (f64, f64) {\n" + " let out1 = y.powi(2);\n" + " let out2 = 4*y;\n" + " (out1, out2)\n" + "}\n" + ) % {'version_line': version_line} + assert source == expected + + +def test_filename_match_prefix(): + name_expr = [ ("foo", [2*x, 3*y]), ("bar", [y**2, 4*y]) ] + result, = codegen(name_expr, "Rust", prefix="baz", header=False, + empty=False) + assert result[0] == "baz.rs" + + +def test_InOutArgument(): + expr = Equality(x, x**2) + name_expr = ("mysqr", expr) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn mysqr(x: f64) -> f64 {\n" + " let x = x.powi(2);\n" + " x\n" + "}\n" + ) + assert source == expected + + +def test_InOutArgument_order(): + # can specify the order as (x, y) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Rust", header=False, + empty=False, argument_sequence=(x,y)) + source = result[1] + expected = ( + "fn test(x: f64, y: f64) -> f64 {\n" + " let x = x.powi(2) + y;\n" + " x\n" + "}\n" + ) + assert source == expected + # make sure it gives (x, y) not (y, x) + expr = Equality(x, x**2 + y) + name_expr = ("test", expr) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test(x: f64, y: f64) -> f64 {\n" + " let x = x.powi(2) + y;\n" + " x\n" + "}\n" + ) + assert source == expected + + +def test_not_supported(): + f = Function('f') + name_expr = ("test", [f(x).diff(x), S.ComplexInfinity]) + result, = codegen(name_expr, "Rust", header=False, empty=False) + source = result[1] + expected = ( + "fn test(x: f64) -> (f64, f64) {\n" + " // unsupported: Derivative(f(x), x)\n" + " // unsupported: zoo\n" + " let out1 = Derivative(f(x), x);\n" + " let out2 = zoo;\n" + " (out1, out2)\n" + "}\n" + ) + assert source == expected + + +def test_global_vars_rust(): + x, y, z, t = symbols("x y z t") + result = codegen(('f', x*y), "Rust", header=False, empty=False, + global_vars=(y,)) + source = result[0][1] + expected = ( + "fn f(x: f64) -> f64 {\n" + " let out1 = x*y;\n" + " out1\n" + "}\n" + ) + assert source == expected + + result = codegen(('f', x*y+z), "Rust", header=False, empty=False, + argument_sequence=(x, y), global_vars=(z, t)) + source = result[0][1] + expected = ( + "fn f(x: f64, y: f64) -> f64 {\n" + " let out1 = x*y + z;\n" + " out1\n" + "}\n" + ) + assert source == expected diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_decorator.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_decorator.py new file mode 100644 index 0000000000000000000000000000000000000000..b1870d4db8f719fdabfeab14120bfb3ce10131a9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_decorator.py @@ -0,0 +1,129 @@ +from functools import wraps + +from sympy.utilities.decorator import threaded, xthreaded, memoize_property, deprecated +from sympy.testing.pytest import warns_deprecated_sympy + +from sympy.core.basic import Basic +from sympy.core.relational import Eq +from sympy.matrices.dense import Matrix + +from sympy.abc import x, y + + +def test_threaded(): + @threaded + def function(expr, *args): + return 2*expr + sum(args) + + assert function(Matrix([[x, y], [1, x]]), 1, 2) == \ + Matrix([[2*x + 3, 2*y + 3], [5, 2*x + 3]]) + + assert function(Eq(x, y), 1, 2) == Eq(2*x + 3, 2*y + 3) + + assert function([x, y], 1, 2) == [2*x + 3, 2*y + 3] + assert function((x, y), 1, 2) == (2*x + 3, 2*y + 3) + + assert function({x, y}, 1, 2) == {2*x + 3, 2*y + 3} + + @threaded + def function(expr, n): + return expr**n + + assert function(x + y, 2) == x**2 + y**2 + assert function(x, 2) == x**2 + + +def test_xthreaded(): + @xthreaded + def function(expr, n): + return expr**n + + assert function(x + y, 2) == (x + y)**2 + + +def test_wraps(): + def my_func(x): + """My function. """ + + my_func.is_my_func = True + + new_my_func = threaded(my_func) + new_my_func = wraps(my_func)(new_my_func) + + assert new_my_func.__name__ == 'my_func' + assert new_my_func.__doc__ == 'My function. ' + assert hasattr(new_my_func, 'is_my_func') + assert new_my_func.is_my_func is True + + +def test_memoize_property(): + class TestMemoize(Basic): + @memoize_property + def prop(self): + return Basic() + + member = TestMemoize() + obj1 = member.prop + obj2 = member.prop + assert obj1 is obj2 + +def test_deprecated(): + @deprecated('deprecated_function is deprecated', + deprecated_since_version='1.10', + # This is the target at the top of the file, which will never + # go away. + active_deprecations_target='active-deprecations') + def deprecated_function(x): + return x + + with warns_deprecated_sympy(): + assert deprecated_function(1) == 1 + + @deprecated('deprecated_class is deprecated', + deprecated_since_version='1.10', + active_deprecations_target='active-deprecations') + class deprecated_class: + pass + + with warns_deprecated_sympy(): + assert isinstance(deprecated_class(), deprecated_class) + + # Ensure the class decorator works even when the class never returns + # itself + @deprecated('deprecated_class_new is deprecated', + deprecated_since_version='1.10', + active_deprecations_target='active-deprecations') + class deprecated_class_new: + def __new__(cls, arg): + return arg + + with warns_deprecated_sympy(): + assert deprecated_class_new(1) == 1 + + @deprecated('deprecated_class_init is deprecated', + deprecated_since_version='1.10', + active_deprecations_target='active-deprecations') + class deprecated_class_init: + def __init__(self, arg): + self.arg = 1 + + with warns_deprecated_sympy(): + assert deprecated_class_init(1).arg == 1 + + @deprecated('deprecated_class_new_init is deprecated', + deprecated_since_version='1.10', + active_deprecations_target='active-deprecations') + class deprecated_class_new_init: + def __new__(cls, arg): + if arg == 0: + return arg + return object.__new__(cls) + + def __init__(self, arg): + self.arg = 1 + + with warns_deprecated_sympy(): + assert deprecated_class_new_init(0) == 0 + + with warns_deprecated_sympy(): + assert deprecated_class_new_init(1).arg == 1 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_deprecated.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_deprecated.py new file mode 100644 index 0000000000000000000000000000000000000000..dd4534ef1abc38ff368011b3ef9d11c497f3675b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_deprecated.py @@ -0,0 +1,13 @@ +from sympy.testing.pytest import warns_deprecated_sympy + +# See https://github.com/sympy/sympy/pull/18095 + +def test_deprecated_utilities(): + with warns_deprecated_sympy(): + import sympy.utilities.pytest # noqa:F401 + with warns_deprecated_sympy(): + import sympy.utilities.runtests # noqa:F401 + with warns_deprecated_sympy(): + import sympy.utilities.randtest # noqa:F401 + with warns_deprecated_sympy(): + import sympy.utilities.tmpfiles # noqa:F401 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_enumerative.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_enumerative.py new file mode 100644 index 0000000000000000000000000000000000000000..357499d5fd400b14e2bcad067f3015b74b0e9003 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_enumerative.py @@ -0,0 +1,179 @@ +import string +from itertools import zip_longest + +from sympy.utilities.enumerative import ( + list_visitor, + MultisetPartitionTraverser, + multiset_partitions_taocp + ) +from sympy.utilities.iterables import _set_partitions + +# first some functions only useful as test scaffolding - these provide +# straightforward, but slow reference implementations against which to +# compare the real versions, and also a comparison to verify that +# different versions are giving identical results. + +def part_range_filter(partition_iterator, lb, ub): + """ + Filters (on the number of parts) a multiset partition enumeration + + Arguments + ========= + + lb, and ub are a range (in the Python slice sense) on the lpart + variable returned from a multiset partition enumeration. Recall + that lpart is 0-based (it points to the topmost part on the part + stack), so if you want to return parts of sizes 2,3,4,5 you would + use lb=1 and ub=5. + """ + for state in partition_iterator: + f, lpart, pstack = state + if lpart >= lb and lpart < ub: + yield state + +def multiset_partitions_baseline(multiplicities, components): + """Enumerates partitions of a multiset + + Parameters + ========== + + multiplicities + list of integer multiplicities of the components of the multiset. + + components + the components (elements) themselves + + Returns + ======= + + Set of partitions. Each partition is tuple of parts, and each + part is a tuple of components (with repeats to indicate + multiplicity) + + Notes + ===== + + Multiset partitions can be created as equivalence classes of set + partitions, and this function does just that. This approach is + slow and memory intensive compared to the more advanced algorithms + available, but the code is simple and easy to understand. Hence + this routine is strictly for testing -- to provide a + straightforward baseline against which to regress the production + versions. (This code is a simplified version of an earlier + production implementation.) + """ + + canon = [] # list of components with repeats + for ct, elem in zip(multiplicities, components): + canon.extend([elem]*ct) + + # accumulate the multiset partitions in a set to eliminate dups + cache = set() + n = len(canon) + for nc, q in _set_partitions(n): + rv = [[] for i in range(nc)] + for i in range(n): + rv[q[i]].append(canon[i]) + canonical = tuple( + sorted([tuple(p) for p in rv])) + cache.add(canonical) + return cache + + +def compare_multiset_w_baseline(multiplicities): + """ + Enumerates the partitions of multiset with AOCP algorithm and + baseline implementation, and compare the results. + + """ + letters = string.ascii_lowercase + bl_partitions = multiset_partitions_baseline(multiplicities, letters) + + # The partitions returned by the different algorithms may have + # their parts in different orders. Also, they generate partitions + # in different orders. Hence the sorting, and set comparison. + + aocp_partitions = set() + for state in multiset_partitions_taocp(multiplicities): + p1 = tuple(sorted( + [tuple(p) for p in list_visitor(state, letters)])) + aocp_partitions.add(p1) + + assert bl_partitions == aocp_partitions + +def compare_multiset_states(s1, s2): + """compare for equality two instances of multiset partition states + + This is useful for comparing different versions of the algorithm + to verify correctness.""" + # Comparison is physical, the only use of semantics is to ignore + # trash off the top of the stack. + f1, lpart1, pstack1 = s1 + f2, lpart2, pstack2 = s2 + + if (lpart1 == lpart2) and (f1[0:lpart1+1] == f2[0:lpart2+1]): + if pstack1[0:f1[lpart1+1]] == pstack2[0:f2[lpart2+1]]: + return True + return False + +def test_multiset_partitions_taocp(): + """Compares the output of multiset_partitions_taocp with a baseline + (set partition based) implementation.""" + + # Test cases should not be too large, since the baseline + # implementation is fairly slow. + multiplicities = [2,2] + compare_multiset_w_baseline(multiplicities) + + multiplicities = [4,3,1] + compare_multiset_w_baseline(multiplicities) + +def test_multiset_partitions_versions(): + """Compares Knuth-based versions of multiset_partitions""" + multiplicities = [5,2,2,1] + m = MultisetPartitionTraverser() + for s1, s2 in zip_longest(m.enum_all(multiplicities), + multiset_partitions_taocp(multiplicities)): + assert compare_multiset_states(s1, s2) + +def subrange_exercise(mult, lb, ub): + """Compare filter-based and more optimized subrange implementations + + Helper for tests, called with both small and larger multisets. + """ + m = MultisetPartitionTraverser() + assert m.count_partitions(mult) == \ + m.count_partitions_slow(mult) + + # Note - multiple traversals from the same + # MultisetPartitionTraverser object cannot execute at the same + # time, hence make several instances here. + ma = MultisetPartitionTraverser() + mc = MultisetPartitionTraverser() + md = MultisetPartitionTraverser() + + # Several paths to compute just the size two partitions + a_it = ma.enum_range(mult, lb, ub) + b_it = part_range_filter(multiset_partitions_taocp(mult), lb, ub) + c_it = part_range_filter(mc.enum_small(mult, ub), lb, sum(mult)) + d_it = part_range_filter(md.enum_large(mult, lb), 0, ub) + + for sa, sb, sc, sd in zip_longest(a_it, b_it, c_it, d_it): + assert compare_multiset_states(sa, sb) + assert compare_multiset_states(sa, sc) + assert compare_multiset_states(sa, sd) + +def test_subrange(): + # Quick, but doesn't hit some of the corner cases + mult = [4,4,2,1] # mississippi + lb = 1 + ub = 2 + subrange_exercise(mult, lb, ub) + + +def test_subrange_large(): + # takes a second or so, depending on cpu, Python version, etc. + mult = [6,3,2,1] + lb = 4 + ub = 7 + subrange_exercise(mult, lb, ub) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_exceptions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..d91e55e95d0ae4ac57cdd1989e0b3d39a55cd07d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_exceptions.py @@ -0,0 +1,12 @@ +from sympy.testing.pytest import raises +from sympy.utilities.exceptions import sympy_deprecation_warning + +# Only test exceptions here because the other cases are tested in the +# warns_deprecated_sympy tests +def test_sympy_deprecation_warning(): + raises(TypeError, lambda: sympy_deprecation_warning('test', + deprecated_since_version=1.10, + active_deprecations_target='active-deprecations')) + + raises(ValueError, lambda: sympy_deprecation_warning('test', + deprecated_since_version="1.10", active_deprecations_target='(active-deprecations)=')) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_iterables.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_iterables.py new file mode 100644 index 0000000000000000000000000000000000000000..1003522bcd556c6f63e04de7da57b43498575fee --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_iterables.py @@ -0,0 +1,945 @@ +from textwrap import dedent +from itertools import islice, product + +from sympy.core.basic import Basic +from sympy.core.numbers import Integer +from sympy.core.sorting import ordered +from sympy.core.symbol import (Dummy, symbols) +from sympy.functions.combinatorial.factorials import factorial +from sympy.matrices.dense import Matrix +from sympy.combinatorics import RGS_enum, RGS_unrank, Permutation +from sympy.utilities.iterables import ( + _partition, _set_partitions, binary_partitions, bracelets, capture, + cartes, common_prefix, common_suffix, connected_components, dict_merge, + filter_symbols, flatten, generate_bell, generate_derangements, + generate_involutions, generate_oriented_forest, group, has_dups, ibin, + iproduct, kbins, minlex, multiset, multiset_combinations, + multiset_partitions, multiset_permutations, necklaces, numbered_symbols, + partitions, permutations, postfixes, + prefixes, reshape, rotate_left, rotate_right, runs, sift, + strongly_connected_components, subsets, take, topological_sort, unflatten, + uniq, variations, ordered_partitions, rotations, is_palindromic, iterable, + NotIterable, multiset_derangements, signed_permutations, + sequence_partitions, sequence_partitions_empty) +from sympy.utilities.enumerative import ( + factoring_visitor, multiset_partitions_taocp ) + +from sympy.core.singleton import S +from sympy.testing.pytest import raises, warns_deprecated_sympy + +w, x, y, z = symbols('w,x,y,z') + + +def test_deprecated_iterables(): + from sympy.utilities.iterables import default_sort_key, ordered + with warns_deprecated_sympy(): + assert list(ordered([y, x])) == [x, y] + with warns_deprecated_sympy(): + assert sorted([y, x], key=default_sort_key) == [x, y] + + +def test_is_palindromic(): + assert is_palindromic('') + assert is_palindromic('x') + assert is_palindromic('xx') + assert is_palindromic('xyx') + assert not is_palindromic('xy') + assert not is_palindromic('xyzx') + assert is_palindromic('xxyzzyx', 1) + assert not is_palindromic('xxyzzyx', 2) + assert is_palindromic('xxyzzyx', 2, -1) + assert is_palindromic('xxyzzyx', 2, 6) + assert is_palindromic('xxyzyx', 1) + assert not is_palindromic('xxyzyx', 2) + assert is_palindromic('xxyzyx', 2, 2 + 3) + + +def test_flatten(): + assert flatten((1, (1,))) == [1, 1] + assert flatten((x, (x,))) == [x, x] + + ls = [[(-2, -1), (1, 2)], [(0, 0)]] + + assert flatten(ls, levels=0) == ls + assert flatten(ls, levels=1) == [(-2, -1), (1, 2), (0, 0)] + assert flatten(ls, levels=2) == [-2, -1, 1, 2, 0, 0] + assert flatten(ls, levels=3) == [-2, -1, 1, 2, 0, 0] + + raises(ValueError, lambda: flatten(ls, levels=-1)) + + class MyOp(Basic): + pass + + assert flatten([MyOp(x, y), z]) == [MyOp(x, y), z] + assert flatten([MyOp(x, y), z], cls=MyOp) == [x, y, z] + + assert flatten({1, 11, 2}) == list({1, 11, 2}) + + +def test_iproduct(): + assert list(iproduct()) == [()] + assert list(iproduct([])) == [] + assert list(iproduct([1,2,3])) == [(1,),(2,),(3,)] + assert sorted(iproduct([1, 2], [3, 4, 5])) == [ + (1,3),(1,4),(1,5),(2,3),(2,4),(2,5)] + assert sorted(iproduct([0,1],[0,1],[0,1])) == [ + (0,0,0),(0,0,1),(0,1,0),(0,1,1),(1,0,0),(1,0,1),(1,1,0),(1,1,1)] + assert iterable(iproduct(S.Integers)) is True + assert iterable(iproduct(S.Integers, S.Integers)) is True + assert (3,) in iproduct(S.Integers) + assert (4, 5) in iproduct(S.Integers, S.Integers) + assert (1, 2, 3) in iproduct(S.Integers, S.Integers, S.Integers) + triples = set(islice(iproduct(S.Integers, S.Integers, S.Integers), 1000)) + for n1, n2, n3 in triples: + assert isinstance(n1, Integer) + assert isinstance(n2, Integer) + assert isinstance(n3, Integer) + for t in set(product(*([range(-2, 3)]*3))): + assert t in iproduct(S.Integers, S.Integers, S.Integers) + + +def test_group(): + assert group([]) == [] + assert group([], multiple=False) == [] + + assert group([1]) == [[1]] + assert group([1], multiple=False) == [(1, 1)] + + assert group([1, 1]) == [[1, 1]] + assert group([1, 1], multiple=False) == [(1, 2)] + + assert group([1, 1, 1]) == [[1, 1, 1]] + assert group([1, 1, 1], multiple=False) == [(1, 3)] + + assert group([1, 2, 1]) == [[1], [2], [1]] + assert group([1, 2, 1], multiple=False) == [(1, 1), (2, 1), (1, 1)] + + assert group([1, 1, 2, 2, 2, 1, 3, 3]) == [[1, 1], [2, 2, 2], [1], [3, 3]] + assert group([1, 1, 2, 2, 2, 1, 3, 3], multiple=False) == [(1, 2), + (2, 3), (1, 1), (3, 2)] + + +def test_subsets(): + # combinations + assert list(subsets([1, 2, 3], 0)) == [()] + assert list(subsets([1, 2, 3], 1)) == [(1,), (2,), (3,)] + assert list(subsets([1, 2, 3], 2)) == [(1, 2), (1, 3), (2, 3)] + assert list(subsets([1, 2, 3], 3)) == [(1, 2, 3)] + l = list(range(4)) + assert list(subsets(l, 0, repetition=True)) == [()] + assert list(subsets(l, 1, repetition=True)) == [(0,), (1,), (2,), (3,)] + assert list(subsets(l, 2, repetition=True)) == [(0, 0), (0, 1), (0, 2), + (0, 3), (1, 1), (1, 2), + (1, 3), (2, 2), (2, 3), + (3, 3)] + assert list(subsets(l, 3, repetition=True)) == [(0, 0, 0), (0, 0, 1), + (0, 0, 2), (0, 0, 3), + (0, 1, 1), (0, 1, 2), + (0, 1, 3), (0, 2, 2), + (0, 2, 3), (0, 3, 3), + (1, 1, 1), (1, 1, 2), + (1, 1, 3), (1, 2, 2), + (1, 2, 3), (1, 3, 3), + (2, 2, 2), (2, 2, 3), + (2, 3, 3), (3, 3, 3)] + assert len(list(subsets(l, 4, repetition=True))) == 35 + + assert list(subsets(l[:2], 3, repetition=False)) == [] + assert list(subsets(l[:2], 3, repetition=True)) == [(0, 0, 0), + (0, 0, 1), + (0, 1, 1), + (1, 1, 1)] + assert list(subsets([1, 2], repetition=True)) == \ + [(), (1,), (2,), (1, 1), (1, 2), (2, 2)] + assert list(subsets([1, 2], repetition=False)) == \ + [(), (1,), (2,), (1, 2)] + assert list(subsets([1, 2, 3], 2)) == \ + [(1, 2), (1, 3), (2, 3)] + assert list(subsets([1, 2, 3], 2, repetition=True)) == \ + [(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)] + + +def test_variations(): + # permutations + l = list(range(4)) + assert list(variations(l, 0, repetition=False)) == [()] + assert list(variations(l, 1, repetition=False)) == [(0,), (1,), (2,), (3,)] + assert list(variations(l, 2, repetition=False)) == [(0, 1), (0, 2), (0, 3), (1, 0), (1, 2), (1, 3), (2, 0), (2, 1), (2, 3), (3, 0), (3, 1), (3, 2)] + assert list(variations(l, 3, repetition=False)) == [(0, 1, 2), (0, 1, 3), (0, 2, 1), (0, 2, 3), (0, 3, 1), (0, 3, 2), (1, 0, 2), (1, 0, 3), (1, 2, 0), (1, 2, 3), (1, 3, 0), (1, 3, 2), (2, 0, 1), (2, 0, 3), (2, 1, 0), (2, 1, 3), (2, 3, 0), (2, 3, 1), (3, 0, 1), (3, 0, 2), (3, 1, 0), (3, 1, 2), (3, 2, 0), (3, 2, 1)] + assert list(variations(l, 0, repetition=True)) == [()] + assert list(variations(l, 1, repetition=True)) == [(0,), (1,), (2,), (3,)] + assert list(variations(l, 2, repetition=True)) == [(0, 0), (0, 1), (0, 2), + (0, 3), (1, 0), (1, 1), + (1, 2), (1, 3), (2, 0), + (2, 1), (2, 2), (2, 3), + (3, 0), (3, 1), (3, 2), + (3, 3)] + assert len(list(variations(l, 3, repetition=True))) == 64 + assert len(list(variations(l, 4, repetition=True))) == 256 + assert list(variations(l[:2], 3, repetition=False)) == [] + assert list(variations(l[:2], 3, repetition=True)) == [ + (0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), + (1, 0, 0), (1, 0, 1), (1, 1, 0), (1, 1, 1) + ] + + +def test_cartes(): + assert list(cartes([1, 2], [3, 4, 5])) == \ + [(1, 3), (1, 4), (1, 5), (2, 3), (2, 4), (2, 5)] + assert list(cartes()) == [()] + assert list(cartes('a')) == [('a',)] + assert list(cartes('a', repeat=2)) == [('a', 'a')] + assert list(cartes(list(range(2)))) == [(0,), (1,)] + + +def test_filter_symbols(): + s = numbered_symbols() + filtered = filter_symbols(s, symbols("x0 x2 x3")) + assert take(filtered, 3) == list(symbols("x1 x4 x5")) + + +def test_numbered_symbols(): + s = numbered_symbols(cls=Dummy) + assert isinstance(next(s), Dummy) + assert next(numbered_symbols('C', start=1, exclude=[symbols('C1')])) == \ + symbols('C2') + + +def test_sift(): + assert sift(list(range(5)), lambda _: _ % 2) == {1: [1, 3], 0: [0, 2, 4]} + assert sift([x, y], lambda _: _.has(x)) == {False: [y], True: [x]} + assert sift([S.One], lambda _: _.has(x)) == {False: [1]} + assert sift([0, 1, 2, 3], lambda x: x % 2, binary=True) == ( + [1, 3], [0, 2]) + assert sift([0, 1, 2, 3], lambda x: x % 3 == 1, binary=True) == ( + [1], [0, 2, 3]) + raises(ValueError, lambda: + sift([0, 1, 2, 3], lambda x: x % 3, binary=True)) + + +def test_take(): + X = numbered_symbols() + + assert take(X, 5) == list(symbols('x0:5')) + assert take(X, 5) == list(symbols('x5:10')) + + assert take([1, 2, 3, 4, 5], 5) == [1, 2, 3, 4, 5] + + +def test_dict_merge(): + assert dict_merge({}, {1: x, y: z}) == {1: x, y: z} + assert dict_merge({1: x, y: z}, {}) == {1: x, y: z} + + assert dict_merge({2: z}, {1: x, y: z}) == {1: x, 2: z, y: z} + assert dict_merge({1: x, y: z}, {2: z}) == {1: x, 2: z, y: z} + + assert dict_merge({1: y, 2: z}, {1: x, y: z}) == {1: x, 2: z, y: z} + assert dict_merge({1: x, y: z}, {1: y, 2: z}) == {1: y, 2: z, y: z} + + +def test_prefixes(): + assert list(prefixes([])) == [] + assert list(prefixes([1])) == [[1]] + assert list(prefixes([1, 2])) == [[1], [1, 2]] + + assert list(prefixes([1, 2, 3, 4, 5])) == \ + [[1], [1, 2], [1, 2, 3], [1, 2, 3, 4], [1, 2, 3, 4, 5]] + + +def test_postfixes(): + assert list(postfixes([])) == [] + assert list(postfixes([1])) == [[1]] + assert list(postfixes([1, 2])) == [[2], [1, 2]] + + assert list(postfixes([1, 2, 3, 4, 5])) == \ + [[5], [4, 5], [3, 4, 5], [2, 3, 4, 5], [1, 2, 3, 4, 5]] + + +def test_topological_sort(): + V = [2, 3, 5, 7, 8, 9, 10, 11] + E = [(7, 11), (7, 8), (5, 11), + (3, 8), (3, 10), (11, 2), + (11, 9), (11, 10), (8, 9)] + + assert topological_sort((V, E)) == [3, 5, 7, 8, 11, 2, 9, 10] + assert topological_sort((V, E), key=lambda v: -v) == \ + [7, 5, 11, 3, 10, 8, 9, 2] + + raises(ValueError, lambda: topological_sort((V, E + [(10, 7)]))) + + +def test_strongly_connected_components(): + assert strongly_connected_components(([], [])) == [] + assert strongly_connected_components(([1, 2, 3], [])) == [[1], [2], [3]] + + V = [1, 2, 3] + E = [(1, 2), (1, 3), (2, 1), (2, 3), (3, 1)] + assert strongly_connected_components((V, E)) == [[1, 2, 3]] + + V = [1, 2, 3, 4] + E = [(1, 2), (2, 3), (3, 2), (3, 4)] + assert strongly_connected_components((V, E)) == [[4], [2, 3], [1]] + + V = [1, 2, 3, 4] + E = [(1, 2), (2, 1), (3, 4), (4, 3)] + assert strongly_connected_components((V, E)) == [[1, 2], [3, 4]] + + +def test_connected_components(): + assert connected_components(([], [])) == [] + assert connected_components(([1, 2, 3], [])) == [[1], [2], [3]] + + V = [1, 2, 3] + E = [(1, 2), (1, 3), (2, 1), (2, 3), (3, 1)] + assert connected_components((V, E)) == [[1, 2, 3]] + + V = [1, 2, 3, 4] + E = [(1, 2), (2, 3), (3, 2), (3, 4)] + assert connected_components((V, E)) == [[1, 2, 3, 4]] + + V = [1, 2, 3, 4] + E = [(1, 2), (3, 4)] + assert connected_components((V, E)) == [[1, 2], [3, 4]] + + +def test_rotate(): + A = [0, 1, 2, 3, 4] + + assert rotate_left(A, 2) == [2, 3, 4, 0, 1] + assert rotate_right(A, 1) == [4, 0, 1, 2, 3] + A = [] + B = rotate_right(A, 1) + assert B == [] + B.append(1) + assert A == [] + B = rotate_left(A, 1) + assert B == [] + B.append(1) + assert A == [] + + +def test_multiset_partitions(): + A = [0, 1, 2, 3, 4] + + assert list(multiset_partitions(A, 5)) == [[[0], [1], [2], [3], [4]]] + assert len(list(multiset_partitions(A, 4))) == 10 + assert len(list(multiset_partitions(A, 3))) == 25 + + assert list(multiset_partitions([1, 1, 1, 2, 2], 2)) == [ + [[1, 1, 1, 2], [2]], [[1, 1, 1], [2, 2]], [[1, 1, 2, 2], [1]], + [[1, 1, 2], [1, 2]], [[1, 1], [1, 2, 2]]] + + assert list(multiset_partitions([1, 1, 2, 2], 2)) == [ + [[1, 1, 2], [2]], [[1, 1], [2, 2]], [[1, 2, 2], [1]], + [[1, 2], [1, 2]]] + + assert list(multiset_partitions([1, 2, 3, 4], 2)) == [ + [[1, 2, 3], [4]], [[1, 2, 4], [3]], [[1, 2], [3, 4]], + [[1, 3, 4], [2]], [[1, 3], [2, 4]], [[1, 4], [2, 3]], + [[1], [2, 3, 4]]] + + assert list(multiset_partitions([1, 2, 2], 2)) == [ + [[1, 2], [2]], [[1], [2, 2]]] + + assert list(multiset_partitions(3)) == [ + [[0, 1, 2]], [[0, 1], [2]], [[0, 2], [1]], [[0], [1, 2]], + [[0], [1], [2]]] + assert list(multiset_partitions(3, 2)) == [ + [[0, 1], [2]], [[0, 2], [1]], [[0], [1, 2]]] + assert list(multiset_partitions([1] * 3, 2)) == [[[1], [1, 1]]] + assert list(multiset_partitions([1] * 3)) == [ + [[1, 1, 1]], [[1], [1, 1]], [[1], [1], [1]]] + a = [3, 2, 1] + assert list(multiset_partitions(a)) == \ + list(multiset_partitions(sorted(a))) + assert list(multiset_partitions(a, 5)) == [] + assert list(multiset_partitions(a, 1)) == [[[1, 2, 3]]] + assert list(multiset_partitions(a + [4], 5)) == [] + assert list(multiset_partitions(a + [4], 1)) == [[[1, 2, 3, 4]]] + assert list(multiset_partitions(2, 5)) == [] + assert list(multiset_partitions(2, 1)) == [[[0, 1]]] + assert list(multiset_partitions('a')) == [[['a']]] + assert list(multiset_partitions('a', 2)) == [] + assert list(multiset_partitions('ab')) == [[['a', 'b']], [['a'], ['b']]] + assert list(multiset_partitions('ab', 1)) == [[['a', 'b']]] + assert list(multiset_partitions('aaa', 1)) == [['aaa']] + assert list(multiset_partitions([1, 1], 1)) == [[[1, 1]]] + ans = [('mpsyy',), ('mpsy', 'y'), ('mps', 'yy'), ('mps', 'y', 'y'), + ('mpyy', 's'), ('mpy', 'sy'), ('mpy', 's', 'y'), ('mp', 'syy'), + ('mp', 'sy', 'y'), ('mp', 's', 'yy'), ('mp', 's', 'y', 'y'), + ('msyy', 'p'), ('msy', 'py'), ('msy', 'p', 'y'), ('ms', 'pyy'), + ('ms', 'py', 'y'), ('ms', 'p', 'yy'), ('ms', 'p', 'y', 'y'), + ('myy', 'ps'), ('myy', 'p', 's'), ('my', 'psy'), ('my', 'ps', 'y'), + ('my', 'py', 's'), ('my', 'p', 'sy'), ('my', 'p', 's', 'y'), + ('m', 'psyy'), ('m', 'psy', 'y'), ('m', 'ps', 'yy'), + ('m', 'ps', 'y', 'y'), ('m', 'pyy', 's'), ('m', 'py', 'sy'), + ('m', 'py', 's', 'y'), ('m', 'p', 'syy'), + ('m', 'p', 'sy', 'y'), ('m', 'p', 's', 'yy'), + ('m', 'p', 's', 'y', 'y')] + assert [tuple("".join(part) for part in p) + for p in multiset_partitions('sympy')] == ans + factorings = [[24], [8, 3], [12, 2], [4, 6], [4, 2, 3], + [6, 2, 2], [2, 2, 2, 3]] + assert [factoring_visitor(p, [2,3]) for + p in multiset_partitions_taocp([3, 1])] == factorings + + +def test_multiset_combinations(): + ans = ['iii', 'iim', 'iip', 'iis', 'imp', 'ims', 'ipp', 'ips', + 'iss', 'mpp', 'mps', 'mss', 'pps', 'pss', 'sss'] + assert [''.join(i) for i in + list(multiset_combinations('mississippi', 3))] == ans + M = multiset('mississippi') + assert [''.join(i) for i in + list(multiset_combinations(M, 3))] == ans + assert [''.join(i) for i in multiset_combinations(M, 30)] == [] + assert list(multiset_combinations([[1], [2, 3]], 2)) == [[[1], [2, 3]]] + assert len(list(multiset_combinations('a', 3))) == 0 + assert len(list(multiset_combinations('a', 0))) == 1 + assert list(multiset_combinations('abc', 1)) == [['a'], ['b'], ['c']] + raises(ValueError, lambda: list(multiset_combinations({0: 3, 1: -1}, 2))) + + +def test_multiset_permutations(): + ans = ['abby', 'abyb', 'aybb', 'baby', 'bayb', 'bbay', 'bbya', 'byab', + 'byba', 'yabb', 'ybab', 'ybba'] + assert [''.join(i) for i in multiset_permutations('baby')] == ans + assert [''.join(i) for i in multiset_permutations(multiset('baby'))] == ans + assert list(multiset_permutations([0, 0, 0], 2)) == [[0, 0]] + assert list(multiset_permutations([0, 2, 1], 2)) == [ + [0, 1], [0, 2], [1, 0], [1, 2], [2, 0], [2, 1]] + assert len(list(multiset_permutations('a', 0))) == 1 + assert len(list(multiset_permutations('a', 3))) == 0 + for nul in ([], {}, ''): + assert list(multiset_permutations(nul)) == [[]] + assert list(multiset_permutations(nul, 0)) == [[]] + # impossible requests give no result + assert list(multiset_permutations(nul, 1)) == [] + assert list(multiset_permutations(nul, -1)) == [] + + def test(): + for i in range(1, 7): + print(i) + for p in multiset_permutations([0, 0, 1, 0, 1], i): + print(p) + assert capture(lambda: test()) == dedent('''\ + 1 + [0] + [1] + 2 + [0, 0] + [0, 1] + [1, 0] + [1, 1] + 3 + [0, 0, 0] + [0, 0, 1] + [0, 1, 0] + [0, 1, 1] + [1, 0, 0] + [1, 0, 1] + [1, 1, 0] + 4 + [0, 0, 0, 1] + [0, 0, 1, 0] + [0, 0, 1, 1] + [0, 1, 0, 0] + [0, 1, 0, 1] + [0, 1, 1, 0] + [1, 0, 0, 0] + [1, 0, 0, 1] + [1, 0, 1, 0] + [1, 1, 0, 0] + 5 + [0, 0, 0, 1, 1] + [0, 0, 1, 0, 1] + [0, 0, 1, 1, 0] + [0, 1, 0, 0, 1] + [0, 1, 0, 1, 0] + [0, 1, 1, 0, 0] + [1, 0, 0, 0, 1] + [1, 0, 0, 1, 0] + [1, 0, 1, 0, 0] + [1, 1, 0, 0, 0] + 6\n''') + raises(ValueError, lambda: list(multiset_permutations({0: 3, 1: -1}))) + + +def test_partitions(): + ans = [[{}], [(0, {})]] + for i in range(2): + assert list(partitions(0, size=i)) == ans[i] + assert list(partitions(1, 0, size=i)) == ans[i] + assert list(partitions(6, 2, 2, size=i)) == ans[i] + assert list(partitions(6, 2, None, size=i)) != ans[i] + assert list(partitions(6, None, 2, size=i)) != ans[i] + assert list(partitions(6, 2, 0, size=i)) == ans[i] + + assert list(partitions(6, k=2)) == [ + {2: 3}, {1: 2, 2: 2}, {1: 4, 2: 1}, {1: 6}] + + assert list(partitions(6, k=3)) == [ + {3: 2}, {1: 1, 2: 1, 3: 1}, {1: 3, 3: 1}, {2: 3}, {1: 2, 2: 2}, + {1: 4, 2: 1}, {1: 6}] + + assert list(partitions(8, k=4, m=3)) == [ + {4: 2}, {1: 1, 3: 1, 4: 1}, {2: 2, 4: 1}, {2: 1, 3: 2}] == [ + i for i in partitions(8, k=4, m=3) if all(k <= 4 for k in i) + and sum(i.values()) <=3] + + assert list(partitions(S(3), m=2)) == [ + {3: 1}, {1: 1, 2: 1}] + + assert list(partitions(4, k=3)) == [ + {1: 1, 3: 1}, {2: 2}, {1: 2, 2: 1}, {1: 4}] == [ + i for i in partitions(4) if all(k <= 3 for k in i)] + + + # Consistency check on output of _partitions and RGS_unrank. + # This provides a sanity test on both routines. Also verifies that + # the total number of partitions is the same in each case. + # (from pkrathmann2) + + for n in range(2, 6): + i = 0 + for m, q in _set_partitions(n): + assert q == RGS_unrank(i, n) + i += 1 + assert i == RGS_enum(n) + + +def test_binary_partitions(): + assert [i[:] for i in binary_partitions(10)] == [[8, 2], [8, 1, 1], + [4, 4, 2], [4, 4, 1, 1], [4, 2, 2, 2], [4, 2, 2, 1, 1], + [4, 2, 1, 1, 1, 1], [4, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2], + [2, 2, 2, 2, 1, 1], [2, 2, 2, 1, 1, 1, 1], [2, 2, 1, 1, 1, 1, 1, 1], + [2, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] + + assert len([j[:] for j in binary_partitions(16)]) == 36 + + +def test_bell_perm(): + assert [len(set(generate_bell(i))) for i in range(1, 7)] == [ + factorial(i) for i in range(1, 7)] + assert list(generate_bell(3)) == [ + (0, 1, 2), (0, 2, 1), (2, 0, 1), (2, 1, 0), (1, 2, 0), (1, 0, 2)] + # generate_bell and trotterjohnson are advertised to return the same + # permutations; this is not technically necessary so this test could + # be removed + for n in range(1, 5): + p = Permutation(range(n)) + b = generate_bell(n) + for bi in b: + assert bi == tuple(p.array_form) + p = p.next_trotterjohnson() + raises(ValueError, lambda: list(generate_bell(0))) # XXX is this consistent with other permutation algorithms? + + +def test_involutions(): + lengths = [1, 2, 4, 10, 26, 76] + for n, N in enumerate(lengths): + i = list(generate_involutions(n + 1)) + assert len(i) == N + assert len({Permutation(j)**2 for j in i}) == 1 + + +def test_derangements(): + assert len(list(generate_derangements(list(range(6))))) == 265 + assert ''.join(''.join(i) for i in generate_derangements('abcde')) == ( + 'badecbaecdbcaedbcdeabceadbdaecbdeacbdecabeacdbedacbedcacabedcadebcaebd' + 'cdaebcdbeacdeabcdebaceabdcebadcedabcedbadabecdaebcdaecbdcaebdcbeadceab' + 'dcebadeabcdeacbdebacdebcaeabcdeadbceadcbecabdecbadecdabecdbaedabcedacb' + 'edbacedbca') + assert list(generate_derangements([0, 1, 2, 3])) == [ + [1, 0, 3, 2], [1, 2, 3, 0], [1, 3, 0, 2], [2, 0, 3, 1], + [2, 3, 0, 1], [2, 3, 1, 0], [3, 0, 1, 2], [3, 2, 0, 1], [3, 2, 1, 0]] + assert list(generate_derangements([0, 1, 2, 2])) == [ + [2, 2, 0, 1], [2, 2, 1, 0]] + assert list(generate_derangements('ba')) == [list('ab')] + # multiset_derangements + D = multiset_derangements + assert list(D('abb')) == [] + assert [''.join(i) for i in D('ab')] == ['ba'] + assert [''.join(i) for i in D('abc')] == ['bca', 'cab'] + assert [''.join(i) for i in D('aabb')] == ['bbaa'] + assert [''.join(i) for i in D('aabbcccc')] == [ + 'ccccaabb', 'ccccabab', 'ccccabba', 'ccccbaab', 'ccccbaba', + 'ccccbbaa'] + assert [''.join(i) for i in D('aabbccc')] == [ + 'cccabba', 'cccabab', 'cccaabb', 'ccacbba', 'ccacbab', + 'ccacabb', 'cbccbaa', 'cbccaba', 'cbccaab', 'bcccbaa', + 'bcccaba', 'bcccaab'] + assert [''.join(i) for i in D('books')] == ['kbsoo', 'ksboo', + 'sbkoo', 'skboo', 'oksbo', 'oskbo', 'okbso', 'obkso', 'oskob', + 'oksob', 'osbok', 'obsok'] + assert list(generate_derangements([[3], [2], [2], [1]])) == [ + [[2], [1], [3], [2]], [[2], [3], [1], [2]]] + + +def test_necklaces(): + def count(n, k, f): + return len(list(necklaces(n, k, f))) + m = [] + for i in range(1, 8): + m.append(( + i, count(i, 2, 0), count(i, 2, 1), count(i, 3, 1))) + assert Matrix(m) == Matrix([ + [1, 2, 2, 3], + [2, 3, 3, 6], + [3, 4, 4, 10], + [4, 6, 6, 21], + [5, 8, 8, 39], + [6, 14, 13, 92], + [7, 20, 18, 198]]) + + +def test_bracelets(): + bc = list(bracelets(2, 4)) + assert Matrix(bc) == Matrix([ + [0, 0], + [0, 1], + [0, 2], + [0, 3], + [1, 1], + [1, 2], + [1, 3], + [2, 2], + [2, 3], + [3, 3] + ]) + bc = list(bracelets(4, 2)) + assert Matrix(bc) == Matrix([ + [0, 0, 0, 0], + [0, 0, 0, 1], + [0, 0, 1, 1], + [0, 1, 0, 1], + [0, 1, 1, 1], + [1, 1, 1, 1] + ]) + + +def test_generate_oriented_forest(): + assert list(generate_oriented_forest(5)) == [[0, 1, 2, 3, 4], + [0, 1, 2, 3, 3], [0, 1, 2, 3, 2], [0, 1, 2, 3, 1], [0, 1, 2, 3, 0], + [0, 1, 2, 2, 2], [0, 1, 2, 2, 1], [0, 1, 2, 2, 0], [0, 1, 2, 1, 2], + [0, 1, 2, 1, 1], [0, 1, 2, 1, 0], [0, 1, 2, 0, 1], [0, 1, 2, 0, 0], + [0, 1, 1, 1, 1], [0, 1, 1, 1, 0], [0, 1, 1, 0, 1], [0, 1, 1, 0, 0], + [0, 1, 0, 1, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 0]] + assert len(list(generate_oriented_forest(10))) == 1842 + + +def test_unflatten(): + r = list(range(10)) + assert unflatten(r) == list(zip(r[::2], r[1::2])) + assert unflatten(r, 5) == [tuple(r[:5]), tuple(r[5:])] + raises(ValueError, lambda: unflatten(list(range(10)), 3)) + raises(ValueError, lambda: unflatten(list(range(10)), -2)) + + +def test_common_prefix_suffix(): + assert common_prefix([], [1]) == [] + assert common_prefix(list(range(3))) == [0, 1, 2] + assert common_prefix(list(range(3)), list(range(4))) == [0, 1, 2] + assert common_prefix([1, 2, 3], [1, 2, 5]) == [1, 2] + assert common_prefix([1, 2, 3], [1, 3, 5]) == [1] + + assert common_suffix([], [1]) == [] + assert common_suffix(list(range(3))) == [0, 1, 2] + assert common_suffix(list(range(3)), list(range(3))) == [0, 1, 2] + assert common_suffix(list(range(3)), list(range(4))) == [] + assert common_suffix([1, 2, 3], [9, 2, 3]) == [2, 3] + assert common_suffix([1, 2, 3], [9, 7, 3]) == [3] + + +def test_minlex(): + assert minlex([1, 2, 0]) == (0, 1, 2) + assert minlex((1, 2, 0)) == (0, 1, 2) + assert minlex((1, 0, 2)) == (0, 2, 1) + assert minlex((1, 0, 2), directed=False) == (0, 1, 2) + assert minlex('aba') == 'aab' + assert minlex(('bb', 'aaa', 'c', 'a'), key=len) == ('c', 'a', 'bb', 'aaa') + + +def test_ordered(): + assert list(ordered((x, y), hash, default=False)) in [[x, y], [y, x]] + assert list(ordered((x, y), hash, default=False)) == \ + list(ordered((y, x), hash, default=False)) + assert list(ordered((x, y))) == [x, y] + + seq, keys = [[[1, 2, 1], [0, 3, 1], [1, 1, 3], [2], [1]], + (lambda x: len(x), lambda x: sum(x))] + assert list(ordered(seq, keys, default=False, warn=False)) == \ + [[1], [2], [1, 2, 1], [0, 3, 1], [1, 1, 3]] + raises(ValueError, lambda: + list(ordered(seq, keys, default=False, warn=True))) + + +def test_runs(): + assert runs([]) == [] + assert runs([1]) == [[1]] + assert runs([1, 1]) == [[1], [1]] + assert runs([1, 1, 2]) == [[1], [1, 2]] + assert runs([1, 2, 1]) == [[1, 2], [1]] + assert runs([2, 1, 1]) == [[2], [1], [1]] + from operator import lt + assert runs([2, 1, 1], lt) == [[2, 1], [1]] + + +def test_reshape(): + seq = list(range(1, 9)) + assert reshape(seq, [4]) == \ + [[1, 2, 3, 4], [5, 6, 7, 8]] + assert reshape(seq, (4,)) == \ + [(1, 2, 3, 4), (5, 6, 7, 8)] + assert reshape(seq, (2, 2)) == \ + [(1, 2, 3, 4), (5, 6, 7, 8)] + assert reshape(seq, (2, [2])) == \ + [(1, 2, [3, 4]), (5, 6, [7, 8])] + assert reshape(seq, ((2,), [2])) == \ + [((1, 2), [3, 4]), ((5, 6), [7, 8])] + assert reshape(seq, (1, [2], 1)) == \ + [(1, [2, 3], 4), (5, [6, 7], 8)] + assert reshape(tuple(seq), ([[1], 1, (2,)],)) == \ + (([[1], 2, (3, 4)],), ([[5], 6, (7, 8)],)) + assert reshape(tuple(seq), ([1], 1, (2,))) == \ + (([1], 2, (3, 4)), ([5], 6, (7, 8))) + assert reshape(list(range(12)), [2, [3], {2}, (1, (3,), 1)]) == \ + [[0, 1, [2, 3, 4], {5, 6}, (7, (8, 9, 10), 11)]] + raises(ValueError, lambda: reshape([0, 1], [-1])) + raises(ValueError, lambda: reshape([0, 1], [3])) + + +def test_uniq(): + assert list(uniq(p for p in partitions(4))) == \ + [{4: 1}, {1: 1, 3: 1}, {2: 2}, {1: 2, 2: 1}, {1: 4}] + assert list(uniq(x % 2 for x in range(5))) == [0, 1] + assert list(uniq('a')) == ['a'] + assert list(uniq('ababc')) == list('abc') + assert list(uniq([[1], [2, 1], [1]])) == [[1], [2, 1]] + assert list(uniq(permutations(i for i in [[1], 2, 2]))) == \ + [([1], 2, 2), (2, [1], 2), (2, 2, [1])] + assert list(uniq([2, 3, 2, 4, [2], [1], [2], [3], [1]])) == \ + [2, 3, 4, [2], [1], [3]] + f = [1] + raises(RuntimeError, lambda: [f.remove(i) for i in uniq(f)]) + f = [[1]] + raises(RuntimeError, lambda: [f.remove(i) for i in uniq(f)]) + + +def test_kbins(): + assert len(list(kbins('1123', 2, ordered=1))) == 24 + assert len(list(kbins('1123', 2, ordered=11))) == 36 + assert len(list(kbins('1123', 2, ordered=10))) == 10 + assert len(list(kbins('1123', 2, ordered=0))) == 5 + assert len(list(kbins('1123', 2, ordered=None))) == 3 + + def test1(): + for orderedval in [None, 0, 1, 10, 11]: + print('ordered =', orderedval) + for p in kbins([0, 0, 1], 2, ordered=orderedval): + print(' ', p) + assert capture(lambda : test1()) == dedent('''\ + ordered = None + [[0], [0, 1]] + [[0, 0], [1]] + ordered = 0 + [[0, 0], [1]] + [[0, 1], [0]] + ordered = 1 + [[0], [0, 1]] + [[0], [1, 0]] + [[1], [0, 0]] + ordered = 10 + [[0, 0], [1]] + [[1], [0, 0]] + [[0, 1], [0]] + [[0], [0, 1]] + ordered = 11 + [[0], [0, 1]] + [[0, 0], [1]] + [[0], [1, 0]] + [[0, 1], [0]] + [[1], [0, 0]] + [[1, 0], [0]]\n''') + + def test2(): + for orderedval in [None, 0, 1, 10, 11]: + print('ordered =', orderedval) + for p in kbins(list(range(3)), 2, ordered=orderedval): + print(' ', p) + assert capture(lambda : test2()) == dedent('''\ + ordered = None + [[0], [1, 2]] + [[0, 1], [2]] + ordered = 0 + [[0, 1], [2]] + [[0, 2], [1]] + [[0], [1, 2]] + ordered = 1 + [[0], [1, 2]] + [[0], [2, 1]] + [[1], [0, 2]] + [[1], [2, 0]] + [[2], [0, 1]] + [[2], [1, 0]] + ordered = 10 + [[0, 1], [2]] + [[2], [0, 1]] + [[0, 2], [1]] + [[1], [0, 2]] + [[0], [1, 2]] + [[1, 2], [0]] + ordered = 11 + [[0], [1, 2]] + [[0, 1], [2]] + [[0], [2, 1]] + [[0, 2], [1]] + [[1], [0, 2]] + [[1, 0], [2]] + [[1], [2, 0]] + [[1, 2], [0]] + [[2], [0, 1]] + [[2, 0], [1]] + [[2], [1, 0]] + [[2, 1], [0]]\n''') + + +def test_has_dups(): + assert has_dups(set()) is False + assert has_dups(list(range(3))) is False + assert has_dups([1, 2, 1]) is True + assert has_dups([[1], [1]]) is True + assert has_dups([[1], [2]]) is False + + +def test__partition(): + assert _partition('abcde', [1, 0, 1, 2, 0]) == [ + ['b', 'e'], ['a', 'c'], ['d']] + assert _partition('abcde', [1, 0, 1, 2, 0], 3) == [ + ['b', 'e'], ['a', 'c'], ['d']] + output = (3, [1, 0, 1, 2, 0]) + assert _partition('abcde', *output) == [['b', 'e'], ['a', 'c'], ['d']] + + +def test_ordered_partitions(): + from sympy.functions.combinatorial.numbers import nT + f = ordered_partitions + assert list(f(0, 1)) == [[]] + assert list(f(1, 0)) == [[]] + for i in range(1, 7): + for j in [None] + list(range(1, i)): + assert ( + sum(1 for p in f(i, j, 1)) == + sum(1 for p in f(i, j, 0)) == + nT(i, j)) + + +def test_rotations(): + assert list(rotations('ab')) == [['a', 'b'], ['b', 'a']] + assert list(rotations(range(3))) == [[0, 1, 2], [1, 2, 0], [2, 0, 1]] + assert list(rotations(range(3), dir=-1)) == [[0, 1, 2], [2, 0, 1], [1, 2, 0]] + + +def test_ibin(): + assert ibin(3) == [1, 1] + assert ibin(3, 3) == [0, 1, 1] + assert ibin(3, str=True) == '11' + assert ibin(3, 3, str=True) == '011' + assert list(ibin(2, 'all')) == [(0, 0), (0, 1), (1, 0), (1, 1)] + assert list(ibin(2, '', str=True)) == ['00', '01', '10', '11'] + raises(ValueError, lambda: ibin(-.5)) + raises(ValueError, lambda: ibin(2, 1)) + + +def test_iterable(): + assert iterable(0) is False + assert iterable(1) is False + assert iterable(None) is False + + class Test1(NotIterable): + pass + + assert iterable(Test1()) is False + + class Test2(NotIterable): + _iterable = True + + assert iterable(Test2()) is True + + class Test3: + pass + + assert iterable(Test3()) is False + + class Test4: + _iterable = True + + assert iterable(Test4()) is True + + class Test5: + def __iter__(self): + yield 1 + + assert iterable(Test5()) is True + + class Test6(Test5): + _iterable = False + + assert iterable(Test6()) is False + + +def test_sequence_partitions(): + assert list(sequence_partitions([1], 1)) == [[[1]]] + assert list(sequence_partitions([1, 2], 1)) == [[[1, 2]]] + assert list(sequence_partitions([1, 2], 2)) == [[[1], [2]]] + assert list(sequence_partitions([1, 2, 3], 1)) == [[[1, 2, 3]]] + assert list(sequence_partitions([1, 2, 3], 2)) == \ + [[[1], [2, 3]], [[1, 2], [3]]] + assert list(sequence_partitions([1, 2, 3], 3)) == [[[1], [2], [3]]] + + # Exceptional cases + assert list(sequence_partitions([], 0)) == [] + assert list(sequence_partitions([], 1)) == [] + assert list(sequence_partitions([1, 2], 0)) == [] + assert list(sequence_partitions([1, 2], 3)) == [] + + +def test_sequence_partitions_empty(): + assert list(sequence_partitions_empty([], 1)) == [[[]]] + assert list(sequence_partitions_empty([], 2)) == [[[], []]] + assert list(sequence_partitions_empty([], 3)) == [[[], [], []]] + assert list(sequence_partitions_empty([1], 1)) == [[[1]]] + assert list(sequence_partitions_empty([1], 2)) == [[[], [1]], [[1], []]] + assert list(sequence_partitions_empty([1], 3)) == \ + [[[], [], [1]], [[], [1], []], [[1], [], []]] + assert list(sequence_partitions_empty([1, 2], 1)) == [[[1, 2]]] + assert list(sequence_partitions_empty([1, 2], 2)) == \ + [[[], [1, 2]], [[1], [2]], [[1, 2], []]] + assert list(sequence_partitions_empty([1, 2], 3)) == [ + [[], [], [1, 2]], [[], [1], [2]], [[], [1, 2], []], + [[1], [], [2]], [[1], [2], []], [[1, 2], [], []] + ] + assert list(sequence_partitions_empty([1, 2, 3], 1)) == [[[1, 2, 3]]] + assert list(sequence_partitions_empty([1, 2, 3], 2)) == \ + [[[], [1, 2, 3]], [[1], [2, 3]], [[1, 2], [3]], [[1, 2, 3], []]] + assert list(sequence_partitions_empty([1, 2, 3], 3)) == [ + [[], [], [1, 2, 3]], [[], [1], [2, 3]], + [[], [1, 2], [3]], [[], [1, 2, 3], []], + [[1], [], [2, 3]], [[1], [2], [3]], + [[1], [2, 3], []], [[1, 2], [], [3]], + [[1, 2], [3], []], [[1, 2, 3], [], []] + ] + + # Exceptional cases + assert list(sequence_partitions([], 0)) == [] + assert list(sequence_partitions([1], 0)) == [] + assert list(sequence_partitions([1, 2], 0)) == [] + + +def test_signed_permutations(): + ans = [(0, 1, 1), (0, -1, 1), (0, 1, -1), (0, -1, -1), + (1, 0, 1), (-1, 0, 1), (1, 0, -1), (-1, 0, -1), + (1, 1, 0), (-1, 1, 0), (1, -1, 0), (-1, -1, 0)] + assert list(signed_permutations((0, 1, 1))) == ans + assert list(signed_permutations((1, 0, 1))) == ans + assert list(signed_permutations((1, 1, 0))) == ans diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_lambdify.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_lambdify.py new file mode 100644 index 0000000000000000000000000000000000000000..b094c67d39f09c24bcb9abc1e755cb5328e143e7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_lambdify.py @@ -0,0 +1,2263 @@ +from itertools import product +import math +import inspect +import linecache +import gc + +import mpmath +import cmath + +from sympy.testing.pytest import raises, warns_deprecated_sympy +from sympy.concrete.summations import Sum +from sympy.core.function import (Function, Lambda, diff) +from sympy.core.numbers import (E, Float, I, Rational, all_close, oo, pi) +from sympy.core.relational import Eq +from sympy.core.singleton import S +from sympy.core.symbol import (Dummy, symbols) +from sympy.functions.combinatorial.factorials import (RisingFactorial, factorial) +from sympy.functions.combinatorial.numbers import bernoulli, harmonic +from sympy.functions.elementary.complexes import Abs, sign +from sympy.functions.elementary.exponential import exp, log +from sympy.functions.elementary.hyperbolic import asinh,acosh,atanh +from sympy.functions.elementary.integers import floor +from sympy.functions.elementary.miscellaneous import (Max, Min, sqrt) +from sympy.functions.elementary.piecewise import Piecewise +from sympy.functions.elementary.trigonometric import (asin, acos, atan, cos, cot, sin, + sinc, tan) +from sympy.functions import sinh,cosh,tanh +from sympy.functions.special.bessel import (besseli, besselj, besselk, bessely, jn, yn) +from sympy.functions.special.beta_functions import (beta, betainc, betainc_regularized) +from sympy.functions.special.delta_functions import (Heaviside) +from sympy.functions.special.error_functions import (Ei, erf, erfc, fresnelc, fresnels, Si, Ci) +from sympy.functions.special.gamma_functions import (digamma, gamma, loggamma, polygamma) +from sympy.functions.special.zeta_functions import zeta +from sympy.integrals.integrals import Integral +from sympy.logic.boolalg import (And, false, ITE, Not, Or, true) +from sympy.matrices.expressions.dotproduct import DotProduct +from sympy.simplify.cse_main import cse +from sympy.tensor.array import derive_by_array, Array +from sympy.tensor.array.expressions import ArraySymbol +from sympy.tensor.indexed import IndexedBase, Idx +from sympy.utilities.lambdify import lambdify +from sympy.utilities.iterables import numbered_symbols +from sympy.vector import CoordSys3D +from sympy.core.expr import UnevaluatedExpr +from sympy.codegen.cfunctions import expm1, log1p, exp2, log2, log10, hypot, isnan, isinf +from sympy.codegen.numpy_nodes import logaddexp, logaddexp2, amin, amax, minimum, maximum +from sympy.codegen.scipy_nodes import cosm1, powm1 +from sympy.functions.elementary.complexes import re, im, arg +from sympy.functions.special.polynomials import \ + chebyshevt, chebyshevu, legendre, hermite, laguerre, gegenbauer, \ + assoc_legendre, assoc_laguerre, jacobi +from sympy.matrices import Matrix, MatrixSymbol, SparseMatrix +from sympy.printing.codeprinter import PrintMethodNotImplementedError +from sympy.printing.lambdarepr import LambdaPrinter +from sympy.printing.numpy import NumPyPrinter +from sympy.utilities.lambdify import implemented_function, lambdastr +from sympy.testing.pytest import skip +from sympy.utilities.decorator import conserve_mpmath_dps +from sympy.utilities.exceptions import ignore_warnings +from sympy.external import import_module +from sympy.functions.special.gamma_functions import uppergamma, lowergamma + + +import sympy + + +MutableDenseMatrix = Matrix + +numpy = import_module('numpy') +scipy = import_module('scipy', import_kwargs={'fromlist': ['sparse']}) +numexpr = import_module('numexpr') +tensorflow = import_module('tensorflow') +cupy = import_module('cupy') +jax = import_module('jax') +numba = import_module('numba') + +if tensorflow: + # Hide Tensorflow warnings + import os + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' + +w, x, y, z = symbols('w,x,y,z') + +#================== Test different arguments ======================= + + +def test_no_args(): + f = lambdify([], 1) + raises(TypeError, lambda: f(-1)) + assert f() == 1 + + +def test_single_arg(): + f = lambdify(x, 2*x) + assert f(1) == 2 + + +def test_list_args(): + f = lambdify([x, y], x + y) + assert f(1, 2) == 3 + + +def test_nested_args(): + f1 = lambdify([[w, x]], [w, x]) + assert f1([91, 2]) == [91, 2] + raises(TypeError, lambda: f1(1, 2)) + + f2 = lambdify([(w, x), (y, z)], [w, x, y, z]) + assert f2((18, 12), (73, 4)) == [18, 12, 73, 4] + raises(TypeError, lambda: f2(3, 4)) + + f3 = lambdify([w, [[[x]], y], z], [w, x, y, z]) + assert f3(10, [[[52]], 31], 44) == [10, 52, 31, 44] + + +def test_str_args(): + f = lambdify('x,y,z', 'z,y,x') + assert f(3, 2, 1) == (1, 2, 3) + assert f(1.0, 2.0, 3.0) == (3.0, 2.0, 1.0) + # make sure correct number of args required + raises(TypeError, lambda: f(0)) + + +def test_own_namespace_1(): + myfunc = lambda x: 1 + f = lambdify(x, sin(x), {"sin": myfunc}) + assert f(0.1) == 1 + assert f(100) == 1 + + +def test_own_namespace_2(): + def myfunc(x): + return 1 + f = lambdify(x, sin(x), {'sin': myfunc}) + assert f(0.1) == 1 + assert f(100) == 1 + + +def test_own_module(): + f = lambdify(x, sin(x), math) + assert f(0) == 0.0 + + p, q, r = symbols("p q r", real=True) + ae = abs(exp(p+UnevaluatedExpr(q+r))) + f = lambdify([p, q, r], [ae, ae], modules=math) + results = f(1.0, 1e18, -1e18) + refvals = [math.exp(1.0)]*2 + for res, ref in zip(results, refvals): + assert abs((res-ref)/ref) < 1e-15 + + +def test_bad_args(): + # no vargs given + raises(TypeError, lambda: lambdify(1)) + # same with vector exprs + raises(TypeError, lambda: lambdify([1, 2])) + + +def test_atoms(): + # Non-Symbol atoms should not be pulled out from the expression namespace + f = lambdify(x, pi + x, {"pi": 3.14}) + assert f(0) == 3.14 + f = lambdify(x, I + x, {"I": 1j}) + assert f(1) == 1 + 1j + +#================== Test different modules ========================= + +# high precision output of sin(0.2*pi) is used to detect if precision is lost unwanted + + +@conserve_mpmath_dps +def test_sympy_lambda(): + mpmath.mp.dps = 50 + sin02 = mpmath.mpf("0.19866933079506121545941262711838975037020672954020") + f = lambdify(x, sin(x), "sympy") + assert f(x) == sin(x) + prec = 1e-15 + assert -prec < f(Rational(1, 5)).evalf() - Float(str(sin02)) < prec + # arctan is in numpy module and should not be available + # The arctan below gives NameError. What is this supposed to test? + # raises(NameError, lambda: lambdify(x, arctan(x), "sympy")) + + +@conserve_mpmath_dps +def test_math_lambda(): + mpmath.mp.dps = 50 + sin02 = mpmath.mpf("0.19866933079506121545941262711838975037020672954020") + f = lambdify(x, sin(x), "math") + prec = 1e-15 + assert -prec < f(0.2) - sin02 < prec + raises(TypeError, lambda: f(x)) + # if this succeeds, it can't be a Python math function + + +@conserve_mpmath_dps +def test_mpmath_lambda(): + mpmath.mp.dps = 50 + sin02 = mpmath.mpf("0.19866933079506121545941262711838975037020672954020") + f = lambdify(x, sin(x), "mpmath") + prec = 1e-49 # mpmath precision is around 50 decimal places + assert -prec < f(mpmath.mpf("0.2")) - sin02 < prec + raises(TypeError, lambda: f(x)) + # if this succeeds, it can't be a mpmath function + + ref2 = (mpmath.mpf("1e-30") + - mpmath.mpf("1e-45")/2 + + 5*mpmath.mpf("1e-60")/6 + - 3*mpmath.mpf("1e-75")/4 + + 33*mpmath.mpf("1e-90")/40 + ) + f2a = lambdify((x, y), x**y - 1, "mpmath") + f2b = lambdify((x, y), powm1(x, y), "mpmath") + f2c = lambdify((x,), expm1(x*log1p(x)), "mpmath") + ans2a = f2a(mpmath.mpf("1")+mpmath.mpf("1e-15"), mpmath.mpf("1e-15")) + ans2b = f2b(mpmath.mpf("1")+mpmath.mpf("1e-15"), mpmath.mpf("1e-15")) + ans2c = f2c(mpmath.mpf("1e-15")) + assert abs(ans2a - ref2) < 1e-51 + assert abs(ans2b - ref2) < 1e-67 + assert abs(ans2c - ref2) < 1e-80 + + +@conserve_mpmath_dps +def test_number_precision(): + mpmath.mp.dps = 50 + sin02 = mpmath.mpf("0.19866933079506121545941262711838975037020672954020") + f = lambdify(x, sin02, "mpmath") + prec = 1e-49 # mpmath precision is around 50 decimal places + assert -prec < f(0) - sin02 < prec + +@conserve_mpmath_dps +def test_mpmath_precision(): + mpmath.mp.dps = 100 + assert str(lambdify((), pi.evalf(100), 'mpmath')()) == str(pi.evalf(100)) + +#================== Test Translations ============================== +# We can only check if all translated functions are valid. It has to be checked +# by hand if they are complete. + + +def test_math_transl(): + from sympy.utilities.lambdify import MATH_TRANSLATIONS + for sym, mat in MATH_TRANSLATIONS.items(): + assert sym in sympy.__dict__ + assert mat in math.__dict__ + + +def test_mpmath_transl(): + from sympy.utilities.lambdify import MPMATH_TRANSLATIONS + for sym, mat in MPMATH_TRANSLATIONS.items(): + assert sym in sympy.__dict__ or sym == 'Matrix' + assert mat in mpmath.__dict__ + + +def test_numpy_transl(): + if not numpy: + skip("numpy not installed.") + + from sympy.utilities.lambdify import NUMPY_TRANSLATIONS + for sym, nump in NUMPY_TRANSLATIONS.items(): + assert sym in sympy.__dict__ + assert nump in numpy.__dict__ + + +def test_scipy_transl(): + if not scipy: + skip("scipy not installed.") + + from sympy.utilities.lambdify import SCIPY_TRANSLATIONS + for sym, scip in SCIPY_TRANSLATIONS.items(): + assert sym in sympy.__dict__ + assert scip in scipy.__dict__ or scip in scipy.special.__dict__ + + +def test_numpy_translation_abs(): + if not numpy: + skip("numpy not installed.") + + f = lambdify(x, Abs(x), "numpy") + assert f(-1) == 1 + assert f(1) == 1 + + +def test_numexpr_printer(): + if not numexpr: + skip("numexpr not installed.") + + # if translation/printing is done incorrectly then evaluating + # a lambdified numexpr expression will throw an exception + from sympy.printing.lambdarepr import NumExprPrinter + + blacklist = ('where', 'complex', 'contains') + arg_tuple = (x, y, z) # some functions take more than one argument + for sym in NumExprPrinter._numexpr_functions.keys(): + if sym in blacklist: + continue + ssym = S(sym) + if hasattr(ssym, '_nargs'): + nargs = ssym._nargs[0] + else: + nargs = 1 + args = arg_tuple[:nargs] + f = lambdify(args, ssym(*args), modules='numexpr') + assert f(*(1, )*nargs) is not None + + +def test_cmath_sqrt(): + f = lambdify(x, sqrt(x), "cmath") + assert f(0) == 0 + assert f(1) == 1 + assert f(4) == 2 + assert abs(f(2) - 1.414) < 0.001 + assert f(-1) == 1j + assert f(-4) == 2j + + +def test_cmath_log(): + f = lambdify(x, log(x), "cmath") + assert abs(f(1) - 0) < 1e-15 + assert abs(f(cmath.e) - 1) < 1e-15 + assert abs(f(-1) - cmath.log(-1)) < 1e-15 + + +def test_cmath_sinh(): + f = lambdify(x, sinh(x), "cmath") + assert abs(f(0) - cmath.sinh(0)) < 1e-15 + assert abs(f(pi) - cmath.sinh(pi)) < 1e-15 + assert abs(f(-pi) - cmath.sinh(-pi)) < 1e-15 + assert abs(f(1j) - cmath.sinh(1j)) < 1e-15 + + +def test_cmath_cosh(): + f = lambdify(x, cosh(x), "cmath") + assert abs(f(0) - cmath.cosh(0)) < 1e-15 + assert abs(f(pi) - cmath.cosh(pi)) < 1e-15 + assert abs(f(-pi) - cmath.cosh(-pi)) < 1e-15 + assert abs(f(1j) - cmath.cosh(1j)) < 1e-15 + + +def test_cmath_tanh(): + f = lambdify(x, tanh(x), "cmath") + assert abs(f(0) - cmath.tanh(0)) < 1e-15 + assert abs(f(pi) - cmath.tanh(pi)) < 1e-15 + assert abs(f(-pi) - cmath.tanh(-pi)) < 1e-15 + assert abs(f(1j) - cmath.tanh(1j)) < 1e-15 + + +def test_cmath_sin(): + f = lambdify(x, sin(x), "cmath") + assert abs(f(0) - cmath.sin(0)) < 1e-15 + assert abs(f(pi) - cmath.sin(pi)) < 1e-15 + assert abs(f(-pi) - cmath.sin(-pi)) < 1e-15 + assert abs(f(1j) - cmath.sin(1j)) < 1e-15 + + +def test_cmath_cos(): + f = lambdify(x, cos(x), "cmath") + assert abs(f(0) - cmath.cos(0)) < 1e-15 + assert abs(f(pi) - cmath.cos(pi)) < 1e-15 + assert abs(f(-pi) - cmath.cos(-pi)) < 1e-15 + assert abs(f(1j) - cmath.cos(1j)) < 1e-15 + + +def test_cmath_tan(): + f = lambdify(x, tan(x), "cmath") + assert abs(f(0) - cmath.tan(0)) < 1e-15 + assert abs(f(1j) - cmath.tan(1j)) < 1e-15 + + +def test_cmath_asin(): + f = lambdify(x, asin(x), "cmath") + assert abs(f(0) - cmath.asin(0)) < 1e-15 + assert abs(f(1) - cmath.asin(1)) < 1e-15 + assert abs(f(-1) - cmath.asin(-1)) < 1e-15 + assert abs(f(2) - cmath.asin(2)) < 1e-15 + assert abs(f(1j) - cmath.asin(1j)) < 1e-15 + + +def test_cmath_acos(): + f = lambdify(x, acos(x), "cmath") + assert abs(f(1) - cmath.acos(1)) < 1e-15 + assert abs(f(-1) - cmath.acos(-1)) < 1e-15 + assert abs(f(2) - cmath.acos(2)) < 1e-15 + assert abs(f(1j) - cmath.acos(1j)) < 1e-15 + + +def test_cmath_atan(): + f = lambdify(x, atan(x), "cmath") + assert abs(f(0) - cmath.atan(0)) < 1e-15 + assert abs(f(1) - cmath.atan(1)) < 1e-15 + assert abs(f(-1) - cmath.atan(-1)) < 1e-15 + assert abs(f(2) - cmath.atan(2)) < 1e-15 + assert abs(f(2j) - cmath.atan(2j)) < 1e-15 + + +def test_cmath_asinh(): + f = lambdify(x, asinh(x), "cmath") + assert abs(f(0) - cmath.asinh(0)) < 1e-15 + assert abs(f(1) - cmath.asinh(1)) < 1e-15 + assert abs(f(-1) - cmath.asinh(-1)) < 1e-15 + assert abs(f(2) - cmath.asinh(2)) < 1e-15 + assert abs(f(2j) - cmath.asinh(2j)) < 1e-15 + + +def test_cmath_acosh(): + f = lambdify(x, acosh(x), "cmath") + assert abs(f(1) - cmath.acosh(1)) < 1e-15 + assert abs(f(2) - cmath.acosh(2)) < 1e-15 + assert abs(f(-1) - cmath.acosh(-1)) < 1e-15 + assert abs(f(2j) - cmath.acosh(2j)) < 1e-15 + + +def test_cmath_atanh(): + f = lambdify(x, atanh(x), "cmath") + assert abs(f(0) - cmath.atanh(0)) < 1e-15 + assert abs(f(0.5) - cmath.atanh(0.5)) < 1e-15 + assert abs(f(-0.5) - cmath.atanh(-0.5)) < 1e-15 + assert abs(f(2) - cmath.atanh(2)) < 1e-15 + assert abs(f(-2) - cmath.atanh(-2)) < 1e-15 + assert abs(f(2j) - cmath.atanh(2j)) < 1e-15 + + +def test_cmath_complex_identities(): + # Define symbol + z = symbols('z') + + # Trigonometric identity using re(z) and im(z) + expr = cos(z) - cos(re(z)) * cosh(im(z)) + I * sin(re(z)) * sinh(im(z)) + func = lambdify([z], expr, modules=["cmath", "math"]) + hpi = math.pi / 2 + assert abs(func(hpi + 1j * hpi)) < 4e-16 + + # Euler's Formula: e^(i*z) = cos(z) + i*sin(z) + func = lambdify([z], exp(I * z) - (cos(z) + I * sin(z)), modules=["cmath", "math"]) + assert abs(func(hpi)) < 4e-16 + + # Exponential Identity: e^z = e^(Re(z)) * (cos(Im(z)) + i*sin(Im(z))) + func_exp = lambdify([z], exp(z) - exp(re(z)) * (cos(im(z)) + I * sin(im(z))), + modules=["cmath", "math"]) + assert abs(func_exp(hpi + 1j * hpi)) < 4e-16 + + # Complex Cosine Identity: cos(z) = cos(Re(z)) * cosh(Im(z)) - i*sin(Re(z)) * sinh(Im(z)) + func_cos = lambdify([z], cos(z) - (cos(re(z)) * cosh(im(z)) - I * sin(re(z)) * sinh(im(z))), + modules=["cmath", "math"]) + assert abs(func_cos(hpi + 1j * hpi)) < 4e-16 + + # Complex Sine Identity: sin(z) = sin(Re(z)) * cosh(Im(z)) + i*cos(Re(z)) * sinh(Im(z)) + func_sin = lambdify([z], sin(z) - (sin(re(z)) * cosh(im(z)) + I * cos(re(z)) * sinh(im(z))), + modules=["cmath", "math"]) + assert abs(func_sin(hpi + 1j * hpi)) < 4e-16 + + # Complex Hyperbolic Cosine Identity: cosh(z) = cosh(Re(z)) * cos(Im(z)) + i*sinh(Re(z)) * sin(Im(z)) + func_cosh_1 = lambdify([z], cosh(z) - (cosh(re(z)) * cos(im(z)) + I * sinh(re(z)) * sin(im(z))), + modules=["cmath", "math"]) + assert abs(func_cosh_1(hpi + 1j * hpi)) < 4e-16 + + # Complex Hyperbolic Sine Identity: sinh(z) = sinh(Re(z)) * cos(Im(z)) + i*cosh(Re(z)) * sin(Im(z)) + func_sinh = lambdify([z], sinh(z) - (sinh(re(z)) * cos(im(z)) + I * cosh(re(z)) * sin(im(z))), + modules=["cmath", "math"]) + assert abs(func_sinh(hpi + 1j * hpi)) < 4e-16 + + # cosh(z) = (e^z + e^(-z)) / 2 + func_cosh_2 = lambdify([z], cosh(z) - (exp(z) + exp(-z)) / 2, modules=["cmath", "math"]) + assert abs(func_cosh_2(hpi)) < 4e-16 + + # Additional expressions testing log and exp with real and imaginary parts + expr1 = log(re(z)) + log(im(z)) - log(re(z) * im(z)) + expr2 = exp(re(z)) * exp(im(z) * I) - exp(z) + expr3 = log(exp(re(z))) - re(z) + expr4 = exp(log(re(z))) - re(z) + expr5 = log(exp(re(z) + im(z))) - (re(z) + im(z)) + expr6 = exp(log(re(z) + im(z))) - (re(z) + im(z)) + func1 = lambdify([z], expr1, modules=["cmath", "math"]) + func2 = lambdify([z], expr2, modules=["cmath", "math"]) + func3 = lambdify([z], expr3, modules=["cmath", "math"]) + func4 = lambdify([z], expr4, modules=["cmath", "math"]) + func5 = lambdify([z], expr5, modules=["cmath", "math"]) + func6 = lambdify([z], expr6, modules=["cmath", "math"]) + test_value = 3 + 4j + assert abs(func1(test_value)) < 4e-16 + assert abs(func2(test_value)) < 4e-16 + assert abs(func3(test_value)) < 4e-16 + assert abs(func4(test_value)) < 4e-16 + assert abs(func5(test_value)) < 4e-16 + assert abs(func6(test_value)) < 4e-16 + + +def test_issue_9334(): + if not numexpr: + skip("numexpr not installed.") + if not numpy: + skip("numpy not installed.") + expr = S('b*a - sqrt(a**2)') + a, b = sorted(expr.free_symbols, key=lambda s: s.name) + func_numexpr = lambdify((a,b), expr, modules=[numexpr], dummify=False) + foo, bar = numpy.random.random((2, 4)) + func_numexpr(foo, bar) + + +def test_issue_12984(): + if not numexpr: + skip("numexpr not installed.") + func_numexpr = lambdify((x,y,z), Piecewise((y, x >= 0), (z, x > -1)), numexpr) + with ignore_warnings(RuntimeWarning): + assert func_numexpr(1, 24, 42) == 24 + assert str(func_numexpr(-1, 24, 42)) == 'nan' + + +def test_empty_modules(): + x, y = symbols('x y') + expr = -(x % y) + + no_modules = lambdify([x, y], expr) + empty_modules = lambdify([x, y], expr, modules=[]) + assert no_modules(3, 7) == empty_modules(3, 7) + assert no_modules(3, 7) == -3 + + +def test_exponentiation(): + f = lambdify(x, x**2) + assert f(-1) == 1 + assert f(0) == 0 + assert f(1) == 1 + assert f(-2) == 4 + assert f(2) == 4 + assert f(2.5) == 6.25 + + +def test_sqrt(): + f = lambdify(x, sqrt(x)) + assert f(0) == 0.0 + assert f(1) == 1.0 + assert f(4) == 2.0 + assert abs(f(2) - 1.414) < 0.001 + assert f(6.25) == 2.5 + + +def test_trig(): + f = lambdify([x], [cos(x), sin(x)], 'math') + d = f(pi) + prec = 1e-11 + assert -prec < d[0] + 1 < prec + assert -prec < d[1] < prec + d = f(3.14159) + prec = 1e-5 + assert -prec < d[0] + 1 < prec + assert -prec < d[1] < prec + + +def test_integral(): + if numpy and not scipy: + skip("scipy not installed.") + f = Lambda(x, exp(-x**2)) + l = lambdify(y, Integral(f(x), (x, y, oo))) + d = l(-oo) + assert 1.77245385 < d < 1.772453851 + + +def test_double_integral(): + if numpy and not scipy: + skip("scipy not installed.") + # example from http://mpmath.org/doc/current/calculus/integration.html + i = Integral(1/(1 - x**2*y**2), (x, 0, 1), (y, 0, z)) + l = lambdify([z], i) + d = l(1) + assert 1.23370055 < d < 1.233700551 + +def test_spherical_bessel(): + if numpy and not scipy: + skip("scipy not installed.") + test_point = 4.2 #randomly selected + x = symbols("x") + jtest = jn(2, x) + assert abs(lambdify(x,jtest)(test_point) - + jtest.subs(x,test_point).evalf()) < 1e-8 + ytest = yn(2, x) + assert abs(lambdify(x,ytest)(test_point) - + ytest.subs(x,test_point).evalf()) < 1e-8 + + +#================== Test vectors =================================== + + +def test_vector_simple(): + f = lambdify((x, y, z), (z, y, x)) + assert f(3, 2, 1) == (1, 2, 3) + assert f(1.0, 2.0, 3.0) == (3.0, 2.0, 1.0) + # make sure correct number of args required + raises(TypeError, lambda: f(0)) + + +def test_vector_discontinuous(): + f = lambdify(x, (-1/x, 1/x)) + raises(ZeroDivisionError, lambda: f(0)) + assert f(1) == (-1.0, 1.0) + assert f(2) == (-0.5, 0.5) + assert f(-2) == (0.5, -0.5) + + +def test_trig_symbolic(): + f = lambdify([x], [cos(x), sin(x)], 'math') + d = f(pi) + assert abs(d[0] + 1) < 0.0001 + assert abs(d[1] - 0) < 0.0001 + + +def test_trig_float(): + f = lambdify([x], [cos(x), sin(x)]) + d = f(3.14159) + assert abs(d[0] + 1) < 0.0001 + assert abs(d[1] - 0) < 0.0001 + + +def test_docs(): + f = lambdify(x, x**2) + assert f(2) == 4 + f = lambdify([x, y, z], [z, y, x]) + assert f(1, 2, 3) == [3, 2, 1] + f = lambdify(x, sqrt(x)) + assert f(4) == 2.0 + f = lambdify((x, y), sin(x*y)**2) + assert f(0, 5) == 0 + + +def test_math(): + f = lambdify((x, y), sin(x), modules="math") + assert f(0, 5) == 0 + + +def test_sin(): + f = lambdify(x, sin(x)**2) + assert isinstance(f(2), float) + f = lambdify(x, sin(x)**2, modules="math") + assert isinstance(f(2), float) + + +def test_matrix(): + A = Matrix([[x, x*y], [sin(z) + 4, x**z]]) + sol = Matrix([[1, 2], [sin(3) + 4, 1]]) + f = lambdify((x, y, z), A, modules="sympy") + assert f(1, 2, 3) == sol + f = lambdify((x, y, z), (A, [A]), modules="sympy") + assert f(1, 2, 3) == (sol, [sol]) + J = Matrix((x, x + y)).jacobian((x, y)) + v = Matrix((x, y)) + sol = Matrix([[1, 0], [1, 1]]) + assert lambdify(v, J, modules='sympy')(1, 2) == sol + assert lambdify(v.T, J, modules='sympy')(1, 2) == sol + + +def test_numpy_matrix(): + if not numpy: + skip("numpy not installed.") + A = Matrix([[x, x*y], [sin(z) + 4, x**z]]) + sol_arr = numpy.array([[1, 2], [numpy.sin(3) + 4, 1]]) + #Lambdify array first, to ensure return to array as default + f = lambdify((x, y, z), A, ['numpy']) + numpy.testing.assert_allclose(f(1, 2, 3), sol_arr) + #Check that the types are arrays and matrices + assert isinstance(f(1, 2, 3), numpy.ndarray) + + # gh-15071 + class dot(Function): + pass + x_dot_mtx = dot(x, Matrix([[2], [1], [0]])) + f_dot1 = lambdify(x, x_dot_mtx) + inp = numpy.zeros((17, 3)) + assert numpy.all(f_dot1(inp) == 0) + + strict_kw = {"allow_unknown_functions": False, "inline": True, "fully_qualified_modules": False} + p2 = NumPyPrinter(dict(user_functions={'dot': 'dot'}, **strict_kw)) + f_dot2 = lambdify(x, x_dot_mtx, printer=p2) + assert numpy.all(f_dot2(inp) == 0) + + p3 = NumPyPrinter(strict_kw) + # The line below should probably fail upon construction (before calling with "(inp)"): + raises(Exception, lambda: lambdify(x, x_dot_mtx, printer=p3)(inp)) + + +def test_numpy_transpose(): + if not numpy: + skip("numpy not installed.") + A = Matrix([[1, x], [0, 1]]) + f = lambdify((x), A.T, modules="numpy") + numpy.testing.assert_array_equal(f(2), numpy.array([[1, 0], [2, 1]])) + + +def test_numpy_dotproduct(): + if not numpy: + skip("numpy not installed") + A = Matrix([x, y, z]) + f1 = lambdify([x, y, z], DotProduct(A, A), modules='numpy') + f2 = lambdify([x, y, z], DotProduct(A, A.T), modules='numpy') + f3 = lambdify([x, y, z], DotProduct(A.T, A), modules='numpy') + f4 = lambdify([x, y, z], DotProduct(A, A.T), modules='numpy') + + assert f1(1, 2, 3) == \ + f2(1, 2, 3) == \ + f3(1, 2, 3) == \ + f4(1, 2, 3) == \ + numpy.array([14]) + + +def test_numpy_inverse(): + if not numpy: + skip("numpy not installed.") + A = Matrix([[1, x], [0, 1]]) + f = lambdify((x), A**-1, modules="numpy") + numpy.testing.assert_array_equal(f(2), numpy.array([[1, -2], [0, 1]])) + + +def test_numpy_old_matrix(): + if not numpy: + skip("numpy not installed.") + A = Matrix([[x, x*y], [sin(z) + 4, x**z]]) + sol_arr = numpy.array([[1, 2], [numpy.sin(3) + 4, 1]]) + f = lambdify((x, y, z), A, [{'ImmutableDenseMatrix': numpy.matrix}, 'numpy']) + with ignore_warnings(PendingDeprecationWarning): + numpy.testing.assert_allclose(f(1, 2, 3), sol_arr) + assert isinstance(f(1, 2, 3), numpy.matrix) + + +def test_scipy_sparse_matrix(): + if not scipy: + skip("scipy not installed.") + A = SparseMatrix([[x, 0], [0, y]]) + f = lambdify((x, y), A, modules="scipy") + B = f(1, 2) + assert isinstance(B, scipy.sparse.coo_matrix) + + +def test_python_div_zero_issue_11306(): + if not numpy: + skip("numpy not installed.") + p = Piecewise((1 / x, y < -1), (x, y < 1), (1 / x, True)) + f = lambdify([x, y], p, modules='numpy') + with numpy.errstate(divide='ignore'): + assert float(f(numpy.array(0), numpy.array(0.5))) == 0 + assert float(f(numpy.array(0), numpy.array(1))) == float('inf') + + +def test_issue9474(): + mods = [None, 'math'] + if numpy: + mods.append('numpy') + if mpmath: + mods.append('mpmath') + for mod in mods: + f = lambdify(x, S.One/x, modules=mod) + assert f(2) == 0.5 + f = lambdify(x, floor(S.One/x), modules=mod) + assert f(2) == 0 + + for absfunc, modules in product([Abs, abs], mods): + f = lambdify(x, absfunc(x), modules=modules) + assert f(-1) == 1 + assert f(1) == 1 + assert f(3+4j) == 5 + + +def test_issue_9871(): + if not numexpr: + skip("numexpr not installed.") + if not numpy: + skip("numpy not installed.") + + r = sqrt(x**2 + y**2) + expr = diff(1/r, x) + + xn = yn = numpy.linspace(1, 10, 16) + # expr(xn, xn) = -xn/(sqrt(2)*xn)^3 + fv_exact = -numpy.sqrt(2.)**-3 * xn**-2 + + fv_numpy = lambdify((x, y), expr, modules='numpy')(xn, yn) + fv_numexpr = lambdify((x, y), expr, modules='numexpr')(xn, yn) + numpy.testing.assert_allclose(fv_numpy, fv_exact, rtol=1e-10) + numpy.testing.assert_allclose(fv_numexpr, fv_exact, rtol=1e-10) + + +def test_numpy_piecewise(): + if not numpy: + skip("numpy not installed.") + pieces = Piecewise((x, x < 3), (x**2, x > 5), (0, True)) + f = lambdify(x, pieces, modules="numpy") + numpy.testing.assert_array_equal(f(numpy.arange(10)), + numpy.array([0, 1, 2, 0, 0, 0, 36, 49, 64, 81])) + # If we evaluate somewhere all conditions are False, we should get back NaN + nodef_func = lambdify(x, Piecewise((x, x > 0), (-x, x < 0))) + numpy.testing.assert_array_equal(nodef_func(numpy.array([-1, 0, 1])), + numpy.array([1, numpy.nan, 1])) + + +def test_numpy_logical_ops(): + if not numpy: + skip("numpy not installed.") + and_func = lambdify((x, y), And(x, y), modules="numpy") + and_func_3 = lambdify((x, y, z), And(x, y, z), modules="numpy") + or_func = lambdify((x, y), Or(x, y), modules="numpy") + or_func_3 = lambdify((x, y, z), Or(x, y, z), modules="numpy") + not_func = lambdify((x), Not(x), modules="numpy") + arr1 = numpy.array([True, True]) + arr2 = numpy.array([False, True]) + arr3 = numpy.array([True, False]) + numpy.testing.assert_array_equal(and_func(arr1, arr2), numpy.array([False, True])) + numpy.testing.assert_array_equal(and_func_3(arr1, arr2, arr3), numpy.array([False, False])) + numpy.testing.assert_array_equal(or_func(arr1, arr2), numpy.array([True, True])) + numpy.testing.assert_array_equal(or_func_3(arr1, arr2, arr3), numpy.array([True, True])) + numpy.testing.assert_array_equal(not_func(arr2), numpy.array([True, False])) + + +def test_numpy_matmul(): + if not numpy: + skip("numpy not installed.") + xmat = Matrix([[x, y], [z, 1+z]]) + ymat = Matrix([[x**2], [Abs(x)]]) + mat_func = lambdify((x, y, z), xmat*ymat, modules="numpy") + numpy.testing.assert_array_equal(mat_func(0.5, 3, 4), numpy.array([[1.625], [3.5]])) + numpy.testing.assert_array_equal(mat_func(-0.5, 3, 4), numpy.array([[1.375], [3.5]])) + # Multiple matrices chained together in multiplication + f = lambdify((x, y, z), xmat*xmat*xmat, modules="numpy") + numpy.testing.assert_array_equal(f(0.5, 3, 4), numpy.array([[72.125, 119.25], + [159, 251]])) + + +def test_numpy_numexpr(): + if not numpy: + skip("numpy not installed.") + if not numexpr: + skip("numexpr not installed.") + a, b, c = numpy.random.randn(3, 128, 128) + # ensure that numpy and numexpr return same value for complicated expression + expr = sin(x) + cos(y) + tan(z)**2 + Abs(z-y)*acos(sin(y*z)) + \ + Abs(y-z)*acosh(2+exp(y-x))- sqrt(x**2+I*y**2) + npfunc = lambdify((x, y, z), expr, modules='numpy') + nefunc = lambdify((x, y, z), expr, modules='numexpr') + assert numpy.allclose(npfunc(a, b, c), nefunc(a, b, c)) + + +def test_numexpr_userfunctions(): + if not numpy: + skip("numpy not installed.") + if not numexpr: + skip("numexpr not installed.") + a, b = numpy.random.randn(2, 10) + uf = type('uf', (Function, ), + {'eval' : classmethod(lambda x, y : y**2+1)}) + func = lambdify(x, 1-uf(x), modules='numexpr') + assert numpy.allclose(func(a), -(a**2)) + + uf = implemented_function(Function('uf'), lambda x, y : 2*x*y+1) + func = lambdify((x, y), uf(x, y), modules='numexpr') + assert numpy.allclose(func(a, b), 2*a*b+1) + + +def test_tensorflow_basic_math(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Max(sin(x), Abs(1/(x+2))) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + a = tensorflow.constant(0, dtype=tensorflow.float32) + assert func(a).eval(session=s) == 0.5 + + +def test_tensorflow_placeholders(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Max(sin(x), Abs(1/(x+2))) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + a = tensorflow.compat.v1.placeholder(dtype=tensorflow.float32) + assert func(a).eval(session=s, feed_dict={a: 0}) == 0.5 + + +def test_tensorflow_variables(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Max(sin(x), Abs(1/(x+2))) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + a = tensorflow.Variable(0, dtype=tensorflow.float32) + s.run(a.initializer) + assert func(a).eval(session=s, feed_dict={a: 0}) == 0.5 + + +def test_tensorflow_logical_operations(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Not(And(Or(x, y), y)) + func = lambdify([x, y], expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + assert func(False, True).eval(session=s) == False + + +def test_tensorflow_piecewise(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Piecewise((0, Eq(x,0)), (-1, x < 0), (1, x > 0)) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + assert func(-1).eval(session=s) == -1 + assert func(0).eval(session=s) == 0 + assert func(1).eval(session=s) == 1 + + +def test_tensorflow_multi_max(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Max(x, -x, x**2) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + assert func(-2).eval(session=s) == 4 + + +def test_tensorflow_multi_min(): + if not tensorflow: + skip("tensorflow not installed.") + expr = Min(x, -x, x**2) + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + assert func(-2).eval(session=s) == -2 + + +def test_tensorflow_relational(): + if not tensorflow: + skip("tensorflow not installed.") + expr = x >= 0 + func = lambdify(x, expr, modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + assert func(1).eval(session=s) == True + + +def test_tensorflow_complexes(): + if not tensorflow: + skip("tensorflow not installed") + + func1 = lambdify(x, re(x), modules="tensorflow") + func2 = lambdify(x, im(x), modules="tensorflow") + func3 = lambdify(x, Abs(x), modules="tensorflow") + func4 = lambdify(x, arg(x), modules="tensorflow") + + with tensorflow.compat.v1.Session() as s: + # For versions before + # https://github.com/tensorflow/tensorflow/issues/30029 + # resolved, using Python numeric types may not work + a = tensorflow.constant(1+2j) + assert func1(a).eval(session=s) == 1 + assert func2(a).eval(session=s) == 2 + + tensorflow_result = func3(a).eval(session=s) + sympy_result = Abs(1 + 2j).evalf() + assert abs(tensorflow_result-sympy_result) < 10**-6 + + tensorflow_result = func4(a).eval(session=s) + sympy_result = arg(1 + 2j).evalf() + assert abs(tensorflow_result-sympy_result) < 10**-6 + + +def test_tensorflow_array_arg(): + # Test for issue 14655 (tensorflow part) + if not tensorflow: + skip("tensorflow not installed.") + + f = lambdify([[x, y]], x*x + y, 'tensorflow') + + with tensorflow.compat.v1.Session() as s: + fcall = f(tensorflow.constant([2.0, 1.0])) + assert fcall.eval(session=s) == 5.0 + + +#================== Test symbolic ================================== + + +def test_sym_single_arg(): + f = lambdify(x, x * y) + assert f(z) == z * y + + +def test_sym_list_args(): + f = lambdify([x, y], x + y + z) + assert f(1, 2) == 3 + z + + +def test_sym_integral(): + f = Lambda(x, exp(-x**2)) + l = lambdify(x, Integral(f(x), (x, -oo, oo)), modules="sympy") + assert l(y) == Integral(exp(-y**2), (y, -oo, oo)) + assert l(y).doit() == sqrt(pi) + + +def test_namespace_order(): + # lambdify had a bug, such that module dictionaries or cached module + # dictionaries would pull earlier namespaces into themselves. + # Because the module dictionaries form the namespace of the + # generated lambda, this meant that the behavior of a previously + # generated lambda function could change as a result of later calls + # to lambdify. + n1 = {'f': lambda x: 'first f'} + n2 = {'f': lambda x: 'second f', + 'g': lambda x: 'function g'} + f = sympy.Function('f') + g = sympy.Function('g') + if1 = lambdify(x, f(x), modules=(n1, "sympy")) + assert if1(1) == 'first f' + if2 = lambdify(x, g(x), modules=(n2, "sympy")) + # previously gave 'second f' + assert if1(1) == 'first f' + + assert if2(1) == 'function g' + + +def test_imps(): + # Here we check if the default returned functions are anonymous - in + # the sense that we can have more than one function with the same name + f = implemented_function('f', lambda x: 2*x) + g = implemented_function('f', lambda x: math.sqrt(x)) + l1 = lambdify(x, f(x)) + l2 = lambdify(x, g(x)) + assert str(f(x)) == str(g(x)) + assert l1(3) == 6 + assert l2(3) == math.sqrt(3) + # check that we can pass in a Function as input + func = sympy.Function('myfunc') + assert not hasattr(func, '_imp_') + my_f = implemented_function(func, lambda x: 2*x) + assert hasattr(my_f, '_imp_') + # Error for functions with same name and different implementation + f2 = implemented_function("f", lambda x: x + 101) + raises(ValueError, lambda: lambdify(x, f(f2(x)))) + + +def test_imps_errors(): + # Test errors that implemented functions can return, and still be able to + # form expressions. + # See: https://github.com/sympy/sympy/issues/10810 + # + # XXX: Removed AttributeError here. This test was added due to issue 10810 + # but that issue was about ValueError. It doesn't seem reasonable to + # "support" catching AttributeError in the same context... + for val, error_class in product((0, 0., 2, 2.0), (TypeError, ValueError)): + + def myfunc(a): + if a == 0: + raise error_class + return 1 + + f = implemented_function('f', myfunc) + expr = f(val) + assert expr == f(val) + + +def test_imps_wrong_args(): + raises(ValueError, lambda: implemented_function(sin, lambda x: x)) + + +def test_lambdify_imps(): + # Test lambdify with implemented functions + # first test basic (sympy) lambdify + f = sympy.cos + assert lambdify(x, f(x))(0) == 1 + assert lambdify(x, 1 + f(x))(0) == 2 + assert lambdify((x, y), y + f(x))(0, 1) == 2 + # make an implemented function and test + f = implemented_function("f", lambda x: x + 100) + assert lambdify(x, f(x))(0) == 100 + assert lambdify(x, 1 + f(x))(0) == 101 + assert lambdify((x, y), y + f(x))(0, 1) == 101 + # Can also handle tuples, lists, dicts as expressions + lam = lambdify(x, (f(x), x)) + assert lam(3) == (103, 3) + lam = lambdify(x, [f(x), x]) + assert lam(3) == [103, 3] + lam = lambdify(x, [f(x), (f(x), x)]) + assert lam(3) == [103, (103, 3)] + lam = lambdify(x, {f(x): x}) + assert lam(3) == {103: 3} + lam = lambdify(x, {f(x): x}) + assert lam(3) == {103: 3} + lam = lambdify(x, {x: f(x)}) + assert lam(3) == {3: 103} + # Check that imp preferred to other namespaces by default + d = {'f': lambda x: x + 99} + lam = lambdify(x, f(x), d) + assert lam(3) == 103 + # Unless flag passed + lam = lambdify(x, f(x), d, use_imps=False) + assert lam(3) == 102 + + +def test_dummification(): + t = symbols('t') + F = Function('F') + G = Function('G') + #"\alpha" is not a valid Python variable name + #lambdify should sub in a dummy for it, and return + #without a syntax error + alpha = symbols(r'\alpha') + some_expr = 2 * F(t)**2 / G(t) + lam = lambdify((F(t), G(t)), some_expr) + assert lam(3, 9) == 2 + lam = lambdify(sin(t), 2 * sin(t)**2) + assert lam(F(t)) == 2 * F(t)**2 + #Test that \alpha was properly dummified + lam = lambdify((alpha, t), 2*alpha + t) + assert lam(2, 1) == 5 + raises(SyntaxError, lambda: lambdify(F(t) * G(t), F(t) * G(t) + 5)) + raises(SyntaxError, lambda: lambdify(2 * F(t), 2 * F(t) + 5)) + raises(SyntaxError, lambda: lambdify(2 * F(t), 4 * F(t) + 5)) + + +def test_lambdify__arguments_with_invalid_python_identifiers(): + # see sympy/sympy#26690 + N = CoordSys3D('N') + xn, yn, zn = N.base_scalars() + expr = xn + yn + f = lambdify([xn, yn], expr) + res = f(0.2, 0.3) + ref = 0.2 + 0.3 + assert abs(res-ref) < 1e-15 + + +def test_curly_matrix_symbol(): + # Issue #15009 + curlyv = sympy.MatrixSymbol("{v}", 2, 1) + lam = lambdify(curlyv, curlyv) + assert lam(1)==1 + lam = lambdify(curlyv, curlyv, dummify=True) + assert lam(1)==1 + + +def test_python_keywords(): + # Test for issue 7452. The automatic dummification should ensure use of + # Python reserved keywords as symbol names will create valid lambda + # functions. This is an additional regression test. + python_if = symbols('if') + expr = python_if / 2 + f = lambdify(python_if, expr) + assert f(4.0) == 2.0 + + +def test_lambdify_docstring(): + func = lambdify((w, x, y, z), w + x + y + z) + ref = ( + "Created with lambdify. Signature:\n\n" + "func(w, x, y, z)\n\n" + "Expression:\n\n" + "w + x + y + z" + ).splitlines() + assert func.__doc__.splitlines()[:len(ref)] == ref + syms = symbols('a1:26') + func = lambdify(syms, sum(syms)) + ref = ( + "Created with lambdify. Signature:\n\n" + "func(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15,\n" + " a16, a17, a18, a19, a20, a21, a22, a23, a24, a25)\n\n" + "Expression:\n\n" + "a1 + a10 + a11 + a12 + a13 + a14 + a15 + a16 + a17 + a18 + a19 + a2 + a20 +..." + ).splitlines() + assert func.__doc__.splitlines()[:len(ref)] == ref + + +def test_lambdify_linecache(): + func = lambdify(x, x + 1) + source = 'def _lambdifygenerated(x):\n return x + 1\n' + assert inspect.getsource(func) == source + filename = inspect.getsourcefile(func) + assert filename.startswith('= (1, 10) + + if have_scipy_1_10plus: + cm2 = lambdify((x, y), powm1(x, y), modules='scipy') + assert abs(cm2(1.2, 1e-9) - 1.82321557e-10) < 1e-17 + + +def test_scipy_bernoulli(): + if not scipy: + skip("scipy not installed") + + bern = lambdify((x,), bernoulli(x), modules='scipy') + assert bern(1) == 0.5 + + +def test_scipy_harmonic(): + if not scipy: + skip("scipy not installed") + + hn = lambdify((x,), harmonic(x), modules='scipy') + assert hn(2) == 1.5 + hnm = lambdify((x, y), harmonic(x, y), modules='scipy') + assert hnm(2, 2) == 1.25 + + +def test_cupy_array_arg(): + if not cupy: + skip("CuPy not installed") + + f = lambdify([[x, y]], x*x + y, 'cupy') + result = f(cupy.array([2.0, 1.0])) + assert result == 5 + assert "cupy" in str(type(result)) + + +def test_cupy_array_arg_using_numpy(): + # numpy functions can be run on cupy arrays + # unclear if we can "officially" support this, + # depends on numpy __array_function__ support + if not cupy: + skip("CuPy not installed") + + f = lambdify([[x, y]], x*x + y, 'numpy') + result = f(cupy.array([2.0, 1.0])) + assert result == 5 + assert "cupy" in str(type(result)) + +def test_cupy_dotproduct(): + if not cupy: + skip("CuPy not installed") + + A = Matrix([x, y, z]) + f1 = lambdify([x, y, z], DotProduct(A, A), modules='cupy') + f2 = lambdify([x, y, z], DotProduct(A, A.T), modules='cupy') + f3 = lambdify([x, y, z], DotProduct(A.T, A), modules='cupy') + f4 = lambdify([x, y, z], DotProduct(A, A.T), modules='cupy') + + assert f1(1, 2, 3) == \ + f2(1, 2, 3) == \ + f3(1, 2, 3) == \ + f4(1, 2, 3) == \ + cupy.array([14]) + + +def test_jax_array_arg(): + if not jax: + skip("JAX not installed") + + f = lambdify([[x, y]], x*x + y, 'jax') + result = f(jax.numpy.array([2.0, 1.0])) + assert result == 5 + assert "jax" in str(type(result)) + + +def test_jax_array_arg_using_numpy(): + if not jax: + skip("JAX not installed") + + f = lambdify([[x, y]], x*x + y, 'numpy') + result = f(jax.numpy.array([2.0, 1.0])) + assert result == 5 + assert "jax" in str(type(result)) + + +def test_jax_dotproduct(): + if not jax: + skip("JAX not installed") + + A = Matrix([x, y, z]) + f1 = lambdify([x, y, z], DotProduct(A, A), modules='jax') + f2 = lambdify([x, y, z], DotProduct(A, A.T), modules='jax') + f3 = lambdify([x, y, z], DotProduct(A.T, A), modules='jax') + f4 = lambdify([x, y, z], DotProduct(A, A.T), modules='jax') + + assert f1(1, 2, 3) == \ + f2(1, 2, 3) == \ + f3(1, 2, 3) == \ + f4(1, 2, 3) == \ + jax.numpy.array([14]) + + +def test_lambdify_cse(): + def no_op_cse(exprs): + return (), exprs + + def dummy_cse(exprs): + from sympy.simplify.cse_main import cse + return cse(exprs, symbols=numbered_symbols(cls=Dummy)) + + def minmem(exprs): + from sympy.simplify.cse_main import cse_release_variables, cse + return cse(exprs, postprocess=cse_release_variables) + + class Case: + def __init__(self, *, args, exprs, num_args, requires_numpy=False): + self.args = args + self.exprs = exprs + self.num_args = num_args + subs_dict = dict(zip(self.args, self.num_args)) + self.ref = [e.subs(subs_dict).evalf() for e in exprs] + self.requires_numpy = requires_numpy + + def lambdify(self, *, cse): + return lambdify(self.args, self.exprs, cse=cse) + + def assertAllClose(self, result, *, abstol=1e-15, reltol=1e-15): + if self.requires_numpy: + assert all(numpy.allclose(result[i], numpy.asarray(r, dtype=float), + rtol=reltol, atol=abstol) + for i, r in enumerate(self.ref)) + return + + for i, r in enumerate(self.ref): + abs_err = abs(result[i] - r) + if r == 0: + assert abs_err < abstol + else: + assert abs_err/abs(r) < reltol + + cases = [ + Case( + args=(x, y, z), + exprs=[ + x + y + z, + x + y - z, + 2*x + 2*y - z, + (x+y)**2 + (y+z)**2, + ], + num_args=(2., 3., 4.) + ), + Case( + args=(x, y, z), + exprs=[ + x + sympy.Heaviside(x), + y + sympy.Heaviside(x), + z + sympy.Heaviside(x, 1), + z/sympy.Heaviside(x, 1) + ], + num_args=(0., 3., 4.) + ), + Case( + args=(x, y, z), + exprs=[ + x + sinc(y), + y + sinc(y), + z - sinc(y) + ], + num_args=(0.1, 0.2, 0.3) + ), + Case( + args=(x, y, z), + exprs=[ + Matrix([[x, x*y], [sin(z) + 4, x**z]]), + x*y+sin(z)-x**z, + Matrix([x*x, sin(z), x**z]) + ], + num_args=(1.,2.,3.), + requires_numpy=True + ), + Case( + args=(x, y), + exprs=[(x + y - 1)**2, x, x + y, + (x + y)/(2*x + 1) + (x + y - 1)**2, (2*x + 1)**(x + y)], + num_args=(1,2) + ) + ] + for case in cases: + if not numpy and case.requires_numpy: + continue + for _cse in [False, True, minmem, no_op_cse, dummy_cse]: + f = case.lambdify(cse=_cse) + result = f(*case.num_args) + case.assertAllClose(result) + +def test_issue_25288(): + syms = numbered_symbols(cls=Dummy) + ok = lambdify(x, [x**2, sin(x**2)], cse=lambda e: cse(e, symbols=syms))(2) + assert ok + + +def test_deprecated_set(): + with warns_deprecated_sympy(): + lambdify({x, y}, x + y) + +def test_issue_13881(): + if not numpy: + skip("numpy not installed.") + + X = MatrixSymbol('X', 3, 1) + + f = lambdify(X, X.T*X, 'numpy') + assert f(numpy.array([1, 2, 3])) == 14 + assert f(numpy.array([3, 2, 1])) == 14 + + f = lambdify(X, X*X.T, 'numpy') + assert f(numpy.array([1, 2, 3])) == 14 + assert f(numpy.array([3, 2, 1])) == 14 + + f = lambdify(X, (X*X.T)*X, 'numpy') + arr1 = numpy.array([[1], [2], [3]]) + arr2 = numpy.array([[14],[28],[42]]) + + assert numpy.array_equal(f(arr1), arr2) + + +def test_23536_lambdify_cse_dummy(): + + f = Function('x')(y) + g = Function('w')(y) + expr = z + (f**4 + g**5)*(f**3 + (g*f)**3) + expr = expr.expand() + eval_expr = lambdify(((f, g), z), expr, cse=True) + ans = eval_expr((1.0, 2.0), 3.0) # shouldn't raise NameError + assert ans == 300.0 # not a list and value is 300 + + +class LambdifyDocstringTestCase: + SIGNATURE = None + EXPR = None + SRC = None + + def __init__(self, docstring_limit, expected_redacted): + self.docstring_limit = docstring_limit + self.expected_redacted = expected_redacted + + @property + def expected_expr(self): + expr_redacted_msg = "EXPRESSION REDACTED DUE TO LENGTH, (see lambdify's `docstring_limit`)" + return self.EXPR if not self.expected_redacted else expr_redacted_msg + + @property + def expected_src(self): + src_redacted_msg = "SOURCE CODE REDACTED DUE TO LENGTH, (see lambdify's `docstring_limit`)" + return self.SRC if not self.expected_redacted else src_redacted_msg + + @property + def expected_docstring(self): + expected_docstring = ( + f'Created with lambdify. Signature:\n\n' + f'func({self.SIGNATURE})\n\n' + f'Expression:\n\n' + f'{self.expected_expr}\n\n' + f'Source code:\n\n' + f'{self.expected_src}\n\n' + f'Imported modules:\n\n' + ) + return expected_docstring + + def __len__(self): + return len(self.expected_docstring) + + def __repr__(self): + return ( + f'{self.__class__.__name__}(' + f'docstring_limit={self.docstring_limit}, ' + f'expected_redacted={self.expected_redacted})' + ) + + +def test_lambdify_docstring_size_limit_simple_symbol(): + + class SimpleSymbolTestCase(LambdifyDocstringTestCase): + SIGNATURE = 'x' + EXPR = 'x' + SRC = ( + 'def _lambdifygenerated(x):\n' + ' return x\n' + ) + + x = symbols('x') + + test_cases = ( + SimpleSymbolTestCase(docstring_limit=None, expected_redacted=False), + SimpleSymbolTestCase(docstring_limit=100, expected_redacted=False), + SimpleSymbolTestCase(docstring_limit=1, expected_redacted=False), + SimpleSymbolTestCase(docstring_limit=0, expected_redacted=True), + SimpleSymbolTestCase(docstring_limit=-1, expected_redacted=True), + ) + for test_case in test_cases: + lambdified_expr = lambdify( + [x], + x, + 'sympy', + docstring_limit=test_case.docstring_limit, + ) + assert lambdified_expr.__doc__ == test_case.expected_docstring + + +def test_lambdify_docstring_size_limit_nested_expr(): + + class ExprListTestCase(LambdifyDocstringTestCase): + SIGNATURE = 'x, y, z' + EXPR = ( + '[x, [y], z, x**3 + 3*x**2*y + 3*x**2*z + 3*x*y**2 + 6*x*y*z ' + '+ 3*x*z**2 +...' + ) + SRC = ( + 'def _lambdifygenerated(x, y, z):\n' + ' return [x, [y], z, x**3 + 3*x**2*y + 3*x**2*z + 3*x*y**2 ' + '+ 6*x*y*z + 3*x*z**2 + y**3 + 3*y**2*z + 3*y*z**2 + z**3]\n' + ) + + x, y, z = symbols('x, y, z') + expr = [x, [y], z, ((x + y + z)**3).expand()] + + test_cases = ( + ExprListTestCase(docstring_limit=None, expected_redacted=False), + ExprListTestCase(docstring_limit=200, expected_redacted=False), + ExprListTestCase(docstring_limit=50, expected_redacted=True), + ExprListTestCase(docstring_limit=0, expected_redacted=True), + ExprListTestCase(docstring_limit=-1, expected_redacted=True), + ) + for test_case in test_cases: + lambdified_expr = lambdify( + [x, y, z], + expr, + 'sympy', + docstring_limit=test_case.docstring_limit, + ) + assert lambdified_expr.__doc__ == test_case.expected_docstring + + +def test_lambdify_docstring_size_limit_matrix(): + + class MatrixTestCase(LambdifyDocstringTestCase): + SIGNATURE = 'x, y, z' + EXPR = ( + 'Matrix([[0, x], [x + y + z, x**3 + 3*x**2*y + 3*x**2*z + 3*x*y**2 ' + '+ 6*x*y*z...' + ) + SRC = ( + 'def _lambdifygenerated(x, y, z):\n' + ' return ImmutableDenseMatrix([[0, x], [x + y + z, x**3 ' + '+ 3*x**2*y + 3*x**2*z + 3*x*y**2 + 6*x*y*z + 3*x*z**2 + y**3 ' + '+ 3*y**2*z + 3*y*z**2 + z**3]])\n' + ) + + x, y, z = symbols('x, y, z') + expr = Matrix([[S.Zero, x], [x + y + z, ((x + y + z)**3).expand()]]) + + test_cases = ( + MatrixTestCase(docstring_limit=None, expected_redacted=False), + MatrixTestCase(docstring_limit=200, expected_redacted=False), + MatrixTestCase(docstring_limit=50, expected_redacted=True), + MatrixTestCase(docstring_limit=0, expected_redacted=True), + MatrixTestCase(docstring_limit=-1, expected_redacted=True), + ) + for test_case in test_cases: + lambdified_expr = lambdify( + [x, y, z], + expr, + 'sympy', + docstring_limit=test_case.docstring_limit, + ) + assert lambdified_expr.__doc__ == test_case.expected_docstring + + +def test_lambdify_empty_tuple(): + a = symbols("a") + expr = ((), (a,)) + f = lambdify(a, expr) + result = f(1) + assert result == ((), (1,)), "Lambdify did not handle the empty tuple correctly." + + +def test_assoc_legendre_numerical_evaluation(): + + tol = 1e-10 + + sympy_result_integer = assoc_legendre(1, 1/2, 0.1).evalf() + sympy_result_complex = assoc_legendre(2, 1, 3).evalf() + mpmath_result_integer = -0.474572528387641 + mpmath_result_complex = -25.45584412271571*I + + assert all_close(sympy_result_integer, mpmath_result_integer, tol) + assert all_close(sympy_result_complex, mpmath_result_complex, tol) + + +def test_Piecewise(): + + modules = [math] + if numpy: + modules.append('numpy') + + for mod in modules: + # test isinf + f = lambdify(x, Piecewise((7.0, isinf(x)), (3.0, True)), mod) + assert f(+float('inf')) == +7.0 + assert f(-float('inf')) == +7.0 + assert f(42.) == 3.0 + + f2 = lambdify(x, Piecewise((7.0*sign(x), isinf(x)), (3.0, True)), mod) + assert f2(+float('inf')) == +7.0 + assert f2(-float('inf')) == -7.0 + assert f2(42.) == 3.0 + + # test isnan (gh-26784) + g = lambdify(x, Piecewise((7.0, isnan(x)), (3.0, True)), mod) + assert g(float('nan')) == 7.0 + assert g(42.) == 3.0 + + +def test_array_symbol(): + if not numpy: + skip("numpy not installed.") + a = ArraySymbol('a', (3,)) + f = lambdify((a), a) + assert numpy.all(f(numpy.array([1,2,3])) == numpy.array([1,2,3])) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_matchpy_connector.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_matchpy_connector.py new file mode 100644 index 0000000000000000000000000000000000000000..3648bd49f9e56ca20fbf428ed46c01429dbe8b15 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_matchpy_connector.py @@ -0,0 +1,164 @@ +import pickle + +from sympy.core.relational import (Eq, Ne) +from sympy.core.singleton import S +from sympy.core.symbol import symbols +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.external import import_module +from sympy.testing.pytest import skip +from sympy.utilities.matchpy_connector import WildDot, WildPlus, WildStar, Replacer + +matchpy = import_module("matchpy") + +x, y, z = symbols("x y z") + + +def _get_first_match(expr, pattern): + from matchpy import ManyToOneMatcher, Pattern + + matcher = ManyToOneMatcher() + matcher.add(Pattern(pattern)) + return next(iter(matcher.match(expr))) + + +def test_matchpy_connector(): + if matchpy is None: + skip("matchpy not installed") + + from multiset import Multiset + from matchpy import Pattern, Substitution + + w_ = WildDot("w_") + w__ = WildPlus("w__") + w___ = WildStar("w___") + + expr = x + y + pattern = x + w_ + p, subst = _get_first_match(expr, pattern) + assert p == Pattern(pattern) + assert subst == Substitution({'w_': y}) + + expr = x + y + z + pattern = x + w__ + p, subst = _get_first_match(expr, pattern) + assert p == Pattern(pattern) + assert subst == Substitution({'w__': Multiset([y, z])}) + + expr = x + y + z + pattern = x + y + z + w___ + p, subst = _get_first_match(expr, pattern) + assert p == Pattern(pattern) + assert subst == Substitution({'w___': Multiset()}) + + +def test_matchpy_optional(): + if matchpy is None: + skip("matchpy not installed") + + from matchpy import Pattern, Substitution + from matchpy import ManyToOneReplacer, ReplacementRule + + p = WildDot("p", optional=1) + q = WildDot("q", optional=0) + + pattern = p*x + q + + expr1 = 2*x + pa, subst = _get_first_match(expr1, pattern) + assert pa == Pattern(pattern) + assert subst == Substitution({'p': 2, 'q': 0}) + + expr2 = x + 3 + pa, subst = _get_first_match(expr2, pattern) + assert pa == Pattern(pattern) + assert subst == Substitution({'p': 1, 'q': 3}) + + expr3 = x + pa, subst = _get_first_match(expr3, pattern) + assert pa == Pattern(pattern) + assert subst == Substitution({'p': 1, 'q': 0}) + + expr4 = x*y + z + pa, subst = _get_first_match(expr4, pattern) + assert pa == Pattern(pattern) + assert subst == Substitution({'p': y, 'q': z}) + + replacer = ManyToOneReplacer() + replacer.add(ReplacementRule(Pattern(pattern), lambda p, q: sin(p)*cos(q))) + assert replacer.replace(expr1) == sin(2)*cos(0) + assert replacer.replace(expr2) == sin(1)*cos(3) + assert replacer.replace(expr3) == sin(1)*cos(0) + assert replacer.replace(expr4) == sin(y)*cos(z) + + +def test_replacer(): + if matchpy is None: + skip("matchpy not installed") + + for info in [True, False]: + for lambdify in [True, False]: + _perform_test_replacer(info, lambdify) + + +def _perform_test_replacer(info, lambdify): + + x1_ = WildDot("x1_") + x2_ = WildDot("x2_") + + a_ = WildDot("a_", optional=S.One) + b_ = WildDot("b_", optional=S.One) + c_ = WildDot("c_", optional=S.Zero) + + replacer = Replacer(common_constraints=[ + matchpy.CustomConstraint(lambda a_: not a_.has(x)), + matchpy.CustomConstraint(lambda b_: not b_.has(x)), + matchpy.CustomConstraint(lambda c_: not c_.has(x)), + ], lambdify=lambdify, info=info) + + # Rewrite the equation into implicit form, unless it's already solved: + replacer.add(Eq(x1_, x2_), Eq(x1_ - x2_, 0), conditions_nonfalse=[Ne(x2_, 0), Ne(x1_, 0), Ne(x1_, x), Ne(x2_, x)], info=1) + + # Simple equation solver for real numbers: + replacer.add(Eq(a_*x + b_, 0), Eq(x, -b_/a_), info=2) + disc = b_**2 - 4*a_*c_ + replacer.add( + Eq(a_*x**2 + b_*x + c_, 0), + Eq(x, (-b_ - sqrt(disc))/(2*a_)) | Eq(x, (-b_ + sqrt(disc))/(2*a_)), + conditions_nonfalse=[disc >= 0], + info=3 + ) + replacer.add( + Eq(a_*x**2 + c_, 0), + Eq(x, sqrt(-c_/a_)) | Eq(x, -sqrt(-c_/a_)), + conditions_nonfalse=[-c_*a_ > 0], + info=4 + ) + + g = lambda expr, infos: (expr, infos) if info else expr + + assert replacer.replace(Eq(3*x, y)) == g(Eq(x, y/3), [1, 2]) + assert replacer.replace(Eq(x**2 + 1, 0)) == g(Eq(x**2 + 1, 0), []) + assert replacer.replace(Eq(x**2, 4)) == g((Eq(x, 2) | Eq(x, -2)), [1, 4]) + assert replacer.replace(Eq(x**2 + 4*y*x + 4*y**2, 0)) == g(Eq(x, -2*y), [3]) + + +def test_matchpy_object_pickle(): + if matchpy is None: + return + + a1 = WildDot("a") + a2 = pickle.loads(pickle.dumps(a1)) + assert a1 == a2 + + a1 = WildDot("a", S(1)) + a2 = pickle.loads(pickle.dumps(a1)) + assert a1 == a2 + + a1 = WildPlus("a", S(1)) + a2 = pickle.loads(pickle.dumps(a1)) + assert a1 == a2 + + a1 = WildStar("a", S(1)) + a2 = pickle.loads(pickle.dumps(a1)) + assert a1 == a2 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_mathml.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_mathml.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a7598f175be34f8bb34c0bd9c003d1c0238c7b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_mathml.py @@ -0,0 +1,33 @@ +import os +from textwrap import dedent +from sympy.external import import_module +from sympy.testing.pytest import skip +from sympy.utilities.mathml import apply_xsl + + + +lxml = import_module('lxml') + +path = os.path.abspath(os.path.join(os.path.dirname(__file__), "test_xxe.py")) + + +def test_xxe(): + assert os.path.isfile(path) + if not lxml: + skip("lxml not installed.") + + mml = dedent( + rf""" + + ]> + + John + &ent; + + """ + ) + xsl = 'mathml/data/simple_mmlctop.xsl' + + res = apply_xsl(mml, xsl) + assert res == \ + '\n\nJohn\n\n\n' diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_misc.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_misc.py new file mode 100644 index 0000000000000000000000000000000000000000..0a3e059419303c33cbd7b770679b5efc1b03486d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_misc.py @@ -0,0 +1,148 @@ +from textwrap import dedent +import sys +from subprocess import Popen, PIPE +import os + +from sympy.core.singleton import S +from sympy.testing.pytest import (raises, warns_deprecated_sympy, + skip_under_pyodide) +from sympy.utilities.misc import (translate, replace, ordinal, rawlines, + strlines, as_int, find_executable) + + +def test_translate(): + abc = 'abc' + assert translate(abc, None, 'a') == 'bc' + assert translate(abc, None, '') == 'abc' + assert translate(abc, {'a': 'x'}, 'c') == 'xb' + assert translate(abc, {'a': 'bc'}, 'c') == 'bcb' + assert translate(abc, {'ab': 'x'}, 'c') == 'x' + assert translate(abc, {'ab': ''}, 'c') == '' + assert translate(abc, {'bc': 'x'}, 'c') == 'ab' + assert translate(abc, {'abc': 'x', 'a': 'y'}) == 'x' + u = chr(4096) + assert translate(abc, 'a', 'x', u) == 'xbc' + assert (u in translate(abc, 'a', u, u)) is True + + +def test_replace(): + assert replace('abc', ('a', 'b')) == 'bbc' + assert replace('abc', {'a': 'Aa'}) == 'Aabc' + assert replace('abc', ('a', 'b'), ('c', 'C')) == 'bbC' + + +def test_ordinal(): + assert ordinal(-1) == '-1st' + assert ordinal(0) == '0th' + assert ordinal(1) == '1st' + assert ordinal(2) == '2nd' + assert ordinal(3) == '3rd' + assert all(ordinal(i).endswith('th') for i in range(4, 21)) + assert ordinal(100) == '100th' + assert ordinal(101) == '101st' + assert ordinal(102) == '102nd' + assert ordinal(103) == '103rd' + assert ordinal(104) == '104th' + assert ordinal(200) == '200th' + assert all(ordinal(i) == str(i) + 'th' for i in range(-220, -203)) + + +def test_rawlines(): + assert rawlines('a a\na') == "dedent('''\\\n a a\n a''')" + assert rawlines('a a') == "'a a'" + assert rawlines(strlines('\\le"ft')) == ( + '(\n' + " '(\\n'\n" + ' \'r\\\'\\\\le"ft\\\'\\n\'\n' + " ')'\n" + ')') + + +def test_strlines(): + q = 'this quote (") is in the middle' + # the following assert rhs was prepared with + # print(rawlines(strlines(q, 10))) + assert strlines(q, 10) == dedent('''\ + ( + 'this quo' + 'te (") i' + 's in the' + ' middle' + )''') + assert q == ( + 'this quo' + 'te (") i' + 's in the' + ' middle' + ) + q = "this quote (') is in the middle" + assert strlines(q, 20) == dedent('''\ + ( + "this quote (') is " + "in the middle" + )''') + assert strlines('\\left') == ( + '(\n' + "r'\\left'\n" + ')') + assert strlines('\\left', short=True) == r"r'\left'" + assert strlines('\\le"ft') == ( + '(\n' + 'r\'\\le"ft\'\n' + ')') + q = 'this\nother line' + assert strlines(q) == rawlines(q) + + +def test_translate_args(): + try: + translate(None, None, None, 'not_none') + except ValueError: + pass # Exception raised successfully + else: + assert False + + assert translate('s', None, None, None) == 's' + + try: + translate('s', 'a', 'bc') + except ValueError: + pass # Exception raised successfully + else: + assert False + + +@skip_under_pyodide("Cannot create subprocess under pyodide.") +def test_debug_output(): + env = os.environ.copy() + env['SYMPY_DEBUG'] = 'True' + cmd = 'from sympy import *; x = Symbol("x"); print(integrate((1-cos(x))/x, x))' + cmdline = [sys.executable, '-c', cmd] + proc = Popen(cmdline, env=env, stdout=PIPE, stderr=PIPE) + out, err = proc.communicate() + out = out.decode('ascii') # utf-8? + err = err.decode('ascii') + expected = 'substituted: -x*(1 - cos(x)), u: 1/x, u_var: _u' + assert expected in err, err + + +def test_as_int(): + raises(ValueError, lambda : as_int(True)) + raises(ValueError, lambda : as_int(1.1)) + raises(ValueError, lambda : as_int([])) + raises(ValueError, lambda : as_int(S.NaN)) + raises(ValueError, lambda : as_int(S.Infinity)) + raises(ValueError, lambda : as_int(S.NegativeInfinity)) + raises(ValueError, lambda : as_int(S.ComplexInfinity)) + # for the following, limited precision makes int(arg) == arg + # but the int value is not necessarily what a user might have + # expected; Q.prime is more nuanced in its response for + # expressions which might be complex representations of an + # integer. This is not -- by design -- as_ints role. + raises(ValueError, lambda : as_int(1e23)) + raises(ValueError, lambda : as_int(S('1.'+'0'*20+'1'))) + assert as_int(True, strict=False) == 1 + +def test_deprecated_find_executable(): + with warns_deprecated_sympy(): + find_executable('python') diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_pickling.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_pickling.py new file mode 100644 index 0000000000000000000000000000000000000000..7ea2d062286cbcf802eb0f42f2d7d130123599af --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_pickling.py @@ -0,0 +1,723 @@ +import inspect +import copy +import pickle + +from sympy.physics.units import meter + +from sympy.testing.pytest import XFAIL, raises, ignore_warnings + +from sympy.core.basic import Atom, Basic +from sympy.core.singleton import SingletonRegistry +from sympy.core.symbol import Str, Dummy, Symbol, Wild +from sympy.core.numbers import (E, I, pi, oo, zoo, nan, Integer, + Rational, Float, AlgebraicNumber) +from sympy.core.relational import (Equality, GreaterThan, LessThan, Relational, + StrictGreaterThan, StrictLessThan, Unequality) +from sympy.core.add import Add +from sympy.core.mul import Mul +from sympy.core.power import Pow +from sympy.core.function import Derivative, Function, FunctionClass, Lambda, \ + WildFunction +from sympy.sets.sets import Interval +from sympy.core.multidimensional import vectorize + +from sympy.external.gmpy import gmpy as _gmpy +from sympy.utilities.exceptions import SymPyDeprecationWarning + +from sympy.core.singleton import S +from sympy.core.symbol import symbols + +from sympy.external import import_module +cloudpickle = import_module('cloudpickle') + + +not_equal_attrs = { + '_assumptions', # This is a local cache that isn't automatically filled on creation + '_mhash', # Cached after __hash__ is called but set to None after creation +} + + +deprecated_attrs = { + 'is_EmptySet', # Deprecated from SymPy 1.5. This can be removed when is_EmptySet is removed. + 'expr_free_symbols', # Deprecated from SymPy 1.9. This can be removed when exr_free_symbols is removed. +} + +dont_check_attrs = { + '_sage_', # Fails because Sage is not installed +} + + +def check(a, exclude=[], check_attr=True, deprecated=()): + """ Check that pickling and copying round-trips. + """ + # Pickling with protocols 0 and 1 is disabled for Basic instances: + if isinstance(a, Basic): + for protocol in [0, 1]: + raises(NotImplementedError, lambda: pickle.dumps(a, protocol)) + + protocols = [2, copy.copy, copy.deepcopy, 3, 4] + if cloudpickle: + protocols.extend([cloudpickle]) + + for protocol in protocols: + if protocol in exclude: + continue + + if callable(protocol): + if isinstance(a, type): + # Classes can't be copied, but that's okay. + continue + b = protocol(a) + elif inspect.ismodule(protocol): + b = protocol.loads(protocol.dumps(a)) + else: + b = pickle.loads(pickle.dumps(a, protocol)) + + d1 = dir(a) + d2 = dir(b) + assert set(d1) == set(d2) + + if not check_attr: + continue + + def c(a, b, d): + for i in d: + if i in dont_check_attrs: + continue + elif i in not_equal_attrs: + if hasattr(a, i): + assert hasattr(b, i), i + elif i in deprecated_attrs or i in deprecated: + with ignore_warnings(SymPyDeprecationWarning): + assert getattr(a, i) == getattr(b, i), i + elif not hasattr(a, i): + continue + else: + attr = getattr(a, i) + if not hasattr(attr, "__call__"): + assert hasattr(b, i), i + assert getattr(b, i) == attr, "%s != %s, protocol: %s" % (getattr(b, i), attr, protocol) + + c(a, b, d1) + c(b, a, d2) + + + +#================== core ========================= + + +def test_core_basic(): + for c in (Atom, Atom(), Basic, Basic(), SingletonRegistry, S): + check(c) + +def test_core_Str(): + check(Str('x')) + +def test_core_symbol(): + # make the Symbol a unique name that doesn't class with any other + # testing variable in this file since after this test the symbol + # having the same name will be cached as noncommutative + for c in (Dummy, Dummy("x", commutative=False), Symbol, + Symbol("_issue_3130", commutative=False), Wild, Wild("x")): + check(c) + + +def test_core_numbers(): + for c in (Integer(2), Rational(2, 3), Float("1.2")): + check(c) + for c in (AlgebraicNumber, AlgebraicNumber(sqrt(3))): + check(c, check_attr=False) + + +def test_core_float_copy(): + # See gh-7457 + y = Symbol("x") + 1.0 + check(y) # does not raise TypeError ("argument is not an mpz") + + +def test_core_relational(): + x = Symbol("x") + y = Symbol("y") + for c in (Equality, Equality(x, y), GreaterThan, GreaterThan(x, y), + LessThan, LessThan(x, y), Relational, Relational(x, y), + StrictGreaterThan, StrictGreaterThan(x, y), StrictLessThan, + StrictLessThan(x, y), Unequality, Unequality(x, y)): + check(c) + + +def test_core_add(): + x = Symbol("x") + for c in (Add, Add(x, 4)): + check(c) + + +def test_core_mul(): + x = Symbol("x") + for c in (Mul, Mul(x, 4)): + check(c) + + +def test_core_power(): + x = Symbol("x") + for c in (Pow, Pow(x, 4)): + check(c) + + +def test_core_function(): + x = Symbol("x") + for f in (Derivative, Derivative(x), Function, FunctionClass, Lambda, + WildFunction): + check(f) + + +def test_core_undefinedfunctions(): + f = Function("f") + check(f) + + +def test_core_appliedundef(): + x = Symbol("_long_unique_name_1") + f = Function("_long_unique_name_2") + check(f(x)) + + +def test_core_interval(): + for c in (Interval, Interval(0, 2)): + check(c) + + +def test_core_multidimensional(): + for c in (vectorize, vectorize(0)): + check(c) + + +def test_Singletons(): + protocols = [0, 1, 2, 3, 4] + copiers = [copy.copy, copy.deepcopy] + copiers += [lambda x: pickle.loads(pickle.dumps(x, proto)) + for proto in protocols] + if cloudpickle: + copiers += [lambda x: cloudpickle.loads(cloudpickle.dumps(x))] + + for obj in (Integer(-1), Integer(0), Integer(1), Rational(1, 2), pi, E, I, + oo, -oo, zoo, nan, S.GoldenRatio, S.TribonacciConstant, + S.EulerGamma, S.Catalan, S.EmptySet, S.IdentityFunction): + for func in copiers: + assert func(obj) is obj + +#================== combinatorics =================== +from sympy.combinatorics.free_groups import FreeGroup + +def test_free_group(): + check(FreeGroup("x, y, z"), check_attr=False) + +#================== functions =================== +from sympy.functions import (Piecewise, lowergamma, acosh, chebyshevu, + chebyshevt, ln, chebyshevt_root, legendre, Heaviside, bernoulli, coth, + tanh, assoc_legendre, sign, arg, asin, DiracDelta, re, rf, Abs, + uppergamma, binomial, sinh, cos, cot, acos, acot, gamma, bell, + hermite, harmonic, LambertW, zeta, log, factorial, asinh, acoth, cosh, + dirichlet_eta, Eijk, loggamma, erf, ceiling, im, fibonacci, + tribonacci, conjugate, tan, chebyshevu_root, floor, atanh, sqrt, sin, + atan, ff, lucas, atan2, polygamma, exp) + + +def test_functions(): + one_var = (acosh, ln, Heaviside, factorial, bernoulli, coth, tanh, + sign, arg, asin, DiracDelta, re, Abs, sinh, cos, cot, acos, acot, + gamma, bell, harmonic, LambertW, zeta, log, factorial, asinh, + acoth, cosh, dirichlet_eta, loggamma, erf, ceiling, im, fibonacci, + tribonacci, conjugate, tan, floor, atanh, sin, atan, lucas, exp) + two_var = (rf, ff, lowergamma, chebyshevu, chebyshevt, binomial, + atan2, polygamma, hermite, legendre, uppergamma) + x, y, z = symbols("x,y,z") + others = (chebyshevt_root, chebyshevu_root, Eijk(x, y, z), + Piecewise( (0, x < -1), (x**2, x <= 1), (x**3, True)), + assoc_legendre) + for cls in one_var: + check(cls) + c = cls(x) + check(c) + for cls in two_var: + check(cls) + c = cls(x, y) + check(c) + for cls in others: + check(cls) + +#================== geometry ==================== +from sympy.geometry.entity import GeometryEntity +from sympy.geometry.point import Point +from sympy.geometry.ellipse import Circle, Ellipse +from sympy.geometry.line import Line, LinearEntity, Ray, Segment +from sympy.geometry.polygon import Polygon, RegularPolygon, Triangle + + +def test_geometry(): + p1 = Point(1, 2) + p2 = Point(2, 3) + p3 = Point(0, 0) + p4 = Point(0, 1) + for c in ( + GeometryEntity, GeometryEntity(), Point, p1, Circle, Circle(p1, 2), + Ellipse, Ellipse(p1, 3, 4), Line, Line(p1, p2), LinearEntity, + LinearEntity(p1, p2), Ray, Ray(p1, p2), Segment, Segment(p1, p2), + Polygon, Polygon(p1, p2, p3, p4), RegularPolygon, + RegularPolygon(p1, 4, 5), Triangle, Triangle(p1, p2, p3)): + check(c, check_attr=False) + +#================== integrals ==================== +from sympy.integrals.integrals import Integral + + +def test_integrals(): + x = Symbol("x") + for c in (Integral, Integral(x)): + check(c) + +#==================== logic ===================== +from sympy.core.logic import Logic + + +def test_logic(): + for c in (Logic, Logic(1)): + check(c) + +#================== matrices ==================== +from sympy.matrices import Matrix, SparseMatrix + + +def test_matrices(): + for c in (Matrix, Matrix([1, 2, 3]), SparseMatrix, SparseMatrix([[1, 2], [3, 4]])): + check(c, deprecated=['_smat', '_mat']) + +#================== ntheory ===================== +from sympy.ntheory.generate import Sieve + + +def test_ntheory(): + for c in (Sieve, Sieve()): + check(c) + +#================== physics ===================== +from sympy.physics.paulialgebra import Pauli +from sympy.physics.units import Unit + + +def test_physics(): + for c in (Unit, meter, Pauli, Pauli(1)): + check(c) + +#================== plotting ==================== +# XXX: These tests are not complete, so XFAIL them + + +@XFAIL +def test_plotting(): + from sympy.plotting.pygletplot.color_scheme import ColorGradient, ColorScheme + from sympy.plotting.pygletplot.managed_window import ManagedWindow + from sympy.plotting.plot import Plot, ScreenShot + from sympy.plotting.pygletplot.plot_axes import PlotAxes, PlotAxesBase, PlotAxesFrame, PlotAxesOrdinate + from sympy.plotting.pygletplot.plot_camera import PlotCamera + from sympy.plotting.pygletplot.plot_controller import PlotController + from sympy.plotting.pygletplot.plot_curve import PlotCurve + from sympy.plotting.pygletplot.plot_interval import PlotInterval + from sympy.plotting.pygletplot.plot_mode import PlotMode + from sympy.plotting.pygletplot.plot_modes import Cartesian2D, Cartesian3D, Cylindrical, \ + ParametricCurve2D, ParametricCurve3D, ParametricSurface, Polar, Spherical + from sympy.plotting.pygletplot.plot_object import PlotObject + from sympy.plotting.pygletplot.plot_surface import PlotSurface + from sympy.plotting.pygletplot.plot_window import PlotWindow + for c in ( + ColorGradient, ColorGradient(0.2, 0.4), ColorScheme, ManagedWindow, + ManagedWindow, Plot, ScreenShot, PlotAxes, PlotAxesBase, + PlotAxesFrame, PlotAxesOrdinate, PlotCamera, PlotController, + PlotCurve, PlotInterval, PlotMode, Cartesian2D, Cartesian3D, + Cylindrical, ParametricCurve2D, ParametricCurve3D, + ParametricSurface, Polar, Spherical, PlotObject, PlotSurface, + PlotWindow): + check(c) + + +@XFAIL +def test_plotting2(): + #from sympy.plotting.color_scheme import ColorGradient + from sympy.plotting.pygletplot.color_scheme import ColorScheme + #from sympy.plotting.managed_window import ManagedWindow + from sympy.plotting.plot import Plot + #from sympy.plotting.plot import ScreenShot + from sympy.plotting.pygletplot.plot_axes import PlotAxes + #from sympy.plotting.plot_axes import PlotAxesBase, PlotAxesFrame, PlotAxesOrdinate + #from sympy.plotting.plot_camera import PlotCamera + #from sympy.plotting.plot_controller import PlotController + #from sympy.plotting.plot_curve import PlotCurve + #from sympy.plotting.plot_interval import PlotInterval + #from sympy.plotting.plot_mode import PlotMode + #from sympy.plotting.plot_modes import Cartesian2D, Cartesian3D, Cylindrical, \ + # ParametricCurve2D, ParametricCurve3D, ParametricSurface, Polar, Spherical + #from sympy.plotting.plot_object import PlotObject + #from sympy.plotting.plot_surface import PlotSurface + # from sympy.plotting.plot_window import PlotWindow + check(ColorScheme("rainbow")) + check(Plot(1, visible=False)) + check(PlotAxes()) + +#================== polys ======================= +from sympy.polys.domains.integerring import ZZ +from sympy.polys.domains.rationalfield import QQ +from sympy.polys.orderings import lex +from sympy.polys.polytools import Poly + +def test_pickling_polys_polytools(): + from sympy.polys.polytools import PurePoly + # from sympy.polys.polytools import GroebnerBasis + x = Symbol('x') + + for c in (Poly, Poly(x, x)): + check(c) + + for c in (PurePoly, PurePoly(x)): + check(c) + + # TODO: fix pickling of Options class (see GroebnerBasis._options) + # for c in (GroebnerBasis, GroebnerBasis([x**2 - 1], x, order=lex)): + # check(c) + +def test_pickling_polys_polyclasses(): + from sympy.polys.polyclasses import DMP, DMF, ANP + + for c in (DMP, DMP([[ZZ(1)], [ZZ(2)], [ZZ(3)]], ZZ)): + check(c, deprecated=['rep']) + for c in (DMF, DMF(([ZZ(1), ZZ(2)], [ZZ(1), ZZ(3)]), ZZ)): + check(c) + for c in (ANP, ANP([QQ(1), QQ(2)], [QQ(1), QQ(2), QQ(3)], QQ)): + check(c) + +@XFAIL +def test_pickling_polys_rings(): + # NOTE: can't use protocols < 2 because we have to execute __new__ to + # make sure caching of rings works properly. + + from sympy.polys.rings import PolyRing + + ring = PolyRing("x,y,z", ZZ, lex) + + for c in (PolyRing, ring): + check(c, exclude=[0, 1]) + + for c in (ring.dtype, ring.one): + check(c, exclude=[0, 1], check_attr=False) # TODO: Py3k + +def test_pickling_polys_fields(): + pass + # NOTE: can't use protocols < 2 because we have to execute __new__ to + # make sure caching of fields works properly. + + # from sympy.polys.fields import FracField + + # field = FracField("x,y,z", ZZ, lex) + + # TODO: AssertionError: assert id(obj) not in self.memo + # for c in (FracField, field): + # check(c, exclude=[0, 1]) + + # TODO: AssertionError: assert id(obj) not in self.memo + # for c in (field.dtype, field.one): + # check(c, exclude=[0, 1]) + +def test_pickling_polys_elements(): + from sympy.polys.domains.pythonrational import PythonRational + #from sympy.polys.domains.pythonfinitefield import PythonFiniteField + #from sympy.polys.domains.mpelements import MPContext + + for c in (PythonRational, PythonRational(1, 7)): + check(c) + + #gf = PythonFiniteField(17) + + # TODO: fix pickling of ModularInteger + # for c in (gf.dtype, gf(5)): + # check(c) + + #mp = MPContext() + + # TODO: fix pickling of RealElement + # for c in (mp.mpf, mp.mpf(1.0)): + # check(c) + + # TODO: fix pickling of ComplexElement + # for c in (mp.mpc, mp.mpc(1.0, -1.5)): + # check(c) + +def test_pickling_polys_domains(): + # from sympy.polys.domains.pythonfinitefield import PythonFiniteField + from sympy.polys.domains.pythonintegerring import PythonIntegerRing + from sympy.polys.domains.pythonrationalfield import PythonRationalField + + # TODO: fix pickling of ModularInteger + # for c in (PythonFiniteField, PythonFiniteField(17)): + # check(c) + + for c in (PythonIntegerRing, PythonIntegerRing()): + check(c, check_attr=False) + + for c in (PythonRationalField, PythonRationalField()): + check(c, check_attr=False) + + if _gmpy is not None: + # from sympy.polys.domains.gmpyfinitefield import GMPYFiniteField + from sympy.polys.domains.gmpyintegerring import GMPYIntegerRing + from sympy.polys.domains.gmpyrationalfield import GMPYRationalField + + # TODO: fix pickling of ModularInteger + # for c in (GMPYFiniteField, GMPYFiniteField(17)): + # check(c) + + for c in (GMPYIntegerRing, GMPYIntegerRing()): + check(c, check_attr=False) + + for c in (GMPYRationalField, GMPYRationalField()): + check(c, check_attr=False) + + #from sympy.polys.domains.realfield import RealField + #from sympy.polys.domains.complexfield import ComplexField + from sympy.polys.domains.algebraicfield import AlgebraicField + #from sympy.polys.domains.polynomialring import PolynomialRing + #from sympy.polys.domains.fractionfield import FractionField + from sympy.polys.domains.expressiondomain import ExpressionDomain + + # TODO: fix pickling of RealElement + # for c in (RealField, RealField(100)): + # check(c) + + # TODO: fix pickling of ComplexElement + # for c in (ComplexField, ComplexField(100)): + # check(c) + + for c in (AlgebraicField, AlgebraicField(QQ, sqrt(3))): + check(c, check_attr=False) + + # TODO: AssertionError + # for c in (PolynomialRing, PolynomialRing(ZZ, "x,y,z")): + # check(c) + + # TODO: AttributeError: 'PolyElement' object has no attribute 'ring' + # for c in (FractionField, FractionField(ZZ, "x,y,z")): + # check(c) + + for c in (ExpressionDomain, ExpressionDomain()): + check(c, check_attr=False) + + +def test_pickling_polys_orderings(): + from sympy.polys.orderings import (LexOrder, GradedLexOrder, + ReversedGradedLexOrder, InverseOrder) + # from sympy.polys.orderings import ProductOrder + + for c in (LexOrder, LexOrder()): + check(c) + + for c in (GradedLexOrder, GradedLexOrder()): + check(c) + + for c in (ReversedGradedLexOrder, ReversedGradedLexOrder()): + check(c) + + # TODO: Argh, Python is so naive. No lambdas nor inner function support in + # pickling module. Maybe someone could figure out what to do with this. + # + # for c in (ProductOrder, ProductOrder((LexOrder(), lambda m: m[:2]), + # (GradedLexOrder(), lambda m: m[2:]))): + # check(c) + + for c in (InverseOrder, InverseOrder(LexOrder())): + check(c) + +def test_pickling_polys_monomials(): + from sympy.polys.monomials import MonomialOps, Monomial + x, y, z = symbols("x,y,z") + + for c in (MonomialOps, MonomialOps(3)): + check(c) + + for c in (Monomial, Monomial((1, 2, 3), (x, y, z))): + check(c) + +def test_pickling_polys_errors(): + from sympy.polys.polyerrors import (HeuristicGCDFailed, + HomomorphismFailed, IsomorphismFailed, ExtraneousFactors, + EvaluationFailed, RefinementFailed, CoercionFailed, NotInvertible, + NotReversible, NotAlgebraic, DomainError, PolynomialError, + UnificationFailed, GeneratorsError, GeneratorsNeeded, + UnivariatePolynomialError, MultivariatePolynomialError, OptionError, + FlagError) + # from sympy.polys.polyerrors import (ExactQuotientFailed, + # OperationNotSupported, ComputationFailed, PolificationFailed) + + # x = Symbol('x') + + # TODO: TypeError: __init__() takes at least 3 arguments (1 given) + # for c in (ExactQuotientFailed, ExactQuotientFailed(x, 3*x, ZZ)): + # check(c) + + # TODO: TypeError: can't pickle instancemethod objects + # for c in (OperationNotSupported, OperationNotSupported(Poly(x), Poly.gcd)): + # check(c) + + for c in (HeuristicGCDFailed, HeuristicGCDFailed()): + check(c) + + for c in (HomomorphismFailed, HomomorphismFailed()): + check(c) + + for c in (IsomorphismFailed, IsomorphismFailed()): + check(c) + + for c in (ExtraneousFactors, ExtraneousFactors()): + check(c) + + for c in (EvaluationFailed, EvaluationFailed()): + check(c) + + for c in (RefinementFailed, RefinementFailed()): + check(c) + + for c in (CoercionFailed, CoercionFailed()): + check(c) + + for c in (NotInvertible, NotInvertible()): + check(c) + + for c in (NotReversible, NotReversible()): + check(c) + + for c in (NotAlgebraic, NotAlgebraic()): + check(c) + + for c in (DomainError, DomainError()): + check(c) + + for c in (PolynomialError, PolynomialError()): + check(c) + + for c in (UnificationFailed, UnificationFailed()): + check(c) + + for c in (GeneratorsError, GeneratorsError()): + check(c) + + for c in (GeneratorsNeeded, GeneratorsNeeded()): + check(c) + + # TODO: PicklingError: Can't pickle at 0x38578c0>: it's not found as __main__. + # for c in (ComputationFailed, ComputationFailed(lambda t: t, 3, None)): + # check(c) + + for c in (UnivariatePolynomialError, UnivariatePolynomialError()): + check(c) + + for c in (MultivariatePolynomialError, MultivariatePolynomialError()): + check(c) + + # TODO: TypeError: __init__() takes at least 3 arguments (1 given) + # for c in (PolificationFailed, PolificationFailed({}, x, x, False)): + # check(c) + + for c in (OptionError, OptionError()): + check(c) + + for c in (FlagError, FlagError()): + check(c) + +#def test_pickling_polys_options(): + #from sympy.polys.polyoptions import Options + + # TODO: fix pickling of `symbols' flag + # for c in (Options, Options((), dict(domain='ZZ', polys=False))): + # check(c) + +# TODO: def test_pickling_polys_rootisolation(): +# RealInterval +# ComplexInterval + +def test_pickling_polys_rootoftools(): + from sympy.polys.rootoftools import CRootOf, RootSum + + x = Symbol('x') + f = x**3 + x + 3 + + for c in (CRootOf, CRootOf(f, 0)): + check(c) + + for c in (RootSum, RootSum(f, exp)): + check(c) + +#================== printing ==================== +from sympy.printing.latex import LatexPrinter +from sympy.printing.mathml import MathMLContentPrinter, MathMLPresentationPrinter +from sympy.printing.pretty.pretty import PrettyPrinter +from sympy.printing.pretty.stringpict import prettyForm, stringPict +from sympy.printing.printer import Printer +from sympy.printing.python import PythonPrinter + + +def test_printing(): + for c in (LatexPrinter, LatexPrinter(), MathMLContentPrinter, + MathMLPresentationPrinter, PrettyPrinter, prettyForm, stringPict, + stringPict("a"), Printer, Printer(), PythonPrinter, + PythonPrinter()): + check(c) + + +@XFAIL +def test_printing1(): + check(MathMLContentPrinter()) + + +@XFAIL +def test_printing2(): + check(MathMLPresentationPrinter()) + + +@XFAIL +def test_printing3(): + check(PrettyPrinter()) + +#================== series ====================== +from sympy.series.limits import Limit +from sympy.series.order import Order + + +def test_series(): + e = Symbol("e") + x = Symbol("x") + for c in (Limit, Limit(e, x, 1), Order, Order(e)): + check(c) + +#================== concrete ================== +from sympy.concrete.products import Product +from sympy.concrete.summations import Sum + + +def test_concrete(): + x = Symbol("x") + for c in (Product, Product(x, (x, 2, 4)), Sum, Sum(x, (x, 2, 4))): + check(c) + +def test_deprecation_warning(): + w = SymPyDeprecationWarning("message", deprecated_since_version='1.0', active_deprecations_target="active-deprecations") + check(w) + +def test_issue_18438(): + assert pickle.loads(pickle.dumps(S.Half)) == S.Half + + +#================= old pickles ================= +def test_unpickle_from_older_versions(): + data = ( + b'\x80\x04\x95^\x00\x00\x00\x00\x00\x00\x00\x8c\x10sympy.core.power' + b'\x94\x8c\x03Pow\x94\x93\x94\x8c\x12sympy.core.numbers\x94\x8c' + b'\x07Integer\x94\x93\x94K\x02\x85\x94R\x94}\x94bh\x03\x8c\x04Half' + b'\x94\x93\x94)R\x94}\x94b\x86\x94R\x94}\x94b.' + ) + assert pickle.loads(data) == sqrt(2) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_source.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_source.py new file mode 100644 index 0000000000000000000000000000000000000000..468185bc579fc325aee21024dfa15ebf14287b5f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_source.py @@ -0,0 +1,11 @@ +from sympy.utilities.source import get_mod_func, get_class + + +def test_get_mod_func(): + assert get_mod_func( + 'sympy.core.basic.Basic') == ('sympy.core.basic', 'Basic') + + +def test_get_class(): + _basic = get_class('sympy.core.basic.Basic') + assert _basic.__name__ == 'Basic' diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_timeutils.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_timeutils.py new file mode 100644 index 0000000000000000000000000000000000000000..14edfd089c7315ee9f39a4298af0289f8919da6b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_timeutils.py @@ -0,0 +1,10 @@ +"""Tests for simple tools for timing functions' execution. """ + +from sympy.utilities.timeutils import timed + +def test_timed(): + result = timed(lambda: 1 + 1, limit=100000) + assert result[0] == 100000 and result[3] == "ns", str(result) + + result = timed("1 + 1", limit=100000) + assert result[0] == 100000 and result[3] == "ns" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_wester.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_wester.py new file mode 100644 index 0000000000000000000000000000000000000000..c5699a4eb0824e507967ecd4ff8f7a5f32cc9a54 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_wester.py @@ -0,0 +1,3104 @@ +""" Tests from Michael Wester's 1999 paper "Review of CAS mathematical +capabilities". + +http://www.math.unm.edu/~wester/cas/book/Wester.pdf +See also http://math.unm.edu/~wester/cas_review.html for detailed output of +each tested system. +""" + +from sympy.assumptions.ask import Q, ask +from sympy.assumptions.refine import refine +from sympy.concrete.products import product +from sympy.core import EulerGamma +from sympy.core.evalf import N +from sympy.core.function import (Derivative, Function, Lambda, Subs, + diff, expand, expand_func) +from sympy.core.mul import Mul +from sympy.core.intfunc import igcd +from sympy.core.numbers import (AlgebraicNumber, E, I, Rational, + nan, oo, pi, zoo) +from sympy.core.relational import Eq, Lt +from sympy.core.singleton import S +from sympy.core.symbol import Dummy, Symbol, symbols +from sympy.functions.combinatorial.factorials import (rf, binomial, + factorial, factorial2) +from sympy.functions.combinatorial.numbers import bernoulli, fibonacci, totient, partition +from sympy.functions.elementary.complexes import (conjugate, im, re, + sign) +from sympy.functions.elementary.exponential import LambertW, exp, log +from sympy.functions.elementary.hyperbolic import (asinh, cosh, sinh, + tanh) +from sympy.functions.elementary.integers import ceiling, floor +from sympy.functions.elementary.miscellaneous import Max, Min, sqrt +from sympy.functions.elementary.piecewise import Piecewise +from sympy.functions.elementary.trigonometric import (acos, acot, asin, + atan, cos, cot, csc, sec, sin, tan) +from sympy.functions.special.bessel import besselj +from sympy.functions.special.delta_functions import DiracDelta +from sympy.functions.special.elliptic_integrals import (elliptic_e, + elliptic_f) +from sympy.functions.special.gamma_functions import gamma, polygamma +from sympy.functions.special.hyper import hyper +from sympy.functions.special.polynomials import (assoc_legendre, + chebyshevt) +from sympy.functions.special.zeta_functions import polylog +from sympy.geometry.util import idiff +from sympy.logic.boolalg import And +from sympy.matrices.dense import hessian, wronskian +from sympy.matrices.expressions.matmul import MatMul +from sympy.ntheory.continued_fraction import ( + continued_fraction_convergents as cf_c, + continued_fraction_iterator as cf_i, continued_fraction_periodic as + cf_p, continued_fraction_reduce as cf_r) +from sympy.ntheory.factor_ import factorint +from sympy.ntheory.generate import primerange +from sympy.polys.domains.integerring import ZZ +from sympy.polys.orthopolys import legendre_poly +from sympy.polys.partfrac import apart +from sympy.polys.polytools import Poly, factor, gcd, resultant +from sympy.series.limits import limit +from sympy.series.order import O +from sympy.series.residues import residue +from sympy.series.series import series +from sympy.sets.fancysets import ImageSet +from sympy.sets.sets import FiniteSet, Intersection, Interval, Union +from sympy.simplify.combsimp import combsimp +from sympy.simplify.hyperexpand import hyperexpand +from sympy.simplify.powsimp import powdenest, powsimp +from sympy.simplify.radsimp import radsimp +from sympy.simplify.simplify import logcombine, simplify +from sympy.simplify.sqrtdenest import sqrtdenest +from sympy.simplify.trigsimp import trigsimp +from sympy.solvers.solvers import solve + +import mpmath +from sympy.functions.combinatorial.numbers import stirling +from sympy.functions.special.delta_functions import Heaviside +from sympy.functions.special.error_functions import Ci, Si, erf +from sympy.functions.special.zeta_functions import zeta +from sympy.testing.pytest import (XFAIL, slow, SKIP, tooslow, raises) +from sympy.utilities.iterables import partitions +from mpmath import mpi, mpc +from sympy.matrices import Matrix, GramSchmidt, eye +from sympy.matrices.expressions.blockmatrix import BlockMatrix, block_collapse +from sympy.matrices.expressions import MatrixSymbol, ZeroMatrix +from sympy.physics.quantum import Commutator +from sympy.polys.rings import PolyRing +from sympy.polys.fields import FracField +from sympy.polys.solvers import solve_lin_sys +from sympy.concrete import Sum +from sympy.concrete.products import Product +from sympy.integrals import integrate +from sympy.integrals.transforms import laplace_transform,\ + inverse_laplace_transform, LaplaceTransform, fourier_transform,\ + mellin_transform, laplace_correspondence, laplace_initial_conds +from sympy.solvers.recurr import rsolve +from sympy.solvers.solveset import solveset, solveset_real, linsolve +from sympy.solvers.ode import dsolve +from sympy.core.relational import Equality +from itertools import islice, takewhile +from sympy.series.formal import fps +from sympy.series.fourier import fourier_series +from sympy.calculus.util import minimum + + +EmptySet = S.EmptySet +R = Rational +x, y, z = symbols('x y z') +i, j, k, l, m, n = symbols('i j k l m n', integer=True) +f = Function('f') +g = Function('g') + +# A. Boolean Logic and Quantifier Elimination +# Not implemented. + +# B. Set Theory + + +def test_B1(): + assert (FiniteSet(i, j, j, k, k, k) | FiniteSet(l, k, j) | + FiniteSet(j, m, j)) == FiniteSet(i, j, k, l, m) + + +def test_B2(): + assert (FiniteSet(i, j, j, k, k, k) & FiniteSet(l, k, j) & + FiniteSet(j, m, j)) == Intersection({j, m}, {i, j, k}, {j, k, l}) + # Previous output below. Not sure why that should be the expected output. + # There should probably be a way to rewrite Intersections that way but I + # don't see why an Intersection should evaluate like that: + # + # == Union({j}, Intersection({m}, Union({j, k}, Intersection({i}, {l})))) + + +def test_B3(): + assert (FiniteSet(i, j, k, l, m) - FiniteSet(j) == + FiniteSet(i, k, l, m)) + + +def test_B4(): + assert (FiniteSet(*(FiniteSet(i, j)*FiniteSet(k, l))) == + FiniteSet((i, k), (i, l), (j, k), (j, l))) + + +# C. Numbers + + +def test_C1(): + assert (factorial(50) == + 30414093201713378043612608166064768844377641568960512000000000000) + + +def test_C2(): + assert (factorint(factorial(50)) == {2: 47, 3: 22, 5: 12, 7: 8, + 11: 4, 13: 3, 17: 2, 19: 2, 23: 2, 29: 1, 31: 1, 37: 1, + 41: 1, 43: 1, 47: 1}) + + +def test_C3(): + assert (factorial2(10), factorial2(9)) == (3840, 945) + + +# Base conversions; not really implemented by SymPy +# Whatever. Take credit! +def test_C4(): + assert 0xABC == 2748 + + +def test_C5(): + assert 123 == int('234', 7) + + +def test_C6(): + assert int('677', 8) == int('1BF', 16) == 447 + + +def test_C7(): + assert log(32768, 8) == 5 + + +def test_C8(): + # Modular multiplicative inverse. Would be nice if divmod could do this. + assert ZZ.invert(5, 7) == 3 + assert ZZ.invert(5, 6) == 5 + + +def test_C9(): + assert igcd(igcd(1776, 1554), 5698) == 74 + + +def test_C10(): + x = 0 + for n in range(2, 11): + x += R(1, n) + assert x == R(4861, 2520) + + +def test_C11(): + assert R(1, 7) == S('0.[142857]') + + +def test_C12(): + assert R(7, 11) * R(22, 7) == 2 + + +def test_C13(): + test = R(10, 7) * (1 + R(29, 1000)) ** R(1, 3) + good = 3 ** R(1, 3) + assert test == good + + +def test_C14(): + assert sqrtdenest(sqrt(2*sqrt(3) + 4)) == 1 + sqrt(3) + + +def test_C15(): + test = sqrtdenest(sqrt(14 + 3*sqrt(3 + 2*sqrt(5 - 12*sqrt(3 - 2*sqrt(2)))))) + good = sqrt(2) + 3 + assert test == good + + +def test_C16(): + test = sqrtdenest(sqrt(10 + 2*sqrt(6) + 2*sqrt(10) + 2*sqrt(15))) + good = sqrt(2) + sqrt(3) + sqrt(5) + assert test == good + + +def test_C17(): + test = radsimp((sqrt(3) + sqrt(2)) / (sqrt(3) - sqrt(2))) + good = 5 + 2*sqrt(6) + assert test == good + + +def test_C18(): + assert simplify((sqrt(-2 + sqrt(-5)) * sqrt(-2 - sqrt(-5))).expand(complex=True)) == 3 + + +@XFAIL +def test_C19(): + assert radsimp(simplify((90 + 34*sqrt(7)) ** R(1, 3))) == 3 + sqrt(7) + + +def test_C20(): + inside = (135 + 78*sqrt(3)) + test = AlgebraicNumber((inside**R(2, 3) + 3) * sqrt(3) / inside**R(1, 3)) + assert simplify(test) == AlgebraicNumber(12) + + +def test_C21(): + assert simplify(AlgebraicNumber((41 + 29*sqrt(2)) ** R(1, 5))) == \ + AlgebraicNumber(1 + sqrt(2)) + + +@XFAIL +def test_C22(): + test = simplify(((6 - 4*sqrt(2))*log(3 - 2*sqrt(2)) + (3 - 2*sqrt(2))*log(17 + - 12*sqrt(2)) + 32 - 24*sqrt(2)) / (48*sqrt(2) - 72)) + good = sqrt(2)/3 - log(sqrt(2) - 1)/3 + assert test == good + + +def test_C23(): + assert 2 * oo - 3 is oo + + +@XFAIL +def test_C24(): + raise NotImplementedError("2**aleph_null == aleph_1") + +# D. Numerical Analysis + + +def test_D1(): + assert 0.0 / sqrt(2) == 0 + + +def test_D2(): + assert str(exp(-1000000).evalf()) == '3.29683147808856e-434295' + + +def test_D3(): + assert exp(pi*sqrt(163)).evalf(50).num.ae(262537412640768744) + + +def test_D4(): + assert floor(R(-5, 3)) == -2 + assert ceiling(R(-5, 3)) == -1 + + +@XFAIL +def test_D5(): + raise NotImplementedError("cubic_spline([1, 2, 4, 5], [1, 4, 2, 3], x)(3) == 27/8") + + +@XFAIL +def test_D6(): + raise NotImplementedError("translate sum(a[i]*x**i, (i,1,n)) to FORTRAN") + + +@XFAIL +def test_D7(): + raise NotImplementedError("translate sum(a[i]*x**i, (i,1,n)) to C") + + +@XFAIL +def test_D8(): + # One way is to cheat by converting the sum to a string, + # and replacing the '[' and ']' with ''. + # E.g., horner(S(str(_).replace('[','').replace(']',''))) + raise NotImplementedError("apply Horner's rule to sum(a[i]*x**i, (i,1,5))") + + +@XFAIL +def test_D9(): + raise NotImplementedError("translate D8 to FORTRAN") + + +@XFAIL +def test_D10(): + raise NotImplementedError("translate D8 to C") + + +@XFAIL +def test_D11(): + #Is there a way to use count_ops? + raise NotImplementedError("flops(sum(product(f[i][k], (i,1,k)), (k,1,n)))") + + +@XFAIL +def test_D12(): + assert (mpi(-4, 2) * x + mpi(1, 3)) ** 2 == mpi(-8, 16)*x**2 + mpi(-24, 12)*x + mpi(1, 9) + + +@XFAIL +def test_D13(): + raise NotImplementedError("discretize a PDE: diff(f(x,t),t) == diff(diff(f(x,t),x),x)") + +# E. Statistics +# See scipy; all of this is numerical. + +# F. Combinatorial Theory. + + +def test_F1(): + assert rf(x, 3) == x*(1 + x)*(2 + x) + + +def test_F2(): + assert expand_func(binomial(n, 3)) == n*(n - 1)*(n - 2)/6 + + +@XFAIL +def test_F3(): + assert combsimp(2**n * factorial(n) * factorial2(2*n - 1)) == factorial(2*n) + + +@XFAIL +def test_F4(): + assert combsimp(2**n * factorial(n) * product(2*k - 1, (k, 1, n))) == factorial(2*n) + + +@XFAIL +def test_F5(): + assert gamma(n + R(1, 2)) / sqrt(pi) / factorial(n) == factorial(2*n)/2**(2*n)/factorial(n)**2 + + +def test_F6(): + partTest = [p.copy() for p in partitions(4)] + partDesired = [{4: 1}, {1: 1, 3: 1}, {2: 2}, {1: 2, 2:1}, {1: 4}] + assert partTest == partDesired + + +def test_F7(): + assert partition(4) == 5 + + +def test_F8(): + assert stirling(5, 2, signed=True) == -50 # if signed, then kind=1 + + +def test_F9(): + assert totient(1776) == 576 + +# G. Number Theory + + +def test_G1(): + assert list(primerange(999983, 1000004)) == [999983, 1000003] + + +@XFAIL +def test_G2(): + raise NotImplementedError("find the primitive root of 191 == 19") + + +@XFAIL +def test_G3(): + raise NotImplementedError("(a+b)**p mod p == a**p + b**p mod p; p prime") + +# ... G14 Modular equations are not implemented. + +def test_G15(): + assert Rational(sqrt(3).evalf()).limit_denominator(15) == R(26, 15) + assert list(takewhile(lambda x: x.q <= 15, cf_c(cf_i(sqrt(3)))))[-1] == \ + R(26, 15) + + +def test_G16(): + assert list(islice(cf_i(pi),10)) == [3, 7, 15, 1, 292, 1, 1, 1, 2, 1] + + +def test_G17(): + assert cf_p(0, 1, 23) == [4, [1, 3, 1, 8]] + + +def test_G18(): + assert cf_p(1, 2, 5) == [[1]] + assert cf_r([[1]]).expand() == S.Half + sqrt(5)/2 + + +@XFAIL +def test_G19(): + s = symbols('s', integer=True, positive=True) + it = cf_i((exp(1/s) - 1)/(exp(1/s) + 1)) + assert list(islice(it, 5)) == [0, 2*s, 6*s, 10*s, 14*s] + + +def test_G20(): + s = symbols('s', integer=True, positive=True) + # Wester erroneously has this as -s + sqrt(s**2 + 1) + assert cf_r([[2*s]]) == s + sqrt(s**2 + 1) + + +@XFAIL +def test_G20b(): + s = symbols('s', integer=True, positive=True) + assert cf_p(s, 1, s**2 + 1) == [[2*s]] + + +# H. Algebra + + +def test_H1(): + assert simplify(2*2**n) == simplify(2**(n + 1)) + assert powdenest(2*2**n) == simplify(2**(n + 1)) + + +def test_H2(): + assert powsimp(4 * 2**n) == 2**(n + 2) + + +def test_H3(): + assert (-1)**(n*(n + 1)) == 1 + + +def test_H4(): + expr = factor(6*x - 10) + assert type(expr) is Mul + assert expr.args[0] == 2 + assert expr.args[1] == 3*x - 5 + +p1 = 64*x**34 - 21*x**47 - 126*x**8 - 46*x**5 - 16*x**60 - 81 +p2 = 72*x**60 - 25*x**25 - 19*x**23 - 22*x**39 - 83*x**52 + 54*x**10 + 81 +q = 34*x**19 - 25*x**16 + 70*x**7 + 20*x**3 - 91*x - 86 + + +def test_H5(): + assert gcd(p1, p2, x) == 1 + + +def test_H6(): + assert gcd(expand(p1 * q), expand(p2 * q)) == q + + +def test_H7(): + p1 = 24*x*y**19*z**8 - 47*x**17*y**5*z**8 + 6*x**15*y**9*z**2 - 3*x**22 + 5 + p2 = 34*x**5*y**8*z**13 + 20*x**7*y**7*z**7 + 12*x**9*y**16*z**4 + 80*y**14*z + assert gcd(p1, p2, x, y, z) == 1 + + +def test_H8(): + p1 = 24*x*y**19*z**8 - 47*x**17*y**5*z**8 + 6*x**15*y**9*z**2 - 3*x**22 + 5 + p2 = 34*x**5*y**8*z**13 + 20*x**7*y**7*z**7 + 12*x**9*y**16*z**4 + 80*y**14*z + q = 11*x**12*y**7*z**13 - 23*x**2*y**8*z**10 + 47*x**17*y**5*z**8 + assert gcd(p1 * q, p2 * q, x, y, z) == q + + +def test_H9(): + x = Symbol('x', zero=False) + p1 = 2*x**(n + 4) - x**(n + 2) + p2 = 4*x**(n + 1) + 3*x**n + assert gcd(p1, p2) == x**n + + +def test_H10(): + p1 = 3*x**4 + 3*x**3 + x**2 - x - 2 + p2 = x**3 - 3*x**2 + x + 5 + assert resultant(p1, p2, x) == 0 + + +def test_H11(): + assert resultant(p1 * q, p2 * q, x) == 0 + + +def test_H12(): + num = x**2 - 4 + den = x**2 + 4*x + 4 + assert simplify(num/den) == (x - 2)/(x + 2) + + +@XFAIL +def test_H13(): + assert simplify((exp(x) - 1) / (exp(x/2) + 1)) == exp(x/2) - 1 + + +def test_H14(): + p = (x + 1) ** 20 + ep = expand(p) + assert ep == (1 + 20*x + 190*x**2 + 1140*x**3 + 4845*x**4 + 15504*x**5 + + 38760*x**6 + 77520*x**7 + 125970*x**8 + 167960*x**9 + 184756*x**10 + + 167960*x**11 + 125970*x**12 + 77520*x**13 + 38760*x**14 + 15504*x**15 + + 4845*x**16 + 1140*x**17 + 190*x**18 + 20*x**19 + x**20) + dep = diff(ep, x) + assert dep == (20 + 380*x + 3420*x**2 + 19380*x**3 + 77520*x**4 + + 232560*x**5 + 542640*x**6 + 1007760*x**7 + 1511640*x**8 + 1847560*x**9 + + 1847560*x**10 + 1511640*x**11 + 1007760*x**12 + 542640*x**13 + + 232560*x**14 + 77520*x**15 + 19380*x**16 + 3420*x**17 + 380*x**18 + + 20*x**19) + assert factor(dep) == 20*(1 + x)**19 + + +def test_H15(): + assert simplify(Mul(*[x - r for r in solveset(x**3 + x**2 - 7)])) == x**3 + x**2 - 7 + + +def test_H16(): + assert factor(x**100 - 1) == ((x - 1)*(x + 1)*(x**2 + 1)*(x**4 - x**3 + + x**2 - x + 1)*(x**4 + x**3 + x**2 + x + 1)*(x**8 - x**6 + x**4 + - x**2 + 1)*(x**20 - x**15 + x**10 - x**5 + 1)*(x**20 + x**15 + x**10 + + x**5 + 1)*(x**40 - x**30 + x**20 - x**10 + 1)) + + +def test_H17(): + assert simplify(factor(expand(p1 * p2)) - p1*p2) == 0 + + +@XFAIL +def test_H18(): + # Factor over complex rationals. + test = factor(4*x**4 + 8*x**3 + 77*x**2 + 18*x + 153) + good = (2*x + 3*I)*(2*x - 3*I)*(x + 1 - 4*I)*(x + 1 + 4*I) + assert test == good + + +def test_H19(): + a = symbols('a') + # The idea is to let a**2 == 2, then solve 1/(a-1). Answer is a+1") + assert Poly(a - 1).invert(Poly(a**2 - 2)) == a + 1 + + +@XFAIL +def test_H20(): + raise NotImplementedError("let a**2==2; (x**3 + (a-2)*x**2 - " + + "(2*a+3)*x - 3*a) / (x**2-2) = (x**2 - 2*x - 3) / (x-a)") + + +@XFAIL +def test_H21(): + raise NotImplementedError("evaluate (b+c)**4 assuming b**3==2, c**2==3. \ + Answer is 2*b + 8*c + 18*b**2 + 12*b*c + 9") + + +def test_H22(): + assert factor(x**4 - 3*x**2 + 1, modulus=5) == (x - 2)**2 * (x + 2)**2 + + +def test_H23(): + f = x**11 + x + 1 + g = (x**2 + x + 1) * (x**9 - x**8 + x**6 - x**5 + x**3 - x**2 + 1) + assert factor(f, modulus=65537) == g + + +def test_H24(): + phi = AlgebraicNumber(S.GoldenRatio.expand(func=True), alias='phi') + assert factor(x**4 - 3*x**2 + 1, extension=phi) == \ + (x - phi)*(x + 1 - phi)*(x - 1 + phi)*(x + phi) + + +def test_H25(): + e = (x - 2*y**2 + 3*z**3) ** 20 + assert factor(expand(e)) == e + + +def test_H26(): + g = expand((sin(x) - 2*cos(y)**2 + 3*tan(z)**3)**20) + assert factor(g, expand=False) == (-sin(x) + 2*cos(y)**2 - 3*tan(z)**3)**20 + + +def test_H27(): + f = 24*x*y**19*z**8 - 47*x**17*y**5*z**8 + 6*x**15*y**9*z**2 - 3*x**22 + 5 + g = 34*x**5*y**8*z**13 + 20*x**7*y**7*z**7 + 12*x**9*y**16*z**4 + 80*y**14*z + h = -2*z*y**7 \ + *(6*x**9*y**9*z**3 + 10*x**7*z**6 + 17*y*x**5*z**12 + 40*y**7) \ + *(3*x**22 + 47*x**17*y**5*z**8 - 6*x**15*y**9*z**2 - 24*x*y**19*z**8 - 5) + assert factor(expand(f*g)) == h + + +@XFAIL +def test_H28(): + raise NotImplementedError("expand ((1 - c**2)**5 * (1 - s**2)**5 * " + + "(c**2 + s**2)**10) with c**2 + s**2 = 1. Answer is c**10*s**10.") + + +@XFAIL +def test_H29(): + assert factor(4*x**2 - 21*x*y + 20*y**2, modulus=3) == (x + y)*(x - y) + + +def test_H30(): + test = factor(x**3 + y**3, extension=sqrt(-3)) + answer = (x + y)*(x + y*(-R(1, 2) - sqrt(3)/2*I))*(x + y*(-R(1, 2) + sqrt(3)/2*I)) + assert answer == test + + +def test_H31(): + f = (x**2 + 2*x + 3)/(x**3 + 4*x**2 + 5*x + 2) + g = 2 / (x + 1)**2 - 2 / (x + 1) + 3 / (x + 2) + assert apart(f) == g + + +@XFAIL +def test_H32(): # issue 6558 + raise NotImplementedError("[A*B*C - (A*B*C)**(-1)]*A*C*B (product \ + of a non-commuting product and its inverse)") + + +def test_H33(): + A, B, C = symbols('A, B, C', commutative=False) + assert (Commutator(A, Commutator(B, C)) + + Commutator(B, Commutator(C, A)) + + Commutator(C, Commutator(A, B))).doit().expand() == 0 + + +# I. Trigonometry + +def test_I1(): + assert tan(pi*R(7, 10)) == -sqrt(1 + 2/sqrt(5)) + + +@XFAIL +def test_I2(): + assert sqrt((1 + cos(6))/2) == -cos(3) + + +def test_I3(): + assert cos(n*pi) + sin((4*n - 1)*pi/2) == (-1)**n - 1 + + +def test_I4(): + assert refine(cos(pi*cos(n*pi)) + sin(pi/2*cos(n*pi)), Q.integer(n)) == (-1)**n - 1 + + +@XFAIL +def test_I5(): + assert sin((n**5/5 + n**4/2 + n**3/3 - n/30) * pi) == 0 + + +@XFAIL +def test_I6(): + raise NotImplementedError("assuming -3*pi pi**E) + + +@XFAIL +def test_N2(): + x = symbols('x', real=True) + assert ask(x**4 - x + 1 > 0) is True + assert ask(x**4 - x + 1 > 1) is False + + +@XFAIL +def test_N3(): + x = symbols('x', real=True) + assert ask(And(Lt(-1, x), Lt(x, 1)), abs(x) < 1 ) + +@XFAIL +def test_N4(): + x, y = symbols('x y', real=True) + assert ask(2*x**2 > 2*y**2, (x > y) & (y > 0)) is True + + +@XFAIL +def test_N5(): + x, y, k = symbols('x y k', real=True) + assert ask(k*x**2 > k*y**2, (x > y) & (y > 0) & (k > 0)) is True + + +@slow +@XFAIL +def test_N6(): + x, y, k, n = symbols('x y k n', real=True) + assert ask(k*x**n > k*y**n, (x > y) & (y > 0) & (k > 0) & (n > 0)) is True + + +@XFAIL +def test_N7(): + x, y = symbols('x y', real=True) + assert ask(y > 0, (x > 1) & (y >= x - 1)) is True + + +@XFAIL +@slow +def test_N8(): + x, y, z = symbols('x y z', real=True) + assert ask(Eq(x, y) & Eq(y, z), + (x >= y) & (y >= z) & (z >= x)) + + +def test_N9(): + x = Symbol('x') + assert solveset(abs(x - 1) > 2, domain=S.Reals) == Union(Interval(-oo, -1, False, True), + Interval(3, oo, True)) + + +def test_N10(): + x = Symbol('x') + p = (x - 1)*(x - 2)*(x - 3)*(x - 4)*(x - 5) + assert solveset(expand(p) < 0, domain=S.Reals) == Union(Interval(-oo, 1, True, True), + Interval(2, 3, True, True), + Interval(4, 5, True, True)) + + +def test_N11(): + x = Symbol('x') + assert solveset(6/(x - 3) <= 3, domain=S.Reals) == Union(Interval(-oo, 3, True, True), Interval(5, oo)) + + +def test_N12(): + x = Symbol('x') + assert solveset(sqrt(x) < 2, domain=S.Reals) == Interval(0, 4, False, True) + + +def test_N13(): + x = Symbol('x') + assert solveset(sin(x) < 2, domain=S.Reals) == S.Reals + + +@XFAIL +def test_N14(): + x = Symbol('x') + # Gives 'Union(Interval(Integer(0), Mul(Rational(1, 2), pi), false, true), + # Interval(Mul(Rational(1, 2), pi), Mul(Integer(2), pi), true, false))' + # which is not the correct answer, but the provided also seems wrong. + assert solveset(sin(x) < 1, x, domain=S.Reals) == Union(Interval(-oo, pi/2, True, True), + Interval(pi/2, oo, True, True)) + + +def test_N15(): + r, t = symbols('r t') + # raises NotImplementedError: only univariate inequalities are supported + solveset(abs(2*r*(cos(t) - 1) + 1) <= 1, r, S.Reals) + + +def test_N16(): + r, t = symbols('r t') + solveset((r**2)*((cos(t) - 4)**2)*sin(t)**2 < 9, r, S.Reals) + + +@XFAIL +def test_N17(): + # currently only univariate inequalities are supported + assert solveset((x + y > 0, x - y < 0), (x, y)) == (abs(x) < y) + + +def test_O1(): + M = Matrix((1 + I, -2, 3*I)) + assert sqrt(expand(M.dot(M.H))) == sqrt(15) + + +def test_O2(): + assert Matrix((2, 2, -3)).cross(Matrix((1, 3, 1))) == Matrix([[11], + [-5], + [4]]) + +# The vector module has no way of representing vectors symbolically (without +# respect to a basis) +@XFAIL +def test_O3(): + # assert (va ^ vb) | (vc ^ vd) == -(va | vc)*(vb | vd) + (va | vd)*(vb | vc) + raise NotImplementedError("""The vector module has no way of representing + vectors symbolically (without respect to a basis)""") + +def test_O4(): + from sympy.vector import CoordSys3D, Del + N = CoordSys3D("N") + delop = Del() + i, j, k = N.base_vectors() + x, y, z = N.base_scalars() + F = i*(x*y*z) + j*((x*y*z)**2) + k*((y**2)*(z**3)) + assert delop.cross(F).doit() == (-2*x**2*y**2*z + 2*y*z**3)*i + x*y*j + (2*x*y**2*z**2 - x*z)*k + +@XFAIL +def test_O5(): + #assert grad|(f^g)-g|(grad^f)+f|(grad^g) == 0 + raise NotImplementedError("""The vector module has no way of representing + vectors symbolically (without respect to a basis)""") + +#testO8-O9 MISSING!! + + +def test_O10(): + L = [Matrix([2, 3, 5]), Matrix([3, 6, 2]), Matrix([8, 3, 6])] + assert GramSchmidt(L) == [Matrix([ + [2], + [3], + [5]]), + Matrix([ + [R(23, 19)], + [R(63, 19)], + [R(-47, 19)]]), + Matrix([ + [R(1692, 353)], + [R(-1551, 706)], + [R(-423, 706)]])] + + +def test_P1(): + assert Matrix(3, 3, lambda i, j: j - i).diagonal(-1) == Matrix( + 1, 2, [-1, -1]) + + +def test_P2(): + M = Matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + M.row_del(1) + M.col_del(2) + assert M == Matrix([[1, 2], + [7, 8]]) + + +def test_P3(): + A = Matrix([ + [11, 12, 13, 14], + [21, 22, 23, 24], + [31, 32, 33, 34], + [41, 42, 43, 44]]) + + A11 = A[0:3, 1:4] + A12 = A[(0, 1, 3), (2, 0, 3)] + A21 = A + A221 = -A[0:2, 2:4] + A222 = -A[(3, 0), (2, 1)] + A22 = BlockMatrix([[A221, A222]]).T + rows = [[-A11, A12], [A21, A22]] + raises(ValueError, lambda: BlockMatrix(rows)) + B = Matrix(rows) + assert B == Matrix([ + [-12, -13, -14, 13, 11, 14], + [-22, -23, -24, 23, 21, 24], + [-32, -33, -34, 43, 41, 44], + [11, 12, 13, 14, -13, -23], + [21, 22, 23, 24, -14, -24], + [31, 32, 33, 34, -43, -13], + [41, 42, 43, 44, -42, -12]]) + + +@XFAIL +def test_P4(): + raise NotImplementedError("Block matrix diagonalization not supported") + + +def test_P5(): + M = Matrix([[7, 11], + [3, 8]]) + assert M % 2 == Matrix([[1, 1], + [1, 0]]) + + +def test_P6(): + M = Matrix([[cos(x), sin(x)], + [-sin(x), cos(x)]]) + assert M.diff(x, 2) == Matrix([[-cos(x), -sin(x)], + [sin(x), -cos(x)]]) + + +def test_P7(): + M = Matrix([[x, y]])*( + z*Matrix([[1, 3, 5], + [2, 4, 6]]) + Matrix([[7, -9, 11], + [-8, 10, -12]])) + assert M == Matrix([[x*(z + 7) + y*(2*z - 8), x*(3*z - 9) + y*(4*z + 10), + x*(5*z + 11) + y*(6*z - 12)]]) + + +def test_P8(): + M = Matrix([[1, -2*I], + [-3*I, 4]]) + assert M.norm(ord=S.Infinity) == 7 + + +def test_P9(): + a, b, c = symbols('a b c', nonzero=True) + M = Matrix([[a/(b*c), 1/c, 1/b], + [1/c, b/(a*c), 1/a], + [1/b, 1/a, c/(a*b)]]) + assert factor(M.norm('fro')) == (a**2 + b**2 + c**2)/(abs(a)*abs(b)*abs(c)) + + +@XFAIL +def test_P10(): + M = Matrix([[1, 2 + 3*I], + [f(4 - 5*I), 6]]) + # conjugate(f(4 - 5*i)) is not simplified to f(4+5*I) + assert M.H == Matrix([[1, f(4 + 5*I)], + [2 + 3*I, 6]]) + + +@XFAIL +def test_P11(): + # raises NotImplementedError("Matrix([[x,y],[1,x*y]]).inv() + # not simplifying to extract common factor") + assert Matrix([[x, y], + [1, x*y]]).inv() == (1/(x**2 - 1))*Matrix([[x, -1], + [-1/y, x/y]]) + + +def test_P11_workaround(): + # This test was changed to inverse method ADJ because it depended on the + # specific form of inverse returned from the 'GE' method which has changed. + M = Matrix([[x, y], [1, x*y]]).inv('ADJ') + c = gcd(tuple(M)) + assert MatMul(c, M/c, evaluate=False) == MatMul(c, Matrix([ + [x*y, -y], + [ -1, x]]), evaluate=False) + + +def test_P12(): + A11 = MatrixSymbol('A11', n, n) + A12 = MatrixSymbol('A12', n, n) + A22 = MatrixSymbol('A22', n, n) + B = BlockMatrix([[A11, A12], + [ZeroMatrix(n, n), A22]]) + assert block_collapse(B.I) == BlockMatrix([[A11.I, (-1)*A11.I*A12*A22.I], + [ZeroMatrix(n, n), A22.I]]) + + +def test_P13(): + M = Matrix([[1, x - 2, x - 3], + [x - 1, x**2 - 3*x + 6, x**2 - 3*x - 2], + [x - 2, x**2 - 8, 2*(x**2) - 12*x + 14]]) + L, U, _ = M.LUdecomposition() + assert simplify(L) == Matrix([[1, 0, 0], + [x - 1, 1, 0], + [x - 2, x - 3, 1]]) + assert simplify(U) == Matrix([[1, x - 2, x - 3], + [0, 4, x - 5], + [0, 0, x - 7]]) + + +def test_P14(): + M = Matrix([[1, 2, 3, 1, 3], + [3, 2, 1, 1, 7], + [0, 2, 4, 1, 1], + [1, 1, 1, 1, 4]]) + R, _ = M.rref() + assert R == Matrix([[1, 0, -1, 0, 2], + [0, 1, 2, 0, -1], + [0, 0, 0, 1, 3], + [0, 0, 0, 0, 0]]) + + +def test_P15(): + M = Matrix([[-1, 3, 7, -5], + [4, -2, 1, 3], + [2, 4, 15, -7]]) + assert M.rank() == 2 + + +def test_P16(): + M = Matrix([[2*sqrt(2), 8], + [6*sqrt(6), 24*sqrt(3)]]) + assert M.rank() == 1 + + +def test_P17(): + t = symbols('t', real=True) + M=Matrix([ + [sin(2*t), cos(2*t)], + [2*(1 - (cos(t)**2))*cos(t), (1 - 2*(sin(t)**2))*sin(t)]]) + assert M.rank() == 1 + + +def test_P18(): + M = Matrix([[1, 0, -2, 0], + [-2, 1, 0, 3], + [-1, 2, -6, 6]]) + assert M.nullspace() == [Matrix([[2], + [4], + [1], + [0]]), + Matrix([[0], + [-3], + [0], + [1]])] + + +def test_P19(): + w = symbols('w') + M = Matrix([[1, 1, 1, 1], + [w, x, y, z], + [w**2, x**2, y**2, z**2], + [w**3, x**3, y**3, z**3]]) + assert M.det() == (w**3*x**2*y - w**3*x**2*z - w**3*x*y**2 + w**3*x*z**2 + + w**3*y**2*z - w**3*y*z**2 - w**2*x**3*y + w**2*x**3*z + + w**2*x*y**3 - w**2*x*z**3 - w**2*y**3*z + w**2*y*z**3 + + w*x**3*y**2 - w*x**3*z**2 - w*x**2*y**3 + w*x**2*z**3 + + w*y**3*z**2 - w*y**2*z**3 - x**3*y**2*z + x**3*y*z**2 + + x**2*y**3*z - x**2*y*z**3 - x*y**3*z**2 + x*y**2*z**3 + ) + + +@XFAIL +def test_P20(): + raise NotImplementedError("Matrix minimal polynomial not supported") + + +def test_P21(): + M = Matrix([[5, -3, -7], + [-2, 1, 2], + [2, -3, -4]]) + assert M.charpoly(x).as_expr() == x**3 - 2*x**2 - 5*x + 6 + + +def test_P22(): + d = 100 + M = (2 - x)*eye(d) + assert M.eigenvals() == {-x + 2: d} + + +def test_P23(): + M = Matrix([ + [2, 1, 0, 0, 0], + [1, 2, 1, 0, 0], + [0, 1, 2, 1, 0], + [0, 0, 1, 2, 1], + [0, 0, 0, 1, 2]]) + assert M.eigenvals() == { + S('1'): 1, + S('2'): 1, + S('3'): 1, + S('sqrt(3) + 2'): 1, + S('-sqrt(3) + 2'): 1} + + +def test_P24(): + M = Matrix([[611, 196, -192, 407, -8, -52, -49, 29], + [196, 899, 113, -192, -71, -43, -8, -44], + [-192, 113, 899, 196, 61, 49, 8, 52], + [ 407, -192, 196, 611, 8, 44, 59, -23], + [ -8, -71, 61, 8, 411, -599, 208, 208], + [ -52, -43, 49, 44, -599, 411, 208, 208], + [ -49, -8, 8, 59, 208, 208, 99, -911], + [ 29, -44, 52, -23, 208, 208, -911, 99]]) + assert M.eigenvals() == { + S('0'): 1, + S('10*sqrt(10405)'): 1, + S('100*sqrt(26) + 510'): 1, + S('1000'): 2, + S('-100*sqrt(26) + 510'): 1, + S('-10*sqrt(10405)'): 1, + S('1020'): 1} + + +def test_P25(): + MF = N(Matrix([[ 611, 196, -192, 407, -8, -52, -49, 29], + [ 196, 899, 113, -192, -71, -43, -8, -44], + [-192, 113, 899, 196, 61, 49, 8, 52], + [ 407, -192, 196, 611, 8, 44, 59, -23], + [ -8, -71, 61, 8, 411, -599, 208, 208], + [ -52, -43, 49, 44, -599, 411, 208, 208], + [ -49, -8, 8, 59, 208, 208, 99, -911], + [ 29, -44, 52, -23, 208, 208, -911, 99]])) + + ev_1 = sorted(MF.eigenvals(multiple=True)) + ev_2 = sorted( + [-1020.0490184299969, 0.0, 0.09804864072151699, 1000.0, 1000.0, + 1019.9019513592784, 1020.0, 1020.0490184299969]) + + for x, y in zip(ev_1, ev_2): + assert abs(x - y) < 1e-12 + + +def test_P26(): + a0, a1, a2, a3, a4 = symbols('a0 a1 a2 a3 a4') + M = Matrix([[-a4, -a3, -a2, -a1, -a0, 0, 0, 0, 0], + [ 1, 0, 0, 0, 0, 0, 0, 0, 0], + [ 0, 1, 0, 0, 0, 0, 0, 0, 0], + [ 0, 0, 1, 0, 0, 0, 0, 0, 0], + [ 0, 0, 0, 1, 0, 0, 0, 0, 0], + [ 0, 0, 0, 0, 0, -1, -1, 0, 0], + [ 0, 0, 0, 0, 0, 1, 0, 0, 0], + [ 0, 0, 0, 0, 0, 0, 1, -1, -1], + [ 0, 0, 0, 0, 0, 0, 0, 1, 0]]) + assert M.eigenvals(error_when_incomplete=False) == { + S('-1/2 - sqrt(3)*I/2'): 2, + S('-1/2 + sqrt(3)*I/2'): 2} + + +def test_P27(): + a = symbols('a') + M = Matrix([[a, 0, 0, 0, 0], + [0, 0, 0, 0, 1], + [0, 0, a, 0, 0], + [0, 0, 0, a, 0], + [0, -2, 0, 0, 2]]) + + assert M.eigenvects() == [ + (a, 3, [ + Matrix([1, 0, 0, 0, 0]), + Matrix([0, 0, 1, 0, 0]), + Matrix([0, 0, 0, 1, 0]) + ]), + (1 - I, 1, [ + Matrix([0, (1 + I)/2, 0, 0, 1]) + ]), + (1 + I, 1, [ + Matrix([0, (1 - I)/2, 0, 0, 1]) + ]), + ] + + +@XFAIL +def test_P28(): + raise NotImplementedError("Generalized eigenvectors not supported \ +https://github.com/sympy/sympy/issues/5293") + + +@XFAIL +def test_P29(): + raise NotImplementedError("Generalized eigenvectors not supported \ +https://github.com/sympy/sympy/issues/5293") + + +def test_P30(): + M = Matrix([[1, 0, 0, 1, -1], + [0, 1, -2, 3, -3], + [0, 0, -1, 2, -2], + [1, -1, 1, 0, 1], + [1, -1, 1, -1, 2]]) + _, J = M.jordan_form() + assert J == Matrix([[-1, 0, 0, 0, 0], + [0, 1, 1, 0, 0], + [0, 0, 1, 0, 0], + [0, 0, 0, 1, 1], + [0, 0, 0, 0, 1]]) + + +@XFAIL +def test_P31(): + raise NotImplementedError("Smith normal form not implemented") + + +def test_P32(): + M = Matrix([[1, -2], + [2, 1]]) + assert exp(M).rewrite(cos).simplify() == Matrix([[E*cos(2), -E*sin(2)], + [E*sin(2), E*cos(2)]]) + + +def test_P33(): + w, t = symbols('w t') + M = Matrix([[0, 1, 0, 0], + [0, 0, 0, 2*w], + [0, 0, 0, 1], + [0, -2*w, 3*w**2, 0]]) + assert exp(M*t).rewrite(cos).expand() == Matrix([ + [1, -3*t + 4*sin(t*w)/w, 6*t*w - 6*sin(t*w), -2*cos(t*w)/w + 2/w], + [0, 4*cos(t*w) - 3, -6*w*cos(t*w) + 6*w, 2*sin(t*w)], + [0, 2*cos(t*w)/w - 2/w, -3*cos(t*w) + 4, sin(t*w)/w], + [0, -2*sin(t*w), 3*w*sin(t*w), cos(t*w)]]) + + +@XFAIL +def test_P34(): + a, b, c = symbols('a b c', real=True) + M = Matrix([[a, 1, 0, 0, 0, 0], + [0, a, 0, 0, 0, 0], + [0, 0, b, 0, 0, 0], + [0, 0, 0, c, 1, 0], + [0, 0, 0, 0, c, 1], + [0, 0, 0, 0, 0, c]]) + # raises exception, sin(M) not supported. exp(M*I) also not supported + # https://github.com/sympy/sympy/issues/6218 + assert sin(M) == Matrix([[sin(a), cos(a), 0, 0, 0, 0], + [0, sin(a), 0, 0, 0, 0], + [0, 0, sin(b), 0, 0, 0], + [0, 0, 0, sin(c), cos(c), -sin(c)/2], + [0, 0, 0, 0, sin(c), cos(c)], + [0, 0, 0, 0, 0, sin(c)]]) + + +@XFAIL +def test_P35(): + M = pi/2*Matrix([[2, 1, 1], + [2, 3, 2], + [1, 1, 2]]) + # raises exception, sin(M) not supported. exp(M*I) also not supported + # https://github.com/sympy/sympy/issues/6218 + assert sin(M) == eye(3) + + +@XFAIL +def test_P36(): + M = Matrix([[10, 7], + [7, 17]]) + assert sqrt(M) == Matrix([[3, 1], + [1, 4]]) + + +def test_P37(): + M = Matrix([[1, 1, 0], + [0, 1, 0], + [0, 0, 1]]) + assert M**S.Half == Matrix([[1, R(1, 2), 0], + [0, 1, 0], + [0, 0, 1]]) + + +@XFAIL +def test_P38(): + M=Matrix([[0, 1, 0], + [0, 0, 0], + [0, 0, 0]]) + + with raises(AssertionError): + # raises ValueError: Matrix det == 0; not invertible + M**S.Half + # if it doesn't raise then this assertion will be + # raised and the test will be flagged as not XFAILing + assert None + +@XFAIL +def test_P39(): + """ + M=Matrix([ + [1, 1], + [2, 2], + [3, 3]]) + M.SVD() + """ + raise NotImplementedError("Singular value decomposition not implemented") + + +def test_P40(): + r, t = symbols('r t', real=True) + M = Matrix([r*cos(t), r*sin(t)]) + assert M.jacobian(Matrix([r, t])) == Matrix([[cos(t), -r*sin(t)], + [sin(t), r*cos(t)]]) + + +def test_P41(): + r, t = symbols('r t', real=True) + assert hessian(r**2*sin(t),(r,t)) == Matrix([[ 2*sin(t), 2*r*cos(t)], + [2*r*cos(t), -r**2*sin(t)]]) + + +def test_P42(): + assert wronskian([cos(x), sin(x)], x).simplify() == 1 + + +def test_P43(): + def __my_jacobian(M, Y): + return Matrix([M.diff(v).T for v in Y]).T + r, t = symbols('r t', real=True) + M = Matrix([r*cos(t), r*sin(t)]) + assert __my_jacobian(M,[r,t]) == Matrix([[cos(t), -r*sin(t)], + [sin(t), r*cos(t)]]) + + +def test_P44(): + def __my_hessian(f, Y): + V = Matrix([diff(f, v) for v in Y]) + return Matrix([V.T.diff(v) for v in Y]) + r, t = symbols('r t', real=True) + assert __my_hessian(r**2*sin(t), (r, t)) == Matrix([ + [ 2*sin(t), 2*r*cos(t)], + [2*r*cos(t), -r**2*sin(t)]]) + + +def test_P45(): + def __my_wronskian(Y, v): + M = Matrix([Matrix(Y).T.diff(x, n) for n in range(0, len(Y))]) + return M.det() + assert __my_wronskian([cos(x), sin(x)], x).simplify() == 1 + +# Q1-Q6 Tensor tests missing + + +@XFAIL +def test_R1(): + i, j, n = symbols('i j n', integer=True, positive=True) + xn = MatrixSymbol('xn', n, 1) + Sm = Sum((xn[i, 0] - Sum(xn[j, 0], (j, 0, n - 1))/n)**2, (i, 0, n - 1)) + # sum does not calculate + # Unknown result + Sm.doit() + raise NotImplementedError('Unknown result') + +@XFAIL +def test_R2(): + m, b = symbols('m b') + i, n = symbols('i n', integer=True, positive=True) + xn = MatrixSymbol('xn', n, 1) + yn = MatrixSymbol('yn', n, 1) + f = Sum((yn[i, 0] - m*xn[i, 0] - b)**2, (i, 0, n - 1)) + f1 = diff(f, m) + f2 = diff(f, b) + # raises TypeError: solveset() takes at most 2 arguments (3 given) + solveset((f1, f2), (m, b), domain=S.Reals) + + +@XFAIL +def test_R3(): + n, k = symbols('n k', integer=True, positive=True) + sk = ((-1)**k) * (binomial(2*n, k))**2 + Sm = Sum(sk, (k, 1, oo)) + T = Sm.doit() + T2 = T.combsimp() + # returns -((-1)**n*factorial(2*n) + # - (factorial(n))**2)*exp_polar(-I*pi)/(factorial(n))**2 + assert T2 == (-1)**n*binomial(2*n, n) + + +@XFAIL +def test_R4(): +# Macsyma indefinite sum test case: +#(c15) /* Check whether the full Gosper algorithm is implemented +# => 1/2^(n + 1) binomial(n, k - 1) */ +#closedform(indefsum(binomial(n, k)/2^n - binomial(n + 1, k)/2^(n + 1), k)); +#Time= 2690 msecs +# (- n + k - 1) binomial(n + 1, k) +#(d15) - -------------------------------- +# n +# 2 2 (n + 1) +# +#(c16) factcomb(makefact(%)); +#Time= 220 msecs +# n! +#(d16) ---------------- +# n +# 2 k! 2 (n - k)! +# Might be possible after fixing https://github.com/sympy/sympy/pull/1879 + raise NotImplementedError("Indefinite sum not supported") + + +@XFAIL +def test_R5(): + a, b, c, n, k = symbols('a b c n k', integer=True, positive=True) + sk = ((-1)**k)*(binomial(a + b, a + k) + *binomial(b + c, b + k)*binomial(c + a, c + k)) + Sm = Sum(sk, (k, 1, oo)) + T = Sm.doit() # hypergeometric series not calculated + assert T == factorial(a+b+c)/(factorial(a)*factorial(b)*factorial(c)) + + +def test_R6(): + n, k = symbols('n k', integer=True, positive=True) + gn = MatrixSymbol('gn', n + 2, 1) + Sm = Sum(gn[k, 0] - gn[k - 1, 0], (k, 1, n + 1)) + assert Sm.doit() == -gn[0, 0] + gn[n + 1, 0] + + +def test_R7(): + n, k = symbols('n k', integer=True, positive=True) + T = Sum(k**3,(k,1,n)).doit() + assert T.factor() == n**2*(n + 1)**2/4 + +@XFAIL +def test_R8(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(k**2*binomial(n, k), (k, 1, n)) + T = Sm.doit() #returns Piecewise function + assert T.combsimp() == n*(n + 1)*2**(n - 2) + + +def test_R9(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(binomial(n, k - 1)/k, (k, 1, n + 1)) + assert Sm.doit().simplify() == (2**(n + 1) - 1)/(n + 1) + + +@XFAIL +def test_R10(): + n, m, r, k = symbols('n m r k', integer=True, positive=True) + Sm = Sum(binomial(n, k)*binomial(m, r - k), (k, 0, r)) + T = Sm.doit() + T2 = T.combsimp().rewrite(factorial) + assert T2 == factorial(m + n)/(factorial(r)*factorial(m + n - r)) + assert T2 == binomial(m + n, r).rewrite(factorial) + # rewrite(binomial) is not working. + # https://github.com/sympy/sympy/issues/7135 + T3 = T2.rewrite(binomial) + assert T3 == binomial(m + n, r) + + +@XFAIL +def test_R11(): + n, k = symbols('n k', integer=True, positive=True) + sk = binomial(n, k)*fibonacci(k) + Sm = Sum(sk, (k, 0, n)) + T = Sm.doit() + # Fibonacci simplification not implemented + # https://github.com/sympy/sympy/issues/7134 + assert T == fibonacci(2*n) + + +@XFAIL +def test_R12(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(fibonacci(k)**2, (k, 0, n)) + T = Sm.doit() + assert T == fibonacci(n)*fibonacci(n + 1) + + +@XFAIL +def test_R13(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(sin(k*x), (k, 1, n)) + T = Sm.doit() # Sum is not calculated + assert T.simplify() == cot(x/2)/2 - cos(x*(2*n + 1)/2)/(2*sin(x/2)) + + +@XFAIL +def test_R14(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(sin((2*k - 1)*x), (k, 1, n)) + T = Sm.doit() # Sum is not calculated + assert T.simplify() == sin(n*x)**2/sin(x) + + +@XFAIL +def test_R15(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(binomial(n - k, k), (k, 0, floor(n/2))) + T = Sm.doit() # Sum is not calculated + assert T.simplify() == fibonacci(n + 1) + + +def test_R16(): + k = symbols('k', integer=True, positive=True) + Sm = Sum(1/k**2 + 1/k**3, (k, 1, oo)) + assert Sm.doit() == zeta(3) + pi**2/6 + + +def test_R17(): + k = symbols('k', integer=True, positive=True) + assert abs(float(Sum(1/k**2 + 1/k**3, (k, 1, oo))) + - 2.8469909700078206) < 1e-15 + + +def test_R18(): + k = symbols('k', integer=True, positive=True) + Sm = Sum(1/(2**k*k**2), (k, 1, oo)) + T = Sm.doit() + assert T.simplify() == -log(2)**2/2 + pi**2/12 + + +@slow +@XFAIL +def test_R19(): + k = symbols('k', integer=True, positive=True) + Sm = Sum(1/((3*k + 1)*(3*k + 2)*(3*k + 3)), (k, 0, oo)) + T = Sm.doit() + # assert fails, T not simplified + assert T.simplify() == -log(3)/4 + sqrt(3)*pi/12 + + +@XFAIL +def test_R20(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(binomial(n, 4*k), (k, 0, oo)) + T = Sm.doit() + # assert fails, T not simplified + assert T.simplify() == 2**(n/2)*cos(pi*n/4)/2 + 2**(n - 1)/2 + + +@XFAIL +def test_R21(): + k = symbols('k', integer=True, positive=True) + Sm = Sum(1/(sqrt(k*(k + 1)) * (sqrt(k) + sqrt(k + 1))), (k, 1, oo)) + T = Sm.doit() # Sum not calculated + assert T.simplify() == 1 + + +# test_R22 answer not available in Wester samples +# Sum(Sum(binomial(n, k)*binomial(n - k, n - 2*k)*x**n*y**(n - 2*k), +# (k, 0, floor(n/2))), (n, 0, oo)) with abs(x*y)<1? + + +@XFAIL +def test_R23(): + n, k = symbols('n k', integer=True, positive=True) + Sm = Sum(Sum((factorial(n)/(factorial(k)**2*factorial(n - 2*k)))* + (x/y)**k*(x*y)**(n - k), (n, 2*k, oo)), (k, 0, oo)) + # Missing how to express constraint abs(x*y)<1? + T = Sm.doit() # Sum not calculated + assert T == -1/sqrt(x**2*y**2 - 4*x**2 - 2*x*y + 1) + + +def test_R24(): + m, k = symbols('m k', integer=True, positive=True) + Sm = Sum(Product(k/(2*k - 1), (k, 1, m)), (m, 2, oo)) + assert Sm.doit() == pi/2 + + +def test_S1(): + k = symbols('k', integer=True, positive=True) + Pr = Product(gamma(k/3), (k, 1, 8)) + assert Pr.doit().simplify() == 640*sqrt(3)*pi**3/6561 + + +def test_S2(): + n, k = symbols('n k', integer=True, positive=True) + assert Product(k, (k, 1, n)).doit() == factorial(n) + + +def test_S3(): + n, k = symbols('n k', integer=True, positive=True) + assert Product(x**k, (k, 1, n)).doit().simplify() == x**(n*(n + 1)/2) + + +def test_S4(): + n, k = symbols('n k', integer=True, positive=True) + assert Product(1 + 1/k, (k, 1, n -1)).doit().simplify() == n + + +def test_S5(): + n, k = symbols('n k', integer=True, positive=True) + assert (Product((2*k - 1)/(2*k), (k, 1, n)).doit().gammasimp() == + gamma(n + S.Half)/(sqrt(pi)*gamma(n + 1))) + + +@XFAIL +def test_S6(): + n, k = symbols('n k', integer=True, positive=True) + # Product does not evaluate + assert (Product(x**2 -2*x*cos(k*pi/n) + 1, (k, 1, n - 1)).doit().simplify() + == (x**(2*n) - 1)/(x**2 - 1)) + + +@XFAIL +def test_S7(): + k = symbols('k', integer=True, positive=True) + Pr = Product((k**3 - 1)/(k**3 + 1), (k, 2, oo)) + T = Pr.doit() # Product does not evaluate + assert T.simplify() == R(2, 3) + + +@XFAIL +def test_S8(): + k = symbols('k', integer=True, positive=True) + Pr = Product(1 - 1/(2*k)**2, (k, 1, oo)) + T = Pr.doit() + # Product does not evaluate + assert T.simplify() == 2/pi + + +@XFAIL +def test_S9(): + k = symbols('k', integer=True, positive=True) + Pr = Product(1 + (-1)**(k + 1)/(2*k - 1), (k, 1, oo)) + T = Pr.doit() + # Product produces 0 + # https://github.com/sympy/sympy/issues/7133 + assert T.simplify() == sqrt(2) + + +@XFAIL +def test_S10(): + k = symbols('k', integer=True, positive=True) + Pr = Product((k*(k + 1) + 1 + I)/(k*(k + 1) + 1 - I), (k, 0, oo)) + T = Pr.doit() + # Product does not evaluate + assert T.simplify() == -1 + + +def test_T1(): + assert limit((1 + 1/n)**n, n, oo) == E + assert limit((1 - cos(x))/x**2, x, 0) == S.Half + + +def test_T2(): + assert limit((3**x + 5**x)**(1/x), x, oo) == 5 + + +def test_T3(): + assert limit(log(x)/(log(x) + sin(x)), x, oo) == 1 + + +def test_T4(): + assert limit((exp(x*exp(-x)/(exp(-x) + exp(-2*x**2/(x + 1)))) + - exp(x))/x, x, oo) == -exp(2) + + +def test_T5(): + assert limit(x*log(x)*log(x*exp(x) - x**2)**2/log(log(x**2 + + 2*exp(exp(3*x**3*log(x))))), x, oo) == R(1, 3) + + +def test_T6(): + assert limit(1/n * factorial(n)**(1/n), n, oo) == exp(-1) + + +def test_T7(): + limit(1/n * gamma(n + 1)**(1/n), n, oo) + + +def test_T8(): + a, z = symbols('a z', positive=True) + assert limit(gamma(z + a)/gamma(z)*exp(-a*log(z)), z, oo) == 1 + + +@XFAIL +def test_T9(): + z, k = symbols('z k', positive=True) + # raises NotImplementedError: + # Don't know how to calculate the mrv of '(1, k)' + assert limit(hyper((1, k), (1,), z/k), k, oo) == exp(z) + + +@XFAIL +def test_T10(): + # No longer raises PoleError, but should return euler-mascheroni constant + assert limit(zeta(x) - 1/(x - 1), x, 1) == integrate(-1/x + 1/floor(x), (x, 1, oo)) + +@XFAIL +def test_T11(): + n, k = symbols('n k', integer=True, positive=True) + # evaluates to 0 + assert limit(n**x/(x*product((1 + x/k), (k, 1, n))), n, oo) == gamma(x) + + +def test_T12(): + x, t = symbols('x t', real=True) + # Does not evaluate the limit but returns an expression with erf + assert limit(x * integrate(exp(-t**2), (t, 0, x))/(1 - exp(-x**2)), + x, 0) == 1 + + +def test_T13(): + x = symbols('x', real=True) + assert [limit(x/abs(x), x, 0, dir='-'), + limit(x/abs(x), x, 0, dir='+')] == [-1, 1] + + +def test_T14(): + x = symbols('x', real=True) + assert limit(atan(-log(x)), x, 0, dir='+') == pi/2 + + +def test_U1(): + x = symbols('x', real=True) + assert diff(abs(x), x) == sign(x) + + +def test_U2(): + f = Lambda(x, Piecewise((-x, x < 0), (x, x >= 0))) + assert diff(f(x), x) == Piecewise((-1, x < 0), (1, x >= 0)) + + +def test_U3(): + f = Lambda(x, Piecewise((x**2 - 1, x == 1), (x**3, x != 1))) + f1 = Lambda(x, diff(f(x), x)) + assert f1(x) == 3*x**2 + assert f1(1) == 3 + + +@XFAIL +def test_U4(): + n = symbols('n', integer=True, positive=True) + x = symbols('x', real=True) + d = diff(x**n, x, n) + assert d.rewrite(factorial) == factorial(n) + + +def test_U5(): + # issue 6681 + t = symbols('t') + ans = ( + Derivative(f(g(t)), g(t))*Derivative(g(t), (t, 2)) + + Derivative(f(g(t)), (g(t), 2))*Derivative(g(t), t)**2) + assert f(g(t)).diff(t, 2) == ans + assert ans.doit() == ans + + +def test_U6(): + h = Function('h') + T = integrate(f(y), (y, h(x), g(x))) + assert T.diff(x) == ( + f(g(x))*Derivative(g(x), x) - f(h(x))*Derivative(h(x), x)) + + +@XFAIL +def test_U7(): + p, t = symbols('p t', real=True) + # Exact differential => d(V(P, T)) => dV/dP DP + dV/dT DT + # raises ValueError: Since there is more than one variable in the + # expression, the variable(s) of differentiation must be supplied to + # differentiate f(p,t) + diff(f(p, t)) + + +def test_U8(): + x, y = symbols('x y', real=True) + eq = cos(x*y) + x + # If SymPy had implicit_diff() function this hack could be avoided + # TODO: Replace solve with solveset, current test fails for solveset + assert idiff(y - eq, y, x) == (-y*sin(x*y) + 1)/(x*sin(x*y) + 1) + + +def test_U9(): + # Wester sample case for Maple: + # O29 := diff(f(x, y), x) + diff(f(x, y), y); + # /d \ /d \ + # |-- f(x, y)| + |-- f(x, y)| + # \dx / \dy / + # + # O30 := factor(subs(f(x, y) = g(x^2 + y^2), %)); + # 2 2 + # 2 D(g)(x + y ) (x + y) + x, y = symbols('x y', real=True) + su = diff(f(x, y), x) + diff(f(x, y), y) + s2 = su.subs(f(x, y), g(x**2 + y**2)) + s3 = s2.doit().factor() + # Subs not performed, s3 = 2*(x + y)*Subs(Derivative( + # g(_xi_1), _xi_1), _xi_1, x**2 + y**2) + # Derivative(g(x*2 + y**2), x**2 + y**2) is not valid in SymPy, + # and probably will remain that way. You can take derivatives with respect + # to other expressions only if they are atomic, like a symbol or a + # function. + # D operator should be added to SymPy + # See https://github.com/sympy/sympy/issues/4719. + assert s3 == (x + y)*Subs(Derivative(g(x), x), x, x**2 + y**2)*2 + + +def test_U10(): + # see issue 2519: + assert residue((z**3 + 5)/((z**4 - 1)*(z + 1)), z, -1) == R(-9, 4) + +@XFAIL +def test_U11(): + # assert (2*dx + dz) ^ (3*dx + dy + dz) ^ (dx + dy + 4*dz) == 8*dx ^ dy ^dz + raise NotImplementedError + + +@XFAIL +def test_U12(): + # Wester sample case: + # (c41) /* d(3 x^5 dy /\ dz + 5 x y^2 dz /\ dx + 8 z dx /\ dy) + # => (15 x^4 + 10 x y + 8) dx /\ dy /\ dz */ + # factor(ext_diff(3*x^5 * dy ~ dz + 5*x*y^2 * dz ~ dx + 8*z * dx ~ dy)); + # 4 + # (d41) (10 x y + 15 x + 8) dx dy dz + raise NotImplementedError( + "External diff of differential form not supported") + + +def test_U13(): + assert minimum(x**4 - x + 1, x) == -3*2**R(1,3)/8 + 1 + + +@XFAIL +def test_U14(): + #f = 1/(x**2 + y**2 + 1) + #assert [minimize(f), maximize(f)] == [0,1] + raise NotImplementedError("minimize(), maximize() not supported") + + +@XFAIL +def test_U15(): + raise NotImplementedError("minimize() not supported and also solve does \ +not support multivariate inequalities") + + +@XFAIL +def test_U16(): + raise NotImplementedError("minimize() not supported in SymPy and also \ +solve does not support multivariate inequalities") + + +@XFAIL +def test_U17(): + raise NotImplementedError("Linear programming, symbolic simplex not \ +supported in SymPy") + + +def test_V1(): + x = symbols('x', real=True) + assert integrate(abs(x), x) == Piecewise((-x**2/2, x <= 0), (x**2/2, True)) + + +def test_V2(): + assert integrate(Piecewise((-x, x < 0), (x, x >= 0)), x + ) == Piecewise((-x**2/2, x < 0), (x**2/2, True)) + + +def test_V3(): + assert integrate(1/(x**3 + 2),x).diff().simplify() == 1/(x**3 + 2) + + +def test_V4(): + assert integrate(2**x/sqrt(1 + 4**x), x) == asinh(2**x)/log(2) + + +@XFAIL +def test_V5(): + # Returns (-45*x**2 + 80*x - 41)/(5*sqrt(2*x - 1)*(4*x**2 - 4*x + 1)) + assert (integrate((3*x - 5)**2/(2*x - 1)**R(7, 2), x).simplify() == + (-41 + 80*x - 45*x**2)/(5*(2*x - 1)**R(5, 2))) + + +@XFAIL +def test_V6(): + # returns RootSum(40*_z**2 - 1, Lambda(_i, _i*log(-4*_i + exp(-m*x))))/m + assert (integrate(1/(2*exp(m*x) - 5*exp(-m*x)), x) == sqrt(10)*( + log(2*exp(m*x) - sqrt(10)) - log(2*exp(m*x) + sqrt(10)))/(20*m)) + + +def test_V7(): + r1 = integrate(sinh(x)**4/cosh(x)**2) + assert r1.simplify() == x*R(-3, 2) + sinh(x)**3/(2*cosh(x)) + 3*tanh(x)/2 + + +@XFAIL +def test_V8_V9(): +#Macsyma test case: +#(c27) /* This example involves several symbolic parameters +# => 1/sqrt(b^2 - a^2) log([sqrt(b^2 - a^2) tan(x/2) + a + b]/ +# [sqrt(b^2 - a^2) tan(x/2) - a - b]) (a^2 < b^2) +# [Gradshteyn and Ryzhik 2.553(3)] */ +#assume(b^2 > a^2)$ +#(c28) integrate(1/(a + b*cos(x)), x); +#(c29) trigsimp(ratsimp(diff(%, x))); +# 1 +#(d29) ------------ +# b cos(x) + a + raise NotImplementedError( + "Integrate with assumption not supported") + + +def test_V10(): + assert integrate(1/(3 + 3*cos(x) + 4*sin(x)), x) == log(4*tan(x/2) + 3)/4 + + +def test_V11(): + r1 = integrate(1/(4 + 3*cos(x) + 4*sin(x)), x) + r2 = factor(r1) + assert (logcombine(r2, force=True) == + log(((tan(x/2) + 1)/(tan(x/2) + 7))**R(1, 3))) + + +def test_V12(): + r1 = integrate(1/(5 + 3*cos(x) + 4*sin(x)), x) + assert r1 == -1/(tan(x/2) + 2) + + +@XFAIL +def test_V13(): + r1 = integrate(1/(6 + 3*cos(x) + 4*sin(x)), x) + # expression not simplified, returns: -sqrt(11)*I*log(tan(x/2) + 4/3 + # - sqrt(11)*I/3)/11 + sqrt(11)*I*log(tan(x/2) + 4/3 + sqrt(11)*I/3)/11 + assert r1.simplify() == 2*sqrt(11)*atan(sqrt(11)*(3*tan(x/2) + 4)/11)/11 + + +@slow +@XFAIL +def test_V14(): + r1 = integrate(log(abs(x**2 - y**2)), x) + # Piecewise result does not simplify to the desired result. + assert (r1.simplify() == x*log(abs(x**2 - y**2)) + + y*log(x + y) - y*log(x - y) - 2*x) + + +def test_V15(): + r1 = integrate(x*acot(x/y), x) + assert simplify(r1 - (x*y + (x**2 + y**2)*acot(x/y))/2) == 0 + + +@XFAIL +def test_V16(): + # Integral not calculated + assert integrate(cos(5*x)*Ci(2*x), x) == Ci(2*x)*sin(5*x)/5 - (Si(3*x) + Si(7*x))/10 + +@XFAIL +def test_V17(): + r1 = integrate((diff(f(x), x)*g(x) + - f(x)*diff(g(x), x))/(f(x)**2 - g(x)**2), x) + # integral not calculated + assert simplify(r1 - (f(x) - g(x))/(f(x) + g(x))/2) == 0 + + +@XFAIL +def test_W1(): + # The function has a pole at y. + # The integral has a Cauchy principal value of zero but SymPy returns -I*pi + # https://github.com/sympy/sympy/issues/7159 + assert integrate(1/(x - y), (x, y - 1, y + 1)) == 0 + + +@XFAIL +def test_W2(): + # The function has a pole at y. + # The integral is divergent but SymPy returns -2 + # https://github.com/sympy/sympy/issues/7160 + # Test case in Macsyma: + # (c6) errcatch(integrate(1/(x - a)^2, x, a - 1, a + 1)); + # Integral is divergent + assert integrate(1/(x - y)**2, (x, y - 1, y + 1)) is zoo + + +@XFAIL +@slow +def test_W3(): + # integral is not calculated + # https://github.com/sympy/sympy/issues/7161 + assert integrate(sqrt(x + 1/x - 2), (x, 0, 1)) == R(4, 3) + + +@XFAIL +@slow +def test_W4(): + # integral is not calculated + assert integrate(sqrt(x + 1/x - 2), (x, 1, 2)) == -2*sqrt(2)/3 + R(4, 3) + + +@XFAIL +@slow +def test_W5(): + # integral is not calculated + assert integrate(sqrt(x + 1/x - 2), (x, 0, 2)) == -2*sqrt(2)/3 + R(8, 3) + + +@XFAIL +@slow +def test_W6(): + # integral is not calculated + assert integrate(sqrt(2 - 2*cos(2*x))/2, (x, pi*R(-3, 4), -pi/4)) == sqrt(2) + + +def test_W7(): + a = symbols('a', positive=True) + r1 = integrate(cos(x)/(x**2 + a**2), (x, -oo, oo)) + assert r1.simplify() == pi*exp(-a)/a + + +@XFAIL +def test_W8(): + # Test case in Mathematica: + # In[19]:= Integrate[t^(a - 1)/(1 + t), {t, 0, Infinity}, + # Assumptions -> 0 < a < 1] + # Out[19]= Pi Csc[a Pi] + raise NotImplementedError( + "Integrate with assumption 0 < a < 1 not supported") + + +@XFAIL +@slow +def test_W9(): + # Integrand with a residue at infinity => -2 pi [sin(pi/5) + sin(2pi/5)] + # (principal value) [Levinson and Redheffer, p. 234] *) + r1 = integrate(5*x**3/(1 + x + x**2 + x**3 + x**4), (x, -oo, oo)) + r2 = r1.doit() + assert r2 == -2*pi*(sqrt(-sqrt(5)/8 + 5/8) + sqrt(sqrt(5)/8 + 5/8)) + + +@XFAIL +def test_W10(): + # integrate(1/[1 + x + x^2 + ... + x^(2 n)], x = -infinity..infinity) = + # 2 pi/(2 n + 1) [1 + cos(pi/[2 n + 1])] csc(2 pi/[2 n + 1]) + # [Levinson and Redheffer, p. 255] => 2 pi/5 [1 + cos(pi/5)] csc(2 pi/5) */ + r1 = integrate(x/(1 + x + x**2 + x**4), (x, -oo, oo)) + r2 = r1.doit() + assert r2 == 2*pi*(sqrt(5)/4 + 5/4)*csc(pi*R(2, 5))/5 + + +@XFAIL +def test_W11(): + # integral not calculated + assert (integrate(sqrt(1 - x**2)/(1 + x**2), (x, -1, 1)) == + pi*(-1 + sqrt(2))) + + +def test_W12(): + p = symbols('p', positive=True) + q = symbols('q', real=True) + r1 = integrate(x*exp(-p*x**2 + 2*q*x), (x, -oo, oo)) + assert r1.simplify() == sqrt(pi)*q*exp(q**2/p)/p**R(3, 2) + + +@XFAIL +def test_W13(): + # Integral not calculated. Expected result is 2*(Euler_mascheroni_constant) + r1 = integrate(1/log(x) + 1/(1 - x) - log(log(1/x)), (x, 0, 1)) + assert r1 == 2*EulerGamma + + +def test_W14(): + assert integrate(sin(x)/x*exp(2*I*x), (x, -oo, oo)) == 0 + + +@XFAIL +def test_W15(): + # integral not calculated + assert integrate(log(gamma(x))*cos(6*pi*x), (x, 0, 1)) == R(1, 12) + + +def test_W16(): + assert integrate((1 + x)**3*legendre_poly(1, x)*legendre_poly(2, x), + (x, -1, 1)) == R(36, 35) + + +def test_W17(): + a, b = symbols('a b', positive=True) + assert integrate(exp(-a*x)*besselj(0, b*x), + (x, 0, oo)) == 1/(b*sqrt(a**2/b**2 + 1)) + + +def test_W18(): + assert integrate((besselj(1, x)/x)**2, (x, 0, oo)) == 4/(3*pi) + + +@XFAIL +def test_W19(): + # Integral not calculated + # Expected result is (cos 7 - 1)/7 [Gradshteyn and Ryzhik 6.782(3)] + assert integrate(Ci(x)*besselj(0, 2*sqrt(7*x)), (x, 0, oo)) == (cos(7) - 1)/7 + + +@XFAIL +def test_W20(): + # integral not calculated + assert (integrate(x**2*polylog(3, 1/(x + 1)), (x, 0, 1)) == + -pi**2/36 - R(17, 108) + zeta(3)/4 + + (-pi**2/2 - 4*log(2) + log(2)**2 + 35/3)*log(2)/9) + + +def test_W21(): + assert abs(N(integrate(x**2*polylog(3, 1/(x + 1)), (x, 0, 1))) + - 0.210882859565594) < 1e-15 + + +def test_W22(): + t, u = symbols('t u', real=True) + s = Lambda(x, Piecewise((1, And(x >= 1, x <= 2)), (0, True))) + assert integrate(s(t)*cos(t), (t, 0, u)) == Piecewise( + (0, u < 0), + (-sin(Min(1, u)) + sin(Min(2, u)), True)) + + +@slow +def test_W23(): + a, b = symbols('a b', positive=True) + r1 = integrate(integrate(x/(x**2 + y**2), (x, a, b)), (y, -oo, oo)) + assert r1.collect(pi).cancel() == -pi*a + pi*b + + +def test_W23b(): + # like W23 but limits are reversed + a, b = symbols('a b', positive=True) + r2 = integrate(integrate(x/(x**2 + y**2), (y, -oo, oo)), (x, a, b)) + assert r2.collect(pi) == pi*(-a + b) + + +@XFAIL +@tooslow +def test_W24(): + # Not that slow, but does not fully evaluate so simplify is slow. + # Maybe also require doit() + x, y = symbols('x y', real=True) + r1 = integrate(integrate(sqrt(x**2 + y**2), (x, 0, 1)), (y, 0, 1)) + assert (r1 - (sqrt(2) + asinh(1))/3).simplify() == 0 + + +@XFAIL +@tooslow +def test_W25(): + a, x, y = symbols('a x y', real=True) + i1 = integrate( + sin(a)*sin(y)/sqrt(1 - sin(a)**2*sin(x)**2*sin(y)**2), + (x, 0, pi/2)) + i2 = integrate(i1, (y, 0, pi/2)) + assert (i2 - pi*a/2).simplify() == 0 + + +def test_W26(): + x, y = symbols('x y', real=True) + assert integrate(integrate(abs(y - x**2), (y, 0, 2)), + (x, -1, 1)) == R(46, 15) + + +def test_W27(): + a, b, c = symbols('a b c') + assert integrate(integrate(integrate(1, (z, 0, c*(1 - x/a - y/b))), + (y, 0, b*(1 - x/a))), + (x, 0, a)) == a*b*c/6 + + +def test_X1(): + v, c = symbols('v c', real=True) + assert (series(1/sqrt(1 - (v/c)**2), v, x0=0, n=8) == + 5*v**6/(16*c**6) + 3*v**4/(8*c**4) + v**2/(2*c**2) + 1 + O(v**8)) + + +def test_X2(): + v, c = symbols('v c', real=True) + s1 = series(1/sqrt(1 - (v/c)**2), v, x0=0, n=8) + assert (1/s1**2).series(v, x0=0, n=8) == -v**2/c**2 + 1 + O(v**8) + + +def test_X3(): + s1 = (sin(x).series()/cos(x).series()).series() + s2 = tan(x).series() + assert s2 == x + x**3/3 + 2*x**5/15 + O(x**6) + assert s1 == s2 + + +def test_X4(): + s1 = log(sin(x)/x).series() + assert s1 == -x**2/6 - x**4/180 + O(x**6) + assert log(series(sin(x)/x)).series() == s1 + + +@XFAIL +def test_X5(): + # test case in Mathematica syntax: + # In[21]:= (* => [a f'(a d) + g(b d) + integrate(h(c y), y = 0..d)] + # + [a^2 f''(a d) + b g'(b d) + h(c d)] (x - d) *) + # In[22]:= D[f[a*x], x] + g[b*x] + Integrate[h[c*y], {y, 0, x}] + # Out[22]= g[b x] + Integrate[h[c y], {y, 0, x}] + a f'[a x] + # In[23]:= Series[%, {x, d, 1}] + # Out[23]= (g[b d] + Integrate[h[c y], {y, 0, d}] + a f'[a d]) + + # 2 2 + # (h[c d] + b g'[b d] + a f''[a d]) (-d + x) + O[-d + x] + h = Function('h') + a, b, c, d = symbols('a b c d', real=True) + # series() raises NotImplementedError: + # The _eval_nseries method should be added to to give terms up to O(x**n) at x=0 + series(diff(f(a*x), x) + g(b*x) + integrate(h(c*y), (y, 0, x)), + x, x0=d, n=2) + # assert missing, until exception is removed + + +def test_X6(): + # Taylor series of nonscalar objects (noncommutative multiplication) + # expected result => (B A - A B) t^2/2 + O(t^3) [Stanly Steinberg] + a, b = symbols('a b', commutative=False, scalar=False) + assert (series(exp((a + b)*x) - exp(a*x) * exp(b*x), x, x0=0, n=3) == + x**2*(-a*b/2 + b*a/2) + O(x**3)) + + +def test_X7(): + # => sum( Bernoulli[k]/k! x^(k - 2), k = 1..infinity ) + # = 1/x^2 - 1/(2 x) + 1/12 - x^2/720 + x^4/30240 + O(x^6) + # [Levinson and Redheffer, p. 173] + assert (series(1/(x*(exp(x) - 1)), x, 0, 7) == x**(-2) - 1/(2*x) + + R(1, 12) - x**2/720 + x**4/30240 - x**6/1209600 + O(x**7)) + + +def test_X8(): + # Puiseux series (terms with fractional degree): + # => 1/sqrt(x - 3/2 pi) + (x - 3/2 pi)^(3/2) / 12 + O([x - 3/2 pi]^(7/2)) + + # see issue 7167: + x = symbols('x', real=True) + assert (series(sqrt(sec(x)), x, x0=pi*3/2, n=4) == + 1/sqrt(x - pi*R(3, 2)) + (x - pi*R(3, 2))**R(3, 2)/12 + + (x - pi*R(3, 2))**R(7, 2)/160 + O((x - pi*R(3, 2))**4, (x, pi*R(3, 2)))) + + +def test_X9(): + assert (series(x**x, x, x0=0, n=4) == 1 + x*log(x) + x**2*log(x)**2/2 + + x**3*log(x)**3/6 + O(x**4*log(x)**4)) + + +def test_X10(): + z, w = symbols('z w') + assert (series(log(sinh(z)) + log(cosh(z + w)), z, x0=0, n=2) == + log(cosh(w)) + log(z) + z*sinh(w)/cosh(w) + O(z**2)) + + +def test_X11(): + z, w = symbols('z w') + assert (series(log(sinh(z) * cosh(z + w)), z, x0=0, n=2) == + log(cosh(w)) + log(z) + z*sinh(w)/cosh(w) + O(z**2)) + + +@XFAIL +def test_X12(): + # Look at the generalized Taylor series around x = 1 + # Result => (x - 1)^a/e^b [1 - (a + 2 b) (x - 1) / 2 + O((x - 1)^2)] + a, b, x = symbols('a b x', real=True) + # series returns O(log(x-1)**2) + # https://github.com/sympy/sympy/issues/7168 + assert (series(log(x)**a*exp(-b*x), x, x0=1, n=2) == + (x - 1)**a/exp(b)*(1 - (a + 2*b)*(x - 1)/2 + O((x - 1)**2))) + + +def test_X13(): + assert series(sqrt(2*x**2 + 1), x, x0=oo, n=1) == sqrt(2)*x + O(1/x, (x, oo)) + + +@XFAIL +def test_X14(): + # Wallis' product => 1/sqrt(pi n) + ... [Knopp, p. 385] + assert series(1/2**(2*n)*binomial(2*n, n), + n, x==oo, n=1) == 1/(sqrt(pi)*sqrt(n)) + O(1/x, (x, oo)) + + +@SKIP("https://github.com/sympy/sympy/issues/7164") +def test_X15(): + # => 0!/x - 1!/x^2 + 2!/x^3 - 3!/x^4 + O(1/x^5) [Knopp, p. 544] + x, t = symbols('x t', real=True) + # raises RuntimeError: maximum recursion depth exceeded + # https://github.com/sympy/sympy/issues/7164 + # 2019-02-17: Raises + # PoleError: + # Asymptotic expansion of Ei around [-oo] is not implemented. + e1 = integrate(exp(-t)/t, (t, x, oo)) + assert (series(e1, x, x0=oo, n=5) == + 6/x**4 + 2/x**3 - 1/x**2 + 1/x + O(x**(-5), (x, oo))) + + +def test_X16(): + # Multivariate Taylor series expansion => 1 - (x^2 + 2 x y + y^2)/2 + O(x^4) + assert (series(cos(x + y), x + y, x0=0, n=4) == 1 - (x + y)**2/2 + + O(x**4 + x**3*y + x**2*y**2 + x*y**3 + y**4, x, y)) + + +@XFAIL +def test_X17(): + # Power series (compute the general formula) + # (c41) powerseries(log(sin(x)/x), x, 0); + # /aquarius/data2/opt/local/macsyma_422/library1/trgred.so being loaded. + # inf + # ==== i1 2 i1 2 i1 + # \ (- 1) 2 bern(2 i1) x + # (d41) > ------------------------------ + # / 2 i1 (2 i1)! + # ==== + # i1 = 1 + # fps does not calculate + assert fps(log(sin(x)/x)) == \ + Sum((-1)**k*2**(2*k - 1)*bernoulli(2*k)*x**(2*k)/(k*factorial(2*k)), (k, 1, oo)) + + +@XFAIL +def test_X18(): + # Power series (compute the general formula). Maple FPS: + # > FormalPowerSeries(exp(-x)*sin(x), x = 0); + # infinity + # ----- (1/2 k) k + # \ 2 sin(3/4 k Pi) x + # ) ------------------------- + # / k! + # ----- + # + # Now, SymPy returns + # oo + # _____ + # \ ` + # \ / k k\ + # \ k |I*(-1 - I) I*(-1 + I) | + # \ x *|----------- - -----------| + # / \ 2 2 / + # / ------------------------------ + # / k! + # /____, + # k = 0 + k = Dummy('k') + assert fps(exp(-x)*sin(x)) == \ + Sum(2**(S.Half*k)*sin(R(3, 4)*k*pi)*x**k/factorial(k), (k, 0, oo)) + + +@XFAIL +def test_X19(): + # (c45) /* Derive an explicit Taylor series solution of y as a function of + # x from the following implicit relation: + # y = x - 1 + (x - 1)^2/2 + 2/3 (x - 1)^3 + (x - 1)^4 + + # 17/10 (x - 1)^5 + ... + # */ + # x = sin(y) + cos(y); + # Time= 0 msecs + # (d45) x = sin(y) + cos(y) + # + # (c46) taylor_revert(%, y, 7); + raise NotImplementedError("Solve using series not supported. \ +Inverse Taylor series expansion also not supported") + + +@XFAIL +def test_X20(): + # Pade (rational function) approximation => (2 - x)/(2 + x) + # > numapprox[pade](exp(-x), x = 0, [1, 1]); + # bytes used=9019816, alloc=3669344, time=13.12 + # 1 - 1/2 x + # --------- + # 1 + 1/2 x + # mpmath support numeric Pade approximant but there is + # no symbolic implementation in SymPy + # https://en.wikipedia.org/wiki/Pad%C3%A9_approximant + raise NotImplementedError("Symbolic Pade approximant not supported") + + +def test_X21(): + """ + Test whether `fourier_series` of x periodical on the [-p, p] interval equals + `- (2 p / pi) sum( (-1)^n / n sin(n pi x / p), n = 1..infinity )`. + """ + p = symbols('p', positive=True) + n = symbols('n', positive=True, integer=True) + s = fourier_series(x, (x, -p, p)) + + # All cosine coefficients are equal to 0 + assert s.an.formula == 0 + + # Check for sine coefficients + assert s.bn.formula.subs(s.bn.variables[0], 0) == 0 + assert s.bn.formula.subs(s.bn.variables[0], n) == \ + -2*p/pi * (-1)**n / n * sin(n*pi*x/p) + + +@XFAIL +def test_X22(): + # (c52) /* => p / 2 + # - (2 p / pi^2) sum( [1 - (-1)^n] cos(n pi x / p) / n^2, + # n = 1..infinity ) */ + # fourier_series(abs(x), x, p); + # p + # (e52) a = - + # 0 2 + # + # %nn + # (2 (- 1) - 2) p + # (e53) a = ------------------ + # %nn 2 2 + # %pi %nn + # + # (e54) b = 0 + # %nn + # + # Time= 5290 msecs + # inf %nn %pi %nn x + # ==== (2 (- 1) - 2) cos(---------) + # \ p + # p > ------------------------------- + # / 2 + # ==== %nn + # %nn = 1 p + # (d54) ----------------------------------------- + - + # 2 2 + # %pi + raise NotImplementedError("Fourier series not supported") + + +def test_Y1(): + t = symbols('t', positive=True) + w = symbols('w', real=True) + s = symbols('s') + F, _, _ = laplace_transform(cos((w - 1)*t), t, s) + assert F == s/(s**2 + (w - 1)**2) + + +def test_Y2(): + t = symbols('t', positive=True) + w = symbols('w', real=True) + s = symbols('s') + f = inverse_laplace_transform(s/(s**2 + (w - 1)**2), s, t, simplify=True) + assert f == cos(t*(w - 1)) + + +def test_Y3(): + t = symbols('t', positive=True) + w = symbols('w', real=True) + s = symbols('s') + F, _, _ = laplace_transform(sinh(w*t)*cosh(w*t), t, s, simplify=True) + assert F == w/(s**2 - 4*w**2) + + +def test_Y4(): + t = symbols('t', positive=True) + s = symbols('s') + F, _, _ = laplace_transform(erf(3/sqrt(t)), t, s, simplify=True) + assert F == 1/s - exp(-6*sqrt(s))/s + + +def test_Y5_Y6(): +# Solve y'' + y = 4 [H(t - 1) - H(t - 2)], y(0) = 1, y'(0) = 0 where H is the +# Heaviside (unit step) function (the RHS describes a pulse of magnitude 4 and +# duration 1). See David A. Sanchez, Richard C. Allen, Jr. and Walter T. +# Kyner, _Differential Equations: An Introduction_, Addison-Wesley Publishing +# Company, 1983, p. 211. First, take the Laplace transform of the ODE +# => s^2 Y(s) - s + Y(s) = 4/s [e^(-s) - e^(-2 s)] +# where Y(s) is the Laplace transform of y(t) + t = symbols('t', real=True) + s = symbols('s') + y = Function('y') + Y = Function('Y') + F = laplace_correspondence(laplace_transform(diff(y(t), t, 2) + y(t) + - 4*(Heaviside(t - 1) - Heaviside(t - 2)), + t, s, noconds=True), {y: Y}) + D = ( + -F + s**2*Y(s) - s*y(0) + Y(s) - Subs(Derivative(y(t), t), t, 0) - + 4*exp(-s)/s + 4*exp(-2*s)/s) + assert D == 0 +# Now, solve for Y(s) and then take the inverse Laplace transform +# => Y(s) = s/(s^2 + 1) + 4 [1/s - s/(s^2 + 1)] [e^(-s) - e^(-2 s)] +# => y(t) = cos t + 4 {[1 - cos(t - 1)] H(t - 1) - [1 - cos(t - 2)] H(t - 2)} + Yf = solve(F, Y(s))[0] + Yf = laplace_initial_conds(Yf, t, {y: [1, 0]}) + assert Yf == (s**2*exp(2*s) + 4*exp(s) - 4)*exp(-2*s)/(s*(s**2 + 1)) + yf = inverse_laplace_transform(Yf, s, t) + yf = yf.collect(Heaviside(t-1)).collect(Heaviside(t-2)) + assert yf == ( + (4 - 4*cos(t - 1))*Heaviside(t - 1) + + (4*cos(t - 2) - 4)*Heaviside(t - 2) + + cos(t)*Heaviside(t)) + + +@XFAIL +def test_Y7(): + # What is the Laplace transform of an infinite square wave? + # => 1/s + 2 sum( (-1)^n e^(- s n a)/s, n = 1..infinity ) + # [Sanchez, Allen and Kyner, p. 213] + t = symbols('t', positive=True) + a = symbols('a', real=True) + s = symbols('s') + F, _, _ = laplace_transform(1 + 2*Sum((-1)**n*Heaviside(t - n*a), + (n, 1, oo)), t, s) + # returns 2*LaplaceTransform(Sum((-1)**n*Heaviside(-a*n + t), + # (n, 1, oo)), t, s) + 1/s + # https://github.com/sympy/sympy/issues/7177 + assert F == 2*Sum((-1)**n*exp(-a*n*s)/s, (n, 1, oo)) + 1/s + + +@XFAIL +def test_Y8(): + assert fourier_transform(1, x, z) == DiracDelta(z) + + +def test_Y9(): + assert (fourier_transform(exp(-9*x**2), x, z) == + sqrt(pi)*exp(-pi**2*z**2/9)/3) + + +def test_Y10(): + assert (fourier_transform(abs(x)*exp(-3*abs(x)), x, z).cancel() == + (-8*pi**2*z**2 + 18)/(16*pi**4*z**4 + 72*pi**2*z**2 + 81)) + + +@SKIP("https://github.com/sympy/sympy/issues/7181") +@slow +def test_Y11(): + # => pi cot(pi s) (0 < Re s < 1) [Gradshteyn and Ryzhik 17.43(5)] + x, s = symbols('x s') + # raises RuntimeError: maximum recursion depth exceeded + # https://github.com/sympy/sympy/issues/7181 + # Update 2019-02-17 raises: + # TypeError: cannot unpack non-iterable MellinTransform object + F, _, _ = mellin_transform(1/(1 - x), x, s) + assert F == pi*cot(pi*s) + + +@XFAIL +def test_Y12(): + # => 2^(s - 4) gamma(s/2)/gamma(4 - s/2) (0 < Re s < 1) + # [Gradshteyn and Ryzhik 17.43(16)] + x, s = symbols('x s') + # returns Wrong value -2**(s - 4)*gamma(s/2 - 3)/gamma(-s/2 + 1) + # https://github.com/sympy/sympy/issues/7182 + F, _, _ = mellin_transform(besselj(3, x)/x**3, x, s) + assert F == -2**(s - 4)*gamma(s/2)/gamma(-s/2 + 4) + + +@XFAIL +def test_Y13(): +# Z[H(t - m T)] => z/[z^m (z - 1)] (H is the Heaviside (unit step) function) z + raise NotImplementedError("z-transform not supported") + + +@XFAIL +def test_Y14(): +# Z[H(t - m T)] => z/[z^m (z - 1)] (H is the Heaviside (unit step) function) + raise NotImplementedError("z-transform not supported") + + +def test_Z1(): + r = Function('r') + assert (rsolve(r(n + 2) - 2*r(n + 1) + r(n) - 2, r(n), + {r(0): 1, r(1): m}).simplify() == n**2 + n*(m - 2) + 1) + + +def test_Z2(): + r = Function('r') + assert (rsolve(r(n) - (5*r(n - 1) - 6*r(n - 2)), r(n), {r(0): 0, r(1): 1}) + == -2**n + 3**n) + + +def test_Z3(): + # => r(n) = Fibonacci[n + 1] [Cohen, p. 83] + r = Function('r') + # recurrence solution is correct, Wester expects it to be simplified to + # fibonacci(n+1), but that is quite hard + expected = ((S(1)/2 - sqrt(5)/2)**n*(S(1)/2 - sqrt(5)/10) + + (S(1)/2 + sqrt(5)/2)**n*(sqrt(5)/10 + S(1)/2)) + sol = rsolve(r(n) - (r(n - 1) + r(n - 2)), r(n), {r(1): 1, r(2): 2}) + assert sol == expected + + +@XFAIL +def test_Z4(): +# => [c^(n+1) [c^(n+1) - 2 c - 2] + (n+1) c^2 + 2 c - n] / [(c-1)^3 (c+1)] +# [Joan Z. Yu and Robert Israel in sci.math.symbolic] + r = Function('r') + c = symbols('c') + # raises ValueError: Polynomial or rational function expected, + # got '(c**2 - c**n)/(c - c**n) + s = rsolve(r(n) - ((1 + c - c**(n-1) - c**(n+1))/(1 - c**n)*r(n - 1) + - c*(1 - c**(n-2))/(1 - c**(n-1))*r(n - 2) + 1), + r(n), {r(1): 1, r(2): (2 + 2*c + c**2)/(1 + c)}) + assert (s - (c*(n + 1)*(c*(n + 1) - 2*c - 2) + + (n + 1)*c**2 + 2*c - n)/((c-1)**3*(c+1)) == 0) + + +@XFAIL +def test_Z5(): + # Second order ODE with initial conditions---solve directly + # transform: f(t) = sin(2 t)/8 - t cos(2 t)/4 + C1, C2 = symbols('C1 C2') + # initial conditions not supported, this is a manual workaround + # https://github.com/sympy/sympy/issues/4720 + eq = Derivative(f(x), x, 2) + 4*f(x) - sin(2*x) + sol = dsolve(eq, f(x)) + f0 = Lambda(x, sol.rhs) + assert f0(x) == C2*sin(2*x) + (C1 - x/4)*cos(2*x) + f1 = Lambda(x, diff(f0(x), x)) + # TODO: Replace solve with solveset, when it works for solveset + const_dict = solve((f0(0), f1(0))) + result = f0(x).subs(C1, const_dict[C1]).subs(C2, const_dict[C2]) + assert result == -x*cos(2*x)/4 + sin(2*x)/8 + # Result is OK, but ODE solving with initial conditions should be + # supported without all this manual work + raise NotImplementedError('ODE solving with initial conditions \ +not supported') + + +@XFAIL +def test_Z6(): + # Second order ODE with initial conditions---solve using Laplace + # transform: f(t) = sin(2 t)/8 - t cos(2 t)/4 + t = symbols('t', positive=True) + s = symbols('s') + eq = Derivative(f(t), t, 2) + 4*f(t) - sin(2*t) + F, _, _ = laplace_transform(eq, t, s) + # Laplace transform for diff() not calculated + # https://github.com/sympy/sympy/issues/7176 + assert (F == s**2*LaplaceTransform(f(t), t, s) + + 4*LaplaceTransform(f(t), t, s) - 2/(s**2 + 4)) + # rest of test case not implemented diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_xxe.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_xxe.py new file mode 100644 index 0000000000000000000000000000000000000000..3936e8aa135dde5f22c71548e2f90ed58ac25cb8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/utilities/tests/test_xxe.py @@ -0,0 +1,3 @@ +# A test file for XXE injection +# Username: Test +# Password: Test diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..685593e65cfc353484f7ac5908dd1ed0736e4b15 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/basisdependent.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/basisdependent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ca1abf9f176bb5ce20a112776e8008aeb479e74 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/basisdependent.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/coordsysrect.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/coordsysrect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c28cfb9c141dad99d5e4674924916589e56439d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/coordsysrect.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/deloperator.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/deloperator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65ae41049306eab2137e400f9fee8cc245e5c584 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/deloperator.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/dyadic.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/dyadic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eec46a0e327d464b011ab871cb3f3b764540faa2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/dyadic.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/functions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c401873dbf805408d2567df06e3e4f1a6cc48764 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/functions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/implicitregion.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/implicitregion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2dc9e10a61033b3d4486f0cc913776f10026b48e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/implicitregion.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/integrals.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/integrals.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5da59d09723775ee7a43c8cfdcc69df4c09c56fa Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/integrals.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/kind.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/kind.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e09b8154f6ff8dd6c7b34c0ee2c0953a9a08015 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/kind.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/operators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/operators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be15c43d46fbe2d0f28be46f5cb006cf79e4ee26 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/operators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/orienters.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/orienters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42adee5bcede56d5eb22076eb64b8a1ee1d6e829 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/orienters.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/parametricregion.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/parametricregion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9f2a03e96798659ba251e688014870b62f33e3f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/parametricregion.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/point.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/point.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5d1e970bf9e1af0320789c7f4f2ad1121f27f81 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/point.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/scalar.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/scalar.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e5deb836b6e815f4fea75d12468264a97ef6c7f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/scalar.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/vector.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/vector.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8c94a1b7cea09225792c2ee60c711d150462271 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/__pycache__/vector.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e99076dca7fa27cd121e783e33038f48a0d92bcf Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_coordsysrect.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_coordsysrect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89c127643f77b62133713258de9c05ca26be1275 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_coordsysrect.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_dyadic.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_dyadic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ad886addc6ab74b861719c33b074311bddd0de0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_dyadic.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_field_functions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_field_functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..630afcd682db1ab6d71e2495d820626684ccef8d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_field_functions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_functions.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d7e72d5f1e2320ac9eae272118c501bdeb9453e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_functions.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_implicitregion.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_implicitregion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8ab5860a66624dc6ade6179233bb8fb663c0f64 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_implicitregion.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_integrals.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_integrals.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..41993c754419cf491ad6fd923ce76087b0a55c02 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_integrals.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_operators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_operators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b74a2b11e35ac22df3f2054fa4f3ba49b5c0530a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_operators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_parametricregion.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_parametricregion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dcafe18f52b3ad5d809e0b221d6fa28f82b554dc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_parametricregion.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_printing.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_printing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c96035901781a7557ac020b02c7ece9aace97708 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_printing.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_vector.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_vector.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0bca15b37c199eb87d75ae33bbaee379fdf188e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/__pycache__/test_vector.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_coordsysrect.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_coordsysrect.py new file mode 100644 index 0000000000000000000000000000000000000000..53eb8c89ec1643a71800efe3e370acff3cb6f9c0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_coordsysrect.py @@ -0,0 +1,464 @@ +from sympy.testing.pytest import raises +from sympy.vector.coordsysrect import CoordSys3D +from sympy.vector.scalar import BaseScalar +from sympy.core.function import expand +from sympy.core.numbers import pi +from sympy.core.symbol import symbols +from sympy.functions.elementary.hyperbolic import (cosh, sinh) +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import (acos, atan2, cos, sin) +from sympy.matrices.dense import zeros +from sympy.matrices.immutable import ImmutableDenseMatrix as Matrix +from sympy.simplify.simplify import simplify +from sympy.vector.functions import express +from sympy.vector.point import Point +from sympy.vector.vector import Vector +from sympy.vector.orienters import (AxisOrienter, BodyOrienter, + SpaceOrienter, QuaternionOrienter) + + +x, y, z = symbols('x y z') +a, b, c, q = symbols('a b c q') +q1, q2, q3, q4 = symbols('q1 q2 q3 q4') + + +def test_func_args(): + A = CoordSys3D('A') + assert A.x.func(*A.x.args) == A.x + expr = 3*A.x + 4*A.y + assert expr.func(*expr.args) == expr + assert A.i.func(*A.i.args) == A.i + v = A.x*A.i + A.y*A.j + A.z*A.k + assert v.func(*v.args) == v + assert A.origin.func(*A.origin.args) == A.origin + + +def test_coordsys3d_equivalence(): + A = CoordSys3D('A') + A1 = CoordSys3D('A') + assert A1 == A + B = CoordSys3D('B') + assert A != B + + +def test_orienters(): + A = CoordSys3D('A') + axis_orienter = AxisOrienter(a, A.k) + body_orienter = BodyOrienter(a, b, c, '123') + space_orienter = SpaceOrienter(a, b, c, '123') + q_orienter = QuaternionOrienter(q1, q2, q3, q4) + assert axis_orienter.rotation_matrix(A) == Matrix([ + [ cos(a), sin(a), 0], + [-sin(a), cos(a), 0], + [ 0, 0, 1]]) + assert body_orienter.rotation_matrix() == Matrix([ + [ cos(b)*cos(c), sin(a)*sin(b)*cos(c) + sin(c)*cos(a), + sin(a)*sin(c) - sin(b)*cos(a)*cos(c)], + [-sin(c)*cos(b), -sin(a)*sin(b)*sin(c) + cos(a)*cos(c), + sin(a)*cos(c) + sin(b)*sin(c)*cos(a)], + [ sin(b), -sin(a)*cos(b), + cos(a)*cos(b)]]) + assert space_orienter.rotation_matrix() == Matrix([ + [cos(b)*cos(c), sin(c)*cos(b), -sin(b)], + [sin(a)*sin(b)*cos(c) - sin(c)*cos(a), + sin(a)*sin(b)*sin(c) + cos(a)*cos(c), sin(a)*cos(b)], + [sin(a)*sin(c) + sin(b)*cos(a)*cos(c), -sin(a)*cos(c) + + sin(b)*sin(c)*cos(a), cos(a)*cos(b)]]) + assert q_orienter.rotation_matrix() == Matrix([ + [q1**2 + q2**2 - q3**2 - q4**2, 2*q1*q4 + 2*q2*q3, + -2*q1*q3 + 2*q2*q4], + [-2*q1*q4 + 2*q2*q3, q1**2 - q2**2 + q3**2 - q4**2, + 2*q1*q2 + 2*q3*q4], + [2*q1*q3 + 2*q2*q4, + -2*q1*q2 + 2*q3*q4, q1**2 - q2**2 - q3**2 + q4**2]]) + + +def test_coordinate_vars(): + """ + Tests the coordinate variables functionality with respect to + reorientation of coordinate systems. + """ + A = CoordSys3D('A') + # Note that the name given on the lhs is different from A.x._name + assert BaseScalar(0, A, 'A_x', r'\mathbf{{x}_{A}}') == A.x + assert BaseScalar(1, A, 'A_y', r'\mathbf{{y}_{A}}') == A.y + assert BaseScalar(2, A, 'A_z', r'\mathbf{{z}_{A}}') == A.z + assert BaseScalar(0, A, 'A_x', r'\mathbf{{x}_{A}}').__hash__() == A.x.__hash__() + assert isinstance(A.x, BaseScalar) and \ + isinstance(A.y, BaseScalar) and \ + isinstance(A.z, BaseScalar) + assert A.x*A.y == A.y*A.x + assert A.scalar_map(A) == {A.x: A.x, A.y: A.y, A.z: A.z} + assert A.x.system == A + assert A.x.diff(A.x) == 1 + B = A.orient_new_axis('B', q, A.k) + assert B.scalar_map(A) == {B.z: A.z, B.y: -A.x*sin(q) + A.y*cos(q), + B.x: A.x*cos(q) + A.y*sin(q)} + assert A.scalar_map(B) == {A.x: B.x*cos(q) - B.y*sin(q), + A.y: B.x*sin(q) + B.y*cos(q), A.z: B.z} + assert express(B.x, A, variables=True) == A.x*cos(q) + A.y*sin(q) + assert express(B.y, A, variables=True) == -A.x*sin(q) + A.y*cos(q) + assert express(B.z, A, variables=True) == A.z + assert expand(express(B.x*B.y*B.z, A, variables=True)) == \ + expand(A.z*(-A.x*sin(q) + A.y*cos(q))*(A.x*cos(q) + A.y*sin(q))) + assert express(B.x*B.i + B.y*B.j + B.z*B.k, A) == \ + (B.x*cos(q) - B.y*sin(q))*A.i + (B.x*sin(q) + \ + B.y*cos(q))*A.j + B.z*A.k + assert simplify(express(B.x*B.i + B.y*B.j + B.z*B.k, A, \ + variables=True)) == \ + A.x*A.i + A.y*A.j + A.z*A.k + assert express(A.x*A.i + A.y*A.j + A.z*A.k, B) == \ + (A.x*cos(q) + A.y*sin(q))*B.i + \ + (-A.x*sin(q) + A.y*cos(q))*B.j + A.z*B.k + assert simplify(express(A.x*A.i + A.y*A.j + A.z*A.k, B, \ + variables=True)) == \ + B.x*B.i + B.y*B.j + B.z*B.k + N = B.orient_new_axis('N', -q, B.k) + assert N.scalar_map(A) == \ + {N.x: A.x, N.z: A.z, N.y: A.y} + C = A.orient_new_axis('C', q, A.i + A.j + A.k) + mapping = A.scalar_map(C) + assert mapping[A.x].equals(C.x*(2*cos(q) + 1)/3 + + C.y*(-2*sin(q + pi/6) + 1)/3 + + C.z*(-2*cos(q + pi/3) + 1)/3) + assert mapping[A.y].equals(C.x*(-2*cos(q + pi/3) + 1)/3 + + C.y*(2*cos(q) + 1)/3 + + C.z*(-2*sin(q + pi/6) + 1)/3) + assert mapping[A.z].equals(C.x*(-2*sin(q + pi/6) + 1)/3 + + C.y*(-2*cos(q + pi/3) + 1)/3 + + C.z*(2*cos(q) + 1)/3) + D = A.locate_new('D', a*A.i + b*A.j + c*A.k) + assert D.scalar_map(A) == {D.z: A.z - c, D.x: A.x - a, D.y: A.y - b} + E = A.orient_new_axis('E', a, A.k, a*A.i + b*A.j + c*A.k) + assert A.scalar_map(E) == {A.z: E.z + c, + A.x: E.x*cos(a) - E.y*sin(a) + a, + A.y: E.x*sin(a) + E.y*cos(a) + b} + assert E.scalar_map(A) == {E.x: (A.x - a)*cos(a) + (A.y - b)*sin(a), + E.y: (-A.x + a)*sin(a) + (A.y - b)*cos(a), + E.z: A.z - c} + F = A.locate_new('F', Vector.zero) + assert A.scalar_map(F) == {A.z: F.z, A.x: F.x, A.y: F.y} + + +def test_rotation_matrix(): + N = CoordSys3D('N') + A = N.orient_new_axis('A', q1, N.k) + B = A.orient_new_axis('B', q2, A.i) + C = B.orient_new_axis('C', q3, B.j) + D = N.orient_new_axis('D', q4, N.j) + E = N.orient_new_space('E', q1, q2, q3, '123') + F = N.orient_new_quaternion('F', q1, q2, q3, q4) + G = N.orient_new_body('G', q1, q2, q3, '123') + assert N.rotation_matrix(C) == Matrix([ + [- sin(q1) * sin(q2) * sin(q3) + cos(q1) * cos(q3), - sin(q1) * + cos(q2), sin(q1) * sin(q2) * cos(q3) + sin(q3) * cos(q1)], \ + [sin(q1) * cos(q3) + sin(q2) * sin(q3) * cos(q1), \ + cos(q1) * cos(q2), sin(q1) * sin(q3) - sin(q2) * cos(q1) * \ + cos(q3)], [- sin(q3) * cos(q2), sin(q2), cos(q2) * cos(q3)]]) + test_mat = D.rotation_matrix(C) - Matrix( + [[cos(q1) * cos(q3) * cos(q4) - sin(q3) * (- sin(q4) * cos(q2) + + sin(q1) * sin(q2) * cos(q4)), - sin(q2) * sin(q4) - sin(q1) * + cos(q2) * cos(q4), sin(q3) * cos(q1) * cos(q4) + cos(q3) * \ + (- sin(q4) * cos(q2) + sin(q1) * sin(q2) * cos(q4))], \ + [sin(q1) * cos(q3) + sin(q2) * sin(q3) * cos(q1), cos(q1) * \ + cos(q2), sin(q1) * sin(q3) - sin(q2) * cos(q1) * cos(q3)], \ + [sin(q4) * cos(q1) * cos(q3) - sin(q3) * (cos(q2) * cos(q4) + \ + sin(q1) * sin(q2) * \ + sin(q4)), sin(q2) * + cos(q4) - sin(q1) * sin(q4) * cos(q2), sin(q3) * \ + sin(q4) * cos(q1) + cos(q3) * (cos(q2) * cos(q4) + \ + sin(q1) * sin(q2) * sin(q4))]]) + assert test_mat.expand() == zeros(3, 3) + assert E.rotation_matrix(N) == Matrix( + [[cos(q2)*cos(q3), sin(q3)*cos(q2), -sin(q2)], + [sin(q1)*sin(q2)*cos(q3) - sin(q3)*cos(q1), \ + sin(q1)*sin(q2)*sin(q3) + cos(q1)*cos(q3), sin(q1)*cos(q2)], \ + [sin(q1)*sin(q3) + sin(q2)*cos(q1)*cos(q3), - \ + sin(q1)*cos(q3) + sin(q2)*sin(q3)*cos(q1), cos(q1)*cos(q2)]]) + assert F.rotation_matrix(N) == Matrix([[ + q1**2 + q2**2 - q3**2 - q4**2, + 2*q1*q4 + 2*q2*q3, -2*q1*q3 + 2*q2*q4],[ -2*q1*q4 + 2*q2*q3, + q1**2 - q2**2 + q3**2 - q4**2, 2*q1*q2 + 2*q3*q4], + [2*q1*q3 + 2*q2*q4, + -2*q1*q2 + 2*q3*q4, + q1**2 - q2**2 - q3**2 + q4**2]]) + assert G.rotation_matrix(N) == Matrix([[ + cos(q2)*cos(q3), sin(q1)*sin(q2)*cos(q3) + sin(q3)*cos(q1), + sin(q1)*sin(q3) - sin(q2)*cos(q1)*cos(q3)], [ + -sin(q3)*cos(q2), -sin(q1)*sin(q2)*sin(q3) + cos(q1)*cos(q3), + sin(q1)*cos(q3) + sin(q2)*sin(q3)*cos(q1)],[ + sin(q2), -sin(q1)*cos(q2), cos(q1)*cos(q2)]]) + + +def test_vector_with_orientation(): + """ + Tests the effects of orientation of coordinate systems on + basic vector operations. + """ + N = CoordSys3D('N') + A = N.orient_new_axis('A', q1, N.k) + B = A.orient_new_axis('B', q2, A.i) + C = B.orient_new_axis('C', q3, B.j) + + # Test to_matrix + v1 = a*N.i + b*N.j + c*N.k + assert v1.to_matrix(A) == Matrix([[ a*cos(q1) + b*sin(q1)], + [-a*sin(q1) + b*cos(q1)], + [ c]]) + + # Test dot + assert N.i.dot(A.i) == cos(q1) + assert N.i.dot(A.j) == -sin(q1) + assert N.i.dot(A.k) == 0 + assert N.j.dot(A.i) == sin(q1) + assert N.j.dot(A.j) == cos(q1) + assert N.j.dot(A.k) == 0 + assert N.k.dot(A.i) == 0 + assert N.k.dot(A.j) == 0 + assert N.k.dot(A.k) == 1 + + assert N.i.dot(A.i + A.j) == -sin(q1) + cos(q1) == \ + (A.i + A.j).dot(N.i) + + assert A.i.dot(C.i) == cos(q3) + assert A.i.dot(C.j) == 0 + assert A.i.dot(C.k) == sin(q3) + assert A.j.dot(C.i) == sin(q2)*sin(q3) + assert A.j.dot(C.j) == cos(q2) + assert A.j.dot(C.k) == -sin(q2)*cos(q3) + assert A.k.dot(C.i) == -cos(q2)*sin(q3) + assert A.k.dot(C.j) == sin(q2) + assert A.k.dot(C.k) == cos(q2)*cos(q3) + + # Test cross + assert N.i.cross(A.i) == sin(q1)*A.k + assert N.i.cross(A.j) == cos(q1)*A.k + assert N.i.cross(A.k) == -sin(q1)*A.i - cos(q1)*A.j + assert N.j.cross(A.i) == -cos(q1)*A.k + assert N.j.cross(A.j) == sin(q1)*A.k + assert N.j.cross(A.k) == cos(q1)*A.i - sin(q1)*A.j + assert N.k.cross(A.i) == A.j + assert N.k.cross(A.j) == -A.i + assert N.k.cross(A.k) == Vector.zero + + assert N.i.cross(A.i) == sin(q1)*A.k + assert N.i.cross(A.j) == cos(q1)*A.k + assert N.i.cross(A.i + A.j) == sin(q1)*A.k + cos(q1)*A.k + assert (A.i + A.j).cross(N.i) == (-sin(q1) - cos(q1))*N.k + + assert A.i.cross(C.i) == sin(q3)*C.j + assert A.i.cross(C.j) == -sin(q3)*C.i + cos(q3)*C.k + assert A.i.cross(C.k) == -cos(q3)*C.j + assert C.i.cross(A.i) == (-sin(q3)*cos(q2))*A.j + \ + (-sin(q2)*sin(q3))*A.k + assert C.j.cross(A.i) == (sin(q2))*A.j + (-cos(q2))*A.k + assert express(C.k.cross(A.i), C).trigsimp() == cos(q3)*C.j + + +def test_orient_new_methods(): + N = CoordSys3D('N') + orienter1 = AxisOrienter(q4, N.j) + orienter2 = SpaceOrienter(q1, q2, q3, '123') + orienter3 = QuaternionOrienter(q1, q2, q3, q4) + orienter4 = BodyOrienter(q1, q2, q3, '123') + D = N.orient_new('D', (orienter1, )) + E = N.orient_new('E', (orienter2, )) + F = N.orient_new('F', (orienter3, )) + G = N.orient_new('G', (orienter4, )) + assert D == N.orient_new_axis('D', q4, N.j) + assert E == N.orient_new_space('E', q1, q2, q3, '123') + assert F == N.orient_new_quaternion('F', q1, q2, q3, q4) + assert G == N.orient_new_body('G', q1, q2, q3, '123') + + +def test_locatenew_point(): + """ + Tests Point class, and locate_new method in CoordSys3D. + """ + A = CoordSys3D('A') + assert isinstance(A.origin, Point) + v = a*A.i + b*A.j + c*A.k + C = A.locate_new('C', v) + assert C.origin.position_wrt(A) == \ + C.position_wrt(A) == \ + C.origin.position_wrt(A.origin) == v + assert A.origin.position_wrt(C) == \ + A.position_wrt(C) == \ + A.origin.position_wrt(C.origin) == -v + assert A.origin.express_coordinates(C) == (-a, -b, -c) + p = A.origin.locate_new('p', -v) + assert p.express_coordinates(A) == (-a, -b, -c) + assert p.position_wrt(C.origin) == p.position_wrt(C) == \ + -2 * v + p1 = p.locate_new('p1', 2*v) + assert p1.position_wrt(C.origin) == Vector.zero + assert p1.express_coordinates(C) == (0, 0, 0) + p2 = p.locate_new('p2', A.i) + assert p1.position_wrt(p2) == 2*v - A.i + assert p2.express_coordinates(C) == (-2*a + 1, -2*b, -2*c) + + +def test_create_new(): + a = CoordSys3D('a') + c = a.create_new('c', transformation='spherical') + assert c._parent == a + assert c.transformation_to_parent() == \ + (c.r*sin(c.theta)*cos(c.phi), c.r*sin(c.theta)*sin(c.phi), c.r*cos(c.theta)) + assert c.transformation_from_parent() == \ + (sqrt(a.x**2 + a.y**2 + a.z**2), acos(a.z/sqrt(a.x**2 + a.y**2 + a.z**2)), atan2(a.y, a.x)) + + +def test_evalf(): + A = CoordSys3D('A') + v = 3*A.i + 4*A.j + a*A.k + assert v.n() == v.evalf() + assert v.evalf(subs={a:1}) == v.subs(a, 1).evalf() + + +def test_lame_coefficients(): + a = CoordSys3D('a', 'spherical') + assert a.lame_coefficients() == (1, a.r, sin(a.theta)*a.r) + a = CoordSys3D('a') + assert a.lame_coefficients() == (1, 1, 1) + a = CoordSys3D('a', 'cartesian') + assert a.lame_coefficients() == (1, 1, 1) + a = CoordSys3D('a', 'cylindrical') + assert a.lame_coefficients() == (1, a.r, 1) + + +def test_transformation_equations(): + + x, y, z = symbols('x y z') + # Str + a = CoordSys3D('a', transformation='spherical', + variable_names=["r", "theta", "phi"]) + r, theta, phi = a.base_scalars() + + assert r == a.r + assert theta == a.theta + assert phi == a.phi + + raises(AttributeError, lambda: a.x) + raises(AttributeError, lambda: a.y) + raises(AttributeError, lambda: a.z) + + assert a.transformation_to_parent() == ( + r*sin(theta)*cos(phi), + r*sin(theta)*sin(phi), + r*cos(theta) + ) + assert a.lame_coefficients() == (1, r, r*sin(theta)) + assert a.transformation_from_parent_function()(x, y, z) == ( + sqrt(x ** 2 + y ** 2 + z ** 2), + acos((z) / sqrt(x**2 + y**2 + z**2)), + atan2(y, x) + ) + a = CoordSys3D('a', transformation='cylindrical', + variable_names=["r", "theta", "z"]) + r, theta, z = a.base_scalars() + assert a.transformation_to_parent() == ( + r*cos(theta), + r*sin(theta), + z + ) + assert a.lame_coefficients() == (1, a.r, 1) + assert a.transformation_from_parent_function()(x, y, z) == (sqrt(x**2 + y**2), + atan2(y, x), z) + + a = CoordSys3D('a', 'cartesian') + assert a.transformation_to_parent() == (a.x, a.y, a.z) + assert a.lame_coefficients() == (1, 1, 1) + assert a.transformation_from_parent_function()(x, y, z) == (x, y, z) + + # Variables and expressions + + # Cartesian with equation tuple: + x, y, z = symbols('x y z') + a = CoordSys3D('a', ((x, y, z), (x, y, z))) + a._calculate_inv_trans_equations() + assert a.transformation_to_parent() == (a.x1, a.x2, a.x3) + assert a.lame_coefficients() == (1, 1, 1) + assert a.transformation_from_parent_function()(x, y, z) == (x, y, z) + r, theta, z = symbols("r theta z") + + # Cylindrical with equation tuple: + a = CoordSys3D('a', [(r, theta, z), (r*cos(theta), r*sin(theta), z)], + variable_names=["r", "theta", "z"]) + r, theta, z = a.base_scalars() + assert a.transformation_to_parent() == ( + r*cos(theta), r*sin(theta), z + ) + assert a.lame_coefficients() == ( + sqrt(sin(theta)**2 + cos(theta)**2), + sqrt(r**2*sin(theta)**2 + r**2*cos(theta)**2), + 1 + ) # ==> this should simplify to (1, r, 1), tests are too slow with `simplify`. + + # Definitions with `lambda`: + + # Cartesian with `lambda` + a = CoordSys3D('a', lambda x, y, z: (x, y, z)) + assert a.transformation_to_parent() == (a.x1, a.x2, a.x3) + assert a.lame_coefficients() == (1, 1, 1) + a._calculate_inv_trans_equations() + assert a.transformation_from_parent_function()(x, y, z) == (x, y, z) + + # Spherical with `lambda` + a = CoordSys3D('a', lambda r, theta, phi: (r*sin(theta)*cos(phi), r*sin(theta)*sin(phi), r*cos(theta)), + variable_names=["r", "theta", "phi"]) + r, theta, phi = a.base_scalars() + assert a.transformation_to_parent() == ( + r*sin(theta)*cos(phi), r*sin(phi)*sin(theta), r*cos(theta) + ) + assert a.lame_coefficients() == ( + sqrt(sin(phi)**2*sin(theta)**2 + sin(theta)**2*cos(phi)**2 + cos(theta)**2), + sqrt(r**2*sin(phi)**2*cos(theta)**2 + r**2*sin(theta)**2 + r**2*cos(phi)**2*cos(theta)**2), + sqrt(r**2*sin(phi)**2*sin(theta)**2 + r**2*sin(theta)**2*cos(phi)**2) + ) # ==> this should simplify to (1, r, sin(theta)*r), `simplify` is too slow. + + # Cylindrical with `lambda` + a = CoordSys3D('a', lambda r, theta, z: + (r*cos(theta), r*sin(theta), z), + variable_names=["r", "theta", "z"] + ) + r, theta, z = a.base_scalars() + assert a.transformation_to_parent() == (r*cos(theta), r*sin(theta), z) + assert a.lame_coefficients() == ( + sqrt(sin(theta)**2 + cos(theta)**2), + sqrt(r**2*sin(theta)**2 + r**2*cos(theta)**2), + 1 + ) # ==> this should simplify to (1, a.x, 1) + + raises(TypeError, lambda: CoordSys3D('a', transformation={ + x: x*sin(y)*cos(z), y:x*sin(y)*sin(z), z: x*cos(y)})) + + +def test_check_orthogonality(): + x, y, z = symbols('x y z') + u,v = symbols('u, v') + a = CoordSys3D('a', transformation=((x, y, z), (x*sin(y)*cos(z), x*sin(y)*sin(z), x*cos(y)))) + assert a._check_orthogonality(a._transformation) is True + a = CoordSys3D('a', transformation=((x, y, z), (x * cos(y), x * sin(y), z))) + assert a._check_orthogonality(a._transformation) is True + a = CoordSys3D('a', transformation=((u, v, z), (cosh(u) * cos(v), sinh(u) * sin(v), z))) + assert a._check_orthogonality(a._transformation) is True + + raises(ValueError, lambda: CoordSys3D('a', transformation=((x, y, z), (x, x, z)))) + raises(ValueError, lambda: CoordSys3D('a', transformation=( + (x, y, z), (x*sin(y/2)*cos(z), x*sin(y)*sin(z), x*cos(y))))) + + +def test_rotation_trans_equations(): + a = CoordSys3D('a') + from sympy.core.symbol import symbols + q0 = symbols('q0') + assert a._rotation_trans_equations(a._parent_rotation_matrix, a.base_scalars()) == (a.x, a.y, a.z) + assert a._rotation_trans_equations(a._inverse_rotation_matrix(), a.base_scalars()) == (a.x, a.y, a.z) + b = a.orient_new_axis('b', 0, -a.k) + assert b._rotation_trans_equations(b._parent_rotation_matrix, b.base_scalars()) == (b.x, b.y, b.z) + assert b._rotation_trans_equations(b._inverse_rotation_matrix(), b.base_scalars()) == (b.x, b.y, b.z) + c = a.orient_new_axis('c', q0, -a.k) + assert c._rotation_trans_equations(c._parent_rotation_matrix, c.base_scalars()) == \ + (-sin(q0) * c.y + cos(q0) * c.x, sin(q0) * c.x + cos(q0) * c.y, c.z) + assert c._rotation_trans_equations(c._inverse_rotation_matrix(), c.base_scalars()) == \ + (sin(q0) * c.y + cos(q0) * c.x, -sin(q0) * c.x + cos(q0) * c.y, c.z) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_dyadic.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_dyadic.py new file mode 100644 index 0000000000000000000000000000000000000000..2e396fcf2a81af897b59c0065f6b15f5c6933222 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_dyadic.py @@ -0,0 +1,134 @@ +from sympy.core.numbers import pi +from sympy.core.symbol import symbols +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.matrices.immutable import ImmutableDenseMatrix as Matrix +from sympy.simplify.simplify import simplify +from sympy.vector import (CoordSys3D, Vector, Dyadic, + DyadicAdd, DyadicMul, DyadicZero, + BaseDyadic, express) + + +A = CoordSys3D('A') + + +def test_dyadic(): + a, b = symbols('a, b') + assert Dyadic.zero != 0 + assert isinstance(Dyadic.zero, DyadicZero) + assert BaseDyadic(A.i, A.j) != BaseDyadic(A.j, A.i) + assert (BaseDyadic(Vector.zero, A.i) == + BaseDyadic(A.i, Vector.zero) == Dyadic.zero) + + d1 = A.i | A.i + d2 = A.j | A.j + d3 = A.i | A.j + + assert isinstance(d1, BaseDyadic) + d_mul = a*d1 + assert isinstance(d_mul, DyadicMul) + assert d_mul.base_dyadic == d1 + assert d_mul.measure_number == a + assert isinstance(a*d1 + b*d3, DyadicAdd) + assert d1 == A.i.outer(A.i) + assert d3 == A.i.outer(A.j) + v1 = a*A.i - A.k + v2 = A.i + b*A.j + assert v1 | v2 == v1.outer(v2) == a * (A.i|A.i) + (a*b) * (A.i|A.j) +\ + - (A.k|A.i) - b * (A.k|A.j) + assert d1 * 0 == Dyadic.zero + assert d1 != Dyadic.zero + assert d1 * 2 == 2 * (A.i | A.i) + assert d1 / 2. == 0.5 * d1 + + assert d1.dot(0 * d1) == Vector.zero + assert d1 & d2 == Dyadic.zero + assert d1.dot(A.i) == A.i == d1 & A.i + + assert d1.cross(Vector.zero) == Dyadic.zero + assert d1.cross(A.i) == Dyadic.zero + assert d1 ^ A.j == d1.cross(A.j) + assert d1.cross(A.k) == - A.i | A.j + assert d2.cross(A.i) == - A.j | A.k == d2 ^ A.i + + assert A.i ^ d1 == Dyadic.zero + assert A.j.cross(d1) == - A.k | A.i == A.j ^ d1 + assert Vector.zero.cross(d1) == Dyadic.zero + assert A.k ^ d1 == A.j | A.i + assert A.i.dot(d1) == A.i & d1 == A.i + assert A.j.dot(d1) == Vector.zero + assert Vector.zero.dot(d1) == Vector.zero + assert A.j & d2 == A.j + + assert d1.dot(d3) == d1 & d3 == A.i | A.j == d3 + assert d3 & d1 == Dyadic.zero + + q = symbols('q') + B = A.orient_new_axis('B', q, A.k) + assert express(d1, B) == express(d1, B, B) + + expr1 = ((cos(q)**2) * (B.i | B.i) + (-sin(q) * cos(q)) * + (B.i | B.j) + (-sin(q) * cos(q)) * (B.j | B.i) + (sin(q)**2) * + (B.j | B.j)) + assert (express(d1, B) - expr1).simplify() == Dyadic.zero + + expr2 = (cos(q)) * (B.i | A.i) + (-sin(q)) * (B.j | A.i) + assert (express(d1, B, A) - expr2).simplify() == Dyadic.zero + + expr3 = (cos(q)) * (A.i | B.i) + (-sin(q)) * (A.i | B.j) + assert (express(d1, A, B) - expr3).simplify() == Dyadic.zero + + assert d1.to_matrix(A) == Matrix([[1, 0, 0], [0, 0, 0], [0, 0, 0]]) + assert d1.to_matrix(A, B) == Matrix([[cos(q), -sin(q), 0], + [0, 0, 0], + [0, 0, 0]]) + assert d3.to_matrix(A) == Matrix([[0, 1, 0], [0, 0, 0], [0, 0, 0]]) + a, b, c, d, e, f = symbols('a, b, c, d, e, f') + v1 = a * A.i + b * A.j + c * A.k + v2 = d * A.i + e * A.j + f * A.k + d4 = v1.outer(v2) + assert d4.to_matrix(A) == Matrix([[a * d, a * e, a * f], + [b * d, b * e, b * f], + [c * d, c * e, c * f]]) + d5 = v1.outer(v1) + C = A.orient_new_axis('C', q, A.i) + for expected, actual in zip(C.rotation_matrix(A) * d5.to_matrix(A) * \ + C.rotation_matrix(A).T, d5.to_matrix(C)): + assert (expected - actual).simplify() == 0 + + +def test_dyadic_simplify(): + x, y, z, k, n, m, w, f, s, A = symbols('x, y, z, k, n, m, w, f, s, A') + N = CoordSys3D('N') + + dy = N.i | N.i + test1 = (1 / x + 1 / y) * dy + assert (N.i & test1 & N.i) != (x + y) / (x * y) + test1 = test1.simplify() + assert test1.simplify() == simplify(test1) + assert (N.i & test1 & N.i) == (x + y) / (x * y) + + test2 = (A**2 * s**4 / (4 * pi * k * m**3)) * dy + test2 = test2.simplify() + assert (N.i & test2 & N.i) == (A**2 * s**4 / (4 * pi * k * m**3)) + + test3 = ((4 + 4 * x - 2 * (2 + 2 * x)) / (2 + 2 * x)) * dy + test3 = test3.simplify() + assert (N.i & test3 & N.i) == 0 + + test4 = ((-4 * x * y**2 - 2 * y**3 - 2 * x**2 * y) / (x + y)**2) * dy + test4 = test4.simplify() + assert (N.i & test4 & N.i) == -2 * y + + +def test_dyadic_srepr(): + from sympy.printing.repr import srepr + N = CoordSys3D('N') + + dy = N.i | N.j + res = "BaseDyadic(CoordSys3D(Str('N'), Tuple(ImmutableDenseMatrix([["\ + "Integer(1), Integer(0), Integer(0)], [Integer(0), Integer(1), "\ + "Integer(0)], [Integer(0), Integer(0), Integer(1)]]), "\ + "VectorZero())).i, CoordSys3D(Str('N'), Tuple(ImmutableDenseMatrix("\ + "[[Integer(1), Integer(0), Integer(0)], [Integer(0), Integer(1), "\ + "Integer(0)], [Integer(0), Integer(0), Integer(1)]]), VectorZero())).j)" + assert srepr(dy) == res diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_field_functions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_field_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..035c2ce0234b81069c5ad8dcb1c74f4de0164a8f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_field_functions.py @@ -0,0 +1,321 @@ +from sympy.core.function import Derivative +from sympy.vector.vector import Vector +from sympy.vector.coordsysrect import CoordSys3D +from sympy.simplify import simplify +from sympy.core.symbol import symbols +from sympy.core import S +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.vector.vector import Dot +from sympy.vector.operators import curl, divergence, gradient, Gradient, Divergence, Cross +from sympy.vector.deloperator import Del +from sympy.vector.functions import (is_conservative, is_solenoidal, + scalar_potential, directional_derivative, + laplacian, scalar_potential_difference) +from sympy.testing.pytest import raises + +C = CoordSys3D('C') +i, j, k = C.base_vectors() +x, y, z = C.base_scalars() +delop = Del() +a, b, c, q = symbols('a b c q') + + +def test_del_operator(): + # Tests for curl + + assert delop ^ Vector.zero == Vector.zero + assert ((delop ^ Vector.zero).doit() == Vector.zero == + curl(Vector.zero)) + assert delop.cross(Vector.zero) == delop ^ Vector.zero + assert (delop ^ i).doit() == Vector.zero + assert delop.cross(2*y**2*j, doit=True) == Vector.zero + assert delop.cross(2*y**2*j) == delop ^ 2*y**2*j + v = x*y*z * (i + j + k) + assert ((delop ^ v).doit() == + (-x*y + x*z)*i + (x*y - y*z)*j + (-x*z + y*z)*k == + curl(v)) + assert delop ^ v == delop.cross(v) + assert (delop.cross(2*x**2*j) == + (Derivative(0, C.y) - Derivative(2*C.x**2, C.z))*C.i + + (-Derivative(0, C.x) + Derivative(0, C.z))*C.j + + (-Derivative(0, C.y) + Derivative(2*C.x**2, C.x))*C.k) + assert (delop.cross(2*x**2*j, doit=True) == 4*x*k == + curl(2*x**2*j)) + + #Tests for divergence + assert delop & Vector.zero is S.Zero == divergence(Vector.zero) + assert (delop & Vector.zero).doit() is S.Zero + assert delop.dot(Vector.zero) == delop & Vector.zero + assert (delop & i).doit() is S.Zero + assert (delop & x**2*i).doit() == 2*x == divergence(x**2*i) + assert (delop.dot(v, doit=True) == x*y + y*z + z*x == + divergence(v)) + assert delop & v == delop.dot(v) + assert delop.dot(1/(x*y*z) * (i + j + k), doit=True) == \ + - 1 / (x*y*z**2) - 1 / (x*y**2*z) - 1 / (x**2*y*z) + v = x*i + y*j + z*k + assert (delop & v == Derivative(C.x, C.x) + + Derivative(C.y, C.y) + Derivative(C.z, C.z)) + assert delop.dot(v, doit=True) == 3 == divergence(v) + assert delop & v == delop.dot(v) + assert simplify((delop & v).doit()) == 3 + + #Tests for gradient + assert (delop.gradient(0, doit=True) == Vector.zero == + gradient(0)) + assert delop.gradient(0) == delop(0) + assert (delop(S.Zero)).doit() == Vector.zero + assert (delop(x) == (Derivative(C.x, C.x))*C.i + + (Derivative(C.x, C.y))*C.j + (Derivative(C.x, C.z))*C.k) + assert (delop(x)).doit() == i == gradient(x) + assert (delop(x*y*z) == + (Derivative(C.x*C.y*C.z, C.x))*C.i + + (Derivative(C.x*C.y*C.z, C.y))*C.j + + (Derivative(C.x*C.y*C.z, C.z))*C.k) + assert (delop.gradient(x*y*z, doit=True) == + y*z*i + z*x*j + x*y*k == + gradient(x*y*z)) + assert delop(x*y*z) == delop.gradient(x*y*z) + assert (delop(2*x**2)).doit() == 4*x*i + assert ((delop(a*sin(y) / x)).doit() == + -a*sin(y)/x**2 * i + a*cos(y)/x * j) + + #Tests for directional derivative + assert (Vector.zero & delop)(a) is S.Zero + assert ((Vector.zero & delop)(a)).doit() is S.Zero + assert ((v & delop)(Vector.zero)).doit() == Vector.zero + assert ((v & delop)(S.Zero)).doit() is S.Zero + assert ((i & delop)(x)).doit() == 1 + assert ((j & delop)(y)).doit() == 1 + assert ((k & delop)(z)).doit() == 1 + assert ((i & delop)(x*y*z)).doit() == y*z + assert ((v & delop)(x)).doit() == x + assert ((v & delop)(x*y*z)).doit() == 3*x*y*z + assert (v & delop)(x + y + z) == C.x + C.y + C.z + assert ((v & delop)(x + y + z)).doit() == x + y + z + assert ((v & delop)(v)).doit() == v + assert ((i & delop)(v)).doit() == i + assert ((j & delop)(v)).doit() == j + assert ((k & delop)(v)).doit() == k + assert ((v & delop)(Vector.zero)).doit() == Vector.zero + + # Tests for laplacian on scalar fields + assert laplacian(x*y*z) is S.Zero + assert laplacian(x**2) == S(2) + assert laplacian(x**2*y**2*z**2) == \ + 2*y**2*z**2 + 2*x**2*z**2 + 2*x**2*y**2 + A = CoordSys3D('A', transformation="spherical", variable_names=["r", "theta", "phi"]) + B = CoordSys3D('B', transformation='cylindrical', variable_names=["r", "theta", "z"]) + assert laplacian(A.r + A.theta + A.phi) == 2/A.r + cos(A.theta)/(A.r**2*sin(A.theta)) + assert laplacian(B.r + B.theta + B.z) == 1/B.r + + # Tests for laplacian on vector fields + assert laplacian(x*y*z*(i + j + k)) == Vector.zero + assert laplacian(x*y**2*z*(i + j + k)) == \ + 2*x*z*i + 2*x*z*j + 2*x*z*k + + +def test_product_rules(): + """ + Tests the six product rules defined with respect to the Del + operator + + References + ========== + + .. [1] https://en.wikipedia.org/wiki/Del + + """ + + #Define the scalar and vector functions + f = 2*x*y*z + g = x*y + y*z + z*x + u = x**2*i + 4*j - y**2*z*k + v = 4*i + x*y*z*k + + # First product rule + lhs = delop(f * g, doit=True) + rhs = (f * delop(g) + g * delop(f)).doit() + assert simplify(lhs) == simplify(rhs) + + # Second product rule + lhs = delop(u & v).doit() + rhs = ((u ^ (delop ^ v)) + (v ^ (delop ^ u)) + \ + ((u & delop)(v)) + ((v & delop)(u))).doit() + assert simplify(lhs) == simplify(rhs) + + # Third product rule + lhs = (delop & (f*v)).doit() + rhs = ((f * (delop & v)) + (v & (delop(f)))).doit() + assert simplify(lhs) == simplify(rhs) + + # Fourth product rule + lhs = (delop & (u ^ v)).doit() + rhs = ((v & (delop ^ u)) - (u & (delop ^ v))).doit() + assert simplify(lhs) == simplify(rhs) + + # Fifth product rule + lhs = (delop ^ (f * v)).doit() + rhs = (((delop(f)) ^ v) + (f * (delop ^ v))).doit() + assert simplify(lhs) == simplify(rhs) + + # Sixth product rule + lhs = (delop ^ (u ^ v)).doit() + rhs = (u * (delop & v) - v * (delop & u) + + (v & delop)(u) - (u & delop)(v)).doit() + assert simplify(lhs) == simplify(rhs) + + +P = C.orient_new_axis('P', q, C.k) # type: ignore +scalar_field = 2*x**2*y*z +grad_field = gradient(scalar_field) +vector_field = y**2*i + 3*x*j + 5*y*z*k +curl_field = curl(vector_field) + + +def test_conservative(): + assert is_conservative(Vector.zero) is True + assert is_conservative(i) is True + assert is_conservative(2 * i + 3 * j + 4 * k) is True + assert (is_conservative(y*z*i + x*z*j + x*y*k) is + True) + assert is_conservative(x * j) is False + assert is_conservative(grad_field) is True + assert is_conservative(curl_field) is False + assert (is_conservative(4*x*y*z*i + 2*x**2*z*j) is + False) + assert is_conservative(z*P.i + P.x*k) is True + + +def test_solenoidal(): + assert is_solenoidal(Vector.zero) is True + assert is_solenoidal(i) is True + assert is_solenoidal(2 * i + 3 * j + 4 * k) is True + assert (is_solenoidal(y*z*i + x*z*j + x*y*k) is + True) + assert is_solenoidal(y * j) is False + assert is_solenoidal(grad_field) is False + assert is_solenoidal(curl_field) is True + assert is_solenoidal((-2*y + 3)*k) is True + assert is_solenoidal(cos(q)*i + sin(q)*j + cos(q)*P.k) is True + assert is_solenoidal(z*P.i + P.x*k) is True + + +def test_directional_derivative(): + assert directional_derivative(C.x*C.y*C.z, 3*C.i + 4*C.j + C.k) == C.x*C.y + 4*C.x*C.z + 3*C.y*C.z + assert directional_derivative(5*C.x**2*C.z, 3*C.i + 4*C.j + C.k) == 5*C.x**2 + 30*C.x*C.z + assert directional_derivative(5*C.x**2*C.z, 4*C.j) is S.Zero + + D = CoordSys3D("D", "spherical", variable_names=["r", "theta", "phi"], + vector_names=["e_r", "e_theta", "e_phi"]) + r, theta, phi = D.base_scalars() + e_r, e_theta, e_phi = D.base_vectors() + assert directional_derivative(r**2*e_r, e_r) == 2*r*e_r + assert directional_derivative(5*r**2*phi, 3*e_r + 4*e_theta + e_phi) == 5*r**2 + 30*r*phi + + +def test_scalar_potential(): + assert scalar_potential(Vector.zero, C) == 0 + assert scalar_potential(i, C) == x + assert scalar_potential(j, C) == y + assert scalar_potential(k, C) == z + assert scalar_potential(y*z*i + x*z*j + x*y*k, C) == x*y*z + assert scalar_potential(grad_field, C) == scalar_field + assert scalar_potential(z*P.i + P.x*k, C) == x*z*cos(q) + y*z*sin(q) + assert scalar_potential(z*P.i + P.x*k, P) == P.x*P.z + raises(ValueError, lambda: scalar_potential(x*j, C)) + + +def test_scalar_potential_difference(): + point1 = C.origin.locate_new('P1', 1*i + 2*j + 3*k) + point2 = C.origin.locate_new('P2', 4*i + 5*j + 6*k) + genericpointC = C.origin.locate_new('RP', x*i + y*j + z*k) + genericpointP = P.origin.locate_new('PP', P.x*P.i + P.y*P.j + P.z*P.k) + assert scalar_potential_difference(S.Zero, C, point1, point2) == 0 + assert (scalar_potential_difference(scalar_field, C, C.origin, + genericpointC) == + scalar_field) + assert (scalar_potential_difference(grad_field, C, C.origin, + genericpointC) == + scalar_field) + assert scalar_potential_difference(grad_field, C, point1, point2) == 948 + assert (scalar_potential_difference(y*z*i + x*z*j + + x*y*k, C, point1, + genericpointC) == + x*y*z - 6) + potential_diff_P = (2*P.z*(P.x*sin(q) + P.y*cos(q))* + (P.x*cos(q) - P.y*sin(q))**2) + assert (scalar_potential_difference(grad_field, P, P.origin, + genericpointP).simplify() == + potential_diff_P.simplify()) + + +def test_differential_operators_curvilinear_system(): + A = CoordSys3D('A', transformation="spherical", variable_names=["r", "theta", "phi"]) + B = CoordSys3D('B', transformation='cylindrical', variable_names=["r", "theta", "z"]) + # Test for spherical coordinate system and gradient + assert gradient(3*A.r + 4*A.theta) == 3*A.i + 4/A.r*A.j + assert gradient(3*A.r*A.phi + 4*A.theta) == 3*A.phi*A.i + 4/A.r*A.j + (3/sin(A.theta))*A.k + assert gradient(0*A.r + 0*A.theta+0*A.phi) == Vector.zero + assert gradient(A.r*A.theta*A.phi) == A.theta*A.phi*A.i + A.phi*A.j + (A.theta/sin(A.theta))*A.k + # Test for spherical coordinate system and divergence + assert divergence(A.r * A.i + A.theta * A.j + A.phi * A.k) == \ + (sin(A.theta)*A.r + cos(A.theta)*A.r*A.theta)/(sin(A.theta)*A.r**2) + 3 + 1/(sin(A.theta)*A.r) + assert divergence(3*A.r*A.phi*A.i + A.theta*A.j + A.r*A.theta*A.phi*A.k) == \ + (sin(A.theta)*A.r + cos(A.theta)*A.r*A.theta)/(sin(A.theta)*A.r**2) + 9*A.phi + A.theta/sin(A.theta) + assert divergence(Vector.zero) == 0 + assert divergence(0*A.i + 0*A.j + 0*A.k) == 0 + # Test for spherical coordinate system and curl + assert curl(A.r*A.i + A.theta*A.j + A.phi*A.k) == \ + (cos(A.theta)*A.phi/(sin(A.theta)*A.r))*A.i + (-A.phi/A.r)*A.j + A.theta/A.r*A.k + assert curl(A.r*A.j + A.phi*A.k) == (cos(A.theta)*A.phi/(sin(A.theta)*A.r))*A.i + (-A.phi/A.r)*A.j + 2*A.k + + # Test for cylindrical coordinate system and gradient + assert gradient(0*B.r + 0*B.theta+0*B.z) == Vector.zero + assert gradient(B.r*B.theta*B.z) == B.theta*B.z*B.i + B.z*B.j + B.r*B.theta*B.k + assert gradient(3*B.r) == 3*B.i + assert gradient(2*B.theta) == 2/B.r * B.j + assert gradient(4*B.z) == 4*B.k + # Test for cylindrical coordinate system and divergence + assert divergence(B.r*B.i + B.theta*B.j + B.z*B.k) == 3 + 1/B.r + assert divergence(B.r*B.j + B.z*B.k) == 1 + # Test for cylindrical coordinate system and curl + assert curl(B.r*B.j + B.z*B.k) == 2*B.k + assert curl(3*B.i + 2/B.r*B.j + 4*B.k) == Vector.zero + +def test_mixed_coordinates(): + # gradient + a = CoordSys3D('a') + b = CoordSys3D('b') + c = CoordSys3D('c') + assert gradient(a.x*b.y) == b.y*a.i + a.x*b.j + assert gradient(3*cos(q)*a.x*b.x+a.y*(a.x+(cos(q)+b.x))) ==\ + (a.y + 3*b.x*cos(q))*a.i + (a.x + b.x + cos(q))*a.j + (3*a.x*cos(q) + a.y)*b.i + # Some tests need further work: + # assert gradient(a.x*(cos(a.x+b.x))) == (cos(a.x + b.x))*a.i + a.x*Gradient(cos(a.x + b.x)) + # assert gradient(cos(a.x + b.x)*cos(a.x + b.z)) == Gradient(cos(a.x + b.x)*cos(a.x + b.z)) + assert gradient(a.x**b.y) == Gradient(a.x**b.y) + # assert gradient(cos(a.x+b.y)*a.z) == None + assert gradient(cos(a.x*b.y)) == Gradient(cos(a.x*b.y)) + assert gradient(3*cos(q)*a.x*b.x*a.z*a.y+ b.y*b.z + cos(a.x+a.y)*b.z) == \ + (3*a.y*a.z*b.x*cos(q) - b.z*sin(a.x + a.y))*a.i + \ + (3*a.x*a.z*b.x*cos(q) - b.z*sin(a.x + a.y))*a.j + (3*a.x*a.y*b.x*cos(q))*a.k + \ + (3*a.x*a.y*a.z*cos(q))*b.i + b.z*b.j + (b.y + cos(a.x + a.y))*b.k + # divergence + assert divergence(a.i*a.x+a.j*a.y+a.z*a.k + b.i*b.x+b.j*b.y+b.z*b.k + c.i*c.x+c.j*c.y+c.z*c.k) == S(9) + # assert divergence(3*a.i*a.x*cos(a.x+b.z) + a.j*b.x*c.z) == None + assert divergence(3*a.i*a.x*a.z + b.j*b.x*c.z + 3*a.j*a.z*a.y) == \ + 6*a.z + b.x*Dot(b.j, c.k) + assert divergence(3*cos(q)*a.x*b.x*b.i*c.x) == \ + 3*a.x*b.x*cos(q)*Dot(b.i, c.i) + 3*a.x*c.x*cos(q) + 3*b.x*c.x*cos(q)*Dot(b.i, a.i) + assert divergence(a.x*b.x*c.x*Cross(a.x*a.i, a.y*b.j)) ==\ + a.x*b.x*c.x*Divergence(Cross(a.x*a.i, a.y*b.j)) + \ + b.x*c.x*Dot(Cross(a.x*a.i, a.y*b.j), a.i) + \ + a.x*c.x*Dot(Cross(a.x*a.i, a.y*b.j), b.i) + \ + a.x*b.x*Dot(Cross(a.x*a.i, a.y*b.j), c.i) + assert divergence(a.x*b.x*c.x*(a.x*a.i + b.x*b.i)) == \ + 4*a.x*b.x*c.x +\ + a.x**2*c.x*Dot(a.i, b.i) +\ + a.x**2*b.x*Dot(a.i, c.i) +\ + b.x**2*c.x*Dot(b.i, a.i) +\ + a.x*b.x**2*Dot(b.i, c.i) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_functions.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..dfdf9821b6c853755ce12d0cbdfa599bd4f312e4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_functions.py @@ -0,0 +1,184 @@ +from sympy.vector.vector import Vector +from sympy.vector.coordsysrect import CoordSys3D +from sympy.vector.functions import express, matrix_to_vector, orthogonalize +from sympy.core.numbers import Rational +from sympy.core.singleton import S +from sympy.core.symbol import symbols +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.matrices.immutable import ImmutableDenseMatrix as Matrix +from sympy.testing.pytest import raises + +N = CoordSys3D('N') +q1, q2, q3, q4, q5 = symbols('q1 q2 q3 q4 q5') +A = N.orient_new_axis('A', q1, N.k) # type: ignore +B = A.orient_new_axis('B', q2, A.i) +C = B.orient_new_axis('C', q3, B.j) + + +def test_express(): + assert express(Vector.zero, N) == Vector.zero + assert express(S.Zero, N) is S.Zero + assert express(A.i, C) == cos(q3)*C.i + sin(q3)*C.k + assert express(A.j, C) == sin(q2)*sin(q3)*C.i + cos(q2)*C.j - \ + sin(q2)*cos(q3)*C.k + assert express(A.k, C) == -sin(q3)*cos(q2)*C.i + sin(q2)*C.j + \ + cos(q2)*cos(q3)*C.k + assert express(A.i, N) == cos(q1)*N.i + sin(q1)*N.j + assert express(A.j, N) == -sin(q1)*N.i + cos(q1)*N.j + assert express(A.k, N) == N.k + assert express(A.i, A) == A.i + assert express(A.j, A) == A.j + assert express(A.k, A) == A.k + assert express(A.i, B) == B.i + assert express(A.j, B) == cos(q2)*B.j - sin(q2)*B.k + assert express(A.k, B) == sin(q2)*B.j + cos(q2)*B.k + assert express(A.i, C) == cos(q3)*C.i + sin(q3)*C.k + assert express(A.j, C) == sin(q2)*sin(q3)*C.i + cos(q2)*C.j - \ + sin(q2)*cos(q3)*C.k + assert express(A.k, C) == -sin(q3)*cos(q2)*C.i + sin(q2)*C.j + \ + cos(q2)*cos(q3)*C.k + # Check to make sure UnitVectors get converted properly + assert express(N.i, N) == N.i + assert express(N.j, N) == N.j + assert express(N.k, N) == N.k + assert express(N.i, A) == (cos(q1)*A.i - sin(q1)*A.j) + assert express(N.j, A) == (sin(q1)*A.i + cos(q1)*A.j) + assert express(N.k, A) == A.k + assert express(N.i, B) == (cos(q1)*B.i - sin(q1)*cos(q2)*B.j + + sin(q1)*sin(q2)*B.k) + assert express(N.j, B) == (sin(q1)*B.i + cos(q1)*cos(q2)*B.j - + sin(q2)*cos(q1)*B.k) + assert express(N.k, B) == (sin(q2)*B.j + cos(q2)*B.k) + assert express(N.i, C) == ( + (cos(q1)*cos(q3) - sin(q1)*sin(q2)*sin(q3))*C.i - + sin(q1)*cos(q2)*C.j + + (sin(q3)*cos(q1) + sin(q1)*sin(q2)*cos(q3))*C.k) + assert express(N.j, C) == ( + (sin(q1)*cos(q3) + sin(q2)*sin(q3)*cos(q1))*C.i + + cos(q1)*cos(q2)*C.j + + (sin(q1)*sin(q3) - sin(q2)*cos(q1)*cos(q3))*C.k) + assert express(N.k, C) == (-sin(q3)*cos(q2)*C.i + sin(q2)*C.j + + cos(q2)*cos(q3)*C.k) + + assert express(A.i, N) == (cos(q1)*N.i + sin(q1)*N.j) + assert express(A.j, N) == (-sin(q1)*N.i + cos(q1)*N.j) + assert express(A.k, N) == N.k + assert express(A.i, A) == A.i + assert express(A.j, A) == A.j + assert express(A.k, A) == A.k + assert express(A.i, B) == B.i + assert express(A.j, B) == (cos(q2)*B.j - sin(q2)*B.k) + assert express(A.k, B) == (sin(q2)*B.j + cos(q2)*B.k) + assert express(A.i, C) == (cos(q3)*C.i + sin(q3)*C.k) + assert express(A.j, C) == (sin(q2)*sin(q3)*C.i + cos(q2)*C.j - + sin(q2)*cos(q3)*C.k) + assert express(A.k, C) == (-sin(q3)*cos(q2)*C.i + sin(q2)*C.j + + cos(q2)*cos(q3)*C.k) + + assert express(B.i, N) == (cos(q1)*N.i + sin(q1)*N.j) + assert express(B.j, N) == (-sin(q1)*cos(q2)*N.i + + cos(q1)*cos(q2)*N.j + sin(q2)*N.k) + assert express(B.k, N) == (sin(q1)*sin(q2)*N.i - + sin(q2)*cos(q1)*N.j + cos(q2)*N.k) + assert express(B.i, A) == A.i + assert express(B.j, A) == (cos(q2)*A.j + sin(q2)*A.k) + assert express(B.k, A) == (-sin(q2)*A.j + cos(q2)*A.k) + assert express(B.i, B) == B.i + assert express(B.j, B) == B.j + assert express(B.k, B) == B.k + assert express(B.i, C) == (cos(q3)*C.i + sin(q3)*C.k) + assert express(B.j, C) == C.j + assert express(B.k, C) == (-sin(q3)*C.i + cos(q3)*C.k) + + assert express(C.i, N) == ( + (cos(q1)*cos(q3) - sin(q1)*sin(q2)*sin(q3))*N.i + + (sin(q1)*cos(q3) + sin(q2)*sin(q3)*cos(q1))*N.j - + sin(q3)*cos(q2)*N.k) + assert express(C.j, N) == ( + -sin(q1)*cos(q2)*N.i + cos(q1)*cos(q2)*N.j + sin(q2)*N.k) + assert express(C.k, N) == ( + (sin(q3)*cos(q1) + sin(q1)*sin(q2)*cos(q3))*N.i + + (sin(q1)*sin(q3) - sin(q2)*cos(q1)*cos(q3))*N.j + + cos(q2)*cos(q3)*N.k) + assert express(C.i, A) == (cos(q3)*A.i + sin(q2)*sin(q3)*A.j - + sin(q3)*cos(q2)*A.k) + assert express(C.j, A) == (cos(q2)*A.j + sin(q2)*A.k) + assert express(C.k, A) == (sin(q3)*A.i - sin(q2)*cos(q3)*A.j + + cos(q2)*cos(q3)*A.k) + assert express(C.i, B) == (cos(q3)*B.i - sin(q3)*B.k) + assert express(C.j, B) == B.j + assert express(C.k, B) == (sin(q3)*B.i + cos(q3)*B.k) + assert express(C.i, C) == C.i + assert express(C.j, C) == C.j + assert express(C.k, C) == C.k == (C.k) + + # Check to make sure Vectors get converted back to UnitVectors + assert N.i == express((cos(q1)*A.i - sin(q1)*A.j), N).simplify() + assert N.j == express((sin(q1)*A.i + cos(q1)*A.j), N).simplify() + assert N.i == express((cos(q1)*B.i - sin(q1)*cos(q2)*B.j + + sin(q1)*sin(q2)*B.k), N).simplify() + assert N.j == express((sin(q1)*B.i + cos(q1)*cos(q2)*B.j - + sin(q2)*cos(q1)*B.k), N).simplify() + assert N.k == express((sin(q2)*B.j + cos(q2)*B.k), N).simplify() + + + assert A.i == express((cos(q1)*N.i + sin(q1)*N.j), A).simplify() + assert A.j == express((-sin(q1)*N.i + cos(q1)*N.j), A).simplify() + + assert A.j == express((cos(q2)*B.j - sin(q2)*B.k), A).simplify() + assert A.k == express((sin(q2)*B.j + cos(q2)*B.k), A).simplify() + + assert A.i == express((cos(q3)*C.i + sin(q3)*C.k), A).simplify() + assert A.j == express((sin(q2)*sin(q3)*C.i + cos(q2)*C.j - + sin(q2)*cos(q3)*C.k), A).simplify() + + assert A.k == express((-sin(q3)*cos(q2)*C.i + sin(q2)*C.j + + cos(q2)*cos(q3)*C.k), A).simplify() + assert B.i == express((cos(q1)*N.i + sin(q1)*N.j), B).simplify() + assert B.j == express((-sin(q1)*cos(q2)*N.i + + cos(q1)*cos(q2)*N.j + sin(q2)*N.k), B).simplify() + + assert B.k == express((sin(q1)*sin(q2)*N.i - + sin(q2)*cos(q1)*N.j + cos(q2)*N.k), B).simplify() + + assert B.j == express((cos(q2)*A.j + sin(q2)*A.k), B).simplify() + assert B.k == express((-sin(q2)*A.j + cos(q2)*A.k), B).simplify() + assert B.i == express((cos(q3)*C.i + sin(q3)*C.k), B).simplify() + assert B.k == express((-sin(q3)*C.i + cos(q3)*C.k), B).simplify() + assert C.i == express((cos(q3)*A.i + sin(q2)*sin(q3)*A.j - + sin(q3)*cos(q2)*A.k), C).simplify() + assert C.j == express((cos(q2)*A.j + sin(q2)*A.k), C).simplify() + assert C.k == express((sin(q3)*A.i - sin(q2)*cos(q3)*A.j + + cos(q2)*cos(q3)*A.k), C).simplify() + assert C.i == express((cos(q3)*B.i - sin(q3)*B.k), C).simplify() + assert C.k == express((sin(q3)*B.i + cos(q3)*B.k), C).simplify() + + +def test_matrix_to_vector(): + m = Matrix([[1], [2], [3]]) + assert matrix_to_vector(m, C) == C.i + 2*C.j + 3*C.k + m = Matrix([[0], [0], [0]]) + assert matrix_to_vector(m, N) == matrix_to_vector(m, C) == \ + Vector.zero + m = Matrix([[q1], [q2], [q3]]) + assert matrix_to_vector(m, N) == q1*N.i + q2*N.j + q3*N.k + + +def test_orthogonalize(): + C = CoordSys3D('C') + a, b = symbols('a b', integer=True) + i, j, k = C.base_vectors() + v1 = i + 2*j + v2 = 2*i + 3*j + v3 = 3*i + 5*j + v4 = 3*i + j + v5 = 2*i + 2*j + v6 = a*i + b*j + v7 = 4*a*i + 4*b*j + assert orthogonalize(v1, v2) == [C.i + 2*C.j, C.i*Rational(2, 5) + -C.j/5] + # from wikipedia + assert orthogonalize(v4, v5, orthonormal=True) == \ + [(3*sqrt(10))*C.i/10 + (sqrt(10))*C.j/10, (-sqrt(10))*C.i/10 + (3*sqrt(10))*C.j/10] + raises(ValueError, lambda: orthogonalize(v1, v2, v3)) + raises(ValueError, lambda: orthogonalize(v6, v7)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_implicitregion.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_implicitregion.py new file mode 100644 index 0000000000000000000000000000000000000000..3686d847a7f165cb5ba9aeb813e5922aaa17e1e0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_implicitregion.py @@ -0,0 +1,90 @@ +from sympy.core.relational import Eq +from sympy.core.singleton import S +from sympy.abc import x, y, z, s, t +from sympy.sets import FiniteSet, EmptySet +from sympy.geometry import Point +from sympy.vector import ImplicitRegion +from sympy.testing.pytest import raises + + +def test_ImplicitRegion(): + ellipse = ImplicitRegion((x, y), (x**2/4 + y**2/16 - 1)) + assert ellipse.equation == x**2/4 + y**2/16 - 1 + assert ellipse.variables == (x, y) + assert ellipse.degree == 2 + r = ImplicitRegion((x, y, z), Eq(x**4 + y**2 - x*y, 6)) + assert r.equation == x**4 + y**2 - x*y - 6 + assert r.variables == (x, y, z) + assert r.degree == 4 + + +def test_regular_point(): + r1 = ImplicitRegion((x,), x**2 - 16) + assert r1.regular_point() == (-4,) + c1 = ImplicitRegion((x, y), x**2 + y**2 - 4) + assert c1.regular_point() == (0, -2) + c2 = ImplicitRegion((x, y), (x - S(5)/2)**2 + y**2 - (S(1)/4)**2) + assert c2.regular_point() == (S(5)/2, -S(1)/4) + c3 = ImplicitRegion((x, y), (y - 5)**2 - 16*(x - 5)) + assert c3.regular_point() == (5, 5) + r2 = ImplicitRegion((x, y), x**2 - 4*x*y - 3*y**2 + 4*x + 8*y - 5) + assert r2.regular_point() == (S(4)/7, S(9)/7) + r3 = ImplicitRegion((x, y), x**2 - 2*x*y + 3*y**2 - 2*x - 5*y + 3/2) + raises(ValueError, lambda: r3.regular_point()) + + +def test_singular_points_and_multiplicty(): + r1 = ImplicitRegion((x, y, z), Eq(x + y + z, 0)) + assert r1.singular_points() == EmptySet + r2 = ImplicitRegion((x, y, z), x*y*z + y**4 -x**2*z**2) + assert r2.singular_points() == FiniteSet((0, 0, z), (x, 0, 0)) + assert r2.multiplicity((0, 0, 0)) == 3 + assert r2.multiplicity((0, 0, 6)) == 2 + r3 = ImplicitRegion((x, y, z), z**2 - x**2 - y**2) + assert r3.singular_points() == FiniteSet((0, 0, 0)) + assert r3.multiplicity((0, 0, 0)) == 2 + r4 = ImplicitRegion((x, y), x**2 + y**2 - 2*x) + assert r4.singular_points() == EmptySet + assert r4.multiplicity(Point(1, 3)) == 0 + + +def test_rational_parametrization(): + p = ImplicitRegion((x,), x - 2) + assert p.rational_parametrization() == (x - 2,) + + line = ImplicitRegion((x, y), Eq(y, 3*x + 2)) + assert line.rational_parametrization() == (x, 3*x + 2) + + circle1 = ImplicitRegion((x, y), (x-2)**2 + (y+3)**2 - 4) + assert circle1.rational_parametrization(parameters=t) == (4*t/(t**2 + 1) + 2, 4*t**2/(t**2 + 1) - 5) + circle2 = ImplicitRegion((x, y), (x - S.Half)**2 + y**2 - (S(1)/2)**2) + + assert circle2.rational_parametrization(parameters=t) == (t/(t**2 + 1) + S(1)/2, t**2/(t**2 + 1) - S(1)/2) + circle3 = ImplicitRegion((x, y), Eq(x**2 + y**2, 2*x)) + assert circle3.rational_parametrization(parameters=(t,)) == (2*t/(t**2 + 1) + 1, 2*t**2/(t**2 + 1) - 1) + + parabola = ImplicitRegion((x, y), (y - 3)**2 - 4*(x + 6)) + assert parabola.rational_parametrization(t) == (-6 + 4/t**2, 3 + 4/t) + + rect_hyperbola = ImplicitRegion((x, y), x*y - 1) + assert rect_hyperbola.rational_parametrization(t) == (-1 + (t + 1)/t, t) + + cubic_curve = ImplicitRegion((x, y), x**3 + x**2 - y**2) + assert cubic_curve.rational_parametrization(parameters=(t)) == (t**2 - 1, t*(t**2 - 1)) + cuspidal = ImplicitRegion((x, y), (x**3 - y**2)) + assert cuspidal.rational_parametrization(t) == (t**2, t**3) + + I = ImplicitRegion((x, y), x**3 + x**2 - y**2) + assert I.rational_parametrization(t) == (t**2 - 1, t*(t**2 - 1)) + + sphere = ImplicitRegion((x, y, z), Eq(x**2 + y**2 + z**2, 2*x)) + assert sphere.rational_parametrization(parameters=(s, t)) == (2/(s**2 + t**2 + 1), 2*t/(s**2 + t**2 + 1), 2*s/(s**2 + t**2 + 1)) + + conic = ImplicitRegion((x, y), Eq(x**2 + 4*x*y + 3*y**2 + x - y + 10, 0)) + assert conic.rational_parametrization(t) == ( + S(17)/2 + 4/(3*t**2 + 4*t + 1), 4*t/(3*t**2 + 4*t + 1) - S(11)/2) + + r1 = ImplicitRegion((x, y), y**2 - x**3 + x) + raises(NotImplementedError, lambda: r1.rational_parametrization()) + r2 = ImplicitRegion((x, y), y**2 - x**3 - x**2 + 1) + raises(NotImplementedError, lambda: r2.rational_parametrization()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_integrals.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_integrals.py new file mode 100644 index 0000000000000000000000000000000000000000..84c900d038e214df1ea59a8cd8fb2929005c3674 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_integrals.py @@ -0,0 +1,106 @@ +from sympy.core.numbers import pi +from sympy.core.singleton import S +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.testing.pytest import raises +from sympy.vector.coordsysrect import CoordSys3D +from sympy.vector.integrals import ParametricIntegral, vector_integrate +from sympy.vector.parametricregion import ParametricRegion +from sympy.vector.implicitregion import ImplicitRegion +from sympy.abc import x, y, z, u, v, r, t, theta, phi +from sympy.geometry import Point, Segment, Curve, Circle, Polygon, Plane + +C = CoordSys3D('C') + +def test_parametric_lineintegrals(): + halfcircle = ParametricRegion((4*cos(theta), 4*sin(theta)), (theta, -pi/2, pi/2)) + assert ParametricIntegral(C.x*C.y**4, halfcircle) == S(8192)/5 + + curve = ParametricRegion((t, t**2, t**3), (t, 0, 1)) + field1 = 8*C.x**2*C.y*C.z*C.i + 5*C.z*C.j - 4*C.x*C.y*C.k + assert ParametricIntegral(field1, curve) == 1 + line = ParametricRegion((4*t - 1, 2 - 2*t, t), (t, 0, 1)) + assert ParametricIntegral(C.x*C.z*C.i - C.y*C.z*C.k, line) == 3 + + assert ParametricIntegral(4*C.x**3, ParametricRegion((1, t), (t, 0, 2))) == 8 + + helix = ParametricRegion((cos(t), sin(t), 3*t), (t, 0, 4*pi)) + assert ParametricIntegral(C.x*C.y*C.z, helix) == -3*sqrt(10)*pi + + field2 = C.y*C.i + C.z*C.j + C.z*C.k + assert ParametricIntegral(field2, ParametricRegion((cos(t), sin(t), t**2), (t, 0, pi))) == -5*pi/2 + pi**4/2 + +def test_parametric_surfaceintegrals(): + + semisphere = ParametricRegion((2*sin(phi)*cos(theta), 2*sin(phi)*sin(theta), 2*cos(phi)),\ + (theta, 0, 2*pi), (phi, 0, pi/2)) + assert ParametricIntegral(C.z, semisphere) == 8*pi + + cylinder = ParametricRegion((sqrt(3)*cos(theta), sqrt(3)*sin(theta), z), (z, 0, 6), (theta, 0, 2*pi)) + assert ParametricIntegral(C.y, cylinder) == 0 + + cone = ParametricRegion((v*cos(u), v*sin(u), v), (u, 0, 2*pi), (v, 0, 1)) + assert ParametricIntegral(C.x*C.i + C.y*C.j + C.z**4*C.k, cone) == pi/3 + + triangle1 = ParametricRegion((x, y), (x, 0, 2), (y, 0, 10 - 5*x)) + triangle2 = ParametricRegion((x, y), (y, 0, 10 - 5*x), (x, 0, 2)) + assert ParametricIntegral(-15.6*C.y*C.k, triangle1) == ParametricIntegral(-15.6*C.y*C.k, triangle2) + assert ParametricIntegral(C.z, triangle1) == 10*C.z + +def test_parametric_volumeintegrals(): + + cube = ParametricRegion((x, y, z), (x, 0, 1), (y, 0, 1), (z, 0, 1)) + assert ParametricIntegral(1, cube) == 1 + + solidsphere1 = ParametricRegion((r*sin(phi)*cos(theta), r*sin(phi)*sin(theta), r*cos(phi)),\ + (r, 0, 2), (theta, 0, 2*pi), (phi, 0, pi)) + solidsphere2 = ParametricRegion((r*sin(phi)*cos(theta), r*sin(phi)*sin(theta), r*cos(phi)),\ + (r, 0, 2), (phi, 0, pi), (theta, 0, 2*pi)) + assert ParametricIntegral(C.x**2 + C.y**2, solidsphere1) == -256*pi/15 + assert ParametricIntegral(C.x**2 + C.y**2, solidsphere2) == 256*pi/15 + + region_under_plane1 = ParametricRegion((x, y, z), (x, 0, 3), (y, 0, -2*x/3 + 2),\ + (z, 0, 6 - 2*x - 3*y)) + region_under_plane2 = ParametricRegion((x, y, z), (x, 0, 3), (z, 0, 6 - 2*x - 3*y),\ + (y, 0, -2*x/3 + 2)) + + assert ParametricIntegral(C.x*C.i + C.j - 100*C.k, region_under_plane1) == \ + ParametricIntegral(C.x*C.i + C.j - 100*C.k, region_under_plane2) + assert ParametricIntegral(2*C.x, region_under_plane2) == -9 + +def test_vector_integrate(): + halfdisc = ParametricRegion((r*cos(theta), r* sin(theta)), (r, -2, 2), (theta, 0, pi)) + assert vector_integrate(C.x**2, halfdisc) == 4*pi + assert vector_integrate(C.x, ParametricRegion((t, t**2), (t, 2, 3))) == -17*sqrt(17)/12 + 37*sqrt(37)/12 + + assert vector_integrate(C.y**3*C.z, (C.x, 0, 3), (C.y, -1, 4)) == 765*C.z/4 + + s1 = Segment(Point(0, 0), Point(0, 1)) + assert vector_integrate(-15*C.y, s1) == S(-15)/2 + s2 = Segment(Point(4, 3, 9), Point(1, 1, 7)) + assert vector_integrate(C.y*C.i, s2) == -6 + + curve = Curve((sin(t), cos(t)), (t, 0, 2)) + assert vector_integrate(5*C.z, curve) == 10*C.z + + c1 = Circle(Point(2, 3), 6) + assert vector_integrate(C.x*C.y, c1) == 72*pi + c2 = Circle(Point(0, 0), Point(1, 1), Point(1, 0)) + assert vector_integrate(1, c2) == c2.circumference + + triangle = Polygon((0, 0), (1, 0), (1, 1)) + assert vector_integrate(C.x*C.i - 14*C.y*C.j, triangle) == 0 + p1, p2, p3, p4 = [(0, 0), (1, 0), (5, 1), (0, 1)] + poly = Polygon(p1, p2, p3, p4) + assert vector_integrate(-23*C.z, poly) == -161*C.z - 23*sqrt(17)*C.z + + point = Point(2, 3) + assert vector_integrate(C.i*C.y, point) == ParametricIntegral(C.y*C.i, ParametricRegion((2, 3))) + + c3 = ImplicitRegion((x, y), x**2 + y**2 - 4) + assert vector_integrate(45, c3) == 180*pi + c4 = ImplicitRegion((x, y), (x - 3)**2 + (y - 4)**2 - 9) + assert vector_integrate(1, c4) == 6*pi + + pl = Plane(Point(1, 1, 1), Point(2, 3, 4), Point(2, 2, 2)) + raises(ValueError, lambda: vector_integrate(C.x*C.z*C.i + C.k, pl)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_operators.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_operators.py new file mode 100644 index 0000000000000000000000000000000000000000..5734edadd00547c67d6f864b50afd966ad8392a6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_operators.py @@ -0,0 +1,43 @@ +from sympy.vector import CoordSys3D, Gradient, Divergence, Curl, VectorZero, Laplacian +from sympy.printing.repr import srepr + +R = CoordSys3D('R') +s1 = R.x*R.y*R.z # type: ignore +s2 = R.x + 3*R.y**2 # type: ignore +s3 = R.x**2 + R.y**2 + R.z**2 # type: ignore +v1 = R.x*R.i + R.z*R.z*R.j # type: ignore +v2 = R.x*R.i + R.y*R.j + R.z*R.k # type: ignore +v3 = R.x**2*R.i + R.y**2*R.j + R.z**2*R.k # type: ignore + + +def test_Gradient(): + assert Gradient(s1) == Gradient(R.x*R.y*R.z) + assert Gradient(s2) == Gradient(R.x + 3*R.y**2) + assert Gradient(s1).doit() == R.y*R.z*R.i + R.x*R.z*R.j + R.x*R.y*R.k + assert Gradient(s2).doit() == R.i + 6*R.y*R.j + + +def test_Divergence(): + assert Divergence(v1) == Divergence(R.x*R.i + R.z*R.z*R.j) + assert Divergence(v2) == Divergence(R.x*R.i + R.y*R.j + R.z*R.k) + assert Divergence(v1).doit() == 1 + assert Divergence(v2).doit() == 3 + # issue 22384 + Rc = CoordSys3D('R', transformation='cylindrical') + assert Divergence(Rc.i).doit() == 1/Rc.r + + +def test_Curl(): + assert Curl(v1) == Curl(R.x*R.i + R.z*R.z*R.j) + assert Curl(v2) == Curl(R.x*R.i + R.y*R.j + R.z*R.k) + assert Curl(v1).doit() == (-2*R.z)*R.i + assert Curl(v2).doit() == VectorZero() + + +def test_Laplacian(): + assert Laplacian(s3) == Laplacian(R.x**2 + R.y**2 + R.z**2) + assert Laplacian(v3) == Laplacian(R.x**2*R.i + R.y**2*R.j + R.z**2*R.k) + assert Laplacian(s3).doit() == 6 + assert Laplacian(v3).doit() == 2*R.i + 2*R.j + 2*R.k + assert srepr(Laplacian(s3)) == \ + 'Laplacian(Add(Pow(R.x, Integer(2)), Pow(R.y, Integer(2)), Pow(R.z, Integer(2))))' diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_parametricregion.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_parametricregion.py new file mode 100644 index 0000000000000000000000000000000000000000..e785b96744f9e2c39e91b997fcb70f8a921256bd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_parametricregion.py @@ -0,0 +1,97 @@ +from sympy.core.numbers import pi +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.vector.coordsysrect import CoordSys3D +from sympy.vector.parametricregion import ParametricRegion, parametric_region_list +from sympy.geometry import Point, Segment, Curve, Ellipse, Line, Parabola, Polygon +from sympy.testing.pytest import raises +from sympy.abc import a, b, r, t, x, y, z, theta, phi + + +C = CoordSys3D('C') + +def test_ParametricRegion(): + + point = ParametricRegion((3, 4)) + assert point.definition == (3, 4) + assert point.parameters == () + assert point.limits == {} + assert point.dimensions == 0 + + # line x = y + line_xy = ParametricRegion((y, y), (y, 1, 5)) + assert line_xy .definition == (y, y) + assert line_xy.parameters == (y,) + assert line_xy.dimensions == 1 + + # line y = z + line_yz = ParametricRegion((x,t,t), x, (t, 1, 2)) + assert line_yz.definition == (x,t,t) + assert line_yz.parameters == (x, t) + assert line_yz.limits == {t: (1, 2)} + assert line_yz.dimensions == 1 + + p1 = ParametricRegion((9*a, -16*b), (a, 0, 2), (b, -1, 5)) + assert p1.definition == (9*a, -16*b) + assert p1.parameters == (a, b) + assert p1.limits == {a: (0, 2), b: (-1, 5)} + assert p1.dimensions == 2 + + p2 = ParametricRegion((t, t**3), t) + assert p2.parameters == (t,) + assert p2.limits == {} + assert p2.dimensions == 0 + + circle = ParametricRegion((r*cos(theta), r*sin(theta)), r, (theta, 0, 2*pi)) + assert circle.definition == (r*cos(theta), r*sin(theta)) + assert circle.dimensions == 1 + + halfdisc = ParametricRegion((r*cos(theta), r*sin(theta)), (r, -2, 2), (theta, 0, pi)) + assert halfdisc.definition == (r*cos(theta), r*sin(theta)) + assert halfdisc.parameters == (r, theta) + assert halfdisc.limits == {r: (-2, 2), theta: (0, pi)} + assert halfdisc.dimensions == 2 + + ellipse = ParametricRegion((a*cos(t), b*sin(t)), (t, 0, 8)) + assert ellipse.parameters == (t,) + assert ellipse.limits == {t: (0, 8)} + assert ellipse.dimensions == 1 + + cylinder = ParametricRegion((r*cos(theta), r*sin(theta), z), (r, 0, 1), (theta, 0, 2*pi), (z, 0, 4)) + assert cylinder.parameters == (r, theta, z) + assert cylinder.dimensions == 3 + + sphere = ParametricRegion((r*sin(phi)*cos(theta),r*sin(phi)*sin(theta), r*cos(phi)), + r, (theta, 0, 2*pi), (phi, 0, pi)) + assert sphere.definition == (r*sin(phi)*cos(theta),r*sin(phi)*sin(theta), r*cos(phi)) + assert sphere.parameters == (r, theta, phi) + assert sphere.dimensions == 2 + + raises(ValueError, lambda: ParametricRegion((a*t**2, 2*a*t), (a, -2))) + raises(ValueError, lambda: ParametricRegion((a, b), (a**2, sin(b)), (a, 2, 4, 6))) + + +def test_parametric_region_list(): + + point = Point(-5, 12) + assert parametric_region_list(point) == [ParametricRegion((-5, 12))] + + e = Ellipse(Point(2, 8), 2, 6) + assert parametric_region_list(e, t) == [ParametricRegion((2*cos(t) + 2, 6*sin(t) + 8), (t, 0, 2*pi))] + + c = Curve((t, t**3), (t, 5, 3)) + assert parametric_region_list(c) == [ParametricRegion((t, t**3), (t, 5, 3))] + + s = Segment(Point(2, 11, -6), Point(0, 2, 5)) + assert parametric_region_list(s, t) == [ParametricRegion((2 - 2*t, 11 - 9*t, 11*t - 6), (t, 0, 1))] + s1 = Segment(Point(0, 0), (1, 0)) + assert parametric_region_list(s1, t) == [ParametricRegion((t, 0), (t, 0, 1))] + s2 = Segment(Point(1, 2, 3), Point(1, 2, 5)) + assert parametric_region_list(s2, t) == [ParametricRegion((1, 2, 2*t + 3), (t, 0, 1))] + s3 = Segment(Point(12, 56), Point(12, 56)) + assert parametric_region_list(s3) == [ParametricRegion((12, 56))] + + poly = Polygon((1,3), (-3, 8), (2, 4)) + assert parametric_region_list(poly, t) == [ParametricRegion((1 - 4*t, 5*t + 3), (t, 0, 1)), ParametricRegion((5*t - 3, 8 - 4*t), (t, 0, 1)), ParametricRegion((2 - t, 4 - t), (t, 0, 1))] + + p1 = Parabola(Point(0, 0), Line(Point(5, 8), Point(7,8))) + raises(ValueError, lambda: parametric_region_list(p1)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_printing.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_printing.py new file mode 100644 index 0000000000000000000000000000000000000000..ae76905e967bdf93485f135c6a69f968e1208986 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_printing.py @@ -0,0 +1,221 @@ +# -*- coding: utf-8 -*- +from sympy.core.function import Function +from sympy.integrals.integrals import Integral +from sympy.printing.latex import latex +from sympy.printing.pretty import pretty as xpretty +from sympy.vector import CoordSys3D, Del, Vector, express +from sympy.abc import a, b, c +from sympy.testing.pytest import XFAIL + + +def pretty(expr): + """ASCII pretty-printing""" + return xpretty(expr, use_unicode=False, wrap_line=False) + + +def upretty(expr): + """Unicode pretty-printing""" + return xpretty(expr, use_unicode=True, wrap_line=False) + + +# Initialize the basic and tedious vector/dyadic expressions +# needed for testing. +# Some of the pretty forms shown denote how the expressions just +# above them should look with pretty printing. +N = CoordSys3D('N') +C = N.orient_new_axis('C', a, N.k) # type: ignore +v = [] +d = [] +v.append(Vector.zero) +v.append(N.i) # type: ignore +v.append(-N.i) # type: ignore +v.append(N.i + N.j) # type: ignore +v.append(a*N.i) # type: ignore +v.append(a*N.i - b*N.j) # type: ignore +v.append((a**2 + N.x)*N.i + N.k) # type: ignore +v.append((a**2 + b)*N.i + 3*(C.y - c)*N.k) # type: ignore +f = Function('f') +v.append(N.j - (Integral(f(b)) - C.x**2)*N.k) # type: ignore +upretty_v_8 = """\ + ⎛ 2 ⌠ ⎞ \n\ +j_N + ⎜x_C - ⎮ f(b) db⎟ k_N\n\ + ⎝ ⌡ ⎠ \ +""" +pretty_v_8 = """\ +j_N + / / \\\n\ + | 2 | |\n\ + |x_C - | f(b) db|\n\ + | | |\n\ + \\ / / \ +""" + +v.append(N.i + C.k) # type: ignore +v.append(express(N.i, C)) # type: ignore +v.append((a**2 + b)*N.i + (Integral(f(b)))*N.k) # type: ignore +upretty_v_11 = """\ +⎛ 2 ⎞ ⎛⌠ ⎞ \n\ +⎝a + b⎠ i_N + ⎜⎮ f(b) db⎟ k_N\n\ + ⎝⌡ ⎠ \ +""" +pretty_v_11 = """\ +/ 2 \\ + / / \\\n\ +\\a + b/ i_N| | |\n\ + | | f(b) db|\n\ + | | |\n\ + \\/ / \ +""" + +for x in v: + d.append(x | N.k) # type: ignore +s = 3*N.x**2*C.y # type: ignore +upretty_s = """\ + 2\n\ +3⋅y_C⋅x_N \ +""" +pretty_s = """\ + 2\n\ +3*y_C*x_N \ +""" + +# This is the pretty form for ((a**2 + b)*N.i + 3*(C.y - c)*N.k) | N.k +upretty_d_7 = """\ +⎛ 2 ⎞ \n\ +⎝a + b⎠ (i_N|k_N) + (3⋅y_C - 3⋅c) (k_N|k_N)\ +""" +pretty_d_7 = """\ +/ 2 \\ (i_N|k_N) + (3*y_C - 3*c) (k_N|k_N)\n\ +\\a + b/ \ +""" + + +def test_str_printing(): + assert str(v[0]) == '0' + assert str(v[1]) == 'N.i' + assert str(v[2]) == '(-1)*N.i' + assert str(v[3]) == 'N.i + N.j' + assert str(v[8]) == 'N.j + (C.x**2 - Integral(f(b), b))*N.k' + assert str(v[9]) == 'C.k + N.i' + assert str(s) == '3*C.y*N.x**2' + assert str(d[0]) == '0' + assert str(d[1]) == '(N.i|N.k)' + assert str(d[4]) == 'a*(N.i|N.k)' + assert str(d[5]) == 'a*(N.i|N.k) + (-b)*(N.j|N.k)' + assert str(d[8]) == ('(N.j|N.k) + (C.x**2 - ' + + 'Integral(f(b), b))*(N.k|N.k)') + + +@XFAIL +def test_pretty_printing_ascii(): + assert pretty(v[0]) == '0' + assert pretty(v[1]) == 'i_N' + assert pretty(v[5]) == '(a) i_N + (-b) j_N' + assert pretty(v[8]) == pretty_v_8 + assert pretty(v[2]) == '(-1) i_N' + assert pretty(v[11]) == pretty_v_11 + assert pretty(s) == pretty_s + assert pretty(d[0]) == '(0|0)' + assert pretty(d[5]) == '(a) (i_N|k_N) + (-b) (j_N|k_N)' + assert pretty(d[7]) == pretty_d_7 + assert pretty(d[10]) == '(cos(a)) (i_C|k_N) + (-sin(a)) (j_C|k_N)' + + +def test_pretty_print_unicode_v(): + assert upretty(v[0]) == '0' + assert upretty(v[1]) == 'i_N' + assert upretty(v[5]) == '(a) i_N + (-b) j_N' + # Make sure the printing works in other objects + assert upretty(v[5].args) == '((a) i_N, (-b) j_N)' + assert upretty(v[8]) == upretty_v_8 + assert upretty(v[2]) == '(-1) i_N' + assert upretty(v[11]) == upretty_v_11 + assert upretty(s) == upretty_s + assert upretty(d[0]) == '(0|0)' + assert upretty(d[5]) == '(a) (i_N|k_N) + (-b) (j_N|k_N)' + assert upretty(d[7]) == upretty_d_7 + assert upretty(d[10]) == '(cos(a)) (i_C|k_N) + (-sin(a)) (j_C|k_N)' + + +def test_latex_printing(): + assert latex(v[0]) == '\\mathbf{\\hat{0}}' + assert latex(v[1]) == '\\mathbf{\\hat{i}_{N}}' + assert latex(v[2]) == '- \\mathbf{\\hat{i}_{N}}' + assert latex(v[5]) == ('\\left(a\\right)\\mathbf{\\hat{i}_{N}} + ' + + '\\left(- b\\right)\\mathbf{\\hat{j}_{N}}') + assert latex(v[6]) == ('\\left(\\mathbf{{x}_{N}} + a^{2}\\right)\\mathbf{\\hat{i}_' + + '{N}} + \\mathbf{\\hat{k}_{N}}') + assert latex(v[8]) == ('\\mathbf{\\hat{j}_{N}} + \\left(\\mathbf{{x}_' + + '{C}}^{2} - \\int f{\\left(b \\right)}\\,' + + ' db\\right)\\mathbf{\\hat{k}_{N}}') + assert latex(s) == '3 \\mathbf{{y}_{C}} \\mathbf{{x}_{N}}^{2}' + assert latex(d[0]) == '(\\mathbf{\\hat{0}}|\\mathbf{\\hat{0}})' + assert latex(d[4]) == ('\\left(a\\right)\\left(\\mathbf{\\hat{i}_{N}}{\\middle|}' + + '\\mathbf{\\hat{k}_{N}}\\right)') + assert latex(d[9]) == ('\\left(\\mathbf{\\hat{k}_{C}}{\\middle|}' + + '\\mathbf{\\hat{k}_{N}}\\right) + \\left(' + + '\\mathbf{\\hat{i}_{N}}{\\middle|}\\mathbf{' + + '\\hat{k}_{N}}\\right)') + assert latex(d[11]) == ('\\left(a^{2} + b\\right)\\left(\\mathbf{\\hat{i}_{N}}' + + '{\\middle|}\\mathbf{\\hat{k}_{N}}\\right) + ' + + '\\left(\\int f{\\left(b \\right)}\\, db\\right)\\left(' + + '\\mathbf{\\hat{k}_{N}}{\\middle|}\\mathbf{' + + '\\hat{k}_{N}}\\right)') + +def test_issue_23058(): + from sympy import symbols, sin, cos, pi, UnevaluatedExpr + + delop = Del() + CC_ = CoordSys3D("C") + y = CC_.y + xhat = CC_.i + + t = symbols("t") + ten = symbols("10", positive=True) + eps, mu = 4*pi*ten**(-11), ten**(-5) + + Bx = 2 * ten**(-4) * cos(ten**5 * t) * sin(ten**(-3) * y) + vecB = Bx * xhat + vecE = (1/eps) * Integral(delop.cross(vecB/mu).doit(), t) + vecE = vecE.doit() + + vecB_str = """\ +⎛ ⎛y_C⎞ ⎛ 5 ⎞⎞ \n\ +⎜2⋅sin⎜───⎟⋅cos⎝10 ⋅t⎠⎟ i_C\n\ +⎜ ⎜ 3⎟ ⎟ \n\ +⎜ ⎝10 ⎠ ⎟ \n\ +⎜─────────────────────⎟ \n\ +⎜ 4 ⎟ \n\ +⎝ 10 ⎠ \ +""" + vecE_str = """\ +⎛ 4 ⎛ 5 ⎞ ⎛y_C⎞ ⎞ \n\ +⎜-10 ⋅sin⎝10 ⋅t⎠⋅cos⎜───⎟ ⎟ k_C\n\ +⎜ ⎜ 3⎟ ⎟ \n\ +⎜ ⎝10 ⎠ ⎟ \n\ +⎜─────────────────────────⎟ \n\ +⎝ 2⋅π ⎠ \ +""" + + assert upretty(vecB) == vecB_str + assert upretty(vecE) == vecE_str + + ten = UnevaluatedExpr(10) + eps, mu = 4*pi*ten**(-11), ten**(-5) + + Bx = 2 * ten**(-4) * cos(ten**5 * t) * sin(ten**(-3) * y) + vecB = Bx * xhat + + vecB_str = """\ +⎛ -4 ⎛ 5⎞ ⎛ -3⎞⎞ \n\ +⎝2⋅10 ⋅cos⎝t⋅10 ⎠⋅sin⎝y_C⋅10 ⎠⎠ i_C \ +""" + assert upretty(vecB) == vecB_str + +def test_custom_names(): + A = CoordSys3D('A', vector_names=['x', 'y', 'z'], + variable_names=['i', 'j', 'k']) + assert A.i.__str__() == 'A.i' + assert A.x.__str__() == 'A.x' + assert A.i._pretty_form == 'i_A' + assert A.x._pretty_form == 'x_A' + assert A.i._latex_form == r'\mathbf{{i}_{A}}' + assert A.x._latex_form == r"\mathbf{\hat{x}_{A}}" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_vector.py b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_vector.py new file mode 100644 index 0000000000000000000000000000000000000000..daba6d6a02c87b41a8bf801eee9b9045897d0003 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/sympy/vector/tests/test_vector.py @@ -0,0 +1,342 @@ +from sympy.core import Rational, S, Add, Mul, I +from sympy.simplify import simplify, trigsimp +from sympy.core.function import (Derivative, Function, diff) +from sympy.core.numbers import pi +from sympy.core.symbol import symbols +from sympy.functions.elementary.miscellaneous import sqrt +from sympy.functions.elementary.trigonometric import (cos, sin) +from sympy.integrals.integrals import Integral +from sympy.matrices.immutable import ImmutableDenseMatrix as Matrix +from sympy.vector.vector import Vector, BaseVector, VectorAdd, \ + VectorMul, VectorZero +from sympy.vector.coordsysrect import CoordSys3D +from sympy.vector.vector import Cross, Dot, cross +from sympy.testing.pytest import raises +from sympy.vector.kind import VectorKind +from sympy.core.kind import NumberKind +from sympy.testing.pytest import XFAIL + + +C = CoordSys3D('C') + +i, j, k = C.base_vectors() +a, b, c = symbols('a b c') + + +def test_cross(): + v1 = C.x * i + C.z * C.z * j + v2 = C.x * i + C.y * j + C.z * k + assert Cross(v1, v2) == Cross(C.x*C.i + C.z**2*C.j, C.x*C.i + C.y*C.j + C.z*C.k) + assert Cross(v1, v2).doit() == C.z**3*C.i + (-C.x*C.z)*C.j + (C.x*C.y - C.x*C.z**2)*C.k + assert cross(v1, v2) == C.z**3*C.i + (-C.x*C.z)*C.j + (C.x*C.y - C.x*C.z**2)*C.k + assert Cross(v1, v2) == -Cross(v2, v1) + # XXX: Cannot use Cross here. See XFAIL test below: + assert cross(v1, v2) + cross(v2, v1) == Vector.zero + + +@XFAIL +def test_cross_xfail(): + v1 = C.x * i + C.z * C.z * j + v2 = C.x * i + C.y * j + C.z * k + assert Cross(v1, v2) + Cross(v2, v1) == Vector.zero + + +def test_dot(): + v1 = C.x * i + C.z * C.z * j + v2 = C.x * i + C.y * j + C.z * k + assert Dot(v1, v2) == Dot(C.x*C.i + C.z**2*C.j, C.x*C.i + C.y*C.j + C.z*C.k) + assert Dot(v1, v2).doit() == C.x**2 + C.y*C.z**2 + assert Dot(v2, v1).doit() == C.x**2 + C.y*C.z**2 + assert Dot(v1, v2) == Dot(v2, v1) + + +def test_vector_sympy(): + """ + Test whether the Vector framework confirms to the hashing + and equality testing properties of SymPy. + """ + v1 = 3*j + assert v1 == j*3 + assert v1.components == {j: 3} + v2 = 3*i + 4*j + 5*k + v3 = 2*i + 4*j + i + 4*k + k + assert v3 == v2 + assert v3.__hash__() == v2.__hash__() + + +def test_kind(): + assert C.i.kind is VectorKind(NumberKind) + assert C.j.kind is VectorKind(NumberKind) + assert C.k.kind is VectorKind(NumberKind) + + assert C.x.kind is NumberKind + assert C.y.kind is NumberKind + assert C.z.kind is NumberKind + + assert Mul._kind_dispatcher(NumberKind, VectorKind(NumberKind)) is VectorKind(NumberKind) + assert Mul(2, C.i).kind is VectorKind(NumberKind) + + v1 = C.x * i + C.z * C.z * j + v2 = C.x * i + C.y * j + C.z * k + assert v1.kind is VectorKind(NumberKind) + assert v2.kind is VectorKind(NumberKind) + + assert (v1 + v2).kind is VectorKind(NumberKind) + assert Add(v1, v2).kind is VectorKind(NumberKind) + assert Cross(v1, v2).doit().kind is VectorKind(NumberKind) + assert VectorAdd(v1, v2).kind is VectorKind(NumberKind) + assert VectorMul(2, v1).kind is VectorKind(NumberKind) + assert VectorZero().kind is VectorKind(NumberKind) + + assert v1.projection(v2).kind is VectorKind(NumberKind) + assert v2.projection(v1).kind is VectorKind(NumberKind) + + +def test_vectoradd(): + assert isinstance(Add(C.i, C.j), VectorAdd) + v1 = C.x * i + C.z * C.z * j + v2 = C.x * i + C.y * j + C.z * k + assert isinstance(Add(v1, v2), VectorAdd) + + # https://github.com/sympy/sympy/issues/26121 + + E = Matrix([C.i, C.j, C.k]).T + a = Matrix([1, 2, 3]) + av = E*a + + assert av[0].kind == VectorKind() + assert isinstance(av[0], VectorAdd) + + +def test_vector(): + assert isinstance(i, BaseVector) + assert i != j + assert j != k + assert k != i + assert i - i == Vector.zero + assert i + Vector.zero == i + assert i - Vector.zero == i + assert Vector.zero != 0 + assert -Vector.zero == Vector.zero + + v1 = a*i + b*j + c*k + v2 = a**2*i + b**2*j + c**2*k + v3 = v1 + v2 + v4 = 2 * v1 + v5 = a * i + + assert isinstance(v1, VectorAdd) + assert v1 - v1 == Vector.zero + assert v1 + Vector.zero == v1 + assert v1.dot(i) == a + assert v1.dot(j) == b + assert v1.dot(k) == c + assert i.dot(v2) == a**2 + assert j.dot(v2) == b**2 + assert k.dot(v2) == c**2 + assert v3.dot(i) == a**2 + a + assert v3.dot(j) == b**2 + b + assert v3.dot(k) == c**2 + c + + assert v1 + v2 == v2 + v1 + assert v1 - v2 == -1 * (v2 - v1) + assert a * v1 == v1 * a + + assert isinstance(v5, VectorMul) + assert v5.base_vector == i + assert v5.measure_number == a + assert isinstance(v4, Vector) + assert isinstance(v4, VectorAdd) + assert isinstance(v4, Vector) + assert isinstance(Vector.zero, VectorZero) + assert isinstance(Vector.zero, Vector) + assert isinstance(v1 * 0, VectorZero) + + assert v1.to_matrix(C) == Matrix([[a], [b], [c]]) + + assert i.components == {i: 1} + assert v5.components == {i: a} + assert v1.components == {i: a, j: b, k: c} + + assert VectorAdd(v1, Vector.zero) == v1 + assert VectorMul(a, v1) == v1*a + assert VectorMul(1, i) == i + assert VectorAdd(v1, Vector.zero) == v1 + assert VectorMul(0, Vector.zero) == Vector.zero + raises(TypeError, lambda: v1.outer(1)) + raises(TypeError, lambda: v1.dot(1)) + + +def test_vector_magnitude_normalize(): + assert Vector.zero.magnitude() == 0 + assert Vector.zero.normalize() == Vector.zero + + assert i.magnitude() == 1 + assert j.magnitude() == 1 + assert k.magnitude() == 1 + assert i.normalize() == i + assert j.normalize() == j + assert k.normalize() == k + + v1 = a * i + assert v1.normalize() == (a/sqrt(a**2))*i + assert v1.magnitude() == sqrt(a**2) + + v2 = a*i + b*j + c*k + assert v2.magnitude() == sqrt(a**2 + b**2 + c**2) + assert v2.normalize() == v2 / v2.magnitude() + + v3 = i + j + assert v3.normalize() == (sqrt(2)/2)*C.i + (sqrt(2)/2)*C.j + + +def test_vector_simplify(): + A, s, k, m = symbols('A, s, k, m') + + test1 = (1 / a + 1 / b) * i + assert (test1 & i) != (a + b) / (a * b) + test1 = simplify(test1) + assert (test1 & i) == (a + b) / (a * b) + assert test1.simplify() == simplify(test1) + + test2 = (A**2 * s**4 / (4 * pi * k * m**3)) * i + test2 = simplify(test2) + assert (test2 & i) == (A**2 * s**4 / (4 * pi * k * m**3)) + + test3 = ((4 + 4 * a - 2 * (2 + 2 * a)) / (2 + 2 * a)) * i + test3 = simplify(test3) + assert (test3 & i) == 0 + + test4 = ((-4 * a * b**2 - 2 * b**3 - 2 * a**2 * b) / (a + b)**2) * i + test4 = simplify(test4) + assert (test4 & i) == -2 * b + + v = (sin(a)+cos(a))**2*i - j + assert trigsimp(v) == (2*sin(a + pi/4)**2)*i + (-1)*j + assert trigsimp(v) == v.trigsimp() + + assert simplify(Vector.zero) == Vector.zero + + +def test_vector_equals(): + assert (2*i).equals(j) is False + assert i.equals(i) is True + + # https://github.com/sympy/sympy/issues/25915 + A = (sqrt(2) + sqrt(6)) / sqrt(sqrt(3) + 2) + assert (A*i).equals(2*i) is True + assert (A*i).equals(3*i) is False + + # Test comparing vectors in different coordinate systems + D = C.orient_new_axis('D', pi/2, C.k) + assert (D.i).equals(C.j) is True + assert (D.i).equals(C.i) is False + + +def test_vector_conjugate(): + # https://github.com/sympy/sympy/issues/27094 + assert (I*i + (1 + I)*j + 2*k).conjugate() == -I*i + (1 - I)*j + 2*k + + +def test_vector_dot(): + assert i.dot(Vector.zero) == 0 + assert Vector.zero.dot(i) == 0 + assert i & Vector.zero == 0 + + assert i.dot(i) == 1 + assert i.dot(j) == 0 + assert i.dot(k) == 0 + assert i & i == 1 + assert i & j == 0 + assert i & k == 0 + + assert j.dot(i) == 0 + assert j.dot(j) == 1 + assert j.dot(k) == 0 + assert j & i == 0 + assert j & j == 1 + assert j & k == 0 + + assert k.dot(i) == 0 + assert k.dot(j) == 0 + assert k.dot(k) == 1 + assert k & i == 0 + assert k & j == 0 + assert k & k == 1 + + raises(TypeError, lambda: k.dot(1)) + + +def test_vector_cross(): + assert i.cross(Vector.zero) == Vector.zero + assert Vector.zero.cross(i) == Vector.zero + + assert i.cross(i) == Vector.zero + assert i.cross(j) == k + assert i.cross(k) == -j + assert i ^ i == Vector.zero + assert i ^ j == k + assert i ^ k == -j + + assert j.cross(i) == -k + assert j.cross(j) == Vector.zero + assert j.cross(k) == i + assert j ^ i == -k + assert j ^ j == Vector.zero + assert j ^ k == i + + assert k.cross(i) == j + assert k.cross(j) == -i + assert k.cross(k) == Vector.zero + assert k ^ i == j + assert k ^ j == -i + assert k ^ k == Vector.zero + + assert k.cross(1) == Cross(k, 1) + + +def test_projection(): + v1 = i + j + k + v2 = 3*i + 4*j + v3 = 0*i + 0*j + assert v1.projection(v1) == i + j + k + assert v1.projection(v2) == Rational(7, 3)*C.i + Rational(7, 3)*C.j + Rational(7, 3)*C.k + assert v1.projection(v1, scalar=True) == S.One + assert v1.projection(v2, scalar=True) == Rational(7, 3) + assert v3.projection(v1) == Vector.zero + assert v3.projection(v1, scalar=True) == S.Zero + + +def test_vector_diff_integrate(): + f = Function('f') + v = f(a)*C.i + a**2*C.j - C.k + assert Derivative(v, a) == Derivative((f(a))*C.i + + a**2*C.j + (-1)*C.k, a) + assert (diff(v, a) == v.diff(a) == Derivative(v, a).doit() == + (Derivative(f(a), a))*C.i + 2*a*C.j) + assert (Integral(v, a) == (Integral(f(a), a))*C.i + + (Integral(a**2, a))*C.j + (Integral(-1, a))*C.k) + + +def test_vector_args(): + raises(ValueError, lambda: BaseVector(3, C)) + raises(TypeError, lambda: BaseVector(0, Vector.zero)) + + +def test_srepr(): + from sympy.printing.repr import srepr + res = "CoordSys3D(Str('C'), Tuple(ImmutableDenseMatrix([[Integer(1), "\ + "Integer(0), Integer(0)], [Integer(0), Integer(1), Integer(0)], "\ + "[Integer(0), Integer(0), Integer(1)]]), VectorZero())).i" + assert srepr(C.i) == res + + +def test_scalar(): + from sympy.vector import CoordSys3D + C = CoordSys3D('C') + v1 = 3*C.i + 4*C.j + 5*C.k + v2 = 3*C.i - 4*C.j + 5*C.k + assert v1.is_Vector is True + assert v1.is_scalar is False + assert (v1.dot(v2)).is_scalar is True + assert (v1.cross(v2)).is_scalar is False diff --git a/TTS/Kokoro/.venv/Lib/site-packages/termcolor-3.3.0.dist-info/licenses/COPYING.txt b/TTS/Kokoro/.venv/Lib/site-packages/termcolor-3.3.0.dist-info/licenses/COPYING.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0b79705c354418108635b67e17ee15443a4a130 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/termcolor-3.3.0.dist-info/licenses/COPYING.txt @@ -0,0 +1,19 @@ +Copyright (c) 2008-2011 Volvox Development Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b711386c3cc703eb72a2322843921d46701448c2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__main__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90e1ed1ce4a320228c48e4f864c0f04c834dda13 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/__main__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/termcolor.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/termcolor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d129ff399966e1596864c9c719795b1c72a971ff Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/termcolor/__pycache__/termcolor.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc-8.3.10.dist-info/licenses/LICENSE b/TTS/Kokoro/.venv/Lib/site-packages/thinc-8.3.10.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6083afcde0e3af84d40cef0f5e0a3e50c3921ba9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc-8.3.10.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (C) 2016 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..458ee6eb28e5d8fe2048cfc41be3c746bd1fd54b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/about.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/about.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ef283613479cd1e3c18961fd22253fc0e3d319d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/about.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/api.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0026ff502899f115fe5586d079c7a55cf94f184 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/api.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/compat.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/compat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af6043441d2e3ffe1ae92e8394dd91ab5535979f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/compat.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/config.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f105c3e70aba1faa6695f5c86e126e776cf9cb7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/config.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/initializers.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/initializers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e2cfd6e89842fdc072c9e574f259624fc5543b1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/initializers.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/loss.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/loss.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11a469a95b308a647a8c511eb5d7c3a8123d5f7f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/loss.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/model.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66c9b847d89bb09367bd3ee8209ef3f28692c908 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/model.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/mypy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/mypy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a027581ae220fcfc3f2fe0ad45db9562434deaf7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/mypy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/optimizers.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/optimizers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..955bba2b753e2ef1ecc7268df6054aaaf63f3b35 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/optimizers.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/schedules.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/schedules.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf3971a7d109d877f025302cb4c4bfae6fe5449e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/schedules.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/types.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a08398601818ec5a9f79c31865e0765a1bc2765 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/types.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/util.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9dcab6cdbeca0c9333164e5c2c80a5ad207d716 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/__pycache__/util.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__init__.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__init__.pxd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e9f65b71f766be4971e7b713c07af83410a77a91 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__init__.py @@ -0,0 +1,183 @@ +import contextlib +import threading +from contextvars import ContextVar +from typing import Any, Callable, Dict, Optional, Type, cast + +from .. import registry +from ..compat import cupy, has_cupy +from ..util import ( + assert_pytorch_installed, + assert_tensorflow_installed, + get_torch_default_device, + is_cupy_array, + require_cpu, +) +from ._cupy_allocators import cupy_pytorch_allocator, cupy_tensorflow_allocator +from ._param_server import ParamServer +from .cupy_ops import CupyOps +from .mps_ops import MPSOps +from .numpy_ops import NumpyOps +from .ops import Ops + +context_ops: ContextVar[Optional[Ops]] = ContextVar("context_ops", default=None) +context_pools: ContextVar[dict] = ContextVar("context_pools", default={}) + +# Internal use of thread-local storage only for detecting cases where a Jupyter +# notebook might not have preserved contextvars across cells. +_GLOBAL_STATE = {"ops": None} + + +def set_gpu_allocator(allocator: str) -> None: # pragma: no cover + """Route GPU memory allocation via PyTorch or tensorflow. + Raise an error if the given argument does not match either of the two. + """ + if allocator == "pytorch": + use_pytorch_for_gpu_memory() + elif allocator == "tensorflow": + use_tensorflow_for_gpu_memory() + else: + raise ValueError( + f"Invalid 'gpu_allocator' argument: '{allocator}'. Available allocators are: 'pytorch', 'tensorflow'" + ) + + +def use_pytorch_for_gpu_memory() -> None: # pragma: no cover + """Route GPU memory allocation via PyTorch. + + This is recommended for using PyTorch and cupy together, as otherwise + OOM errors can occur when there's available memory sitting in the other + library's pool. + + We'd like to support routing Tensorflow memory allocation via PyTorch as well + (or vice versa), but do not currently have an implementation for it. + """ + assert_pytorch_installed() + + if get_torch_default_device().type != "cuda": + return + + pools = context_pools.get() + if "pytorch" not in pools: + pools["pytorch"] = cupy.cuda.MemoryPool(allocator=cupy_pytorch_allocator) + cupy.cuda.set_allocator(pools["pytorch"].malloc) + + +def use_tensorflow_for_gpu_memory() -> None: # pragma: no cover + """Route GPU memory allocation via TensorFlow. + + This is recommended for using TensorFlow and cupy together, as otherwise + OOM errors can occur when there's available memory sitting in the other + library's pool. + + We'd like to support routing PyTorch memory allocation via Tensorflow as + well (or vice versa), but do not currently have an implementation for it. + """ + assert_tensorflow_installed() + pools = context_pools.get() + if "tensorflow" not in pools: + pools["tensorflow"] = cupy.cuda.MemoryPool(allocator=cupy_tensorflow_allocator) + cupy.cuda.set_allocator(pools["tensorflow"].malloc) + + +def _import_extra_cpu_backends(): + try: + from thinc_apple_ops import AppleOps + except ImportError: + pass + try: + from thinc_bigendian_ops import BigEndianOps + except ImportError: + pass + + +def get_ops(name: str, **kwargs) -> Ops: + """Get a backend object. + + The special name "cpu" returns the best available CPU backend.""" + + ops_by_name = {ops_cls.name: ops_cls for ops_cls in registry.ops.get_all().values()} # type: ignore + + cls: Optional[Callable[..., Ops]] = None + if name == "cpu": + _import_extra_cpu_backends() + cls = ops_by_name.get("numpy") + cls = ops_by_name.get("apple", cls) + cls = ops_by_name.get("bigendian", cls) + else: + cls = ops_by_name.get(name) + + if cls is None: + raise ValueError(f"Invalid backend: {name}") + + return cls(**kwargs) + + +def get_array_ops(arr): + """Return CupyOps for a cupy array, NumpyOps otherwise.""" + if is_cupy_array(arr): + return CupyOps() + else: + return NumpyOps() + + +@contextlib.contextmanager +def use_ops(name: str, **kwargs): + """Change the backend to execute on for the scope of the block.""" + current_ops = get_current_ops() + set_current_ops(get_ops(name, **kwargs)) + try: + yield + finally: + set_current_ops(current_ops) + + +def get_current_ops() -> Ops: + """Get the current backend object.""" + if context_ops.get() is None: + require_cpu() + return cast(Ops, context_ops.get()) + + +def set_current_ops(ops: Ops) -> None: + """Change the current backend object.""" + context_ops.set(ops) + _get_thread_state().ops = ops + + +def contextvars_eq_thread_ops() -> bool: + current_ops = context_ops.get() + thread_ops = _get_thread_state().ops + if type(current_ops) == type(thread_ops): + return True + return False + + +def _get_thread_state(): + """Get a thread-specific state variable that inherits from a global + state when it's created.""" + thread: threading.Thread = threading.current_thread() + if not hasattr(thread, "__local"): + thread.__local = _create_thread_local(_GLOBAL_STATE) + return thread.__local + + +def _create_thread_local( + attrs: Dict[str, Any], local_class: Type[threading.local] = threading.local +): + obj = local_class() + for name, value in attrs.items(): + setattr(obj, name, value) + return obj + + +__all__ = [ + "set_current_ops", + "get_current_ops", + "use_ops", + "ParamServer", + "Ops", + "CupyOps", + "MPSOps", + "NumpyOps", + "has_cupy", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93d75e0f41eff56f2bef099ad35eb76e98b2abb5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_cupy_allocators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_cupy_allocators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff0a58d164f04ecf9b198f9ce4b15227d767e462 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_cupy_allocators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_custom_kernels.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_custom_kernels.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6442b9f4e8d62f9c5f844fc80446215e7295bbfc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_custom_kernels.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_param_server.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_param_server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..abd6feb3d09257364cc006c779d41aae6561311c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/_param_server.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/cupy_ops.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/cupy_ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4d2fe85ba96c0bb1a7bc7f1d9167d76f8bd1f18 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/cupy_ops.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/mps_ops.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/mps_ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa8842dad9d3b99fa2d3c70dfd510995cd8bcab0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/mps_ops.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/ops.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d4e5295f7fce693f505ad92cced35b5ee97fd51 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/__pycache__/ops.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_cupy_allocators.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_cupy_allocators.py new file mode 100644 index 0000000000000000000000000000000000000000..9049ef6c5c910965ad629b19ec4cfcb5656c072b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_cupy_allocators.py @@ -0,0 +1,46 @@ +from typing import cast + +from ..compat import cupy, tensorflow, torch +from ..types import ArrayXd +from ..util import get_torch_default_device, tensorflow2xp + + +def cupy_tensorflow_allocator(size_in_bytes: int): + """Function that can be passed into cupy.cuda.set_allocator, to have cupy + allocate memory via TensorFlow. This is important when using the two libraries + together, as otherwise OOM errors can occur when there's available memory + sitting in the other library's pool. + """ + size_in_bytes = max(1024, size_in_bytes) + tensor = tensorflow.zeros((size_in_bytes // 4,), dtype=tensorflow.dtypes.float32) # type: ignore + # We convert to cupy via dlpack, so that we can get a memory pointer. + cupy_array = cast(ArrayXd, tensorflow2xp(tensor)) + address = int(cupy_array.data) + # cupy has a neat class to help us here. Otherwise it will try to free. + memory = cupy.cuda.memory.UnownedMemory(address, size_in_bytes, cupy_array) + # Now return a new memory pointer. + return cupy.cuda.memory.MemoryPointer(memory, 0) + + +def cupy_pytorch_allocator(size_in_bytes: int): + device = get_torch_default_device() + """Function that can be passed into cupy.cuda.set_allocator, to have cupy + allocate memory via PyTorch. This is important when using the two libraries + together, as otherwise OOM errors can occur when there's available memory + sitting in the other library's pool. + """ + # Cupy was having trouble with very small allocations? + size_in_bytes = max(1024, size_in_bytes) + # We use pytorch's underlying FloatStorage type to avoid overhead from + # creating a whole Tensor. + # This turns out to be way faster than making FloatStorage? Maybe + # a Python vs C++ thing I guess? + torch_tensor = torch.zeros( + (size_in_bytes // 4,), requires_grad=False, device=device + ) + # cupy has a neat class to help us here. Otherwise it will try to free. + # I think this is a private API? It's not in the types. + address = torch_tensor.data_ptr() # type: ignore + memory = cupy.cuda.memory.UnownedMemory(address, size_in_bytes, torch_tensor) + # Now return a new memory pointer. + return cupy.cuda.memory.MemoryPointer(memory, 0) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.cu b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.cu new file mode 100644 index 0000000000000000000000000000000000000000..cb39b47bcc649a3673361ebb4016e7a8c56a25d3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.cu @@ -0,0 +1,658 @@ +// Use grid strided loops, described here: +// https://devblogs.nvidia.com/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/ +// This pattern ensures that all of the loop values are visited once, no matter +// what grid parameters are used for the function. + +// We cannot include CUDA header for mathematical constants, since it requires +// that the development headers of the CUDA toolkit are installed. + +template +struct Constants {}; + +template <> +struct Constants { + static constexpr double INV_SQRT_2 = 0.7071067811865475; + static constexpr double INV_SQRT_2PI = 0.3989422804014327; +}; + +template <> +struct Constants { + static constexpr float INV_SQRT_2 = 0.70710677; + static constexpr float INV_SQRT_2PI = 0.3989423; +}; + + +template +__global__ void gather_add(U* out_bo, const U* table_to, const int* indices_bk, + int T, int O, int B, int K) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int b = _loop_start; b < B; b += _loop_stride) { + for (int k = 0; k < K; ++k) { + int idx = indices_bk[b * K + k]; + const U* table = table_to + idx * O; + U* out = out_bo + b * O; + for (int o = 0; o < O; ++o) { + out[o] += table[o]; + } + } + } +} + + +template +__global__ void seq2col(T* output, const T* X, const int* lengths, + int nW, int B, int I, int nL) +{ + // Let's say nW is 1 (it usually is). Then we want to take: + + // 1a 1b 1c + // 2a 2b 2c + // 3a 3b 3c + + // And make + + // __ __ __ 1a 1b 1c 2a 2b 2c + // 1a 1b 1c 2a 2b 2c 3a 3b 3c + // 2a 2b 2c 3a 3b 3c __ __ __ + + // Where __ is padding. + + // Now let's say nW is 2. Then we want to take: + + // 1a 1b 1c + // 2a 2b 2c + // 3a 3b 3c + + // And make + + // __ __ __ __ __ __ 1a 1b 1c 2a 2b 2c 3a 3b 3c + // __ __ __ 1a 1b 1c 2a 2b 2c 3a 3b 3c __ __ __ + // 1a 1b 1c 2a 2b 2c 3a 3b 3c __ __ __ __ __ __ + + // * x_start=-6, x_end=9 : (0-2) * 3, (0+2+1) * 3 + // * x_start=-3, x_end=13 : (1-2) * 3, (1+2+1) * 3 + // * x_start=0, x_end=16 : (2-2) * 3, (2+2+1) * 3 + // + // If lengths > 1, then the sequence lengths dictate + // the boundaries/padding rather than the begin/end + // of X. + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + int nF = nW * 2 + 1; + + int seq = 0; + int seq_start = 0; + for (int b = _loop_start; b < B; b += _loop_stride) + { + // Find sequence sequence in which b lies. + for (; seq < nL; ++seq) { + if (b < seq_start + lengths[seq]) { + break; + } + seq_start += lengths[seq]; + } + + // Calculate the bounds of the sequence wherein b lies. + int seq_end = seq_start + lengths[seq]; + + // Find the unconstrained window around b, which + // may be out of the sequence bounds. + int window_start = b - nW; + int window_end = b + nW + 1; + + // Find the sequence-constrained window around b. + int x_start = max(seq_start, window_start); + int x_end = min(seq_end, window_end); + int n_elems = x_end - x_start; + + // If the left window is cut short, we want to start by + // the same amount in the output. + int out_offset = x_start - window_start; + + for (int i = 0; i < n_elems * I; i++) { + output[(b * I * nF) + (out_offset * I) + i] = + X[(x_start * I) + i]; + } + } +} + + +template +__global__ void pad(T* out, T const **seqs, int const *lengths, int stride, int N, int L) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < L * stride; i += _loop_stride) { + for (int j = 0; j < N; ++j) { + T const *seq = seqs[j]; + if (i < lengths[j] * stride) { + out[j * L * stride + i] = seq[i]; + } else { + out[j * L * stride + i] = T(); + } + } + } +} + + +template +__global__ void maxout(T* best, int* which, const T* cands, int B, int O, int P) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int bo = _loop_start; bo < B * O; bo += _loop_stride) + { + // Go to the candidates at the output we're working on + const T* cands_bo = &cands[bo * P]; + + int best_idx = 0; + T best_val = cands_bo[0]; + for (int p = 1; p < P; ++p) + { + if (cands_bo[p] > best_val) { + best_idx = p; + best_val = cands_bo[p]; + } + } + + which[bo] = best_idx; + best[bo] = best_val; + } +} + + +template +__global__ void clipped_linear(T* Y, const T* X, double slope, double offset, double min_val, double max_val, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T y = X[i] * slope + offset; + Y[i] = min(max(y, min_val), max_val); + } +} + + +template +__global__ void dish(T* Y, const T* X, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + Y[i] = 0.5 * x * (x / sqrt(1 + x * x) + 1); + } +} + + +template +__global__ void gelu(T* Y, const T* X, double threshold, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + if (x >= threshold) { + Y[i] = x; + } else if (x <= -threshold) { + Y[i] = 0.0; + } else { + T cdf = 0.5 * (1.0 + erf(Constants::INV_SQRT_2 * x)); + Y[i] = x * cdf; + } + } +} + + +template +__global__ void mish(T* Y, const T* X, double threshold, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + T one = 1.; + for (int i = _loop_start; i < N; i += _loop_stride) + { + if (X[i] >= threshold) + Y[i] = X[i]; + else + Y[i] = X[i] * tanh(log(one + exp(X[i]))); + } +} + + +template +__global__ void swish(T* Y, const T* X, double threshold, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + if (X[i] >= threshold) { + Y[i] = X[i]; + } else if (X[i] <= -threshold) { + Y[i] = 0.0; + } else { + T logistic_cdf = 1.0 / (1.0 + exp(-X[i])); + Y[i] = X[i] * logistic_cdf; + } + } +} + + +template +__global__ void reduce_sum(U* output, const U* X, + const int* lengths, int B, int T, int O) +{ + // Compute sums of a batch of concatenated sequences + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int b = _loop_start; b < B; b += _loop_stride) + { + // Go to the regions we're working on + U* output_b = &output[b*O]; + // Find the sequence item we're working on + int t = 0; + for (int i=0; i < b; ++i) { + t += lengths[i]; + } + int length = lengths[b]; + // Each invocation of the kernel sums one batch. + for (int i=0; i < length; ++i) // Iterate over rows + { + const U* X_t = &X[(t+i)*O]; + for (int j=0; j < O; ++j) + { + output_b[j] += X_t[j]; + } + } + } +} + + +template +__global__ void reduce_max(U* maxes, int* which, + const U* X, const int* lengths, int B, int T, int O) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int b = _loop_start; b < B; b += _loop_stride) + { + + // Go to the regions we're working on + U* maxes_b = &maxes[b*O]; + int* which_b = &which[b*O]; + // Find the sequence item we're working on + const U* X_t = X; + for (int i=0; i < b; ++i) { + X_t += lengths[i] * O; + } + // Each invocation of the kernel maxes one sequence. + // Start by assuming maxes are the first element. + for (int i=0; i < O; ++i) { + maxes_b[i] = X_t[i]; + which_b[i] = 0; + } + int length = lengths[b]; + for (int i=1; i < length; ++i) // Iterate over rows + { + X_t += O; + for (int j=0; j < O; ++j) + { + if (X_t[j] > maxes_b[j]) + { + maxes_b[j] = X_t[j]; + which_b[j] = i; + } + } + } + + } +} + + +template +__global__ void backprop_seq2col(T* d_seqs, const T* d_cols, const int* lengths, + int nW, int B, int I, int nL) +{ + // Here's what we're doing, if we had 2d indexing. + //for i in range(B): + // d_seq[i] += d_cols[i-2, 4] + // d_seq[i] += d_cols[i-1, 3] + // d_seq[i] += d_cols[i, 2] + // d_seq[i] += d_cols[i+1, 1] + // d_seq[i] += d_cols[i+2, 0] + + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + int nF = nW * 2 + 1; + int seq = 0; + int seq_start = 0; + for (int b = _loop_start; b < B; b += _loop_stride) + { + // Find sequence offset in which b lies. + // Fixme: do not restart offset search for every b. + for (; seq < nL; ++seq) { + if (b < seq_start + lengths[seq]) { + break; + } + seq_start += lengths[seq]; + } + + // Calculate the bounds of the sequence wherein b lies. + int seq_end = seq_start + lengths[seq]; + + // Find the unconstrained window around b, which + // may be out of the sequence bounds. + int window_start = b - nW; + int window_end = b + nW + 1; + + // Find the sequence-constrained window around b. + int d_seqs_start = max(seq_start, window_start); + int d_seqs_end = min(seq_end, window_end); + + + // The here update proceeds differently than the other seq2col + // implementations. We have to do all the updates for the b in this loop + // iteration, otherwise we get data races due to parallelism in CUDA. + // + // A batch item b occurs, given nw=1, in: + // + // position 0 in b - 1 (if present) <- window_start + // position 1 in b + // position 2 in b + 1 (if present) <- window_end + // + // The following loop sums the gradients for those occurrences. + // b_w loops over [b - 1, b, b + 1] and computes the position + // of b within the column gradients of [b - 1 ... b + 1]. + for (int b_w = d_seqs_start; b_w < d_seqs_end; ++b_w) { + int position = (2 * nW) - (b_w - window_start); + int start = (b_w * I * nF) + (position * I); + for (int i = 0; i < I; ++i) { + d_seqs[(b*I + i)] += d_cols[start + i]; + } + } + } +} + + +template +__global__ void backprop_clipped_linear(T* dX, const T* dY, const T* X, double slope, double offset, double min_val, double max_val, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + T low = (min_val - offset) / slope; + T high = (max_val - offset) / slope; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + + if (low < x && x < high) { + dX[i] = dY[i] * slope; + } else { + dX[i] = 0; + } + } +} + + +template +__global__ void backprop_hard_swish(T* dX, const T* dY, const T* X, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + if (X[i] > 2.5) { + dX[i] = dY[i]; + } else if (X[i] < -2.5) { + dX[i] = 0; + } else { + dX[i] = dY[i] * (X[i] * 0.4 + 0.5); + } + } +} + + +template +__global__ void backprop_hard_swish_mobilenet(T* dX, const T* dY, const T* X, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + if (X[i] > 3.0) { + dX[i] = dY[i]; + } else if (X[i] < -3.0) { + dX[i] = 0; + } else { + dX[i] = dY[i] * ((X[i] * 2.0 + 3.0) / 6.0); + } + } +} + + +template +__global__ void backprop_dish(T* dX, const T* dY, const T* X, int N) +{ + + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + T x_sq = x * x; + T x_sq_plus_one = x_sq + 1.0; + dX[i] = dY[i] * (x/sqrt(x_sq_plus_one) - (0.5 * x * x_sq) + / pow(x_sq_plus_one, static_cast(1.5)) + 0.5); + } +} + + +template +__global__ void backprop_gelu(T* dX, const T* dY, const T* X, + double threshold, int N) +{ + + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + + if (x >= threshold) { + dX[i] = dY[i]; + } else if (x <= -threshold) { + dX[i] = 0.0; + } else { + T cdf = 0.5 * (1.0 + erf(Constants::INV_SQRT_2 * x)); + T pdf = Constants::INV_SQRT_2PI * exp(-0.5 * x * x); + dX[i] = dY[i] * (cdf + x * pdf); + } + } +} + + +template +__global__ void backprop_maxout(T* dX, + const T* dY, const int* which, int B, int O, int P) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int b = _loop_start; b < B; b += _loop_stride) + { + // Go to the regions we're working on + T* dX_b = &dX[b*O*P]; + const T* dY_b = &dY[b*O]; + const int* which_b = &which[b*O]; + for (int i=0; i < O; ++i) + dX_b[(i*P)+which_b[i]] = dY_b[i]; + } +} + + +template +__global__ void backprop_mish(T* dX, + const T* dY, const T* X, double threshold, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + if (x >= threshold) + { + dX[i] = dY[i]; + } else + { + T exp_x = exp(x); + T exp_2x = exp(2*x); + T exp_3x = exp(3*x); + + T omega = (4. * (x+1)) + (4 * exp_2x) + exp_3x + exp_x * (4.*x+6); + T delta = 2 * exp_x + exp_2x + 2; + dX[i] = dY[i] * ((exp_x * omega) / (delta * delta)); + } + } +} + + +template +__global__ void backprop_swish(T* dX, const T* dY, const T* X, + const T* Y, double threshold, int N) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + + for (int i = _loop_start; i < N; i += _loop_stride) + { + T x = X[i]; + T y = Y[i]; + + if (x >= threshold) { + dX[i] = dY[i]; + } else if (x <= -threshold) { + dX[i] = 0.0; + } else { + T cdf = 1.0 / (1 + exp(-x)); + T d = y + cdf * (1 - y); + dX[i] = dY[i] * d; + } + } +} + + +template +__global__ void backprop_reduce_sum(U* dX, const U* d_sum, const int* lengths, + int B, int T, int O) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + int seq_start = 0; + int b = 0; + for (int t = _loop_start; t < T; t += _loop_stride) + { + // Find the sequence item we're working on + while ((b < B) && (seq_start+lengths[b]) <= t) + { + seq_start += lengths[b]; + b += 1; + } + if (lengths[b] == 0) + continue; + + for (int i=0; i < O; ++i) + { + dX[t * O + i] = d_sum[b * O + i]; + } + } +} + + +template +__global__ void backprop_reduce_mean(U* dX, const U* d_mean, const int* lengths, + int B, int T, int O) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + int seq_start = 0; + int b = 0; + for (int t = _loop_start; t < T; t += _loop_stride) + { + // Find the sequence item we're working on + while ((b < B) && (seq_start+lengths[b]) <= t) + { + seq_start += lengths[b]; + b += 1; + } + if (lengths[b] == 0) + continue; + + U* dX_t = &dX[t * O]; + const U* d_mean_b = &d_mean[b * O]; + int lengths_b = lengths[b]; + for (int i=0; i < O; ++i) + { + dX_t[i] = d_mean_b[i] / lengths_b; + } + } +} + + +template +__global__ void backprop_reduce_max(U* dX, const U* d_maxes, + const int* which, const int* lengths, int B, int T, int O) +{ + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + int seq_start = 0; + int b = 0; + for (int t = _loop_start; t < T; t += _loop_stride) + { + // We're calculating the gradient of the unpooled sequences, from + // the gradient of the maxes. In this loop, we're getting the gradient + // of a single sequence item, t. We need to know the sequence index, + // b. + while ((b < B) && (seq_start+lengths[b]) <= t) + { + seq_start += lengths[b]; + b += 1; + } + if (lengths[b] == 0) + continue; + + // The "which" array tells us which rows were selected as the max. + // So we need to find the index of our t in the sequence. + int index_of_t = t-seq_start; + // Get the rows we're dealing with, to avoid cluttering the loop + // with the index math. + U* dX_t = &dX[t*O]; + const U* d_maxes_b = &d_maxes[b*O]; + const int* which_b = &which[b*O]; + // Now loop over our row. + for (int i=0; i < O; ++i) + { + // If we used the value for this cell, + // pass the gradient + if (which_b[i] == index_of_t) + dX_t[i] = d_maxes_b[i]; + } + } +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.py new file mode 100644 index 0000000000000000000000000000000000000000..ae11b2050f5e73088ecedc2c74e602bcc33ab99c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_custom_kernels.py @@ -0,0 +1,883 @@ +import operator +import re +from collections import defaultdict +from functools import reduce +from pathlib import Path +from typing import Callable, Optional, Tuple + +import numpy + +from ..compat import cupy, has_cupy_gpu + +PWD = Path(__file__).parent +KERNELS_SRC = (PWD / "_custom_kernels.cu").read_text(encoding="utf8") +KERNELS_LIST = [ + "backprop_clipped_linear", + "backprop_clipped_linear", + "backprop_dish", + "backprop_dish", + "backprop_gelu", + "backprop_gelu", + "backprop_hard_swish", + "backprop_hard_swish", + "backprop_hard_swish_mobilenet", + "backprop_hard_swish_mobilenet", + "backprop_maxout", + "backprop_maxout", + "backprop_mish", + "backprop_mish", + "backprop_reduce_max", + "backprop_reduce_max", + "backprop_reduce_mean", + "backprop_reduce_mean", + "backprop_reduce_sum", + "backprop_reduce_sum", + "backprop_seq2col", + "backprop_seq2col", + "backprop_swish", + "backprop_swish", + "clipped_linear", + "clipped_linear", + "dish", + "dish", + "gather_add", + "gather_add", + "gelu", + "gelu", + "maxout", + "maxout", + "mish", + "mish", + "pad", + "pad", + "pad", + "pad", + "reduce_max", + "reduce_max", + "reduce_sum", + "reduce_sum", + "seq2col", + "seq2col", + "swish", + "swish", +] +KERNELS = ( + cupy.RawModule( + code=KERNELS_SRC, options=("--std=c++11",), name_expressions=KERNELS_LIST + ) + if has_cupy_gpu + else None +) + + +class LazyKernel: + """Wraps around `cupy.RawModule` and `cupy.RawKernel` to verify CuPy availability + and lazily compile the latter on first invocation. + + The default CuPy behaviour triggers the compilation as soon as the `cupy.RawKernel` object + is accessed.""" + + name: str + _kernel: Optional["cupy.RawKernel"] + _compile_callback: Optional[Callable[[], "cupy.RawKernel"]] + + __slots__ = ["name", "_kernel", "_compile_callback"] + + def __init__( + self, + name: str, + *, + compile_callback: Optional[Callable[[], "cupy.RawKernel"]] = None, + ) -> None: + self.name = name + self._kernel = None + self._compile_callback = compile_callback + + def __call__(self, *args, **kwargs): + self._compile_kernel() + self._kernel(*args, **kwargs) + + def _compile_kernel(self): + if self._kernel is not None: + return + + if self._compile_callback is not None: + self._kernel = self._compile_callback() + elif KERNELS is not None: + self._kernel = KERNELS.get_function(self.name) + + if self._kernel is None: + raise ValueError(f"couldn't compile Cupy kernel '{self.name}'") + + +def compile_mmh(): + if not has_cupy_gpu: + return None + return cupy.RawKernel((PWD / "_murmur3.cu").read_text(encoding="utf8"), "hash_data") + + +clipped_linear_kernel_float = LazyKernel("clipped_linear") +clipped_linear_kernel_double = LazyKernel("clipped_linear") +dish_kernel_float = LazyKernel("dish") +dish_kernel_double = LazyKernel("dish") +gather_add_kernel_float = LazyKernel("gather_add") +gather_add_kernel_double = LazyKernel("gather_add") +gelu_kernel_float = LazyKernel("gelu") +gelu_kernel_double = LazyKernel("gelu") +hash_data_kernel = LazyKernel("hash_data", compile_callback=compile_mmh) +maxout_kernel_float = LazyKernel("maxout") +maxout_kernel_double = LazyKernel("maxout") +mish_kernel_float = LazyKernel("mish") +mish_kernel_double = LazyKernel("mish") +pad_kernel_float = LazyKernel("pad") +pad_kernel_double = LazyKernel("pad") +pad_kernel_int32 = LazyKernel("pad") +pad_kernel_int64 = LazyKernel("pad") +reduce_max_kernel_float = LazyKernel("reduce_max") +reduce_max_kernel_double = LazyKernel("reduce_max") +reduce_sum_kernel_float = LazyKernel("reduce_sum") +reduce_sum_kernel_double = LazyKernel("reduce_sum") +seq2col_kernel_float = LazyKernel("seq2col") +seq2col_kernel_double = LazyKernel("seq2col") +swish_kernel_float = LazyKernel("swish") +swish_kernel_double = LazyKernel("swish") + +backprop_clipped_linear_kernel_double = LazyKernel("backprop_clipped_linear") +backprop_clipped_linear_kernel_float = LazyKernel("backprop_clipped_linear") +backprop_dish_kernel_double = LazyKernel("backprop_dish") +backprop_dish_kernel_float = LazyKernel("backprop_dish") +backprop_gelu_kernel_double = LazyKernel("backprop_gelu") +backprop_gelu_kernel_float = LazyKernel("backprop_gelu") +backprop_hard_swish_kernel_double = LazyKernel("backprop_hard_swish") +backprop_hard_swish_kernel_float = LazyKernel("backprop_hard_swish") +backprop_hard_swish_mobilenet_kernel_double = LazyKernel( + "backprop_hard_swish_mobilenet" +) +backprop_hard_swish_mobilenet_kernel_float = LazyKernel( + "backprop_hard_swish_mobilenet" +) +backprop_maxout_kernel_double = LazyKernel("backprop_maxout") +backprop_maxout_kernel_float = LazyKernel("backprop_maxout") +backprop_mish_kernel_double = LazyKernel("backprop_mish") +backprop_mish_kernel_float = LazyKernel("backprop_mish") +backprop_reduce_max_kernel_double = LazyKernel("backprop_reduce_max") +backprop_reduce_max_kernel_float = LazyKernel("backprop_reduce_max") +backprop_reduce_mean_kernel_double = LazyKernel("backprop_reduce_mean") +backprop_reduce_mean_kernel_float = LazyKernel("backprop_reduce_mean") +backprop_reduce_sum_kernel_double = LazyKernel("backprop_reduce_sum") +backprop_reduce_sum_kernel_float = LazyKernel("backprop_reduce_sum") +backprop_seq2col_kernel_double = LazyKernel("backprop_seq2col") +backprop_seq2col_kernel_float = LazyKernel("backprop_seq2col") +backprop_swish_kernel_double = LazyKernel("backprop_swish") +backprop_swish_kernel_float = LazyKernel("backprop_swish") + + +def _alloc(shape, dtype, *, zeros: bool = True): + if zeros: + return cupy.zeros(shape, dtype) + else: + return cupy.empty(shape, dtype) + + +def _alloc_like(array, zeros: bool = True): + if zeros: + return cupy.zeros_like(array) + else: + return cupy.empty_like(array) + + +def pad(seqs, round_to=1, *, threads_per_block=128, num_blocks=128): + if round_to < 1: + raise ValueError(f"Rounding for padding must at least be 1, was: {round_to}") + for seq in seqs: + _is_float_or_int_array(seq) + + seq_lens = [len(seq) for seq in seqs] + max_seq_len = max(seq_lens) + # Round the length to nearest bucket -- helps on GPU, to make similar + # array sizes. + max_seq_len += -max_seq_len % round_to + seq_lens = cupy.array(seq_lens, dtype="int32") + final_shape = (len(seqs), max_seq_len) + seqs[0].shape[1:] + out = cupy.empty(final_shape, dtype=seqs[0].dtype) + + # Extract pointers from CuPy arrays, so that we can address + # them in the CUDA kernel. + ptrs = numpy.empty( + ( + len( + seqs, + ) + ), + "int64", + ) + for idx, seq in enumerate(seqs): + ptrs[idx] = seq.data.ptr + ptrs = cupy.array(ptrs) + + stride = reduce(operator.mul, seqs[0].shape[1:], 1) + + if out.dtype == "float32": + pad_kernel_float( + (num_blocks,), + (threads_per_block,), + (out, ptrs, seq_lens, stride, len(seqs), max_seq_len), + ) + elif out.dtype == "float64": + pad_kernel_double( + (num_blocks,), + (threads_per_block,), + (out, ptrs, seq_lens, stride, len(seqs), max_seq_len), + ) + elif out.dtype == "int32": + pad_kernel_int32( + (num_blocks,), + (threads_per_block,), + (out, ptrs, seq_lens, stride, len(seqs), max_seq_len), + ) + elif out.dtype == "int64": + pad_kernel_int64( + (num_blocks,), + (threads_per_block,), + (out, ptrs, seq_lens, stride, len(seqs), max_seq_len), + ) + + return out + + +def clipped_linear( + X, + *, + inplace=False, + slope=1.0, + offset=0.0, + min_val=0.0, + max_val=1.0, + threads_per_block=128, + num_blocks=128, +): + _is_float_array(X) + + out = X + if not inplace: + out = _alloc_like(X, zeros=False) + if X.dtype == "float32": + clipped_linear_kernel_float( + (num_blocks,), + (threads_per_block,), + (out, X, slope, offset, min_val, max_val, X.size), + ) + else: + clipped_linear_kernel_double( + (num_blocks,), + (threads_per_block,), + (out, X, slope, offset, min_val, max_val, X.size), + ) + return out + + +def gather_add(table, indices, *, threads_per_block=128, num_blocks=128): + if table.ndim != 2: + raise ValueError( + f"gather_add expects table with dimensionality 2, was: {table.ndim}" + ) + if indices.ndim != 2: + raise ValueError( + f"gather_add expects indices with dimensionality 2, was: {indices.ndim}" + ) + _is_float_array(table) + indices = indices.astype("int32") + _check_indices(indices, table.shape[0]) + + B = indices.shape[0] + K = indices.shape[1] + T = table.shape[0] + O = table.shape[1] + + out = _alloc((B, O), dtype=table.dtype, zeros=True) + if table.dtype == "float32": + gather_add_kernel_float( + (num_blocks,), (threads_per_block,), (out, table, indices, T, O, B, K) + ) + else: + gather_add_kernel_double( + (num_blocks,), (threads_per_block,), (out, table, indices, T, O, B, K) + ) + return out + + +def dish(X, *, inplace=False, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + out = X + if not inplace: + out = _alloc_like(X, zeros=False) + if X.dtype == "float32": + dish_kernel_float((num_blocks,), (threads_per_block,), (out, X, X.size)) + else: + dish_kernel_double((num_blocks,), (threads_per_block,), (out, X, X.size)) + return out + + +def gelu(X, *, inplace=False, threshold=6.0, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + out = X + if not inplace: + out = _alloc_like(X, zeros=False) + if X.dtype == "float32": + gelu_kernel_float( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + else: + gelu_kernel_double( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + return out + + +def check_seq2col_lengths(lengths, B): + if lengths is None: + lengths = cupy.array([B], dtype="int32") + else: + _check_lengths(lengths, B) + return lengths + + +def seq2col(seq, nW, *, lengths=None, threads_per_block=128, num_blocks=128): + _is_float_array(seq) + + B = seq.shape[0] + nF = nW * 2 + 1 + I = seq.shape[1] + + lengths = check_seq2col_lengths(lengths, B) + nL = lengths.shape[0] + + out = _alloc((B, I * nF), dtype=seq.dtype, zeros=True) + + if seq.size != 0 and lengths.size != 0: + if seq.dtype == "float32": + seq2col_kernel_float( + (num_blocks,), (threads_per_block,), (out, seq, lengths, nW, B, I, nL) + ) + else: + seq2col_kernel_double( + (num_blocks,), (threads_per_block,), (out, seq, lengths, nW, B, I, nL) + ) + + return out + + +def maxout(X, *, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + B, I, P = X.shape + + out_shape = (B, I) + best = _alloc(out_shape, dtype=X.dtype, zeros=False) + which = _alloc(out_shape, dtype="i", zeros=False) + + if X.dtype == "float32": + maxout_kernel_float( + (num_blocks,), (threads_per_block,), (best, which, X, B, I, P) + ) + else: + maxout_kernel_double( + (num_blocks,), (threads_per_block,), (best, which, X, B, I, P) + ) + + return best, which + + +def mish(X, *, inplace=False, threshold=5, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + out = X + if not inplace: + out = _alloc_like(X, zeros=False) + + if X.dtype == "float32": + mish_kernel_float( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + else: + mish_kernel_double( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + + return out + + +def reduce_sum(X, lengths, *, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + B = len(lengths) + T = X.shape[0] + O = X.shape[1] + + _check_lengths(lengths, T) + + out = _alloc((B, O), dtype=X.dtype, zeros=True) + + if X.dtype == "float32": + reduce_sum_kernel_float( + (num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O) + ) + else: + reduce_sum_kernel_double( + (num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O) + ) + + return out + + +def reduce_mean(X, lengths, *, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + B = len(lengths) + T = X.shape[0] + O = X.shape[1] + + _check_lengths(lengths, T) + + out = _alloc((B, O), dtype=X.dtype, zeros=True) + + if X.dtype == "float32": + reduce_sum_kernel_float( + (num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O) + ) + else: + reduce_sum_kernel_double( + (num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O) + ) + + # Avoid divide by zero + out /= lengths.reshape((-1, 1)) + 1e-10 + return out + + +def reduce_max(X, lengths, *, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + B = len(lengths) + T = X.shape[0] + O = X.shape[1] + + _check_lengths(lengths, T, min_length=1) + + out_shape = (B, O) + maxes = _alloc(out_shape, dtype=X.dtype, zeros=False) + which = _alloc(out_shape, dtype="i", zeros=False) + + if X.dtype == "float32": + reduce_max_kernel_float( + (num_blocks,), (threads_per_block,), (maxes, which, X, lengths, B, T, O) + ) + else: + reduce_max_kernel_double( + (num_blocks,), (threads_per_block,), (maxes, which, X, lengths, B, T, O) + ) + + return maxes, which + + +def swish(X, *, inplace=False, threshold=17.0, threads_per_block=128, num_blocks=128): + _is_float_array(X) + + out = X + if not inplace: + out = _alloc_like(X, zeros=False) + if X.dtype == "float32": + swish_kernel_float( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + else: + swish_kernel_double( + (num_blocks,), (threads_per_block,), (out, X, threshold, X.size) + ) + return out + + +def backprop_seq2col(dY, nW, *, lengths=None, threads_per_block=128, num_blocks=128): + _is_float_array(dY) + + B = dY.shape[0] + nF = nW * 2 + 1 + I = dY.shape[1] // nF + + lengths = check_seq2col_lengths(lengths, B) + nL = lengths.shape[0] + + out = _alloc((B, I), dtype=dY.dtype, zeros=True) + + if dY.size != 0 and lengths.size != 0: + if dY.dtype == "float32": + backprop_seq2col_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, lengths, nW, B, I, nL) + ) + else: + backprop_seq2col_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, lengths, nW, B, I, nL) + ) + + return out + + +def backprop_clipped_linear( + dY, + X, + *, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, + inplace: bool = False, + threads_per_block=128, + num_blocks=128, +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_clipped_linear_kernel_float( + (num_blocks,), + (threads_per_block,), + (out, dY, X, slope, offset, min_val, max_val, out.size), + ) + else: + backprop_clipped_linear_kernel_double( + (num_blocks,), + (threads_per_block,), + (out, dY, X, slope, offset, min_val, max_val, out.size), + ) + + return out + + +def backprop_hard_swish( + dY, X, *, inplace: bool = False, threads_per_block=128, num_blocks=128 +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_hard_swish_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + else: + backprop_hard_swish_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + + return out + + +def backprop_hard_swish_mobilenet( + dY, X, *, inplace: bool = False, threads_per_block=128, num_blocks=128 +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_hard_swish_mobilenet_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + else: + backprop_hard_swish_mobilenet_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + + return out + + +def backprop_dish( + dY, + X, + *, + inplace: bool = False, + threads_per_block=128, + num_blocks=128, +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_dish_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + else: + backprop_dish_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, out.size) + ) + + return out + + +def backprop_gelu( + dY, + X, + *, + inplace: bool = False, + threshold=6.0, + threads_per_block=128, + num_blocks=128, +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_gelu_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, threshold, out.size) + ) + else: + backprop_gelu_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, threshold, out.size) + ) + + return out + + +def backprop_maxout(dY, which, P, *, threads_per_block=128, num_blocks=128): + _is_float_array(dY) + + B = dY.shape[0] + I = dY.shape[1] + + out = _alloc((B, I, P), dtype=dY.dtype, zeros=True) + + _check_which_maxout(which, B, I, P) + + if dY.dtype == "float32": + backprop_maxout_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, which, B, I, P) + ) + else: + backprop_maxout_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, which, B, I, P) + ) + + return out + + +def backprop_mish( + dY, X, *, inplace: bool = False, threshold=5, threads_per_block=128, num_blocks=128 +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_mish_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, threshold, dY.size) + ) + else: + backprop_mish_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, threshold, dY.size) + ) + + return out + + +def backprop_reduce_sum(d_sums, lengths, *, threads_per_block=128, num_blocks=128): + _is_float_array(d_sums) + + B = len(lengths) + T = int(lengths.sum()) + O = d_sums.shape[1] + _check_lengths(lengths, T) + + out = _alloc((T, O), dtype=d_sums.dtype, zeros=False) + + if d_sums.dtype == "float32": + backprop_reduce_sum_kernel_float( + (num_blocks,), (threads_per_block,), (out, d_sums, lengths, B, T, O) + ) + else: + backprop_reduce_sum_kernel_double( + (num_blocks,), (threads_per_block,), (out, d_sums, lengths, B, T, O) + ) + + return out + + +def backprop_reduce_mean(d_means, lengths, *, threads_per_block=128, num_blocks=128): + _is_float_array(d_means) + + B = len(lengths) + T = int(lengths.sum()) + O = d_means.shape[1] + _check_lengths(lengths, T) + + out = _alloc((T, O), dtype=d_means.dtype, zeros=False) + + if d_means.dtype == "float32": + backprop_reduce_mean_kernel_float( + (num_blocks,), (threads_per_block,), (out, d_means, lengths, B, T, O) + ) + else: + backprop_reduce_mean_kernel_double( + (num_blocks,), (threads_per_block,), (out, d_means, lengths, B, T, O) + ) + + return out + + +def backprop_reduce_max( + d_maxes, which, lengths, *, threads_per_block=128, num_blocks=128 +): + _is_float_array(d_maxes) + + B = len(lengths) + T = int(lengths.sum()) + O = d_maxes.shape[1] + _check_lengths(lengths, T, min_length=1) + + out = _alloc((T, O), dtype=d_maxes.dtype, zeros=True) + + _check_which_reduce_max(which, (B, O), lengths) + + if d_maxes.dtype == "float32": + backprop_reduce_max_kernel_float( + (num_blocks,), (threads_per_block,), (out, d_maxes, which, lengths, B, T, O) + ) + else: + backprop_reduce_max_kernel_double( + (num_blocks,), (threads_per_block,), (out, d_maxes, which, lengths, B, T, O) + ) + + return out + + +def backprop_swish( + dY, X, Y, *, inplace=False, threshold=17.0, threads_per_block=128, num_blocks=128 +): + _is_float_array(dY) + _is_float_array(X, shape=dY.shape) + _is_float_array(Y, shape=dY.shape) + + out = dY + if not inplace: + out = _alloc_like(dY, zeros=False) + + if dY.dtype == "float32": + backprop_swish_kernel_float( + (num_blocks,), (threads_per_block,), (out, dY, X, Y, threshold, out.size) + ) + else: + backprop_swish_kernel_double( + (num_blocks,), (threads_per_block,), (out, dY, X, Y, threshold, out.size) + ) + + return out + + +def hash(ids, seed, *, threads_per_block=128, num_blocks=128): + out = _alloc((ids.shape[0], 4), dtype="uint32", zeros=True) + + # sizeof(uint32_t) * 4 + out_size = 4 * 4 + in_size = 8 # sizeof(uint64_t) + # T = ids.shape[0] + hash_data_kernel( + (num_blocks,), + (threads_per_block,), + (out, ids, out_size, in_size, ids.shape[0], seed), + ) + return out + + +def _is_float_array(out, *, shape: Optional[Tuple] = None): + assert out.dtype in ( + "float32", + "float64", + ), "CUDA kernel can only handle float32 and float64" + if shape is not None and out.shape != shape: + msg = f"array has incorrect shape, expected: {shape}, was: {out.shape}" + raise ValueError(msg) + + +def _is_float_or_int_array(out, *, shape: Optional[Tuple] = None): + assert out.dtype in ( + "float32", + "float64", + "int32", + "int64", + ), "CUDA kernel can only handle float32, float64, int32 and int64" + if shape is not None and out.shape != shape: + msg = f"array has incorrect shape, expected: {shape}, was: {out.shape}" + raise ValueError(msg) + + +def _check_lengths(lengths, n_elems: int, *, min_length=0): + assert lengths.dtype == "int32", "lengths should be encoded as 32-bit integers" + if not cupy.all(lengths >= min_length): + raise ValueError(f"all sequence lengths must be >= {min_length}") + if cupy.sum(lengths) != n_elems: + raise IndexError("lengths must sum up to the batch size") + + +def _check_indices(indices, n: int): + assert indices.dtype == "int32", "indices should be encoded as 32-bit integers" + + if not _values_within_range(indices, 0, n): + raise IndexError(f"index out of bounds, must be >= 0 && < {n}") + + +def _check_which_maxout(which, B: int, I: int, P: int): + shape = (B, I) + msg = "maximum index (which) should be encoded as 32-bit integers" + assert which.dtype == "int32", msg + if which.shape != shape: + msg = f"maximum index (which) has incorrect shape, expected: {shape}, was: {which.shape}" + raise ValueError(msg) + if not _values_within_range(which, 0, P): + raise IndexError("maximum index (which) value out of bounds") + + +_values_within_range = ( + cupy.ReductionKernel( + "T x, T lower, T upper", + "bool r", + "x >= lower && x < upper", + "a & b", + "r = a", + "true", + "within_range", + ) + if has_cupy_gpu + else None +) + + +def _check_which_reduce_max(which, shape: Tuple, lengths): + msg = "maximum index (which) should be encoded as 32-bit integers" + assert which.dtype == "int32", msg + if which.shape != shape: + msg = f"maximum index (which) has incorrect shape, expected: {shape}, was: {which.shape}" + raise ValueError(msg) + if not cupy.all((which >= 0) & (which < cupy.expand_dims(lengths, -1))): + raise IndexError("maximum index (which) value out of bounds") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_murmur3.cu b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_murmur3.cu new file mode 100644 index 0000000000000000000000000000000000000000..d8b4daf97a4cf8f71e0af8007efbeb9688c96005 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_murmur3.cu @@ -0,0 +1,188 @@ +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +/* + * This version is taken from https://github.com/PeterScott/murmur3 + * and modified to work with CUDA. + */ +// Including stdint.h is a pain in cupy, so just put the declarations in. +// Beware that long int is not 64bit on all platforms! +// e.g. Windows requires a 'long long' to get to 64 bit. + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef signed char int8_t; +typedef int int32_t; + +const unsigned long int test_var = 0; +const int size = sizeof(test_var); + +#if size == 64 + +typedef unsigned long int uint64_t; +typedef long int int64_t; + +#else + +typedef unsigned long long uint64_t; +typedef long long int64_t; + +#endif + + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +#define FORCE_INLINE + +__device__ static inline FORCE_INLINE uint64_t rotl64 ( uint64_t x, int8_t r ) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL64(x,y) rotl64(x,y) +#define BIG_CONSTANT(x) (x##LLU) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +#define getblock(p, i) (p[i]) + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +__device__ static inline FORCE_INLINE uint32_t fmix32 ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//---------- + +__device__ static inline FORCE_INLINE uint64_t fmix64 ( uint64_t k ) +{ + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +//----------------------------------------------------------------------------- + +__device__ void MurmurHash3_x64_128 ( const void * key, const int len, + const uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + int i; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + //---------- + // body + + const uint64_t * blocks = (const uint64_t *)(data); + + for(i = 0; i < nblocks; i++) + { + uint64_t k1 = getblock(blocks,i*2+0); + uint64_t k2 = getblock(blocks,i*2+1); + + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + + h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; + + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch(len & 15) + { + case 15: k2 ^= (uint64_t)(tail[14]) << 48; + case 14: k2 ^= (uint64_t)(tail[13]) << 40; + case 13: k2 ^= (uint64_t)(tail[12]) << 32; + case 12: k2 ^= (uint64_t)(tail[11]) << 24; + case 11: k2 ^= (uint64_t)(tail[10]) << 16; + case 10: k2 ^= (uint64_t)(tail[ 9]) << 8; + case 9: k2 ^= (uint64_t)(tail[ 8]) << 0; + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= (uint64_t)(tail[ 7]) << 56; + case 7: k1 ^= (uint64_t)(tail[ 6]) << 48; + case 6: k1 ^= (uint64_t)(tail[ 5]) << 40; + case 5: k1 ^= (uint64_t)(tail[ 4]) << 32; + case 4: k1 ^= (uint64_t)(tail[ 3]) << 24; + case 3: k1 ^= (uint64_t)(tail[ 2]) << 16; + case 2: k1 ^= (uint64_t)(tail[ 1]) << 8; + case 1: k1 ^= (uint64_t)(tail[ 0]) << 0; + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + ((uint64_t*)out)[0] = h1; + ((uint64_t*)out)[1] = h2; +} + +//----------------------------------------------------------------------------- + +// Write a stream of hash values for an input stream. Each output may have up to 128 bits +// of entropy. Input size should be specified in bytes. +extern "C" __global__ +void hash_data(char* dest, + const char* src, size_t out_size, size_t in_size, size_t n_items, uint32_t seed) +{ + char entropy[16]; // 128/8=16 + int _loop_start = blockIdx.x * blockDim.x + threadIdx.x; + int _loop_stride = blockDim.x * gridDim.x; + for (int i = _loop_start; i < n_items; i += _loop_stride) + { + + const char* src_i = &src[i*in_size]; + char* dest_i = &dest[i*out_size]; + + MurmurHash3_x64_128(src_i, in_size, seed, entropy); + for (int j=0; j < out_size; ++j) + dest_i[j] = entropy[j]; + } +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_param_server.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_param_server.py new file mode 100644 index 0000000000000000000000000000000000000000..24a7c27d205837630e48e8c5bae3bfa940a19d61 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/_param_server.py @@ -0,0 +1,79 @@ +from typing import Any, Dict, Optional, Tuple + +from ..types import FloatsXd +from ..util import get_array_module + +KeyT = Tuple[int, str] + + +class ParamServer: + """Serve parameters for a single process.""" + + _params: Dict[KeyT, FloatsXd] = {} + _grads: Dict[KeyT, FloatsXd] = {} + proxy: Optional[Any] + + def __init__( + self, + params: Dict[KeyT, FloatsXd] = {}, + grads: Dict[KeyT, FloatsXd] = {}, + *, + proxy=None + ): + self._params = dict(params) + self._grads = dict(grads) + # Allow a 'proxy' to be provided to support remote parameters. This + # is experimental, it's the mechanism we use in the Ray integration. + self.proxy = proxy + + @property + def param_keys(self) -> Tuple[KeyT, ...]: + """Get the names of registered parameter (including unset).""" + return tuple(self._params.keys()) + + @property + def grad_keys(self) -> Tuple[KeyT, ...]: + return tuple([key for key in self.param_keys if self.has_grad(*key)]) + + def has_param(self, model_id: int, name: str) -> bool: + return (model_id, name) in self._params + + def has_grad(self, model_id: int, name: str) -> bool: + return (model_id, name) in self._grads + + def get_param(self, model_id: int, name: str) -> FloatsXd: + key = (model_id, name) + if self.proxy is not None: + self._params[key] = self.proxy.get_param(model_id, name) + return self._params[key] + + def get_grad(self, model_id: int, name: str) -> FloatsXd: + key = (model_id, name) + return self._grads[key] + + def set_param(self, model_id: int, name: str, value: FloatsXd) -> None: + if self.proxy is not None: + self.proxy.set_param(model_id, name, value) + self._params[(model_id, name)] = value + + def set_grad(self, model_id: int, name: str, value: FloatsXd) -> None: + if self.proxy is not None: + self.proxy.set_grad(model_id, name, value) + else: + self._grads[(model_id, name)] = value + + def inc_grad(self, model_id: int, name: str, value: FloatsXd) -> None: + key = (model_id, name) + if self.proxy is not None: + self.proxy.inc_grad(model_id, name, value) + elif not self.has_grad(model_id, name): # pragma: no cover + if hasattr(value, "copy"): + # Adjustment for Jax + self._grads[key] = value.copy() + elif not value.flags["C_CONTIGUOUS"]: + xp = get_array_module(value) + self._grads[(model_id, name)] = xp.ascontiguousarray(value) + else: + self._grads[(model_id, name)] = value + else: + self._grads[(model_id, name)] += value diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.cpp new file mode 100644 index 0000000000000000000000000000000000000000..22b138faae1bfc54d629022574751dcfb1f9d723 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.cpp @@ -0,0 +1,27141 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "include_dirs": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.backends.cblas", + "sources": [ + "thinc/backends/cblas.pyx" + ] + }, + "module_name": "thinc.backends.cblas" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__backends__cblas +#define __PYX_HAVE_API__thinc__backends__cblas +/* Early includes */ +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include +#include "pythread.h" +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/backends/cblas.pyx", + "", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* BufferFormatStructs.proto */ +struct __Pyx_StructField_; +#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0) +typedef struct { + const char* name; + const struct __Pyx_StructField_* fields; + size_t size; + size_t arraysize[8]; + int ndim; + char typegroup; + char is_unsigned; + int flags; +} __Pyx_TypeInfo; +typedef struct __Pyx_StructField_ { + const __Pyx_TypeInfo* type; + const char* name; + size_t offset; +} __Pyx_StructField; +typedef struct { + const __Pyx_StructField* field; + size_t parent_offset; +} __Pyx_BufFmt_StackElem; +typedef struct { + __Pyx_StructField root; + __Pyx_BufFmt_StackElem* head; + size_t fmt_offset; + size_t new_count, enc_count; + size_t struct_alignment; + int is_complex; + char enc_type; + char new_packmode; + char enc_packmode; + char is_valid_array; +} __Pyx_BufFmt_Context; + +/* MemviewSliceStruct.proto */ +struct __pyx_memoryview_obj; +typedef struct { + struct __pyx_memoryview_obj *memview; + char *data; + Py_ssize_t shape[8]; + Py_ssize_t strides[8]; + Py_ssize_t suboffsets[8]; +} __Pyx_memviewslice; +#define __Pyx_MemoryView_Len(m) (m.shape[0]) +#define __Pyx_MEMVIEW_DIRECT 1 +#define __Pyx_MEMVIEW_PTR 2 +#define __Pyx_MEMVIEW_FULL 4 +#define __Pyx_MEMVIEW_CONTIG 8 +#define __Pyx_MEMVIEW_STRIDED 16 +#define __Pyx_MEMVIEW_FOLLOW 32 +#define __Pyx_IS_C_CONTIG 1 +#define __Pyx_IS_F_CONTIG 2 +#define __Pyx_MEMSLICE_INIT { 0, 0, { 0 }, { 0 }, { 0 } } +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "blis/cy.pxd":58 + * + * + * ctypedef int64_t dim_t # <<<<<<<<<<<<<< + * ctypedef int64_t inc_t + * ctypedef int64_t doff_t +*/ +typedef int64_t __pyx_t_4blis_2cy_dim_t; + +/* "blis/cy.pxd":59 + * + * ctypedef int64_t dim_t + * ctypedef int64_t inc_t # <<<<<<<<<<<<<< + * ctypedef int64_t doff_t + * +*/ +typedef int64_t __pyx_t_4blis_2cy_inc_t; + +/* "blis/cy.pxd":60 + * ctypedef int64_t dim_t + * ctypedef int64_t inc_t + * ctypedef int64_t doff_t # <<<<<<<<<<<<<< + * + * +*/ +typedef int64_t __pyx_t_4blis_2cy_doff_t; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5thinc_8backends_5cblas_CBlas; +struct __pyx_array_obj; +struct __pyx_MemviewEnum_obj; +struct __pyx_memoryview_obj; +struct __pyx_memoryviewslice_obj; + +/* "blis/cy.pxd":66 + * # the header into our header. + * # We get some piece of mind from checking the values on init. + * cpdef enum trans_t: # <<<<<<<<<<<<<< + * NO_TRANSPOSE = 0 + * TRANSPOSE = 8 +*/ +enum __pyx_t_4blis_2cy_trans_t { + __pyx_e_4blis_2cy_NO_TRANSPOSE = 0, + __pyx_e_4blis_2cy_TRANSPOSE = 8, + __pyx_e_4blis_2cy_CONJ_NO_TRANSPOSE = 16, + __pyx_e_4blis_2cy_CONJ_TRANSPOSE = 24 +}; + +/* "blis/cy.pxd":73 + * + * + * cpdef enum conj_t: # <<<<<<<<<<<<<< + * NO_CONJUGATE = 0 + * CONJUGATE = 16 +*/ +enum __pyx_t_4blis_2cy_conj_t { + __pyx_e_4blis_2cy_NO_CONJUGATE = 0, + __pyx_e_4blis_2cy_CONJUGATE = 16 +}; + +/* "blis/cy.pxd":78 + * + * + * cpdef enum side_t: # <<<<<<<<<<<<<< + * LEFT = 0 + * RIGHT = 1 +*/ +enum __pyx_t_4blis_2cy_side_t { + __pyx_e_4blis_2cy_LEFT = 0, + __pyx_e_4blis_2cy_RIGHT = 1 +}; + +/* "blis/cy.pxd":83 + * + * + * cpdef enum uplo_t: # <<<<<<<<<<<<<< + * LOWER = 192 + * UPPER = 96 +*/ +enum __pyx_t_4blis_2cy_uplo_t { + __pyx_e_4blis_2cy_LOWER = 0xC0, + __pyx_e_4blis_2cy_UPPER = 96, + __pyx_e_4blis_2cy_DENSE = 0xE0 +}; + +/* "blis/cy.pxd":89 + * + * + * cpdef enum diag_t: # <<<<<<<<<<<<<< + * NONUNIT_DIAG = 0 + * UNIT_DIAG = 256 +*/ +enum __pyx_t_4blis_2cy_diag_t { + __pyx_e_4blis_2cy_NONUNIT_DIAG = 0, + __pyx_e_4blis_2cy_UNIT_DIAG = 0x100 +}; + +/* "blis/cy.pxd":6 + * + * + * ctypedef float[::1] float1d_t # <<<<<<<<<<<<<< + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_float1d_t; + +/* "blis/cy.pxd":7 + * + * ctypedef float[::1] float1d_t + * ctypedef double[::1] double1d_t # <<<<<<<<<<<<<< + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_double1d_t; + +/* "blis/cy.pxd":8 + * ctypedef float[::1] float1d_t + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t # <<<<<<<<<<<<<< + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_float2d_t; + +/* "blis/cy.pxd":9 + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t # <<<<<<<<<<<<<< + * ctypedef float* floats_t + * ctypedef double* doubles_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_double2d_t; + +/* "blis/cy.pxd":10 + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t # <<<<<<<<<<<<<< + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t +*/ +typedef float *__pyx_t_4blis_2cy_floats_t; + +/* "blis/cy.pxd":11 + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t + * ctypedef double* doubles_t # <<<<<<<<<<<<<< + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t +*/ +typedef double *__pyx_t_4blis_2cy_doubles_t; + +/* "blis/cy.pxd":12 + * ctypedef float* floats_t + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t # <<<<<<<<<<<<<< + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_float1d_t; + +/* "blis/cy.pxd":13 + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t # <<<<<<<<<<<<<< + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_double1d_t; + +/* "blis/cy.pxd":14 + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t # <<<<<<<<<<<<<< + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_float2d_t; + +/* "blis/cy.pxd":15 + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t # <<<<<<<<<<<<<< + * ctypedef const float* const_floats_t + * ctypedef const double* const_doubles_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_double2d_t; + +/* "blis/cy.pxd":16 + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t # <<<<<<<<<<<<<< + * ctypedef const double* const_doubles_t + * +*/ +typedef float const *__pyx_t_4blis_2cy_const_floats_t; + +/* "blis/cy.pxd":17 + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t + * ctypedef const double* const_doubles_t # <<<<<<<<<<<<<< + * + * +*/ +typedef double const *__pyx_t_4blis_2cy_const_doubles_t; +struct __pyx_t_5thinc_8backends_5cblas_BlasFuncs; + +/* "thinc/backends/cblas.pxd":3 + * from libcpp.memory cimport shared_ptr + * + * ctypedef void (*sgemm_ptr)(bint transA, bint transB, int M, int N, int K, # <<<<<<<<<<<<<< + * float alpha, const float* A, int lda, const float *B, + * int ldb, float beta, float* C, int ldc) noexcept nogil +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_sgemm_ptr)(int, int, int, int, int, float, float const *, int, float const *, int, float, float *, int); + +/* "thinc/backends/cblas.pxd":8 + * + * + * ctypedef void (*saxpy_ptr)(int N, float alpha, const float* X, int incX, # <<<<<<<<<<<<<< + * float *Y, int incY) noexcept nogil + * +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_saxpy_ptr)(int, float, float const *, int, float *, int); + +/* "thinc/backends/cblas.pxd":12 + * + * + * ctypedef void (*daxpy_ptr)(int N, double alpha, const double* X, int incX, # <<<<<<<<<<<<<< + * double *Y, int incY) noexcept nogil + * +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_daxpy_ptr)(int, double, double const *, int, double *, int); + +/* "thinc/backends/cblas.pxd":19 + * # that consumers of the CBlas class cannot become dependent on its size or + * # ordering. + * cdef struct BlasFuncs # <<<<<<<<<<<<<< + * + * +*/ +struct __pyx_t_5thinc_8backends_5cblas_BlasFuncs { + __pyx_t_5thinc_8backends_5cblas_daxpy_ptr daxpy; + __pyx_t_5thinc_8backends_5cblas_saxpy_ptr saxpy; + __pyx_t_5thinc_8backends_5cblas_sgemm_ptr sgemm; +}; + +/* "thinc/backends/cblas.pxd":22 + * + * + * cdef class CBlas: # <<<<<<<<<<<<<< + * cdef shared_ptr[BlasFuncs] ptr + * +*/ +struct __pyx_obj_5thinc_8backends_5cblas_CBlas { + PyObject_HEAD + std::shared_ptr ptr; +}; + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ +struct __pyx_array_obj { + PyObject_HEAD + struct __pyx_vtabstruct_array *__pyx_vtab; + char *data; + Py_ssize_t len; + char *format; + int ndim; + Py_ssize_t *_shape; + Py_ssize_t *_strides; + Py_ssize_t itemsize; + PyObject *mode; + PyObject *_format; + void (*callback_free_data)(void *); + int free_data; + int dtype_is_object; +}; + + +/* "View.MemoryView":299 + * + * + * @cname('__pyx_MemviewEnum') # <<<<<<<<<<<<<< + * cdef class Enum(object): + * cdef object name +*/ +struct __pyx_MemviewEnum_obj { + PyObject_HEAD + PyObject *name; +}; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ +struct __pyx_memoryview_obj { + PyObject_HEAD + struct __pyx_vtabstruct_memoryview *__pyx_vtab; + PyObject *obj; + PyObject *_size; + void *_unused; + PyThread_type_lock lock; + __pyx_atomic_int_type acquisition_count; + Py_buffer view; + int flags; + int dtype_is_object; + __Pyx_TypeInfo const *typeinfo; +}; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ +struct __pyx_memoryviewslice_obj { + struct __pyx_memoryview_obj __pyx_base; + __Pyx_memviewslice from_slice; + PyObject *from_object; + PyObject *(*to_object_func)(char *); + int (*to_dtype_func)(char *, PyObject *); +}; + + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ + +struct __pyx_vtabstruct_array { + PyObject *(*get_memview)(struct __pyx_array_obj *); +}; +static struct __pyx_vtabstruct_array *__pyx_vtabptr_array; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ + +struct __pyx_vtabstruct_memoryview { + char *(*get_item_pointer)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*is_slice)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_slice_assignment)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*setitem_slice_assign_scalar)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_indexed)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*convert_item_to_object)(struct __pyx_memoryview_obj *, char *); + PyObject *(*assign_item_from_object)(struct __pyx_memoryview_obj *, char *, PyObject *); + PyObject *(*_get_base)(struct __pyx_memoryview_obj *); +}; +static struct __pyx_vtabstruct_memoryview *__pyx_vtabptr_memoryview; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ + +struct __pyx_vtabstruct__memoryviewslice { + struct __pyx_vtabstruct_memoryview __pyx_base; +}; +static struct __pyx_vtabstruct__memoryviewslice *__pyx_vtabptr__memoryviewslice; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallCFunction.proto (used by CallUnboundCMethod0) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallOneArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectGetAttrStr.proto (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyValueError_Check.proto */ +#define __Pyx_PyExc_ValueError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ValueError) + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* PyObjectFastCallMethod.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL) +#else +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf); +#endif + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* PyMemoryError_Check.proto */ +#define __Pyx_PyExc_MemoryError_Check(obj) __Pyx_TypeCheck(obj, PyExc_MemoryError) + +/* BuildPyUnicode.proto (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char); + +/* COrdinalToPyUnicode.proto (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value); +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t width, char padding_char); + +/* GCCDiagnostics.proto (used by CIntToPyUnicode) */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* IncludeStdlibH.proto (used by CIntToPyUnicode) */ +#include + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_int(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_int(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_int(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char); + +/* JoinPyUnicode.export */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyObjectFormatSimple.proto */ +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#elif CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) :\ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) :\ + PyObject_Format(s, f)) +#else + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#endif + +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *); /*proto*/ +/* GetAttr.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ObjectGetItem.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key); +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/* RejectKeywords.export */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds); + +/* PyTypeError_Check.proto */ +#define __Pyx_PyExc_TypeError_Check(obj) __Pyx_TypeCheck(obj, PyExc_TypeError) + +/* DivInt[Py_ssize_t].proto */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t, Py_ssize_t, int b_is_constant); + +/* UnaryNegOverflows.proto */ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x)\ + (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* RaiseNoneIterError.proto */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); + +/* ExtTypeTest.proto */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* PySequenceMultiply.proto */ +#define __Pyx_PySequence_Multiply_Left(mul, seq) __Pyx_PySequence_Multiply(seq, mul) +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul); + +/* PyObjectFormatAndDecref.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f); +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f); + +/* PyObjectFormat.proto */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* s, PyObject* f); +#else +#define __Pyx_PyObject_Format(s, f) PyObject_Format(s, f) +#endif + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* PyIndexError_Check.proto */ +#define __Pyx_PyExc_IndexError_Check(obj) __Pyx_TypeCheck(obj, PyExc_IndexError) + +/* DivInt[long].proto */ +static CYTHON_INLINE long __Pyx_div_long(long, long, int b_is_constant); + +/* MoveIfSupported.proto */ +#if CYTHON_USE_CPP_STD_MOVE + #include + #define __PYX_STD_MOVE_IF_SUPPORTED(x) std::move(x) +#else + #define __PYX_STD_MOVE_IF_SUPPORTED(x) x +#endif + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* DefaultPlacementNew.proto */ +#include +template +void __Pyx_default_placement_construct(T* x) { + new (static_cast(x)) T(); +} + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(PyObject *api_dict, const char *name, void (*f)(void), const char *sig); + +/* GetApiDict.proto */ +static PyObject *__Pyx_ApiExport_GetApiDict(void); + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* DelItemOnTypeDict.proto (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k); +#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k) + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* FunctionImport.proto */ +static int __Pyx_ImportFunction_3_2_1(PyObject *module, const char *funcname, void (**f)(void), const char *sig); + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* BufferStructDeclare.proto */ +typedef struct { + Py_ssize_t shape, strides, suboffsets; +} __Pyx_Buf_DimInfo; +typedef struct { + size_t refcount; + Py_buffer pybuffer; +} __Pyx_Buffer; +typedef struct { + __Pyx_Buffer *rcbuffer; + char *data; + __Pyx_Buf_DimInfo diminfo[8]; +} __Pyx_LocalBuf_ND; + +/* MemviewRefcount.proto */ +static CYTHON_INLINE int __pyx_add_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +#define __pyx_get_slice_count_pointer(memview) (&memview->acquisition_count) +#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) +static CYTHON_INLINE void __Pyx_INC_MEMVIEW(__Pyx_memviewslice *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *, int, int); + +/* MemviewSliceIsContig.proto */ +static int __pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim); + +/* OverlappingSlices.proto */ +static int __pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize); + +/* MemviewSliceInit.proto */ +static int __Pyx_init_memviewslice( + struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference); + +/* MemviewSliceCopy.proto */ +static __Pyx_memviewslice +__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object); + +/* PyObjectCall2Args.proto (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethod1.proto (used by UpdateUnpickledDict) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* UpdateUnpickledDict.proto */ +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index); + +/* CheckUnpickleChecksum.proto */ +static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* PyObjectVectorCallKwBuilder.proto (used by CIntToPy) */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self); /* proto*/ +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto*/ +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self); /* proto*/ +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto*/ + +/* Module declarations from "libcpp" */ + +/* Module declarations from "libcpp.memory" */ + +/* Module declarations from "cython.view" */ + +/* Module declarations from "cython.dataclasses" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "blis.cy" */ +static void (*__pyx_f_4blis_2cy_sgemm)(int, int, int, int, int, float, float const *, int, float const *, int, float, float *, int); /*proto*/ +static void (*__pyx_f_4blis_2cy_daxpy)(int, double, double const *, int, double *, int); /*proto*/ +static void (*__pyx_f_4blis_2cy_saxpy)(int, float, float const *, int, float *, int); /*proto*/ + +/* Module declarations from "blis" */ + +/* Module declarations from "thinc.backends.cblas" */ +static PyObject *__pyx_collections_abc_Sequence = 0; +static PyObject *generic = 0; +static PyObject *strided = 0; +static PyObject *indirect = 0; +static PyObject *contiguous = 0; +static PyObject *indirect_contiguous = 0; +static int __pyx_memoryview_thread_locks_used; +static PyThread_type_lock __pyx_memoryview_thread_locks[8]; +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *); /*proto*/ +static struct __pyx_array_obj *__pyx_array_new(PyObject *, Py_ssize_t, char *, char const *, char *); /*proto*/ +static PyObject *__pyx_memoryview_new(PyObject *, int, int, __Pyx_TypeInfo const *); /*proto*/ +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *); /*proto*/ +static PyObject *_unellipsify(PyObject *, int); /*proto*/ +static int assert_direct_dimensions(Py_ssize_t *, int); /*proto*/ +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *, PyObject *); /*proto*/ +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int, int); /*proto*/ +static char *__pyx_pybuffer_index(Py_buffer *, char *, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memslice_transpose(__Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice, int, PyObject *(*)(char *), int (*)(char *, PyObject *), int); /*proto*/ +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static Py_ssize_t abs_py_ssize_t(Py_ssize_t); /*proto*/ +static char __pyx_get_best_slice_order(__Pyx_memviewslice *, int); /*proto*/ +static void _copy_strided_to_strided(char *, Py_ssize_t *, char *, Py_ssize_t *, Py_ssize_t *, Py_ssize_t *, int, size_t); /*proto*/ +static void copy_strided_to_strided(__Pyx_memviewslice *, __Pyx_memviewslice *, int, size_t); /*proto*/ +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *, int); /*proto*/ +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *, Py_ssize_t *, Py_ssize_t, int, char); /*proto*/ +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *, __Pyx_memviewslice *, char, int); /*proto*/ +static int __pyx_memoryview_err_extents(int, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memoryview_err_dim(PyObject *, PyObject *, int); /*proto*/ +static int __pyx_memoryview_err(PyObject *, PyObject *); /*proto*/ +static int __pyx_memoryview_err_no_memory(void); /*proto*/ +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice, __Pyx_memviewslice, int, int, int); /*proto*/ +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *, int, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *, int, size_t, void *, int); /*proto*/ +static void __pyx_memoryview__slice_assign_scalar(char *, Py_ssize_t *, Py_ssize_t *, int, size_t, void *); /*proto*/ +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.backends.cblas" +extern int __pyx_module_is_main_thinc__backends__cblas; +int __pyx_module_is_main_thinc__backends__cblas = 0; + +/* Implementation of "thinc.backends.cblas" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin___import__; +static PyObject *__pyx_builtin_enumerate; +static PyObject *__pyx_builtin_Ellipsis; +static PyObject *__pyx_builtin_id; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object); /* proto */ +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_pf_5thinc_8backends_5cblas_5CBlas___init__(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_tp_new_5thinc_8backends_5cblas_CBlas(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyObject *__pyx_type_5thinc_8backends_5cblas_CBlas; + PyObject *__pyx_type___pyx_array; + PyObject *__pyx_type___pyx_MemviewEnum; + PyObject *__pyx_type___pyx_memoryview; + PyObject *__pyx_type___pyx_memoryviewslice; + PyTypeObject *__pyx_ptype_5thinc_8backends_5cblas_CBlas; + PyTypeObject *__pyx_array_type; + PyTypeObject *__pyx_MemviewEnum_type; + PyTypeObject *__pyx_memoryview_type; + PyTypeObject *__pyx_memoryviewslice_type; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_slice[1]; + PyObject *__pyx_tuple[2]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[124]; + PyObject *__pyx_number_tab[4]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_All_dimensions_preceding_dimensi __pyx_string_tab[1] +#define __pyx_kp_u_Buffer_view_does_not_expose_stri __pyx_string_tab[2] +#define __pyx_kp_u_Can_only_create_a_buffer_that_is __pyx_string_tab[3] +#define __pyx_kp_u_Cannot_assign_to_read_only_memor __pyx_string_tab[4] +#define __pyx_kp_u_Cannot_create_writable_memory_vi __pyx_string_tab[5] +#define __pyx_kp_u_Cannot_index_with_type __pyx_string_tab[6] +#define __pyx_kp_u_Cannot_transpose_memoryview_with __pyx_string_tab[7] +#define __pyx_kp_u_Dimension_d_is_not_direct __pyx_string_tab[8] +#define __pyx_kp_u_Empty_shape_tuple_for_cython_arr __pyx_string_tab[9] +#define __pyx_kp_u_Index_out_of_bounds_axis_d __pyx_string_tab[10] +#define __pyx_kp_u_Indirect_dimensions_not_supporte __pyx_string_tab[11] +#define __pyx_kp_u_Invalid_mode_expected_c_or_fortr __pyx_string_tab[12] +#define __pyx_kp_u_Invalid_shape_in_axis __pyx_string_tab[13] +#define __pyx_kp_u_MemoryView_of __pyx_string_tab[14] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[15] +#define __pyx_kp_u_Out_of_bounds_on_buffer_access_a __pyx_string_tab[16] +#define __pyx_kp_u_Step_may_not_be_zero_axis_d __pyx_string_tab[17] +#define __pyx_kp_u_Unable_to_convert_item_to_object __pyx_string_tab[18] +#define __pyx_kp_u__2 __pyx_string_tab[19] +#define __pyx_kp_u__3 __pyx_string_tab[20] +#define __pyx_kp_u__4 __pyx_string_tab[21] +#define __pyx_kp_u__5 __pyx_string_tab[22] +#define __pyx_kp_u__6 __pyx_string_tab[23] +#define __pyx_kp_u_add_note __pyx_string_tab[24] +#define __pyx_kp_u_and __pyx_string_tab[25] +#define __pyx_kp_u_at_0x __pyx_string_tab[26] +#define __pyx_kp_u_collections_abc __pyx_string_tab[27] +#define __pyx_kp_u_contiguous_and_direct __pyx_string_tab[28] +#define __pyx_kp_u_contiguous_and_indirect __pyx_string_tab[29] +#define __pyx_kp_u_disable __pyx_string_tab[30] +#define __pyx_kp_u_enable __pyx_string_tab[31] +#define __pyx_kp_u_gc __pyx_string_tab[32] +#define __pyx_kp_u_got __pyx_string_tab[33] +#define __pyx_kp_u_got_differing_extents_in_dimensi __pyx_string_tab[34] +#define __pyx_kp_u_isenabled __pyx_string_tab[35] +#define __pyx_kp_u_itemsize_0_for_cython_array __pyx_string_tab[36] +#define __pyx_kp_u_no_default___reduce___due_to_non __pyx_string_tab[37] +#define __pyx_kp_u_object __pyx_string_tab[38] +#define __pyx_kp_u_self_name_is_not_None __pyx_string_tab[39] +#define __pyx_kp_u_self_ptr_cannot_be_converted_to __pyx_string_tab[40] +#define __pyx_kp_u_strided_and_direct __pyx_string_tab[41] +#define __pyx_kp_u_strided_and_direct_or_indirect __pyx_string_tab[42] +#define __pyx_kp_u_strided_and_indirect __pyx_string_tab[43] +#define __pyx_kp_u_stringsource __pyx_string_tab[44] +#define __pyx_kp_u_unable_to_allocate_array_data __pyx_string_tab[45] +#define __pyx_kp_u_unable_to_allocate_shape_and_str __pyx_string_tab[46] +#define __pyx_n_u_ASCII __pyx_string_tab[47] +#define __pyx_n_u_CBlas __pyx_string_tab[48] +#define __pyx_n_u_CBlas___reduce_cython __pyx_string_tab[49] +#define __pyx_n_u_CBlas___setstate_cython __pyx_string_tab[50] +#define __pyx_n_u_Ellipsis __pyx_string_tab[51] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[52] +#define __pyx_n_u_Sequence __pyx_string_tab[53] +#define __pyx_n_u_View_MemoryView __pyx_string_tab[54] +#define __pyx_n_u_abc __pyx_string_tab[55] +#define __pyx_n_u_allocate_buffer __pyx_string_tab[56] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[57] +#define __pyx_n_u_base __pyx_string_tab[58] +#define __pyx_n_u_c __pyx_string_tab[59] +#define __pyx_n_u_class __pyx_string_tab[60] +#define __pyx_n_u_class_getitem __pyx_string_tab[61] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[62] +#define __pyx_n_u_count __pyx_string_tab[63] +#define __pyx_n_u_dict __pyx_string_tab[64] +#define __pyx_n_u_dtype_is_object __pyx_string_tab[65] +#define __pyx_n_u_encode __pyx_string_tab[66] +#define __pyx_n_u_enumerate __pyx_string_tab[67] +#define __pyx_n_u_error __pyx_string_tab[68] +#define __pyx_n_u_flags __pyx_string_tab[69] +#define __pyx_n_u_format __pyx_string_tab[70] +#define __pyx_n_u_fortran __pyx_string_tab[71] +#define __pyx_n_u_func __pyx_string_tab[72] +#define __pyx_n_u_getstate __pyx_string_tab[73] +#define __pyx_n_u_id __pyx_string_tab[74] +#define __pyx_n_u_import __pyx_string_tab[75] +#define __pyx_n_u_index __pyx_string_tab[76] +#define __pyx_n_u_is_coroutine __pyx_string_tab[77] +#define __pyx_n_u_items __pyx_string_tab[78] +#define __pyx_n_u_itemsize __pyx_string_tab[79] +#define __pyx_n_u_main __pyx_string_tab[80] +#define __pyx_n_u_memview __pyx_string_tab[81] +#define __pyx_n_u_mode __pyx_string_tab[82] +#define __pyx_n_u_module __pyx_string_tab[83] +#define __pyx_n_u_name __pyx_string_tab[84] +#define __pyx_n_u_name_2 __pyx_string_tab[85] +#define __pyx_n_u_ndim __pyx_string_tab[86] +#define __pyx_n_u_new __pyx_string_tab[87] +#define __pyx_n_u_obj __pyx_string_tab[88] +#define __pyx_n_u_pack __pyx_string_tab[89] +#define __pyx_n_u_pop __pyx_string_tab[90] +#define __pyx_n_u_pyx_capi __pyx_string_tab[91] +#define __pyx_n_u_pyx_checksum __pyx_string_tab[92] +#define __pyx_n_u_pyx_state __pyx_string_tab[93] +#define __pyx_n_u_pyx_type __pyx_string_tab[94] +#define __pyx_n_u_pyx_unpickle_Enum __pyx_string_tab[95] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[96] +#define __pyx_n_u_qualname __pyx_string_tab[97] +#define __pyx_n_u_reduce __pyx_string_tab[98] +#define __pyx_n_u_reduce_cython __pyx_string_tab[99] +#define __pyx_n_u_reduce_ex __pyx_string_tab[100] +#define __pyx_n_u_register __pyx_string_tab[101] +#define __pyx_n_u_self __pyx_string_tab[102] +#define __pyx_n_u_set_name __pyx_string_tab[103] +#define __pyx_n_u_setdefault __pyx_string_tab[104] +#define __pyx_n_u_setstate __pyx_string_tab[105] +#define __pyx_n_u_setstate_cython __pyx_string_tab[106] +#define __pyx_n_u_shape __pyx_string_tab[107] +#define __pyx_n_u_size __pyx_string_tab[108] +#define __pyx_n_u_slots __pyx_string_tab[109] +#define __pyx_n_u_start __pyx_string_tab[110] +#define __pyx_n_u_step __pyx_string_tab[111] +#define __pyx_n_u_stop __pyx_string_tab[112] +#define __pyx_n_u_struct __pyx_string_tab[113] +#define __pyx_n_u_test __pyx_string_tab[114] +#define __pyx_n_u_thinc_backends_cblas __pyx_string_tab[115] +#define __pyx_n_u_unpack __pyx_string_tab[116] +#define __pyx_n_u_update __pyx_string_tab[117] +#define __pyx_n_u_values __pyx_string_tab[118] +#define __pyx_n_u_x __pyx_string_tab[119] +#define __pyx_kp_b_iso88591_Q __pyx_string_tab[120] +#define __pyx_kp_b_pyx_t_5thinc_8backends_5cblas __pyx_string_tab[121] +#define __pyx_kp_b_void_int_double_double_const_int __pyx_string_tab[122] +#define __pyx_n_b_O __pyx_string_tab[123] +#define __pyx_int_0 __pyx_number_tab[0] +#define __pyx_int_neg_1 __pyx_number_tab[1] +#define __pyx_int_1 __pyx_number_tab[2] +#define __pyx_int_136983863 __pyx_number_tab[3] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_5cblas_CBlas); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_5cblas_CBlas); + Py_CLEAR(clear_module_state->__pyx_array_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_array); + Py_CLEAR(clear_module_state->__pyx_MemviewEnum_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_MemviewEnum); + Py_CLEAR(clear_module_state->__pyx_memoryview_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryview); + Py_CLEAR(clear_module_state->__pyx_memoryviewslice_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_slice[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<124; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<4; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_5cblas_CBlas); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_5cblas_CBlas); + Py_VISIT(traverse_module_state->__pyx_array_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_array); + Py_VISIT(traverse_module_state->__pyx_MemviewEnum_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_MemviewEnum); + Py_VISIT(traverse_module_state->__pyx_memoryview_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryview); + Py_VISIT(traverse_module_state->__pyx_memoryviewslice_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_slice[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<124; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<4; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + +/* Python wrapper */ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_shape = 0; + Py_ssize_t __pyx_v_itemsize; + PyObject *__pyx_v_format = 0; + PyObject *__pyx_v_mode = 0; + int __pyx_v_allocate_buffer; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_shape,&__pyx_mstate_global->__pyx_n_u_itemsize,&__pyx_mstate_global->__pyx_n_u_format,&__pyx_mstate_global->__pyx_n_u_mode,&__pyx_mstate_global->__pyx_n_u_allocate_buffer,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 129, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 129, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, i); __PYX_ERR(1, 129, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + } + __pyx_v_shape = ((PyObject*)values[0]); + __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_v_format = values[2]; + __pyx_v_mode = values[3]; + if (values[4]) { + __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) + } else { + + /* "View.MemoryView":130 + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, + * mode="c", bint allocate_buffer=True): # <<<<<<<<<<<<<< + * + * cdef int idx +*/ + __pyx_v_allocate_buffer = ((int)1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, __pyx_nargs); __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 129, __pyx_L1_error) + if (unlikely(((PyObject *)__pyx_v_format) == Py_None)) { + PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 129, __pyx_L1_error) + } + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v_shape, __pyx_v_itemsize, __pyx_v_format, __pyx_v_mode, __pyx_v_allocate_buffer); + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer) { + int __pyx_v_idx; + Py_ssize_t __pyx_v_dim; + char __pyx_v_order; + int __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11[5]; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + __Pyx_INCREF(__pyx_v_format); + + /* "View.MemoryView":135 + * cdef Py_ssize_t dim + * + * self.ndim = len(shape) # <<<<<<<<<<<<<< + * self.itemsize = itemsize + * +*/ + if (unlikely(__pyx_v_shape == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 135, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 135, __pyx_L1_error) + __pyx_v_self->ndim = ((int)__pyx_t_1); + + /* "View.MemoryView":136 + * + * self.ndim = len(shape) + * self.itemsize = itemsize # <<<<<<<<<<<<<< + * + * if not self.ndim: +*/ + __pyx_v_self->itemsize = __pyx_v_itemsize; + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + __pyx_t_2 = (!(__pyx_v_self->ndim != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":139 + * + * if not self.ndim: + * raise ValueError, "Empty shape tuple for cython.array" # <<<<<<<<<<<<<< + * + * if itemsize <= 0: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Empty_shape_tuple_for_cython_arr, 0, 0); + __PYX_ERR(1, 139, __pyx_L1_error) + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + } + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + __pyx_t_2 = (__pyx_v_itemsize <= 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":142 + * + * if itemsize <= 0: + * raise ValueError, "itemsize <= 0 for cython.array" # <<<<<<<<<<<<<< + * + * if not isinstance(format, bytes): +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_itemsize_0_for_cython_array, 0, 0); + __PYX_ERR(1, 142, __pyx_L1_error) + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + } + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + __pyx_t_2 = PyBytes_Check(__pyx_v_format); + __pyx_t_3 = (!__pyx_t_2); + if (__pyx_t_3) { + + /* "View.MemoryView":145 + * + * if not isinstance(format, bytes): + * format = format.encode('ASCII') # <<<<<<<<<<<<<< + * self._format = format # keep a reference to the byte string + * self.format = self._format +*/ + __pyx_t_5 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ASCII}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_encode, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF_SET(__pyx_v_format, __pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + } + + /* "View.MemoryView":146 + * if not isinstance(format, bytes): + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string # <<<<<<<<<<<<<< + * self.format = self._format + * +*/ + __pyx_t_4 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_4); + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_v_self->_format); + __Pyx_DECREF(__pyx_v_self->_format); + __pyx_v_self->_format = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":147 + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string + * self.format = self._format # <<<<<<<<<<<<<< + * + * +*/ + if (unlikely(__pyx_v_self->_format == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(1, 147, __pyx_L1_error) + } + __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) + __pyx_v_self->format = __pyx_t_7; + + /* "View.MemoryView":150 + * + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) # <<<<<<<<<<<<<< + * self._strides = self._shape + self.ndim + * +*/ + __pyx_v_self->_shape = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * __pyx_v_self->ndim) * 2))); + + /* "View.MemoryView":151 + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) + * self._strides = self._shape + self.ndim # <<<<<<<<<<<<<< + * + * if not self._shape: +*/ + __pyx_v_self->_strides = (__pyx_v_self->_shape + __pyx_v_self->ndim); + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + __pyx_t_3 = (!(__pyx_v_self->_shape != 0)); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":154 + * + * if not self._shape: + * raise MemoryError, "unable to allocate shape and strides." # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_shape_and_str, 0, 0); + __PYX_ERR(1, 154, __pyx_L1_error) + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + } + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + __pyx_t_8 = 0; + __pyx_t_4 = __pyx_v_shape; __Pyx_INCREF(__pyx_t_4); + __pyx_t_1 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 157, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_4, __pyx_t_1); + #endif + ++__pyx_t_1; + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_9; + __pyx_v_idx = __pyx_t_8; + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + __pyx_t_3 = (__pyx_v_dim <= 0); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":159 + * for idx, dim in enumerate(shape): + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." # <<<<<<<<<<<<<< + * self._shape[idx] = dim + * +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_int(__pyx_v_idx, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11[0] = __pyx_mstate_global->__pyx_kp_u_Invalid_shape_in_axis; + __pyx_t_11[1] = __pyx_t_5; + __pyx_t_11[2] = __pyx_mstate_global->__pyx_kp_u_; + __pyx_t_11[3] = __pyx_t_10; + __pyx_t_11[4] = __pyx_mstate_global->__pyx_kp_u__2; + __pyx_t_12 = __Pyx_PyUnicode_Join(__pyx_t_11, 5, 22 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_10) + 1, 127); + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 159, __pyx_L1_error) + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + } + + /* "View.MemoryView":160 + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim # <<<<<<<<<<<<<< + * + * cdef char order +*/ + (__pyx_v_self->_shape[__pyx_v_idx]) = __pyx_v_dim; + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 163, __pyx_L1_error) + if (__pyx_t_3) { + + /* "View.MemoryView":164 + * cdef char order + * if mode == 'c': + * order = b'C' # <<<<<<<<<<<<<< + * self.mode = u'c' + * elif mode == 'fortran': +*/ + __pyx_v_order = 'C'; + + /* "View.MemoryView":165 + * if mode == 'c': + * order = b'C' + * self.mode = u'c' # <<<<<<<<<<<<<< + * elif mode == 'fortran': + * order = b'F' +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_c; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 166, __pyx_L1_error) + if (likely(__pyx_t_3)) { + + /* "View.MemoryView":167 + * self.mode = u'c' + * elif mode == 'fortran': + * order = b'F' # <<<<<<<<<<<<<< + * self.mode = u'fortran' + * else: +*/ + __pyx_v_order = 'F'; + + /* "View.MemoryView":168 + * elif mode == 'fortran': + * order = b'F' + * self.mode = u'fortran' # <<<<<<<<<<<<<< + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_fortran; + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":170 + * self.mode = u'fortran' + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" # <<<<<<<<<<<<<< + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) +*/ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_mode, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_12 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_Invalid_mode_expected_c_or_fortr, __pyx_t_4); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 170, __pyx_L1_error) + } + __pyx_L11:; + + /* "View.MemoryView":172 + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) # <<<<<<<<<<<<<< + * + * self.free_data = allocate_buffer +*/ + __pyx_v_self->len = __pyx_fill_contig_strides_array(__pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_itemsize, __pyx_v_self->ndim, __pyx_v_order); + + /* "View.MemoryView":174 + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) + * + * self.free_data = allocate_buffer # <<<<<<<<<<<<<< + * self.dtype_is_object = format == b'O' + * +*/ + __pyx_v_self->free_data = __pyx_v_allocate_buffer; + + /* "View.MemoryView":175 + * + * self.free_data = allocate_buffer + * self.dtype_is_object = format == b'O' # <<<<<<<<<<<<<< + * + * if allocate_buffer: +*/ + __pyx_t_12 = PyObject_RichCompare(__pyx_v_format, __pyx_mstate_global->__pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_12); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_12); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 175, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_v_self->dtype_is_object = __pyx_t_3; + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + if (__pyx_v_allocate_buffer) { + + /* "View.MemoryView":178 + * + * if allocate_buffer: + * _allocate_buffer(self) # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_t_8 = __pyx_array_allocate_buffer(__pyx_v_self); if (unlikely(__pyx_t_8 == ((int)-1))) __PYX_ERR(1, 178, __pyx_L1_error) + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + } + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_format); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(((struct __pyx_array_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_bufmode; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + char *__pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":182 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 # <<<<<<<<<<<<<< + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": +*/ + __pyx_v_bufmode = -1; + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_t_1 = ((__pyx_v_flags & ((PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS) | PyBUF_ANY_CONTIGUOUS)) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 184, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":185 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_v_bufmode = (PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 186, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":187 + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." +*/ + __pyx_v_bufmode = (PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + } + __pyx_L4:; + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + __pyx_t_1 = (!((__pyx_v_flags & __pyx_v_bufmode) != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":189 + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." # <<<<<<<<<<<<<< + * info.buf = self.data + * info.len = self.len +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Can_only_create_a_buffer_that_is, 0, 0); + __PYX_ERR(1, 189, __pyx_L1_error) + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + } + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + } + + /* "View.MemoryView":190 + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data # <<<<<<<<<<<<<< + * info.len = self.len + * +*/ + __pyx_t_2 = __pyx_v_self->data; + __pyx_v_info->buf = __pyx_t_2; + + /* "View.MemoryView":191 + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data + * info.len = self.len # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + __pyx_t_3 = __pyx_v_self->len; + __pyx_v_info->len = __pyx_t_3; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":194 + * + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim # <<<<<<<<<<<<<< + * info.shape = self._shape + * info.strides = self._strides +*/ + __pyx_t_4 = __pyx_v_self->ndim; + __pyx_v_info->ndim = __pyx_t_4; + + /* "View.MemoryView":195 + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim + * info.shape = self._shape # <<<<<<<<<<<<<< + * info.strides = self._strides + * else: +*/ + __pyx_t_5 = __pyx_v_self->_shape; + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":196 + * info.ndim = self.ndim + * info.shape = self._shape + * info.strides = self._strides # <<<<<<<<<<<<<< + * else: + * info.ndim = 1 +*/ + __pyx_t_5 = __pyx_v_self->_strides; + __pyx_v_info->strides = __pyx_t_5; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":198 + * info.strides = self._strides + * else: + * info.ndim = 1 # <<<<<<<<<<<<<< + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL +*/ + /*else*/ { + __pyx_v_info->ndim = 1; + + /* "View.MemoryView":199 + * else: + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL # <<<<<<<<<<<<<< + * info.strides = NULL + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + __pyx_t_5 = (&__pyx_v_self->len); + } else { + __pyx_t_5 = NULL; + } + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":200 + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL # <<<<<<<<<<<<<< + * + * info.suboffsets = NULL +*/ + __pyx_v_info->strides = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":202 + * info.strides = NULL + * + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * info.itemsize = self.itemsize + * info.readonly = 0 +*/ + __pyx_v_info->suboffsets = NULL; + + /* "View.MemoryView":203 + * + * info.suboffsets = NULL + * info.itemsize = self.itemsize # <<<<<<<<<<<<<< + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL +*/ + __pyx_t_3 = __pyx_v_self->itemsize; + __pyx_v_info->itemsize = __pyx_t_3; + + /* "View.MemoryView":204 + * info.suboffsets = NULL + * info.itemsize = self.itemsize + * info.readonly = 0 # <<<<<<<<<<<<<< + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self +*/ + __pyx_v_info->readonly = 0; + + /* "View.MemoryView":205 + * info.itemsize = self.itemsize + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + __pyx_t_2 = __pyx_v_self->format; + } else { + __pyx_t_2 = NULL; + } + __pyx_v_info->format = __pyx_t_2; + + /* "View.MemoryView":206 + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self # <<<<<<<<<<<<<< + * + * def __dealloc__(array self): +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + +/* Python wrapper */ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + __pyx_t_1 = (__pyx_v_self->callback_free_data != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":210 + * def __dealloc__(array self): + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) # <<<<<<<<<<<<<< + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: +*/ + __pyx_v_self->callback_free_data(__pyx_v_self->data); + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + if (__pyx_v_self->free_data) { + } else { + __pyx_t_1 = __pyx_v_self->free_data; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_self->data != NULL); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":213 + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) # <<<<<<<<<<<<<< + * free(self.data) + * PyObject_Free(self._shape) +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_self->data, __pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_self->ndim, 0); + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + } + + /* "View.MemoryView":214 + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) # <<<<<<<<<<<<<< + * PyObject_Free(self._shape) + * +*/ + free(__pyx_v_self->data); + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + } + __pyx_L3:; + + /* "View.MemoryView":215 + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) + * PyObject_Free(self._shape) # <<<<<<<<<<<<<< + * + * @property +*/ + PyObject_Free(__pyx_v_self->_shape); + + /* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_5array_7memview___get__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":219 + * @property + * def memview(self): + * return self.get_memview() # <<<<<<<<<<<<<< + * + * @cname('get_memview') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.memview.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_memview", 0); + + /* "View.MemoryView":223 + * @cname('get_memview') + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE # <<<<<<<<<<<<<< + * return memoryview(self, flags, self.dtype_is_object) + * +*/ + __pyx_v_flags = ((PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT) | PyBUF_WRITABLE); + + /* "View.MemoryView":224 + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE + * return memoryview(self, flags, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, ((PyObject *)__pyx_v_self), __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_r = ((PyObject *)__pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array.get_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + + /* "View.MemoryView":227 + * + * def __len__(self): + * return self._shape[0] # <<<<<<<<<<<<<< + * + * def __getattr__(self, attr): +*/ + __pyx_r = (__pyx_v_self->_shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr); /*proto*/ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getattr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_attr)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getattr__", 0); + + /* "View.MemoryView":230 + * + * def __getattr__(self, attr): + * return getattr(self.memview, attr) # <<<<<<<<<<<<<< + * + * def __getitem__(self, item): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getattr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item); /*proto*/ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":233 + * + * def __getitem__(self, item): + * return self.memview[item] # <<<<<<<<<<<<<< + * + * def __setitem__(self, item, value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + +/* Python wrapper */ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + + /* "View.MemoryView":236 + * + * def __setitem__(self, item, value): + * self.memview[item] = value # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely((PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0))) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_array___reduce_cython__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_array_2__setstate_cython__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_v_i; + PyObject **__pyx_v_p; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":252 + * cdef PyObject **p + * + * self.free_data = True # <<<<<<<<<<<<<< + * self.data = malloc(self.len) + * if not self.data: +*/ + __pyx_v_self->free_data = 1; + + /* "View.MemoryView":253 + * + * self.free_data = True + * self.data = malloc(self.len) # <<<<<<<<<<<<<< + * if not self.data: + * raise MemoryError, "unable to allocate array data." +*/ + __pyx_v_self->data = ((char *)malloc(__pyx_v_self->len)); + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + __pyx_t_1 = (!(__pyx_v_self->data != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":255 + * self.data = malloc(self.len) + * if not self.data: + * raise MemoryError, "unable to allocate array data." # <<<<<<<<<<<<<< + * + * if self.dtype_is_object: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_array_data, 0, 0); + __PYX_ERR(1, 255, __pyx_L1_error) + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":258 + * + * if self.dtype_is_object: + * p = self.data # <<<<<<<<<<<<<< + * for i in range(self.len // self.itemsize): + * p[i] = Py_None +*/ + __pyx_v_p = ((PyObject **)__pyx_v_self->data); + + /* "View.MemoryView":259 + * if self.dtype_is_object: + * p = self.data + * for i in range(self.len // self.itemsize): # <<<<<<<<<<<<<< + * p[i] = Py_None + * Py_INCREF(Py_None) +*/ + if (unlikely(__pyx_v_self->itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_self->itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_self->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_div_Py_ssize_t(__pyx_v_self->len, __pyx_v_self->itemsize, 0); + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":260 + * p = self.data + * for i in range(self.len // self.itemsize): + * p[i] = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * return 0 +*/ + (__pyx_v_p[__pyx_v_i]) = Py_None; + + /* "View.MemoryView":261 + * for i in range(self.len // self.itemsize): + * p[i] = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * return 0 + * +*/ + Py_INCREF(Py_None); + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + } + + /* "View.MemoryView":262 + * p[i] = Py_None + * Py_INCREF(Py_None) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._allocate_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + +static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, char *__pyx_v_format, char const *__pyx_v_c_mode, char *__pyx_v_buf) { + struct __pyx_array_obj *__pyx_v_result = 0; + PyObject *__pyx_v_mode = 0; + struct __pyx_array_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("array_cwrapper", 0); + + /* "View.MemoryView":268 + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. # <<<<<<<<<<<<<< + * + * if buf is NULL: +*/ + __pyx_t_2 = ((__pyx_v_c_mode[0]) == 'f'); + if (__pyx_t_2) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_fortran; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_c; + } + __pyx_v_mode = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + __pyx_t_2 = (__pyx_v_buf == NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":271 + * + * if buf is NULL: + * result = array.__new__(array, shape, itemsize, format, mode) # <<<<<<<<<<<<<< + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) +*/ + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_3) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_4, NULL)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":273 + * result = array.__new__(array, shape, itemsize, format, mode) + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) # <<<<<<<<<<<<<< + * result.data = buf + * +*/ + /*else*/ { + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_3) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_allocate_buffer, Py_False) < (0)) __PYX_ERR(1, 273, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_1, __pyx_t_4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":274 + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) + * result.data = buf # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->data = __pyx_v_buf; + } + __pyx_L3:; + + /* "View.MemoryView":276 + * result.data = buf + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_mode); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + +/* Python wrapper */ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_name = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 302, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(1, 302, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, i); __PYX_ERR(1, 302, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + } + __pyx_v_name = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 302, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v_name); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__", 0); + + /* "View.MemoryView":303 + * cdef object name + * def __init__(self, name): + * self.name = name # <<<<<<<<<<<<<< + * def __repr__(self): + * return self.name +*/ + __Pyx_INCREF(__pyx_v_name); + __Pyx_GIVEREF(__pyx_v_name); + __Pyx_GOTREF(__pyx_v_self->name); + __Pyx_DECREF(__pyx_v_self->name); + __pyx_v_self->name = __pyx_v_name; + + /* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":305 + * self.name = name + * def __repr__(self): + * return self.name # <<<<<<<<<<<<<< + * + * cdef generic = Enum("") +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->name); + __pyx_r = __pyx_v_self->name; + goto __pyx_L0; + + /* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_MemviewEnum___reduce_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_v_state = 0; + PyObject *__pyx_v__dict = 0; + int __pyx_v_use_setstate; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":5 + * cdef object _dict + * cdef bint use_setstate + * state = (self.name,) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_self->name); + __Pyx_GIVEREF(__pyx_v_self->name); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->name) != (0)) __PYX_ERR(1, 5, __pyx_L1_error); + __pyx_v_state = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "(tree fragment)":6 + * cdef bint use_setstate + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None and _dict: + * state += (_dict,) +*/ + __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v__dict = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + __pyx_t_3 = (__pyx_v__dict != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_2) { + + /* "(tree fragment)":8 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(1, 8, __pyx_L1_error); + __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "(tree fragment)":9 + * if _dict is not None and _dict: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = ('self.name is not None',) +*/ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + goto __pyx_L3; + } + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + /*else*/ { + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_mstate_global->__pyx_tuple[0]); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_v_use_setstate = __pyx_t_2; + } + __pyx_L3:; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + if (__pyx_v_use_setstate) { + + /* "(tree fragment)":13 + * use_setstate = ('self.name is not None',) + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + } + + /* "(tree fragment)":15 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.Enum.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 16, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 16, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 16, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 16, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_MemviewEnum_2__setstate_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":17 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) # <<<<<<<<<<<<<< +*/ + __pyx_t_1 = __pyx_v___pyx_state; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(1, 17, __pyx_L1_error) + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 17, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + +/* Python wrapper */ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + int __pyx_v_flags; + int __pyx_v_dtype_is_object; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_obj,&__pyx_mstate_global->__pyx_n_u_flags,&__pyx_mstate_global->__pyx_n_u_dtype_is_object,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 348, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 348, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, i); __PYX_ERR(1, 348, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_obj = values[0]; + __pyx_v_flags = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + if (values[2]) { + __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + } else { + __pyx_v_dtype_is_object = ((int)0); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, __pyx_nargs); __PYX_ERR(1, 348, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_obj, __pyx_v_flags, __pyx_v_dtype_is_object); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + Py_intptr_t __pyx_t_4; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "View.MemoryView":349 + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj # <<<<<<<<<<<<<< + * self.flags = flags + * if type(self) is memoryview or obj is not None: +*/ + __Pyx_INCREF(__pyx_v_obj); + __Pyx_GIVEREF(__pyx_v_obj); + __Pyx_GOTREF(__pyx_v_self->obj); + __Pyx_DECREF(__pyx_v_self->obj); + __pyx_v_self->obj = __pyx_v_obj; + + /* "View.MemoryView":350 + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj + * self.flags = flags # <<<<<<<<<<<<<< + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) +*/ + __pyx_v_self->flags = __pyx_v_flags; + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + __pyx_t_2 = (((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))) == ((PyObject *)__pyx_mstate_global->__pyx_memoryview_type)); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_obj != Py_None); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":352 + * self.flags = flags + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) # <<<<<<<<<<<<<< + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None +*/ + __pyx_t_3 = PyObject_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 352, __pyx_L1_error) + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_self->view.obj) == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":354 + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = Py_None; + + /* "View.MemoryView":355 + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + } + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + __pyx_t_1 = (!__PYX_CYTHON_ATOMICS_ENABLED()); + if (__pyx_t_1) { + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + __pyx_t_2 = (__pyx_memoryview_thread_locks_used < 8); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L9_bool_binop_done; + } + + /* "View.MemoryView":361 + * if (__pyx_memoryview_thread_locks_used < 8 and + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 +*/ + __pyx_t_2 = (!__PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()); + __pyx_t_1 = __pyx_t_2; + __pyx_L9_bool_binop_done:; + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":362 + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: +*/ + __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + + /* "View.MemoryView":363 + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":365 + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< + * if self.lock is NULL: + * raise MemoryError +*/ + __pyx_v_self->lock = PyThread_allocate_lock(); + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":367 + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 367, __pyx_L1_error) + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + } + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":370 + * + * if flags & PyBUF_FORMAT: + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') # <<<<<<<<<<<<<< + * else: + * self.dtype_is_object = dtype_is_object +*/ + __pyx_t_2 = ((__pyx_v_self->view.format[0]) == 'O'); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_self->view.format[1]) == '\x00'); + __pyx_t_1 = __pyx_t_2; + __pyx_L14_bool_binop_done:; + __pyx_v_self->dtype_is_object = __pyx_t_1; + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":372 + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: + * self.dtype_is_object = dtype_is_object # <<<<<<<<<<<<<< + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 +*/ + /*else*/ { + __pyx_v_self->dtype_is_object = __pyx_v_dtype_is_object; + } + __pyx_L13:; + + /* "View.MemoryView":374 + * self.dtype_is_object = dtype_is_object + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 # <<<<<<<<<<<<<< + * self.typeinfo = NULL + * +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_4 = ((Py_intptr_t)((void *)(&__pyx_v_self->acquisition_count))); + __pyx_t_5 = (sizeof(__pyx_atomic_int_type)); + if (unlikely(__pyx_t_5 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 374, __pyx_L1_error) + } + __pyx_t_1 = ((__pyx_t_4 % __pyx_t_5) == 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 374, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 374, __pyx_L1_error) + #endif + + /* "View.MemoryView":375 + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 + * self.typeinfo = NULL # <<<<<<<<<<<<<< + * + * def __dealloc__(memoryview self): +*/ + __pyx_v_self->typeinfo = NULL; + + /* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + +/* Python wrapper */ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self) { + int __pyx_v_i; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + PyThread_type_lock __pyx_t_5; + PyThread_type_lock __pyx_t_6; + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + __pyx_t_1 = (__pyx_v_self->obj != Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":379 + * def __dealloc__(memoryview self): + * if self.obj is not None: + * PyBuffer_Release(&self.view) # <<<<<<<<<<<<<< + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * +*/ + PyBuffer_Release((&__pyx_v_self->view)); + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + __pyx_t_1 = (((Py_buffer *)(&__pyx_v_self->view))->obj == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":382 + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * + * (<__pyx_buffer *> &self.view).obj = NULL # <<<<<<<<<<<<<< + * Py_DECREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = NULL; + + /* "View.MemoryView":383 + * + * (<__pyx_buffer *> &self.view).obj = NULL + * Py_DECREF(Py_None) # <<<<<<<<<<<<<< + * + * cdef int i +*/ + Py_DECREF(Py_None); + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + } + __pyx_L3:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + __pyx_t_1 = (__pyx_v_self->lock != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":388 + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): # <<<<<<<<<<<<<< + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 +*/ + __pyx_t_1 = __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING(); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = __pyx_memoryview_thread_locks_used; + } + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + __pyx_t_1 = ((__pyx_memoryview_thread_locks[__pyx_v_i]) == __pyx_v_self->lock); + if (__pyx_t_1) { + + /* "View.MemoryView":390 + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 # <<<<<<<<<<<<<< + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used - 1); + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + __pyx_t_1 = (__pyx_v_i != __pyx_memoryview_thread_locks_used); + if (__pyx_t_1) { + + /* "View.MemoryView":393 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_t_5 = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + __pyx_t_6 = (__pyx_memoryview_thread_locks[__pyx_v_i]); + + /* "View.MemoryView":392 + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break +*/ + (__pyx_memoryview_thread_locks[__pyx_v_i]) = __pyx_t_5; + (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]) = __pyx_t_6; + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + } + + /* "View.MemoryView":394 + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break # <<<<<<<<<<<<<< + * else: + * PyThread_free_lock(self.lock) +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + } + } + /*else*/ { + + /* "View.MemoryView":396 + * break + * else: + * PyThread_free_lock(self.lock) # <<<<<<<<<<<<<< + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: +*/ + PyThread_free_lock(__pyx_v_self->lock); + } + __pyx_L6_break:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + } + + /* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + Py_ssize_t __pyx_v_dim; + char *__pyx_v_itemp; + PyObject *__pyx_v_idx = NULL; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_item_pointer", 0); + + /* "View.MemoryView":400 + * cdef char *get_item_pointer(memoryview self, object index) except NULL: + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf # <<<<<<<<<<<<<< + * + * for dim, idx in enumerate(index): +*/ + __pyx_v_itemp = ((char *)__pyx_v_self->view.buf); + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + __pyx_t_1 = 0; + if (likely(PyList_CheckExact(__pyx_v_index)) || PyTuple_CheckExact(__pyx_v_index)) { + __pyx_t_2 = __pyx_v_index; __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 402, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 402, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_3, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 402, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 402, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_1; + __pyx_t_1 = (__pyx_t_1 + 1); + + /* "View.MemoryView":403 + * + * for dim, idx in enumerate(index): + * itemp = pybuffer_index(&self.view, itemp, idx, dim) # <<<<<<<<<<<<<< + * + * return itemp +*/ + __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_7; + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":405 + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * + * return itemp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_itemp; + goto __pyx_L0; + + /* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.get_item_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index); /*proto*/ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_indices = NULL; + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + char *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + __pyx_t_1 = (__pyx_v_index == __pyx_builtin_Ellipsis); + if (__pyx_t_1) { + + /* "View.MemoryView":410 + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: + * return self # <<<<<<<<<<<<<< + * + * have_slices, indices = _unellipsify(index, self.view.ndim) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_self); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + } + + /* "View.MemoryView":412 + * return self + * + * have_slices, indices = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * cdef char *itemp +*/ + __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (likely(__pyx_t_2 != Py_None)) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 412, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 412, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_indices = __pyx_t_4; + __pyx_t_4 = 0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 415, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":416 + * cdef char *itemp + * if have_slices: + * return memview_slice(self, indices) # <<<<<<<<<<<<<< + * else: + * itemp = self.get_item_pointer(indices) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + } + + /* "View.MemoryView":418 + * return memview_slice(self, indices) + * else: + * itemp = self.get_item_pointer(indices) # <<<<<<<<<<<<<< + * return self.convert_item_to_object(itemp) + * +*/ + /*else*/ { + __pyx_t_5 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_5; + + /* "View.MemoryView":419 + * else: + * itemp = self.get_item_pointer(indices) + * return self.convert_item_to_object(itemp) # <<<<<<<<<<<<<< + * + * def __setitem__(memoryview self, object index, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + +/* Python wrapper */ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_obj = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + __Pyx_INCREF(__pyx_v_index); + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + if (unlikely(__pyx_v_self->view.readonly)) { + + /* "View.MemoryView":423 + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" # <<<<<<<<<<<<<< + * + * have_slices, index = _unellipsify(index, self.view.ndim) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_Cannot_assign_to_read_only_memor, 0, 0); + __PYX_ERR(1, 423, __pyx_L1_error) + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + } + + /* "View.MemoryView":425 + * raise TypeError, "Cannot assign to read-only memoryview" + * + * have_slices, index = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * if have_slices: +*/ + __pyx_t_1 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (likely(__pyx_t_1 != Py_None)) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 425, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 425, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_DECREF_SET(__pyx_v_index, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(1, 427, __pyx_L1_error) + if (__pyx_t_4) { + + /* "View.MemoryView":428 + * + * if have_slices: + * obj = self.is_slice(value) # <<<<<<<<<<<<<< + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 428, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_obj = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + __pyx_t_4 = (__pyx_v_obj != Py_None); + if (__pyx_t_4) { + + /* "View.MemoryView":430 + * obj = self.is_slice(value) + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) # <<<<<<<<<<<<<< + * else: + * self.setitem_slice_assign_scalar(self[index], value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_1, __pyx_v_obj); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":432 + * self.setitem_slice_assignment(self[index], obj) + * else: + * self.setitem_slice_assign_scalar(self[index], value) # <<<<<<<<<<<<<< + * else: + * self.setitem_indexed(index, value) +*/ + /*else*/ { + __pyx_t_3 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 432, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_3), __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L5:; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":434 + * self.setitem_slice_assign_scalar(self[index], value) + * else: + * self.setitem_indexed(index, value) # <<<<<<<<<<<<<< + * + * cdef is_slice(self, obj): +*/ + /*else*/ { + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L4:; + + /* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_slice", 0); + __Pyx_INCREF(__pyx_v_obj); + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_obj, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + /*try:*/ { + + /* "View.MemoryView":439 + * if not isinstance(obj, memoryview): + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< + * self.dtype_is_object) + * except TypeError: +*/ + __pyx_t_7 = NULL; + __pyx_t_8 = __Pyx_PyLong_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + + /* "View.MemoryView":440 + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) # <<<<<<<<<<<<<< + * except TypeError: + * return None +*/ + __pyx_t_9 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 440, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_7, __pyx_v_obj, __pyx_t_8, __pyx_t_9}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_10, (4-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF((PyObject *)__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_obj, ((PyObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L9_try_end; + __pyx_L4_error:; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":441 + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) + * except TypeError: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_11) { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":442 + * self.dtype_is_object) + * except TypeError: + * return None # <<<<<<<<<<<<<< + * + * return obj +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L7_except_return; + } + goto __pyx_L6_except_error; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + __pyx_L6_except_error:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L1_error; + __pyx_L7_except_return:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L0; + __pyx_L9_try_end:; + } + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + } + + /* "View.MemoryView":444 + * return None + * + * return obj # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assignment(self, dst, src): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_obj); + __pyx_r = __pyx_v_obj; + goto __pyx_L0; + + /* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src) { + __Pyx_memviewslice __pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_src_slice; + __Pyx_memviewslice __pyx_v_msrc; + __Pyx_memviewslice __pyx_v_mdst; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assignment", 0); + + /* "View.MemoryView":449 + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * +*/ + if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_v_msrc = (__pyx_t_1[0]); + + /* "View.MemoryView":450 + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] # <<<<<<<<<<<<<< + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) +*/ + if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_v_mdst = (__pyx_t_1[0]); + + /* "View.MemoryView":452 + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __pyx_memoryview_copy_contents(__pyx_v_msrc, __pyx_v_mdst, __pyx_t_3, __pyx_t_4, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 452, __pyx_L1_error) + + /* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assignment", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value) { + int __pyx_v_array[128]; + void *__pyx_v_tmp; + void *__pyx_v_item; + __Pyx_memviewslice *__pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_tmp_slice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + char const *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assign_scalar", 0); + + /* "View.MemoryView":456 + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): + * cdef int array[128] + * cdef void *tmp = NULL # <<<<<<<<<<<<<< + * cdef void *item + * +*/ + __pyx_v_tmp = NULL; + + /* "View.MemoryView":461 + * cdef __Pyx_memviewslice *dst_slice + * cdef __Pyx_memviewslice tmp_slice + * dst_slice = get_slice_from_memview(dst, &tmp_slice) # <<<<<<<<<<<<<< + * + * if self.view.itemsize > sizeof(array): +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 461, __pyx_L1_error) + __pyx_v_dst_slice = __pyx_t_1; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + __pyx_t_2 = (((size_t)__pyx_v_self->view.itemsize) > (sizeof(__pyx_v_array))); + if (__pyx_t_2) { + + /* "View.MemoryView":464 + * + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) # <<<<<<<<<<<<<< + * if tmp == NULL: + * raise MemoryError +*/ + __pyx_v_tmp = PyMem_Malloc(__pyx_v_self->view.itemsize); + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + __pyx_t_2 = (__pyx_v_tmp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":466 + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * item = tmp + * else: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 466, __pyx_L1_error) + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + } + + /* "View.MemoryView":467 + * if tmp == NULL: + * raise MemoryError + * item = tmp # <<<<<<<<<<<<<< + * else: + * item = array +*/ + __pyx_v_item = __pyx_v_tmp; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":469 + * item = tmp + * else: + * item = array # <<<<<<<<<<<<<< + * + * try: +*/ + /*else*/ { + __pyx_v_item = ((void *)__pyx_v_array); + } + __pyx_L3:; + + /* "View.MemoryView":471 + * item = array + * + * try: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * ( item)[0] = value +*/ + /*try:*/ { + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":473 + * try: + * if self.dtype_is_object: + * ( item)[0] = value # <<<<<<<<<<<<<< + * else: + * self.assign_item_from_object( item, value) +*/ + (((PyObject **)__pyx_v_item)[0]) = ((PyObject *)__pyx_v_value); + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":475 + * ( item)[0] = value + * else: + * self.assign_item_from_object( item, value) # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 475, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L8:; + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + __pyx_t_2 = (__pyx_v_self->view.suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":480 + * + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) # <<<<<<<<<<<<<< + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, + * item, self.dtype_is_object) +*/ + __pyx_t_4 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 480, __pyx_L6_error) + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + } + + /* "View.MemoryView":481 + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, # <<<<<<<<<<<<<< + * item, self.dtype_is_object) + * finally: +*/ + __pyx_memoryview_slice_assign_scalar(__pyx_v_dst_slice, __pyx_v_dst->view.ndim, __pyx_v_self->view.itemsize, __pyx_v_item, __pyx_v_self->dtype_is_object); + } + + /* "View.MemoryView":484 + * item, self.dtype_is_object) + * finally: + * PyMem_Free(tmp) # <<<<<<<<<<<<<< + * + * cdef setitem_indexed(self, index, value): +*/ + /*finally:*/ { + /*normal exit:*/{ + PyMem_Free(__pyx_v_tmp); + goto __pyx_L7; + } + __pyx_L6_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_ExceptionSwap(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); + if ( unlikely(__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0)) __Pyx_ErrFetch(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_XGOTREF(__pyx_t_12); + __pyx_t_4 = __pyx_lineno; __pyx_t_5 = __pyx_clineno; __pyx_t_6 = __pyx_filename; + { + PyMem_Free(__pyx_v_tmp); + } + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ErrRestore(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_5; __pyx_filename = __pyx_t_6; + goto __pyx_L1_error; + } + __pyx_L7:; + } + + /* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assign_scalar", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_indexed", 0); + + /* "View.MemoryView":487 + * + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) # <<<<<<<<<<<<<< + * self.assign_item_from_object(itemp, value) + * +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 487, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_1; + + /* "View.MemoryView":488 + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 488, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_indexed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_v_struct = NULL; + PyObject *__pyx_v_bytesitem = 0; + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":493 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef bytes bytesitem + * +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 493, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":496 + * cdef bytes bytesitem + * + * bytesitem = itemp[:self.view.itemsize] # <<<<<<<<<<<<<< + * try: + * result = struct.unpack(self.view.format, bytesitem) +*/ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 496, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_bytesitem = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_2, &__pyx_t_3, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + /*try:*/ { + + /* "View.MemoryView":498 + * bytesitem = itemp[:self.view.itemsize] + * try: + * result = struct.unpack(self.view.format, bytesitem) # <<<<<<<<<<<<<< + * except struct.error: + * raise ValueError, "Unable to convert item to object" +*/ + __pyx_t_5 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_t_6, __pyx_v_bytesitem}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_unpack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_result = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + } + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + /*else:*/ { + __pyx_t_8 = __Pyx_ssize_strlen(__pyx_v_self->view.format); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 502, __pyx_L5_except_error) + __pyx_t_9 = (__pyx_t_8 == 1); + if (__pyx_t_9) { + + /* "View.MemoryView":503 + * else: + * if len(self.view.format) == 1: + * return result[0] # <<<<<<<<<<<<<< + * return result + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 503, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L6_except_return; + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + } + + /* "View.MemoryView":504 + * if len(self.view.format) == 1: + * return result[0] + * return result # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L6_except_return; + } + __pyx_L3_error:; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "View.MemoryView":499 + * try: + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: # <<<<<<<<<<<<<< + * raise ValueError, "Unable to convert item to object" + * else: +*/ + __Pyx_ErrFetch(&__pyx_t_1, &__pyx_t_6, &__pyx_t_5); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_error); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_1, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_ErrRestore(__pyx_t_1, __pyx_t_6, __pyx_t_5); + __pyx_t_1 = 0; __pyx_t_6 = 0; __pyx_t_5 = 0; + if (__pyx_t_11) { + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_1) < 0) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_1); + + /* "View.MemoryView":500 + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: + * raise ValueError, "Unable to convert item to object" # <<<<<<<<<<<<<< + * else: + * if len(self.view.format) == 1: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Unable_to_convert_item_to_object, 0, 0); + __PYX_ERR(1, 500, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L1_error; + __pyx_L6_except_return:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L0; + } + + /* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesitem); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_v_struct = NULL; + char __pyx_v_c; + PyObject *__pyx_v_bytesvalue = 0; + Py_ssize_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + char *__pyx_t_10; + char *__pyx_t_11; + Py_ssize_t __pyx_t_12; + char *__pyx_t_13; + char *__pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":509 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef char c + * cdef bytes bytesvalue +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 509, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + __pyx_t_3 = PyTuple_Check(__pyx_v_value); + if (__pyx_t_3) { + + /* "View.MemoryView":515 + * + * if isinstance(value, tuple): + * bytesvalue = struct.pack(self.view.format, *value) # <<<<<<<<<<<<<< + * else: + * bytesvalue = struct.pack(self.view.format, value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 515, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 515, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":517 + * bytesvalue = struct.pack(self.view.format, *value) + * else: + * bytesvalue = struct.pack(self.view.format, value) # <<<<<<<<<<<<<< + * + * for i, c in enumerate(bytesvalue): +*/ + /*else*/ { + __pyx_t_6 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_t_1, __pyx_v_value}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_pack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 517, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = 0; + if (unlikely(__pyx_v_bytesvalue == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(1, 519, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytesvalue); + __pyx_t_9 = __pyx_v_bytesvalue; + __pyx_t_11 = __Pyx_PyBytes_AsWritableString(__pyx_t_9); if (unlikely(__pyx_t_11 == ((char *)NULL))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyBytes_GET_SIZE(__pyx_t_9); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_13 = (__pyx_t_11 + __pyx_t_12); + for (__pyx_t_14 = __pyx_t_11; __pyx_t_14 < __pyx_t_13; __pyx_t_14++) { + __pyx_t_10 = __pyx_t_14; + __pyx_v_c = (__pyx_t_10[0]); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_v_i = __pyx_t_8; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + (__pyx_v_itemp[__pyx_v_i]) = __pyx_v_c; + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t *__pyx_t_3; + char *__pyx_t_4; + void *__pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + __pyx_t_2 = ((__pyx_v_flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_L4_bool_binop_done:; + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":525 + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: + * raise ValueError, "Cannot create writable memory view from read-only memoryview" # <<<<<<<<<<<<<< + * + * if flags & PyBUF_ND: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Cannot_create_writable_memory_vi, 0, 0); + __PYX_ERR(1, 525, __pyx_L1_error) + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + } + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":528 + * + * if flags & PyBUF_ND: + * info.shape = self.view.shape # <<<<<<<<<<<<<< + * else: + * info.shape = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.shape; + __pyx_v_info->shape = __pyx_t_3; + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":530 + * info.shape = self.view.shape + * else: + * info.shape = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + /*else*/ { + __pyx_v_info->shape = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":533 + * + * if flags & PyBUF_STRIDES: + * info.strides = self.view.strides # <<<<<<<<<<<<<< + * else: + * info.strides = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.strides; + __pyx_v_info->strides = __pyx_t_3; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":535 + * info.strides = self.view.strides + * else: + * info.strides = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_INDIRECT: +*/ + /*else*/ { + __pyx_v_info->strides = NULL; + } + __pyx_L7:; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_INDIRECT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":538 + * + * if flags & PyBUF_INDIRECT: + * info.suboffsets = self.view.suboffsets # <<<<<<<<<<<<<< + * else: + * info.suboffsets = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.suboffsets; + __pyx_v_info->suboffsets = __pyx_t_3; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":540 + * info.suboffsets = self.view.suboffsets + * else: + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + /*else*/ { + __pyx_v_info->suboffsets = NULL; + } + __pyx_L8:; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":543 + * + * if flags & PyBUF_FORMAT: + * info.format = self.view.format # <<<<<<<<<<<<<< + * else: + * info.format = NULL +*/ + __pyx_t_4 = __pyx_v_self->view.format; + __pyx_v_info->format = __pyx_t_4; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":545 + * info.format = self.view.format + * else: + * info.format = NULL # <<<<<<<<<<<<<< + * + * info.buf = self.view.buf +*/ + /*else*/ { + __pyx_v_info->format = NULL; + } + __pyx_L9:; + + /* "View.MemoryView":547 + * info.format = NULL + * + * info.buf = self.view.buf # <<<<<<<<<<<<<< + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize +*/ + __pyx_t_5 = __pyx_v_self->view.buf; + __pyx_v_info->buf = __pyx_t_5; + + /* "View.MemoryView":548 + * + * info.buf = self.view.buf + * info.ndim = self.view.ndim # <<<<<<<<<<<<<< + * info.itemsize = self.view.itemsize + * info.len = self.view.len +*/ + __pyx_t_6 = __pyx_v_self->view.ndim; + __pyx_v_info->ndim = __pyx_t_6; + + /* "View.MemoryView":549 + * info.buf = self.view.buf + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize # <<<<<<<<<<<<<< + * info.len = self.view.len + * info.readonly = self.view.readonly +*/ + __pyx_t_7 = __pyx_v_self->view.itemsize; + __pyx_v_info->itemsize = __pyx_t_7; + + /* "View.MemoryView":550 + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize + * info.len = self.view.len # <<<<<<<<<<<<<< + * info.readonly = self.view.readonly + * info.obj = self +*/ + __pyx_t_7 = __pyx_v_self->view.len; + __pyx_v_info->len = __pyx_t_7; + + /* "View.MemoryView":551 + * info.itemsize = self.view.itemsize + * info.len = self.view.len + * info.readonly = self.view.readonly # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_v_info->readonly = __pyx_t_1; + + /* "View.MemoryView":552 + * info.len = self.view.len + * info.readonly = self.view.readonly + * info.obj = self # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":557 + * @property + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) # <<<<<<<<<<<<<< + * transpose_memslice(&result.from_slice) + * return result +*/ + __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 557, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 557, __pyx_L1_error) + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":558 + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 558, __pyx_L1_error) + + /* "View.MemoryView":559 + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) + * return result # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.T.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":563 + * @property + * def base(self): + * return self._get_base() # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->_get_base(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 563, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.base.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":566 + * + * cdef _get_base(self): + * return self.obj # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->obj); + __pyx_r = __pyx_v_self->obj; + goto __pyx_L0; + + /* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_7genexpr__pyx_v_length; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":570 + * @property + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_7genexpr__pyx_v_length = (__pyx_t_2[0]); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_7genexpr__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } /* exit inner scope */ + __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.shape.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr1__pyx_v_stride; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + __pyx_t_1 = (__pyx_v_self->view.strides == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":576 + * if self.view.strides == NULL: + * + * raise ValueError, "Buffer view does not expose strides" # <<<<<<<<<<<<<< + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Buffer_view_does_not_expose_stri, 0, 0); + __PYX_ERR(1, 576, __pyx_L1_error) + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + } + + /* "View.MemoryView":578 + * raise ValueError, "Buffer view does not expose strides" + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.strides + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.strides; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr1__pyx_v_stride = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr1__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.strides.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr2__pyx_v_suboffset; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + __pyx_t_1 = (__pyx_v_self->view.suboffsets == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PySequence_Multiply(__pyx_mstate_global->__pyx_tuple[1], __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + } + + /* "View.MemoryView":585 + * return (-1,) * self.view.ndim + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.suboffsets + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.suboffsets; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr2__pyx_v_suboffset = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr2__pyx_v_suboffset); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.suboffsets.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":589 + * @property + * def ndim(self): + * return self.view.ndim # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 589, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.ndim.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":593 + * @property + * def itemsize(self): + * return self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 593, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.itemsize.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":597 + * @property + * def nbytes(self): + * return self.size * self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.nbytes.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + __pyx_t_1 = (__pyx_v_self->_size == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":602 + * def size(self): + * if self._size is None: + * result = 1 # <<<<<<<<<<<<<< + * + * for length in self.view.shape[:self.view.ndim]: +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __pyx_v_result = __pyx_mstate_global->__pyx_int_1; + + /* "View.MemoryView":604 + * result = 1 + * + * for length in self.view.shape[:self.view.ndim]: # <<<<<<<<<<<<<< + * result *= length + * +*/ + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_t_5 = PyLong_FromSsize_t((__pyx_t_2[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 604, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_5); + __pyx_t_5 = 0; + + /* "View.MemoryView":605 + * + * for length in self.view.shape[:self.view.ndim]: + * result *= length # <<<<<<<<<<<<<< + * + * self._size = result +*/ + __pyx_t_5 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 605, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_result, __pyx_t_5); + __pyx_t_5 = 0; + } + + /* "View.MemoryView":607 + * result *= length + * + * self._size = result # <<<<<<<<<<<<<< + * + * return self._size +*/ + __Pyx_INCREF(__pyx_v_result); + __Pyx_GIVEREF(__pyx_v_result); + __Pyx_GOTREF(__pyx_v_self->_size); + __Pyx_DECREF(__pyx_v_self->_size); + __pyx_v_self->_size = __pyx_v_result; + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + } + + /* "View.MemoryView":609 + * self._size = result + * + * return self._size # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->_size); + __pyx_r = __pyx_v_self->_size; + goto __pyx_L0; + + /* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.size.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + __pyx_t_1 = (__pyx_v_self->view.ndim >= 1); + if (__pyx_t_1) { + + /* "View.MemoryView":613 + * def __len__(self): + * if self.view.ndim >= 1: + * return self.view.shape[0] # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_r = (__pyx_v_self->view.shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + } + + /* "View.MemoryView":615 + * return self.view.shape[0] + * + * return 0 # <<<<<<<<<<<<<< + * + * def __repr__(self): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[5]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":619 + * def __repr__(self): + * return "" % (self.base.__class__.__name__, + * id(self)) # <<<<<<<<<<<<<< + * + * def __str__(self): +*/ + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_Format(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_x); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_4[1] = __pyx_t_2; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_at_0x; + __pyx_t_4[3] = __pyx_t_3; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u__3; + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_4, 5, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3[3]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__str__", 0); + + /* "View.MemoryView":622 + * + * def __str__(self): + * return "" % (self.base.__class__.__name__,) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_3[1] = __pyx_t_2; + __pyx_t_3[2] = __pyx_mstate_global->__pyx_kp_u_object; + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_3, 3, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 8, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_c_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_c_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_c_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_c_contig", 0); + + /* "View.MemoryView":628 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 628, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":629 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'C', self.view.ndim) # <<<<<<<<<<<<<< + * + * def is_f_contig(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 629, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_c_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_f_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_f_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_f_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_f_contig", 0); + + /* "View.MemoryView":634 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 634, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":635 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'F', self.view.ndim) # <<<<<<<<<<<<<< + * + * def copy(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 635, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_f_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_mslice; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy", 0); + + /* "View.MemoryView":639 + * def copy(self): + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &mslice) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_F_CONTIGUOUS)); + + /* "View.MemoryView":641 + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS + * + * slice_copy(self, &mslice) # <<<<<<<<<<<<<< + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_mslice)); + + /* "View.MemoryView":642 + * + * slice_copy(self, &mslice) + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_C_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char const *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 642, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":647 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &mslice) # <<<<<<<<<<<<<< + * + * def copy_fortran(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 647, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy_fortran (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy_fortran", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy_fortran", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy_fortran", 0); + + /* "View.MemoryView":651 + * def copy_fortran(self): + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &src) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_C_CONTIGUOUS)); + + /* "View.MemoryView":653 + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS + * + * slice_copy(self, &src) # <<<<<<<<<<<<<< + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_src)); + + /* "View.MemoryView":654 + * + * slice_copy(self, &src) + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_F_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char const *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 654, __pyx_L1_error) + __pyx_v_dst = __pyx_t_1; + + /* "View.MemoryView":659 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &dst) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 659, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy_fortran", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryview___reduce_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryview_2__setstate_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + +static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, int __pyx_v_dtype_is_object, __Pyx_TypeInfo const *__pyx_v_typeinfo) { + struct __pyx_memoryview_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_cwrapper", 0); + + /* "View.MemoryView":664 + * @cname('__pyx_memoryview_new') + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) # <<<<<<<<<<<<<< + * result.typeinfo = typeinfo + * return result +*/ + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, __pyx_v_o, __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_v_result = ((struct __pyx_memoryview_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":665 + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_v_result->typeinfo = __pyx_v_typeinfo; + + /* "View.MemoryView":666 + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_check') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { + int __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":670 + * @cname('__pyx_memoryview_check') + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) # <<<<<<<<<<<<<< + * + * cdef tuple _unellipsify(object index, int ndim): +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_o, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + +static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_idx; + PyObject *__pyx_v_tup = NULL; + PyObject *__pyx_v_result = NULL; + int __pyx_v_have_slices; + int __pyx_v_seen_ellipsis; + PyObject *__pyx_v_item = NULL; + Py_ssize_t __pyx_v_nslices; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6[3]; + PyObject *__pyx_t_7 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_unellipsify", 0); + + /* "View.MemoryView":678 + * """ + * cdef Py_ssize_t idx + * tup = index if isinstance(index, tuple) else (index,) # <<<<<<<<<<<<<< + * + * result = [slice(None)] * ndim +*/ + __pyx_t_2 = PyTuple_Check(__pyx_v_index); + if (__pyx_t_2) { + __Pyx_INCREF(((PyObject*)__pyx_v_index)); + __pyx_t_1 = __pyx_v_index; + } else { + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_index); + __Pyx_GIVEREF(__pyx_v_index); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_index) != (0)) __PYX_ERR(1, 678, __pyx_L1_error); + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_v_tup = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_t_1 = PyList_New(1 * ((__pyx_v_ndim<0) ? 0:__pyx_v_ndim)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_ndim; __pyx_temp++) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, __pyx_temp, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(1, 680, __pyx_L1_error); + } + } + __pyx_v_result = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":681 + * + * result = [slice(None)] * ndim + * have_slices = False # <<<<<<<<<<<<<< + * seen_ellipsis = False + * idx = 0 +*/ + __pyx_v_have_slices = 0; + + /* "View.MemoryView":682 + * result = [slice(None)] * ndim + * have_slices = False + * seen_ellipsis = False # <<<<<<<<<<<<<< + * idx = 0 + * for item in tup: +*/ + __pyx_v_seen_ellipsis = 0; + + /* "View.MemoryView":683 + * have_slices = False + * seen_ellipsis = False + * idx = 0 # <<<<<<<<<<<<<< + * for item in tup: + * if item is Ellipsis: +*/ + __pyx_v_idx = 0; + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(1, 684, __pyx_L1_error) + } + __pyx_t_1 = __pyx_v_tup; __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 684, __pyx_L1_error) + #endif + if (__pyx_t_4 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_4); + #endif + ++__pyx_t_4; + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 684, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + __pyx_t_2 = (__pyx_v_item == __pyx_builtin_Ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + __pyx_t_2 = (!__pyx_v_seen_ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":687 + * if item is Ellipsis: + * if not seen_ellipsis: + * idx += ndim - len(tup) # <<<<<<<<<<<<<< + * seen_ellipsis = True + * have_slices = True +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 687, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(__pyx_v_tup); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 687, __pyx_L1_error) + __pyx_v_idx = (__pyx_v_idx + (__pyx_v_ndim - __pyx_t_5)); + + /* "View.MemoryView":688 + * if not seen_ellipsis: + * idx += ndim - len(tup) + * seen_ellipsis = True # <<<<<<<<<<<<<< + * have_slices = True + * else: +*/ + __pyx_v_seen_ellipsis = 1; + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + } + + /* "View.MemoryView":689 + * idx += ndim - len(tup) + * seen_ellipsis = True + * have_slices = True # <<<<<<<<<<<<<< + * else: + * if isinstance(item, slice): +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + /*else*/ { + __pyx_t_2 = PySlice_Check(__pyx_v_item); + if (__pyx_t_2) { + + /* "View.MemoryView":692 + * else: + * if isinstance(item, slice): + * have_slices = True # <<<<<<<<<<<<<< + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + __pyx_t_2 = (!(PyIndex_Check(__pyx_v_item) != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":694 + * have_slices = True + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" # <<<<<<<<<<<<<< + * result[idx] = item + * idx += 1 +*/ + __pyx_t_3 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_item)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6[0] = __pyx_mstate_global->__pyx_kp_u_Cannot_index_with_type; + __pyx_t_6[1] = __pyx_t_3; + __pyx_t_6[2] = __pyx_mstate_global->__pyx_kp_u__4; + __pyx_t_7 = __Pyx_PyUnicode_Join(__pyx_t_6, 3, 24 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_t_7, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 694, __pyx_L1_error) + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + } + __pyx_L7:; + + /* "View.MemoryView":695 + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item # <<<<<<<<<<<<<< + * idx += 1 + * +*/ + if (unlikely((__Pyx_SetItemInt(__pyx_v_result, __pyx_v_idx, __pyx_v_item, Py_ssize_t, 1, PyLong_FromSsize_t, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(1, 695, __pyx_L1_error) + } + __pyx_L5:; + + /* "View.MemoryView":696 + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item + * idx += 1 # <<<<<<<<<<<<<< + * + * nslices = ndim - idx +*/ + __pyx_v_idx = (__pyx_v_idx + 1); + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":698 + * idx += 1 + * + * nslices = ndim - idx # <<<<<<<<<<<<<< + * return have_slices or nslices, tuple(result) + * +*/ + __pyx_v_nslices = (__pyx_v_ndim - __pyx_v_idx); + + /* "View.MemoryView":699 + * + * nslices = ndim - idx + * return have_slices or nslices, tuple(result) # <<<<<<<<<<<<<< + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: +*/ + __Pyx_XDECREF(__pyx_r); + if (!__pyx_v_have_slices) { + } else { + __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_7 = PyLong_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_L9_bool_binop_done:; + __pyx_t_7 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_7 = 0; + __pyx_r = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("View.MemoryView._unellipsify", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_tup); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_item); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + +static int assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_suboffset; + int __pyx_r; + Py_ssize_t *__pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":702 + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" +*/ + __pyx_t_2 = (__pyx_v_suboffsets + __pyx_v_ndim); + for (__pyx_t_3 = __pyx_v_suboffsets; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_suboffset = (__pyx_t_1[0]); + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + __pyx_t_4 = (__pyx_v_suboffset >= 0); + if (unlikely(__pyx_t_4)) { + + /* "View.MemoryView":704 + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" # <<<<<<<<<<<<<< + * return 0 # return type just used as an error flag + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Indirect_dimensions_not_supporte, 0, 0); + __PYX_ERR(1, 704, __pyx_L1_error) + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + } + } + + /* "View.MemoryView":705 + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.assert_direct_dimensions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *__pyx_v_memview, PyObject *__pyx_v_indices) { + int __pyx_v_new_ndim; + int __pyx_v_suboffset_dim; + int __pyx_v_dim; + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + __Pyx_memviewslice *__pyx_v_p_src; + struct __pyx_memoryviewslice_obj *__pyx_v_memviewsliceobj = 0; + __Pyx_memviewslice *__pyx_v_p_dst; + int *__pyx_v_p_suboffset_dim; + Py_ssize_t __pyx_v_start; + Py_ssize_t __pyx_v_stop; + Py_ssize_t __pyx_v_step; + Py_ssize_t __pyx_v_cindex; + int __pyx_v_have_start; + int __pyx_v_have_stop; + int __pyx_v_have_step; + PyObject *__pyx_v_index = NULL; + struct __pyx_memoryview_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + struct __pyx_memoryview_obj *__pyx_t_3; + char *__pyx_t_4; + int __pyx_t_5; + Py_ssize_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memview_slice", 0); + + /* "View.MemoryView":713 + * @cname('__pyx_memview_slice') + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim # <<<<<<<<<<<<<< + * cdef bint negative_step + * cdef __Pyx_memviewslice src, dst +*/ + __pyx_v_new_ndim = 0; + __pyx_v_suboffset_dim = -1; + + /* "View.MemoryView":720 + * + * + * memset(&dst, 0, sizeof(dst)) # <<<<<<<<<<<<<< + * + * cdef _memoryviewslice memviewsliceobj +*/ + (void)(memset((&__pyx_v_dst), 0, (sizeof(__pyx_v_dst)))); + + /* "View.MemoryView":724 + * cdef _memoryviewslice memviewsliceobj + * + * assert memview.view.ndim > 0 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_memview->view.ndim > 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 724, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 724, __pyx_L1_error) + #endif + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":727 + * + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview # <<<<<<<<<<<<<< + * p_src = &memviewsliceobj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 727, __pyx_L1_error) + __pyx_v_memviewsliceobj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":728 + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, &src) +*/ + __pyx_v_p_src = (&__pyx_v_memviewsliceobj->from_slice); + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":730 + * p_src = &memviewsliceobj.from_slice + * else: + * slice_copy(memview, &src) # <<<<<<<<<<<<<< + * p_src = &src + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_src)); + + /* "View.MemoryView":731 + * else: + * slice_copy(memview, &src) + * p_src = &src # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_p_src = (&__pyx_v_src); + } + __pyx_L3:; + + /* "View.MemoryView":737 + * + * + * dst.memview = p_src.memview # <<<<<<<<<<<<<< + * dst.data = p_src.data + * +*/ + __pyx_t_3 = __pyx_v_p_src->memview; + __pyx_v_dst.memview = __pyx_t_3; + + /* "View.MemoryView":738 + * + * dst.memview = p_src.memview + * dst.data = p_src.data # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_v_p_src->data; + __pyx_v_dst.data = __pyx_t_4; + + /* "View.MemoryView":743 + * + * + * cdef __Pyx_memviewslice *p_dst = &dst # <<<<<<<<<<<<<< + * cdef int *p_suboffset_dim = &suboffset_dim + * cdef Py_ssize_t start, stop, step, cindex +*/ + __pyx_v_p_dst = (&__pyx_v_dst); + + /* "View.MemoryView":744 + * + * cdef __Pyx_memviewslice *p_dst = &dst + * cdef int *p_suboffset_dim = &suboffset_dim # <<<<<<<<<<<<<< + * cdef Py_ssize_t start, stop, step, cindex + * cdef bint have_start, have_stop, have_step +*/ + __pyx_v_p_suboffset_dim = (&__pyx_v_suboffset_dim); + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_indices)) || PyTuple_CheckExact(__pyx_v_indices)) { + __pyx_t_2 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 748, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 748, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_6)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 748, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_2); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 748, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_index, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_dim = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + __pyx_t_1 = (PyIndex_Check(__pyx_v_index) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":750 + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): + * cindex = index # <<<<<<<<<<<<<< + * slice_memviewslice( + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], +*/ + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 750, __pyx_L1_error) + __pyx_v_cindex = __pyx_t_9; + + /* "View.MemoryView":751 + * if PyIndex_Check(index): + * cindex = index + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_cindex, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 751, __pyx_L1_error) + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + __pyx_t_1 = (__pyx_v_index == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":758 + * False) + * elif index is None: + * p_dst.shape[new_ndim] = 1 # <<<<<<<<<<<<<< + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 +*/ + (__pyx_v_p_dst->shape[__pyx_v_new_ndim]) = 1; + + /* "View.MemoryView":759 + * elif index is None: + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 # <<<<<<<<<<<<<< + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 +*/ + (__pyx_v_p_dst->strides[__pyx_v_new_ndim]) = 0; + + /* "View.MemoryView":760 + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 # <<<<<<<<<<<<<< + * new_ndim += 1 + * else: +*/ + (__pyx_v_p_dst->suboffsets[__pyx_v_new_ndim]) = -1L; + + /* "View.MemoryView":761 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 # <<<<<<<<<<<<<< + * else: + * start = index.start or 0 +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":763 + * new_ndim += 1 + * else: + * start = index.start or 0 # <<<<<<<<<<<<<< + * stop = index.stop or 0 + * step = index.step or 0 +*/ + /*else*/ { + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 763, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 763, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 763, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L7_bool_binop_done:; + __pyx_v_start = __pyx_t_9; + + /* "View.MemoryView":764 + * else: + * start = index.start or 0 + * stop = index.stop or 0 # <<<<<<<<<<<<<< + * step = index.step or 0 + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 764, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 764, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 764, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L9_bool_binop_done:; + __pyx_v_stop = __pyx_t_9; + + /* "View.MemoryView":765 + * start = index.start or 0 + * stop = index.stop or 0 + * step = index.step or 0 # <<<<<<<<<<<<<< + * + * have_start = index.start is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 765, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 765, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 765, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L11_bool_binop_done:; + __pyx_v_step = __pyx_t_9; + + /* "View.MemoryView":767 + * step = index.step or 0 + * + * have_start = index.start is not None # <<<<<<<<<<<<<< + * have_stop = index.stop is not None + * have_step = index.step is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 767, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_start = __pyx_t_1; + + /* "View.MemoryView":768 + * + * have_start = index.start is not None + * have_stop = index.stop is not None # <<<<<<<<<<<<<< + * have_step = index.step is not None + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 768, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_stop = __pyx_t_1; + + /* "View.MemoryView":769 + * have_start = index.start is not None + * have_stop = index.stop is not None + * have_step = index.step is not None # <<<<<<<<<<<<<< + * + * slice_memviewslice( +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_step = __pyx_t_1; + + /* "View.MemoryView":771 + * have_step = index.step is not None + * + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 771, __pyx_L1_error) + + /* "View.MemoryView":777 + * have_start, have_stop, have_step, + * True) + * new_ndim += 1 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + } + __pyx_L6:; + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":781 + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, # <<<<<<<<<<<<<< + * memviewsliceobj.to_dtype_func, + * memview.dtype_is_object) +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 781, __pyx_L1_error) } + + /* "View.MemoryView":782 + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * else: +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 782, __pyx_L1_error) } + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 780, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + } + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + /*else*/ { + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":786 + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 785, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 785, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("View.MemoryView.memview_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_memviewsliceobj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, Py_ssize_t __pyx_v_shape, Py_ssize_t __pyx_v_stride, Py_ssize_t __pyx_v_suboffset, int __pyx_v_dim, int __pyx_v_new_ndim, int *__pyx_v_suboffset_dim, Py_ssize_t __pyx_v_start, Py_ssize_t __pyx_v_stop, Py_ssize_t __pyx_v_step, int __pyx_v_have_start, int __pyx_v_have_stop, int __pyx_v_have_step, int __pyx_v_is_slice) { + Py_ssize_t __pyx_v_new_shape; + int __pyx_v_negative_step; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + __pyx_t_1 = (!__pyx_v_is_slice); + if (__pyx_t_1) { + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + __pyx_t_1 = (__pyx_v_start < 0); + if (__pyx_t_1) { + + /* "View.MemoryView":817 + * + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + } + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + __pyx_t_1 = (0 <= __pyx_v_start); + if (__pyx_t_1) { + __pyx_t_1 = (__pyx_v_start < __pyx_v_shape); + } + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":819 + * start += shape + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_Index_out_of_bounds_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 819, __pyx_L1_error) + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + /*else*/ { + __pyx_t_2 = (__pyx_v_have_step != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":823 + * + * if have_step: + * negative_step = step < 0 # <<<<<<<<<<<<<< + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) +*/ + __pyx_v_negative_step = (__pyx_v_step < 0); + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + __pyx_t_2 = (__pyx_v_step == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":825 + * negative_step = step < 0 + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * negative_step = False +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Step_may_not_be_zero_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 825, __pyx_L1_error) + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":827 + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: + * negative_step = False # <<<<<<<<<<<<<< + * step = 1 + * +*/ + /*else*/ { + __pyx_v_negative_step = 0; + + /* "View.MemoryView":828 + * else: + * negative_step = False + * step = 1 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_step = 1; + } + __pyx_L6:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + __pyx_t_2 = (__pyx_v_have_start != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":833 + * if have_start: + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if start < 0: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":835 + * start += shape + * if start < 0: + * start = 0 # <<<<<<<<<<<<<< + * elif start >= shape: + * if negative_step: +*/ + __pyx_v_start = 0; + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + } + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + __pyx_t_2 = (__pyx_v_start >= __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + if (__pyx_v_negative_step) { + + /* "View.MemoryView":838 + * elif start >= shape: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = shape +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":840 + * start = shape - 1 + * else: + * start = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + /*else*/ { + __pyx_v_start = __pyx_v_shape; + } + __pyx_L11:; + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + } + __pyx_L9:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":843 + * else: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":845 + * start = shape - 1 + * else: + * start = 0 # <<<<<<<<<<<<<< + * + * if have_stop: +*/ + /*else*/ { + __pyx_v_start = 0; + } + __pyx_L12:; + } + __pyx_L8:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + __pyx_t_2 = (__pyx_v_have_stop != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":849 + * if have_stop: + * if stop < 0: + * stop += shape # <<<<<<<<<<<<<< + * if stop < 0: + * stop = 0 +*/ + __pyx_v_stop = (__pyx_v_stop + __pyx_v_shape); + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":851 + * stop += shape + * if stop < 0: + * stop = 0 # <<<<<<<<<<<<<< + * elif stop > shape: + * stop = shape +*/ + __pyx_v_stop = 0; + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + } + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + goto __pyx_L14; + } + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + __pyx_t_2 = (__pyx_v_stop > __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":853 + * stop = 0 + * elif stop > shape: + * stop = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + __pyx_v_stop = __pyx_v_shape; + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + } + __pyx_L14:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":856 + * else: + * if negative_step: + * stop = -1 # <<<<<<<<<<<<<< + * else: + * stop = shape +*/ + __pyx_v_stop = -1L; + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + goto __pyx_L16; + } + + /* "View.MemoryView":858 + * stop = -1 + * else: + * stop = shape # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_v_stop = __pyx_v_shape; + } + __pyx_L16:; + } + __pyx_L13:; + + /* "View.MemoryView":862 + * + * with cython.cdivision(True): + * new_shape = (stop - start) // step # <<<<<<<<<<<<<< + * + * if (stop - start) - step * new_shape: +*/ + __pyx_v_new_shape = ((__pyx_v_stop - __pyx_v_start) / __pyx_v_step); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + __pyx_t_2 = (((__pyx_v_stop - __pyx_v_start) - (__pyx_v_step * __pyx_v_new_shape)) != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":865 + * + * if (stop - start) - step * new_shape: + * new_shape += 1 # <<<<<<<<<<<<<< + * + * if new_shape < 0: +*/ + __pyx_v_new_shape = (__pyx_v_new_shape + 1); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + } + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + __pyx_t_2 = (__pyx_v_new_shape < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":868 + * + * if new_shape < 0: + * new_shape = 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_new_shape = 0; + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + } + + /* "View.MemoryView":871 + * + * + * dst.strides[new_ndim] = stride * step # <<<<<<<<<<<<<< + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset +*/ + (__pyx_v_dst->strides[__pyx_v_new_ndim]) = (__pyx_v_stride * __pyx_v_step); + + /* "View.MemoryView":872 + * + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape # <<<<<<<<<<<<<< + * dst.suboffsets[new_ndim] = suboffset + * +*/ + (__pyx_v_dst->shape[__pyx_v_new_ndim]) = __pyx_v_new_shape; + + /* "View.MemoryView":873 + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_dst->suboffsets[__pyx_v_new_ndim]) = __pyx_v_suboffset; + } + __pyx_L3:; + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + __pyx_t_2 = ((__pyx_v_suboffset_dim[0]) < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":877 + * + * if suboffset_dim[0] < 0: + * dst.data += start * stride # <<<<<<<<<<<<<< + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride +*/ + __pyx_v_dst->data = (__pyx_v_dst->data + (__pyx_v_start * __pyx_v_stride)); + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + goto __pyx_L19; + } + + /* "View.MemoryView":879 + * dst.data += start * stride + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride # <<<<<<<<<<<<<< + * + * if suboffset >= 0: +*/ + /*else*/ { + __pyx_t_3 = (__pyx_v_suboffset_dim[0]); + (__pyx_v_dst->suboffsets[__pyx_t_3]) = ((__pyx_v_dst->suboffsets[__pyx_t_3]) + (__pyx_v_start * __pyx_v_stride)); + } + __pyx_L19:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + __pyx_t_2 = (!__pyx_v_is_slice); + if (__pyx_t_2) { + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + __pyx_t_2 = (__pyx_v_new_ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":884 + * if not is_slice: + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset # <<<<<<<<<<<<<< + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " +*/ + __pyx_v_dst->data = ((((char **)__pyx_v_dst->data)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + goto __pyx_L22; + } + + /* "View.MemoryView":886 + * dst.data = ( dst.data)[0] + suboffset + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " # <<<<<<<<<<<<<< + * "must be indexed and not sliced", dim) + * else: +*/ + /*else*/ { + + /* "View.MemoryView":887 + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " + * "must be indexed and not sliced", dim) # <<<<<<<<<<<<<< + * else: + * suboffset_dim[0] = new_ndim +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_All_dimensions_preceding_dimensi, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 886, __pyx_L1_error) + } + __pyx_L22:; + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + goto __pyx_L21; + } + + /* "View.MemoryView":889 + * "must be indexed and not sliced", dim) + * else: + * suboffset_dim[0] = new_ndim # <<<<<<<<<<<<<< + * + * return 0 +*/ + /*else*/ { + (__pyx_v_suboffset_dim[0]) = __pyx_v_new_ndim; + } + __pyx_L21:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + } + + /* "View.MemoryView":891 + * suboffset_dim[0] = new_ndim + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.slice_memviewslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + +static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, Py_ssize_t __pyx_v_index, Py_ssize_t __pyx_v_dim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_suboffset; + Py_ssize_t __pyx_v_itemsize; + char *__pyx_v_resultp; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[3]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("pybuffer_index", 0); + + /* "View.MemoryView":899 + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 # <<<<<<<<<<<<<< + * cdef Py_ssize_t itemsize = view.itemsize + * cdef char *resultp +*/ + __pyx_v_suboffset = -1L; + + /* "View.MemoryView":900 + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 + * cdef Py_ssize_t itemsize = view.itemsize # <<<<<<<<<<<<<< + * cdef char *resultp + * +*/ + __pyx_t_1 = __pyx_v_view->itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + __pyx_t_2 = (__pyx_v_view->ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":904 + * + * if view.ndim == 0: + * shape = view.len // itemsize # <<<<<<<<<<<<<< + * stride = itemsize + * else: +*/ + if (unlikely(__pyx_v_itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_view->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + __pyx_v_shape = __Pyx_div_Py_ssize_t(__pyx_v_view->len, __pyx_v_itemsize, 0); + + /* "View.MemoryView":905 + * if view.ndim == 0: + * shape = view.len // itemsize + * stride = itemsize # <<<<<<<<<<<<<< + * else: + * shape = view.shape[dim] +*/ + __pyx_v_stride = __pyx_v_itemsize; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":907 + * stride = itemsize + * else: + * shape = view.shape[dim] # <<<<<<<<<<<<<< + * stride = view.strides[dim] + * if view.suboffsets != NULL: +*/ + /*else*/ { + __pyx_v_shape = (__pyx_v_view->shape[__pyx_v_dim]); + + /* "View.MemoryView":908 + * else: + * shape = view.shape[dim] + * stride = view.strides[dim] # <<<<<<<<<<<<<< + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] +*/ + __pyx_v_stride = (__pyx_v_view->strides[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + __pyx_t_2 = (__pyx_v_view->suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":910 + * stride = view.strides[dim] + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] # <<<<<<<<<<<<<< + * + * if index < 0: +*/ + __pyx_v_suboffset = (__pyx_v_view->suboffsets[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + } + } + __pyx_L3:; + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":913 + * + * if index < 0: + * index += view.shape[dim] # <<<<<<<<<<<<<< + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" +*/ + __pyx_v_index = (__pyx_v_index + (__pyx_v_view->shape[__pyx_v_dim])); + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":915 + * index += view.shape[dim] + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * if index >= shape: +*/ + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_3; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 915, __pyx_L1_error) + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + } + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index >= __pyx_v_shape); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":918 + * + * if index >= shape: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * resultp = bufp + index * stride +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_5; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 1, 127); + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 918, __pyx_L1_error) + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":920 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * resultp = bufp + index * stride # <<<<<<<<<<<<<< + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset +*/ + __pyx_v_resultp = (__pyx_v_bufp + (__pyx_v_index * __pyx_v_stride)); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":922 + * resultp = bufp + index * stride + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset # <<<<<<<<<<<<<< + * + * return resultp +*/ + __pyx_v_resultp = ((((char **)__pyx_v_resultp)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + } + + /* "View.MemoryView":924 + * resultp = ( resultp)[0] + suboffset + * + * return resultp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_resultp; + goto __pyx_L0; + + /* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.pybuffer_index", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + +static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { + int __pyx_v_ndim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + int __pyx_v_i; + int __pyx_v_j; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + long __pyx_t_3; + long __pyx_t_4; + Py_ssize_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":931 + * @cname('__pyx_memslice_transpose') + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim # <<<<<<<<<<<<<< + * + * cdef Py_ssize_t *shape = memslice.shape +*/ + __pyx_t_1 = __pyx_v_memslice->memview->view.ndim; + __pyx_v_ndim = __pyx_t_1; + + /* "View.MemoryView":933 + * cdef int ndim = memslice.memview.view.ndim + * + * cdef Py_ssize_t *shape = memslice.shape # <<<<<<<<<<<<<< + * cdef Py_ssize_t *strides = memslice.strides + * +*/ + __pyx_t_2 = __pyx_v_memslice->shape; + __pyx_v_shape = __pyx_t_2; + + /* "View.MemoryView":934 + * + * cdef Py_ssize_t *shape = memslice.shape + * cdef Py_ssize_t *strides = memslice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_v_memslice->strides; + __pyx_v_strides = __pyx_t_2; + + /* "View.MemoryView":938 + * + * cdef int i, j + * for i in range(ndim // 2): # <<<<<<<<<<<<<< + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] +*/ + __pyx_t_3 = __Pyx_div_long(__pyx_v_ndim, 2, 1); + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_4; __pyx_t_1+=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":939 + * cdef int i, j + * for i in range(ndim // 2): + * j = ndim - 1 - i # <<<<<<<<<<<<<< + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] +*/ + __pyx_v_j = ((__pyx_v_ndim - 1) - __pyx_v_i); + + /* "View.MemoryView":940 + * for i in range(ndim // 2): + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] # <<<<<<<<<<<<<< + * shape[i], shape[j] = shape[j], shape[i] + * +*/ + __pyx_t_5 = (__pyx_v_strides[__pyx_v_j]); + __pyx_t_6 = (__pyx_v_strides[__pyx_v_i]); + (__pyx_v_strides[__pyx_v_i]) = __pyx_t_5; + (__pyx_v_strides[__pyx_v_j]) = __pyx_t_6; + + /* "View.MemoryView":941 + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] # <<<<<<<<<<<<<< + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: +*/ + __pyx_t_6 = (__pyx_v_shape[__pyx_v_j]); + __pyx_t_5 = (__pyx_v_shape[__pyx_v_i]); + (__pyx_v_shape[__pyx_v_i]) = __pyx_t_6; + (__pyx_v_shape[__pyx_v_j]) = __pyx_t_5; + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_i]) >= 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_j]) >= 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L6_bool_binop_done:; + if (__pyx_t_7) { + + /* "View.MemoryView":944 + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_t_9 = __pyx_memoryview_err(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Cannot_transpose_memoryview_with); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 944, __pyx_L1_error) + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + } + } + + /* "View.MemoryView":946 + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.transpose_memslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + +/* Python wrapper */ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + + /* "View.MemoryView":965 + * + * def __dealloc__(self): + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_self->from_slice), 1); + + /* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_object_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":969 + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) # <<<<<<<<<<<<<< + * else: + * return memoryview.convert_item_to_object(self, itemp) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 969, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + } + + /* "View.MemoryView":971 + * return self.to_object_func(itemp) + * else: + * return memoryview.convert_item_to_object(self, itemp) # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 971, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_dtype_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":975 + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) # <<<<<<<<<<<<<< + * else: + * memoryview.assign_item_from_object(self, itemp, value) +*/ + __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 975, __pyx_L1_error) + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":977 + * self.to_dtype_func(itemp, value) + * else: + * memoryview.assign_item_from_object(self, itemp, value) # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + /*else*/ { + __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 977, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":980 + * + * cdef _get_base(self): + * return self.from_object # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->from_object); + __pyx_r = __pyx_v_self->from_object; + goto __pyx_L0; + + /* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryviewslice___reduce_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryviewslice_2__setstate_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewslice, int __pyx_v_ndim, PyObject *(*__pyx_v_to_object_func)(char *), int (*__pyx_v_to_dtype_func)(char *, PyObject *), int __pyx_v_dtype_is_object) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + Py_ssize_t __pyx_v_suboffset; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_TypeInfo const *__pyx_t_4; + Py_buffer __pyx_t_5; + Py_ssize_t *__pyx_t_6; + Py_ssize_t *__pyx_t_7; + Py_ssize_t *__pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_fromslice", 0); + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_memviewslice.memview) == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":1009 + * + * if memviewslice.memview == Py_None: + * return None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + } + + /* "View.MemoryView":1014 + * + * + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) # <<<<<<<<<<<<<< + * + * result.from_slice = memviewslice +*/ + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, Py_None) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_0) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_2 = ((PyObject *)__pyx_tp_new__memoryviewslice(((PyTypeObject *)__pyx_mstate_global->__pyx_memoryviewslice_type), __pyx_t_3, NULL)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1016 + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) + * + * result.from_slice = memviewslice # <<<<<<<<<<<<<< + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * +*/ + __pyx_v_result->from_slice = __pyx_v_memviewslice; + + /* "View.MemoryView":1017 + * + * result.from_slice = memviewslice + * __PYX_INC_MEMVIEW(&memviewslice, 1) # <<<<<<<<<<<<<< + * + * result.from_object = ( memviewslice.memview)._get_base() +*/ + __PYX_INC_MEMVIEW((&__pyx_v_memviewslice), 1); + + /* "View.MemoryView":1019 + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * + * result.from_object = ( memviewslice.memview)._get_base() # <<<<<<<<<<<<<< + * result.typeinfo = memviewslice.memview.typeinfo + * +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->__pyx_vtab)->_get_base(((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1019, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_result->from_object); + __Pyx_DECREF(__pyx_v_result->from_object); + __pyx_v_result->from_object = __pyx_t_2; + __pyx_t_2 = 0; + + /* "View.MemoryView":1020 + * + * result.from_object = ( memviewslice.memview)._get_base() + * result.typeinfo = memviewslice.memview.typeinfo # <<<<<<<<<<<<<< + * + * result.view = memviewslice.memview.view +*/ + __pyx_t_4 = __pyx_v_memviewslice.memview->typeinfo; + __pyx_v_result->__pyx_base.typeinfo = __pyx_t_4; + + /* "View.MemoryView":1022 + * result.typeinfo = memviewslice.memview.typeinfo + * + * result.view = memviewslice.memview.view # <<<<<<<<<<<<<< + * result.view.buf = memviewslice.data + * result.view.ndim = ndim +*/ + __pyx_t_5 = __pyx_v_memviewslice.memview->view; + __pyx_v_result->__pyx_base.view = __pyx_t_5; + + /* "View.MemoryView":1023 + * + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data # <<<<<<<<<<<<<< + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None +*/ + __pyx_v_result->__pyx_base.view.buf = ((void *)__pyx_v_memviewslice.data); + + /* "View.MemoryView":1024 + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data + * result.view.ndim = ndim # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_v_result->__pyx_base.view.ndim = __pyx_v_ndim; + + /* "View.MemoryView":1025 + * result.view.buf = memviewslice.data + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_result->__pyx_base.view))->obj = Py_None; + + /* "View.MemoryView":1026 + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + __pyx_t_1 = ((((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1029 + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: + * result.flags = PyBUF_RECORDS # <<<<<<<<<<<<<< + * else: + * result.flags = PyBUF_RECORDS_RO +*/ + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS; + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1031 + * result.flags = PyBUF_RECORDS + * else: + * result.flags = PyBUF_RECORDS_RO # <<<<<<<<<<<<<< + * + * result.view.shape = result.from_slice.shape +*/ + /*else*/ { + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS_RO; + } + __pyx_L4:; + + /* "View.MemoryView":1033 + * result.flags = PyBUF_RECORDS_RO + * + * result.view.shape = result.from_slice.shape # <<<<<<<<<<<<<< + * result.view.strides = result.from_slice.strides + * +*/ + __pyx_v_result->__pyx_base.view.shape = ((Py_ssize_t *)__pyx_v_result->from_slice.shape); + + /* "View.MemoryView":1034 + * + * result.view.shape = result.from_slice.shape + * result.view.strides = result.from_slice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_result->__pyx_base.view.strides = ((Py_ssize_t *)__pyx_v_result->from_slice.strides); + + /* "View.MemoryView":1037 + * + * + * result.view.suboffsets = NULL # <<<<<<<<<<<<<< + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: +*/ + __pyx_v_result->__pyx_base.view.suboffsets = NULL; + + /* "View.MemoryView":1038 + * + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets +*/ + __pyx_t_7 = (__pyx_v_result->from_slice.suboffsets + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->from_slice.suboffsets; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_v_suboffset = (__pyx_t_6[0]); + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + __pyx_t_1 = (__pyx_v_suboffset >= 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1040 + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_result->__pyx_base.view.suboffsets = ((Py_ssize_t *)__pyx_v_result->from_slice.suboffsets); + + /* "View.MemoryView":1041 + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets + * break # <<<<<<<<<<<<<< + * + * result.view.len = result.view.itemsize +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + } + } + __pyx_L6_break:; + + /* "View.MemoryView":1043 + * break + * + * result.view.len = result.view.itemsize # <<<<<<<<<<<<<< + * for length in result.view.shape[:ndim]: + * result.view.len *= length +*/ + __pyx_t_9 = __pyx_v_result->__pyx_base.view.itemsize; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + + /* "View.MemoryView":1044 + * + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: # <<<<<<<<<<<<<< + * result.view.len *= length + * +*/ + __pyx_t_7 = (__pyx_v_result->__pyx_base.view.shape + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->__pyx_base.view.shape; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_t_2 = PyLong_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1044, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1045 + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: + * result.view.len *= length # <<<<<<<<<<<<<< + * + * result.to_object_func = to_object_func +*/ + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + } + + /* "View.MemoryView":1047 + * result.view.len *= length + * + * result.to_object_func = to_object_func # <<<<<<<<<<<<<< + * result.to_dtype_func = to_dtype_func + * +*/ + __pyx_v_result->to_object_func = __pyx_v_to_object_func; + + /* "View.MemoryView":1048 + * + * result.to_object_func = to_object_func + * result.to_dtype_func = to_dtype_func # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->to_dtype_func = __pyx_v_to_dtype_func; + + /* "View.MemoryView":1050 + * result.to_dtype_func = to_dtype_func + * + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview_fromslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_mslice) { + struct __pyx_memoryviewslice_obj *__pyx_v_obj = 0; + __Pyx_memviewslice *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_slice_from_memview", 0); + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1057 + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): + * obj = memview # <<<<<<<<<<<<<< + * return &obj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 1057, __pyx_L1_error) + __pyx_v_obj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1058 + * if isinstance(memview, _memoryviewslice): + * obj = memview + * return &obj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, mslice) +*/ + __pyx_r = (&__pyx_v_obj->from_slice); + goto __pyx_L0; + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + } + + /* "View.MemoryView":1060 + * return &obj.from_slice + * else: + * slice_copy(memview, mslice) # <<<<<<<<<<<<<< + * return mslice + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, __pyx_v_mslice); + + /* "View.MemoryView":1061 + * else: + * slice_copy(memview, mslice) + * return mslice # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_slice_copy') +*/ + __pyx_r = __pyx_v_mslice; + goto __pyx_L0; + } + + /* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.get_slice_from_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_obj); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_dst) { + int __pyx_v_dim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + Py_ssize_t *__pyx_v_suboffsets; + Py_ssize_t *__pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + + /* "View.MemoryView":1068 + * cdef (Py_ssize_t*) shape, strides, suboffsets + * + * shape = memview.view.shape # <<<<<<<<<<<<<< + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets +*/ + __pyx_t_1 = __pyx_v_memview->view.shape; + __pyx_v_shape = __pyx_t_1; + + /* "View.MemoryView":1069 + * + * shape = memview.view.shape + * strides = memview.view.strides # <<<<<<<<<<<<<< + * suboffsets = memview.view.suboffsets + * +*/ + __pyx_t_1 = __pyx_v_memview->view.strides; + __pyx_v_strides = __pyx_t_1; + + /* "View.MemoryView":1070 + * shape = memview.view.shape + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets # <<<<<<<<<<<<<< + * + * dst.memview = <__pyx_memoryview *> memview +*/ + __pyx_t_1 = __pyx_v_memview->view.suboffsets; + __pyx_v_suboffsets = __pyx_t_1; + + /* "View.MemoryView":1072 + * suboffsets = memview.view.suboffsets + * + * dst.memview = <__pyx_memoryview *> memview # <<<<<<<<<<<<<< + * dst.data = memview.view.buf + * +*/ + __pyx_v_dst->memview = ((struct __pyx_memoryview_obj *)__pyx_v_memview); + + /* "View.MemoryView":1073 + * + * dst.memview = <__pyx_memoryview *> memview + * dst.data = memview.view.buf # <<<<<<<<<<<<<< + * + * for dim in range(memview.view.ndim): +*/ + __pyx_v_dst->data = ((char *)__pyx_v_memview->view.buf); + + /* "View.MemoryView":1075 + * dst.data = memview.view.buf + * + * for dim in range(memview.view.ndim): # <<<<<<<<<<<<<< + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] +*/ + __pyx_t_2 = __pyx_v_memview->view.ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_dim = __pyx_t_4; + + /* "View.MemoryView":1076 + * + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] # <<<<<<<<<<<<<< + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 +*/ + (__pyx_v_dst->shape[__pyx_v_dim]) = (__pyx_v_shape[__pyx_v_dim]); + + /* "View.MemoryView":1077 + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] # <<<<<<<<<<<<<< + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * +*/ + (__pyx_v_dst->strides[__pyx_v_dim]) = (__pyx_v_strides[__pyx_v_dim]); + + /* "View.MemoryView":1078 + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object') +*/ + __pyx_t_6 = (__pyx_v_suboffsets != 0); + if (__pyx_t_6) { + __pyx_t_5 = (__pyx_v_suboffsets[__pyx_v_dim]); + } else { + __pyx_t_5 = -1L; + } + (__pyx_v_dst->suboffsets[__pyx_v_dim]) = __pyx_t_5; + } + + /* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx_v_memview) { + __Pyx_memviewslice __pyx_v_memviewslice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy", 0); + + /* "View.MemoryView":1084 + * "Create a new memoryview object" + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) # <<<<<<<<<<<<<< + * return memoryview_copy_from_slice(memview, &memviewslice) + * +*/ + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_memviewslice)); + + /* "View.MemoryView":1085 + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) + * return memoryview_copy_from_slice(memview, &memviewslice) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object_from_slice') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1085, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_memviewslice) { + PyObject *(*__pyx_v_to_object_func)(char *); + int (*__pyx_v_to_dtype_func)(char *, PyObject *); + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *(*__pyx_t_2)(char *); + int (*__pyx_t_3)(char *, PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy_from_slice", 0); + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1096 + * + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func # <<<<<<<<<<<<<< + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: +*/ + __pyx_t_2 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_object_func; + __pyx_v_to_object_func = __pyx_t_2; + + /* "View.MemoryView":1097 + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func # <<<<<<<<<<<<<< + * else: + * to_object_func = NULL +*/ + __pyx_t_3 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_dtype_func; + __pyx_v_to_dtype_func = __pyx_t_3; + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1099 + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: + * to_object_func = NULL # <<<<<<<<<<<<<< + * to_dtype_func = NULL + * +*/ + /*else*/ { + __pyx_v_to_object_func = NULL; + + /* "View.MemoryView":1100 + * else: + * to_object_func = NULL + * to_dtype_func = NULL # <<<<<<<<<<<<<< + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, +*/ + __pyx_v_to_dtype_func = NULL; + } + __pyx_L3:; + + /* "View.MemoryView":1102 + * to_dtype_func = NULL + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, # <<<<<<<<<<<<<< + * to_object_func, to_dtype_func, + * memview.dtype_is_object) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "View.MemoryView":1104 + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, + * to_object_func, to_dtype_func, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_from_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + +static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":1111 + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: + * return -arg if arg < 0 else arg # <<<<<<<<<<<<<< + * + * @cname('__pyx_get_best_slice_order') +*/ + __pyx_t_2 = (__pyx_v_arg < 0); + if (__pyx_t_2) { + __pyx_t_1 = (-__pyx_v_arg); + } else { + __pyx_t_1 = __pyx_v_arg; + } + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + +static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim) { + int __pyx_v_i; + Py_ssize_t __pyx_v_c_stride; + Py_ssize_t __pyx_v_f_stride; + char __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1119 + * """ + * cdef int i + * cdef Py_ssize_t c_stride = 0 # <<<<<<<<<<<<<< + * cdef Py_ssize_t f_stride = 0 + * +*/ + __pyx_v_c_stride = 0; + + /* "View.MemoryView":1120 + * cdef int i + * cdef Py_ssize_t c_stride = 0 + * cdef Py_ssize_t f_stride = 0 # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_f_stride = 0; + + /* "View.MemoryView":1122 + * cdef Py_ssize_t f_stride = 0 + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1124 + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_c_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1125 + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + goto __pyx_L4_break; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L4_break:; + + /* "View.MemoryView":1127 + * break + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] +*/ + __pyx_t_1 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_1; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1129 + * for i in range(ndim): + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_f_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1130 + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): +*/ + goto __pyx_L7_break; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L7_break:; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + __pyx_t_2 = (abs_py_ssize_t(__pyx_v_c_stride) <= abs_py_ssize_t(__pyx_v_f_stride)); + if (__pyx_t_2) { + + /* "View.MemoryView":1133 + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): + * return 'C' # <<<<<<<<<<<<<< + * else: + * return 'F' +*/ + __pyx_r = 'C'; + goto __pyx_L0; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + } + + /* "View.MemoryView":1135 + * return 'C' + * else: + * return 'F' # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + /*else*/ { + __pyx_r = 'F'; + goto __pyx_L0; + } + + /* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + +static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v_src_strides, char *__pyx_v_dst_data, Py_ssize_t *__pyx_v_dst_strides, Py_ssize_t *__pyx_v_src_shape, Py_ssize_t *__pyx_v_dst_shape, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + CYTHON_UNUSED Py_ssize_t __pyx_v_src_extent; + Py_ssize_t __pyx_v_dst_extent; + Py_ssize_t __pyx_v_src_stride; + Py_ssize_t __pyx_v_dst_stride; + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + + /* "View.MemoryView":1145 + * + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] +*/ + __pyx_v_src_extent = (__pyx_v_src_shape[0]); + + /* "View.MemoryView":1146 + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] +*/ + __pyx_v_dst_extent = (__pyx_v_dst_shape[0]); + + /* "View.MemoryView":1147 + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_stride = dst_strides[0] + * +*/ + __pyx_v_src_stride = (__pyx_v_src_strides[0]); + + /* "View.MemoryView":1148 + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_dst_stride = (__pyx_v_dst_strides[0]); + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + __pyx_t_2 = (__pyx_v_src_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_dst_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + + /* "View.MemoryView":1152 + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: +*/ + __pyx_t_2 = (((size_t)__pyx_v_src_stride) == __pyx_v_itemsize); + if (__pyx_t_2) { + __pyx_t_2 = (__pyx_v_itemsize == ((size_t)__pyx_v_dst_stride)); + } + __pyx_t_1 = __pyx_t_2; + __pyx_L5_bool_binop_done:; + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":1153 + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, (__pyx_v_itemsize * __pyx_v_dst_extent))); + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1155 + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1156 + * else: + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) # <<<<<<<<<<<<<< + * src_data += src_stride + * dst_data += dst_stride +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, __pyx_v_itemsize)); + + /* "View.MemoryView":1157 + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * else: +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1158 + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L4:; + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1160 + * dst_data += dst_stride + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * _copy_strided_to_strided(src_data, src_strides + 1, + * dst_data, dst_strides + 1, +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1161 + * else: + * for i in range(dst_extent): + * _copy_strided_to_strided(src_data, src_strides + 1, # <<<<<<<<<<<<<< + * dst_data, dst_strides + 1, + * src_shape + 1, dst_shape + 1, +*/ + _copy_strided_to_strided(__pyx_v_src_data, (__pyx_v_src_strides + 1), __pyx_v_dst_data, (__pyx_v_dst_strides + 1), (__pyx_v_src_shape + 1), (__pyx_v_dst_shape + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize); + + /* "View.MemoryView":1165 + * src_shape + 1, dst_shape + 1, + * ndim - 1, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1166 + * ndim - 1, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + +static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + + /* "View.MemoryView":1171 + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: + * _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, # <<<<<<<<<<<<<< + * src.shape, dst.shape, ndim, itemsize) + * +*/ + _copy_strided_to_strided(__pyx_v_src->data, __pyx_v_src->strides, __pyx_v_dst->data, __pyx_v_dst->strides, __pyx_v_src->shape, __pyx_v_dst->shape, __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_src, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_size; + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + + /* "View.MemoryView":1177 + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" + * cdef Py_ssize_t shape, size = src.memview.view.itemsize # <<<<<<<<<<<<<< + * + * for shape in src.shape[:ndim]: +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_size = __pyx_t_1; + + /* "View.MemoryView":1179 + * cdef Py_ssize_t shape, size = src.memview.view.itemsize + * + * for shape in src.shape[:ndim]: # <<<<<<<<<<<<<< + * size *= shape + * +*/ + __pyx_t_3 = (__pyx_v_src->shape + __pyx_v_ndim); + for (__pyx_t_4 = __pyx_v_src->shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_v_shape = (__pyx_t_2[0]); + + /* "View.MemoryView":1180 + * + * for shape in src.shape[:ndim]: + * size *= shape # <<<<<<<<<<<<<< + * + * return size +*/ + __pyx_v_size = (__pyx_v_size * __pyx_v_shape); + } + + /* "View.MemoryView":1182 + * size *= shape + * + * return size # <<<<<<<<<<<<<< + * + * @cname('__pyx_fill_contig_strides_array') +*/ + __pyx_r = __pyx_v_size; + goto __pyx_L0; + + /* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, Py_ssize_t __pyx_v_stride, int __pyx_v_ndim, char __pyx_v_order) { + int __pyx_v_idx; + Py_ssize_t __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + __pyx_t_1 = (__pyx_v_order == 'F'); + if (__pyx_t_1) { + + /* "View.MemoryView":1195 + * + * if order == 'F': + * for idx in range(ndim): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + __pyx_t_2 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_idx = __pyx_t_4; + + /* "View.MemoryView":1196 + * if order == 'F': + * for idx in range(ndim): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * else: +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1197 + * for idx in range(ndim): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * else: + * for idx in range(ndim - 1, -1, -1): +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1199 + * stride *= shape[idx] + * else: + * for idx in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + /*else*/ { + for (__pyx_t_2 = (__pyx_v_ndim - 1); __pyx_t_2 > -1; __pyx_t_2-=1) { + __pyx_v_idx = __pyx_t_2; + + /* "View.MemoryView":1200 + * else: + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1201 + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * + * return stride +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + } + __pyx_L3:; + + /* "View.MemoryView":1203 + * stride *= shape[idx] + * + * return stride # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_data_to_temp') +*/ + __pyx_r = __pyx_v_stride; + goto __pyx_L0; + + /* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_tmpslice, char __pyx_v_order, int __pyx_v_ndim) { + int __pyx_v_i; + void *__pyx_v_result; + size_t __pyx_v_itemsize; + size_t __pyx_v_size; + void *__pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + struct __pyx_memoryview_obj *__pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1217 + * cdef void *result + * + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef size_t size = slice_get_size(src, ndim) + * +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1218 + * + * cdef size_t itemsize = src.memview.view.itemsize + * cdef size_t size = slice_get_size(src, ndim) # <<<<<<<<<<<<<< + * + * result = malloc(size) +*/ + __pyx_v_size = __pyx_memoryview_slice_get_size(__pyx_v_src, __pyx_v_ndim); + + /* "View.MemoryView":1220 + * cdef size_t size = slice_get_size(src, ndim) + * + * result = malloc(size) # <<<<<<<<<<<<<< + * if not result: + * _err_no_memory() +*/ + __pyx_v_result = malloc(__pyx_v_size); + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + __pyx_t_2 = (!(__pyx_v_result != 0)); + if (__pyx_t_2) { + + /* "View.MemoryView":1222 + * result = malloc(size) + * if not result: + * _err_no_memory() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __pyx_memoryview_err_no_memory(); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1222, __pyx_L1_error) + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + } + + /* "View.MemoryView":1225 + * + * + * tmpslice.data = result # <<<<<<<<<<<<<< + * tmpslice.memview = src.memview + * for i in range(ndim): +*/ + __pyx_v_tmpslice->data = ((char *)__pyx_v_result); + + /* "View.MemoryView":1226 + * + * tmpslice.data = result + * tmpslice.memview = src.memview # <<<<<<<<<<<<<< + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] +*/ + __pyx_t_4 = __pyx_v_src->memview; + __pyx_v_tmpslice->memview = __pyx_t_4; + + /* "View.MemoryView":1227 + * tmpslice.data = result + * tmpslice.memview = src.memview + * for i in range(ndim): # <<<<<<<<<<<<<< + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1228 + * tmpslice.memview = src.memview + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] # <<<<<<<<<<<<<< + * tmpslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_tmpslice->shape[__pyx_v_i]) = (__pyx_v_src->shape[__pyx_v_i]); + + /* "View.MemoryView":1229 + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) +*/ + (__pyx_v_tmpslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1231 + * tmpslice.suboffsets[i] = -1 + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # <<<<<<<<<<<<<< + * + * +*/ + (void)(__pyx_fill_contig_strides_array((&(__pyx_v_tmpslice->shape[0])), (&(__pyx_v_tmpslice->strides[0])), __pyx_v_itemsize, __pyx_v_ndim, __pyx_v_order)); + + /* "View.MemoryView":1234 + * + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + __pyx_t_2 = ((__pyx_v_tmpslice->shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1236 + * for i in range(ndim): + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 # <<<<<<<<<<<<<< + * + * if slice_is_contig(src[0], order, ndim): +*/ + (__pyx_v_tmpslice->strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + } + } + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig((__pyx_v_src[0]), __pyx_v_order, __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1239 + * + * if slice_is_contig(src[0], order, ndim): + * memcpy(result, src.data, size) # <<<<<<<<<<<<<< + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) +*/ + (void)(memcpy(__pyx_v_result, __pyx_v_src->data, __pyx_v_size)); + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":1241 + * memcpy(result, src.data, size) + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) # <<<<<<<<<<<<<< + * + * return result +*/ + /*else*/ { + copy_strided_to_strided(__pyx_v_src, __pyx_v_tmpslice, __pyx_v_ndim, __pyx_v_itemsize); + } + __pyx_L9:; + + /* "View.MemoryView":1243 + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.copy_data_to_temp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + +static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent1, Py_ssize_t __pyx_v_extent2) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[7]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_extents", 0); + + /* "View.MemoryView":1250 + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_dim') +*/ + __pyx_t_1 = __Pyx_PyUnicode_From_int(__pyx_v_i, 0, ' ', 'd'); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent1, 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent2, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_got_differing_extents_in_dimensi; + __pyx_t_4[1] = __pyx_t_1; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_got; + __pyx_t_4[3] = __pyx_t_2; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_4[5] = __pyx_t_3; + __pyx_t_4[6] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 7, 35 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_1) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 1250, __pyx_L1_error) + + /* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView._err_extents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + +static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, PyObject *__pyx_v_msg, int __pyx_v_dim) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_dim", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1254 + * @cname('__pyx_memoryview_err_dim') + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err') +*/ + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_FormatSafe(__pyx_v_msg, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_t_2, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 1254, __pyx_L1_error) + + /* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._err_dim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + +static int __pyx_memoryview_err(PyObject *__pyx_v_error, PyObject *__pyx_v_msg) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1258 + * @cname('__pyx_memoryview_err') + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_no_memory') +*/ + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_v_msg, 0, 0); + __PYX_ERR(1, 1258, __pyx_L1_error) + + /* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + +static int __pyx_memoryview_err_no_memory(void) { + int __pyx_r; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1262 + * @cname('__pyx_memoryview_err_no_memory') + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError # <<<<<<<<<<<<<< + * + * +*/ + PyErr_NoMemory(); __PYX_ERR(1, 1262, __pyx_L1_error) + + /* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err_no_memory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_memviewslice __pyx_v_dst, int __pyx_v_src_ndim, int __pyx_v_dst_ndim, int __pyx_v_dtype_is_object) { + void *__pyx_v_tmpdata; + size_t __pyx_v_itemsize; + int __pyx_v_i; + char __pyx_v_order; + int __pyx_v_broadcasting; + int __pyx_v_direct_copy; + __Pyx_memviewslice __pyx_v_tmp; + int __pyx_v_ndim; + int __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + void *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1274 + * Check for overlapping memory and verify the shapes. + * """ + * cdef void *tmpdata = NULL # <<<<<<<<<<<<<< + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i +*/ + __pyx_v_tmpdata = NULL; + + /* "View.MemoryView":1275 + * """ + * cdef void *tmpdata = NULL + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) +*/ + __pyx_t_1 = __pyx_v_src.memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1277 + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) # <<<<<<<<<<<<<< + * cdef bint broadcasting = False + * cdef bint direct_copy = False +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_src), __pyx_v_src_ndim); + + /* "View.MemoryView":1278 + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False # <<<<<<<<<<<<<< + * cdef bint direct_copy = False + * cdef __Pyx_memviewslice tmp +*/ + __pyx_v_broadcasting = 0; + + /* "View.MemoryView":1279 + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False + * cdef bint direct_copy = False # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice tmp + * +*/ + __pyx_v_direct_copy = 0; + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + __pyx_t_2 = (__pyx_v_src_ndim < __pyx_v_dst_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1283 + * + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_src), __pyx_v_src_ndim, __pyx_v_dst_ndim); + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + __pyx_t_2 = (__pyx_v_dst_ndim < __pyx_v_src_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1285 + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) # <<<<<<<<<<<<<< + * + * cdef int ndim = max(src_ndim, dst_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_dst), __pyx_v_dst_ndim, __pyx_v_src_ndim); + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + } + __pyx_L3:; + + /* "View.MemoryView":1287 + * broadcast_leading(&dst, dst_ndim, src_ndim) + * + * cdef int ndim = max(src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + __pyx_t_3 = __pyx_v_dst_ndim; + __pyx_t_4 = __pyx_v_src_ndim; + __pyx_t_2 = (__pyx_t_3 > __pyx_t_4); + if (__pyx_t_2) { + __pyx_t_5 = __pyx_t_3; + } else { + __pyx_t_5 = __pyx_t_4; + } + __pyx_v_ndim = __pyx_t_5; + + /* "View.MemoryView":1289 + * cdef int ndim = max(src_ndim, dst_ndim) + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: +*/ + __pyx_t_5 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_5; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) != (__pyx_v_dst.shape[__pyx_v_i])); + if (__pyx_t_2) { + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1292 + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: + * broadcasting = True # <<<<<<<<<<<<<< + * src.strides[i] = 0 + * else: +*/ + __pyx_v_broadcasting = 1; + + /* "View.MemoryView":1293 + * if src.shape[i] == 1: + * broadcasting = True + * src.strides[i] = 0 # <<<<<<<<<<<<<< + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) +*/ + (__pyx_v_src.strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":1295 + * src.strides[i] = 0 + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) # <<<<<<<<<<<<<< + * + * if src.suboffsets[i] >= 0: +*/ + /*else*/ { + __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1295, __pyx_L1_error) + } + __pyx_L7:; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + } + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + __pyx_t_2 = ((__pyx_v_src.suboffsets[__pyx_v_i]) >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":1298 + * + * if src.suboffsets[i] >= 0: + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) # <<<<<<<<<<<<<< + * + * if slices_overlap(&src, &dst, ndim, itemsize): +*/ + __pyx_t_6 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Dimension_d_is_not_direct, __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1298, __pyx_L1_error) + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + } + } + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + __pyx_t_2 = __pyx_slices_overlap((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + if (__pyx_t_2) { + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + __pyx_t_2 = (!__pyx_memviewslice_is_contig(__pyx_v_src, __pyx_v_order, __pyx_v_ndim)); + if (__pyx_t_2) { + + /* "View.MemoryView":1303 + * + * if not slice_is_contig(src, order, ndim): + * order = get_best_order(&dst, ndim) # <<<<<<<<<<<<<< + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim); + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + } + + /* "View.MemoryView":1305 + * order = get_best_order(&dst, ndim) + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) # <<<<<<<<<<<<<< + * src = tmp + * +*/ + __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1305, __pyx_L1_error) + __pyx_v_tmpdata = __pyx_t_7; + + /* "View.MemoryView":1306 + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) + * src = tmp # <<<<<<<<<<<<<< + * + * if not broadcasting: +*/ + __pyx_v_src = __pyx_v_tmp; + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (!__pyx_v_broadcasting); + if (__pyx_t_2) { + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'C', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1312 + * + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) # <<<<<<<<<<<<<< + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'C', __pyx_v_ndim); + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'F', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1314 + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) # <<<<<<<<<<<<<< + * + * if direct_copy: +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'F', __pyx_v_ndim); + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + } + __pyx_L12:; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + if (__pyx_v_direct_copy) { + + /* "View.MemoryView":1318 + * if direct_copy: + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1319 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) +*/ + (void)(memcpy(__pyx_v_dst.data, __pyx_v_src.data, __pyx_memoryview_slice_get_size((&__pyx_v_src), __pyx_v_ndim))); + + /* "View.MemoryView":1320 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * free(tmpdata) + * return 0 +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1321 + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1322 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * if order == 'F' == get_best_order(&dst, ndim): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (__pyx_v_order == 'F'); + if (__pyx_t_2) { + __pyx_t_2 = ('F' == __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim)); + } + if (__pyx_t_2) { + + /* "View.MemoryView":1327 + * + * + * transpose_memslice(&src) # <<<<<<<<<<<<<< + * transpose_memslice(&dst) + * +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1327, __pyx_L1_error) + + /* "View.MemoryView":1328 + * + * transpose_memslice(&src) + * transpose_memslice(&dst) # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1328, __pyx_L1_error) + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1330 + * transpose_memslice(&dst) + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1331 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * +*/ + copy_strided_to_strided((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1332 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * free(tmpdata) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1334 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1335 + * + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_broadcast_leading') +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_contents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim, int __pyx_v_ndim_other) { + int __pyx_v_i; + int __pyx_v_offset; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + + /* "View.MemoryView":1342 + * int ndim_other) noexcept nogil: + * cdef int i + * cdef int offset = ndim_other - ndim # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_offset = (__pyx_v_ndim_other - __pyx_v_ndim); + + /* "View.MemoryView":1344 + * cdef int offset = ndim_other - ndim + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1345 + * + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] # <<<<<<<<<<<<<< + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] +*/ + (__pyx_v_mslice->shape[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->shape[__pyx_v_i]); + + /* "View.MemoryView":1346 + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] # <<<<<<<<<<<<<< + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * +*/ + (__pyx_v_mslice->strides[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1347 + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] # <<<<<<<<<<<<<< + * + * for i in range(offset): +*/ + (__pyx_v_mslice->suboffsets[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->suboffsets[__pyx_v_i]); + } + + /* "View.MemoryView":1349 + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * + * for i in range(offset): # <<<<<<<<<<<<<< + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] +*/ + __pyx_t_1 = __pyx_v_offset; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1350 + * + * for i in range(offset): + * mslice.shape[i] = 1 # <<<<<<<<<<<<<< + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 +*/ + (__pyx_v_mslice->shape[__pyx_v_i]) = 1; + + /* "View.MemoryView":1351 + * for i in range(offset): + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] # <<<<<<<<<<<<<< + * mslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_mslice->strides[__pyx_v_i]) = (__pyx_v_mslice->strides[0]); + + /* "View.MemoryView":1352 + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_mslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_dtype_is_object, int __pyx_v_ndim, int __pyx_v_inc) { + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + if (__pyx_v_dtype_is_object) { + + /* "View.MemoryView":1363 + * + * if dtype_is_object: + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') +*/ + __pyx_memoryview_refcount_objects_in_slice_with_gil(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + } + + /* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1369 + * Py_ssize_t *strides, int ndim, + * bint inc) noexcept with gil: + * refcount_objects_in_slice(data, shape, strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, __pyx_v_shape, __pyx_v_strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ + __Pyx_PyGILState_Release(__pyx_gilstate_save); +} + +/* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + +static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1375 + * Py_ssize_t *strides, int ndim, bint inc) noexcept: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * + * for i in range(shape[0]): +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1377 + * cdef Py_ssize_t stride = strides[0] + * + * for i in range(shape[0]): # <<<<<<<<<<<<<< + * if ndim == 1: + * if inc: +*/ + __pyx_t_1 = (__pyx_v_shape[0]); + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + __pyx_t_4 = (__pyx_v_ndim == 1); + if (__pyx_t_4) { + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + if (__pyx_v_inc) { + + /* "View.MemoryView":1380 + * if ndim == 1: + * if inc: + * Py_INCREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * Py_DECREF(( data)[0]) +*/ + Py_INCREF((((PyObject **)__pyx_v_data)[0])); + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":1382 + * Py_INCREF(( data)[0]) + * else: + * Py_DECREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) +*/ + /*else*/ { + Py_DECREF((((PyObject **)__pyx_v_data)[0])); + } + __pyx_L6:; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":1384 + * Py_DECREF(( data)[0]) + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) # <<<<<<<<<<<<<< + * + * data += stride +*/ + /*else*/ { + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_inc); + } + __pyx_L5:; + + /* "View.MemoryView":1386 + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) + * + * data += stride # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item, int __pyx_v_dtype_is_object) { + + /* "View.MemoryView":1395 + * size_t itemsize, void *item, + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1396 + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) # <<<<<<<<<<<<<< + * refcount_copying(dst, dtype_is_object, ndim, inc=True) + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1397 + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_extent; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + + /* "View.MemoryView":1405 + * size_t itemsize, void *item) noexcept nogil: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t extent = shape[0] + * +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1406 + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] + * cdef Py_ssize_t extent = shape[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_extent = (__pyx_v_shape[0]); + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1409 + * + * if ndim == 1: + * for i in range(extent): # <<<<<<<<<<<<<< + * memcpy(data, item, itemsize) + * data += stride +*/ + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1410 + * if ndim == 1: + * for i in range(extent): + * memcpy(data, item, itemsize) # <<<<<<<<<<<<<< + * data += stride + * else: +*/ + (void)(memcpy(__pyx_v_data, __pyx_v_item, __pyx_v_itemsize)); + + /* "View.MemoryView":1411 + * for i in range(extent): + * memcpy(data, item, itemsize) + * data += stride # <<<<<<<<<<<<<< + * else: + * for i in range(extent): +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1413 + * data += stride + * else: + * for i in range(extent): # <<<<<<<<<<<<<< + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride +*/ + /*else*/ { + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1414 + * else: + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) # <<<<<<<<<<<<<< + * data += stride + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1415 + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride # <<<<<<<<<<<<<< + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ +} + +/* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum = {"__pyx_unpickle_Enum", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 4, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_Enum", 0) < (0)) __PYX_ERR(1, 4, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, i); __PYX_ERR(1, 4, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_v___pyx_state = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, __pyx_nargs); __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_r = __pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum", 0); + + /* "(tree fragment)":6 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') # <<<<<<<<<<<<<< + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: +*/ + __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, ((char const *)"name")); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(1, 6, __pyx_L1_error) + + /* "(tree fragment)":7 + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) +*/ + __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_MemviewEnum_type); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v___pyx_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None)); + if (__pyx_t_5) { + + /* "(tree fragment)":9 + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): +*/ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 9, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(((struct __pyx_MemviewEnum_obj *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + } + + /* "(tree fragment)":10 + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum__set_state", 0); + + /* "(tree fragment)":12 + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] # <<<<<<<<<<<<<< + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->name); + __Pyx_DECREF(__pyx_v___pyx_result->name); + __pyx_v___pyx_result->name = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":13 + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) # <<<<<<<<<<<<<< +*/ + __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 13, __pyx_L1_error) + + /* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/cblas.pyx":16 + * __slots__ = [] + * + * def __init__(self): # <<<<<<<<<<<<<< + * """Construct a CBlas instance set to use BLIS implementations of the + * supported BLAS functions.""" +*/ + +/* Python wrapper */ +static int __pyx_pw_5thinc_8backends_5cblas_5CBlas_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_5cblas_5CBlas___init__, "Construct a CBlas instance set to use BLIS implementations of the\n supported BLAS functions."); +#if CYTHON_UPDATE_DESCRIPTOR_DOC +struct wrapperbase __pyx_wrapperbase_5thinc_8backends_5cblas_5CBlas___init__; +#endif +static int __pyx_pw_5thinc_8backends_5cblas_5CBlas_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, __pyx_nargs); return -1; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return -1; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__init__", __pyx_kwds); return -1;} + __pyx_r = __pyx_pf_5thinc_8backends_5cblas_5CBlas___init__(((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5thinc_8backends_5cblas_5CBlas___init__(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self) { + struct __pyx_t_5thinc_8backends_5cblas_BlasFuncs __pyx_v_funcs; + int __pyx_r; + std::shared_ptr __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "thinc/backends/cblas.pyx":20 + * supported BLAS functions.""" + * cdef BlasFuncs funcs + * funcs.daxpy = blis.cy.daxpy # <<<<<<<<<<<<<< + * funcs.saxpy = blis.cy.saxpy + * funcs.sgemm = blis.cy.sgemm +*/ + __pyx_v_funcs.daxpy = __pyx_f_4blis_2cy_daxpy; + + /* "thinc/backends/cblas.pyx":21 + * cdef BlasFuncs funcs + * funcs.daxpy = blis.cy.daxpy + * funcs.saxpy = blis.cy.saxpy # <<<<<<<<<<<<<< + * funcs.sgemm = blis.cy.sgemm + * self.ptr = make_shared[BlasFuncs](funcs) +*/ + __pyx_v_funcs.saxpy = __pyx_f_4blis_2cy_saxpy; + + /* "thinc/backends/cblas.pyx":22 + * funcs.daxpy = blis.cy.daxpy + * funcs.saxpy = blis.cy.saxpy + * funcs.sgemm = blis.cy.sgemm # <<<<<<<<<<<<<< + * self.ptr = make_shared[BlasFuncs](funcs) + * +*/ + __pyx_v_funcs.sgemm = __pyx_f_4blis_2cy_sgemm; + + /* "thinc/backends/cblas.pyx":23 + * funcs.saxpy = blis.cy.saxpy + * funcs.sgemm = blis.cy.sgemm + * self.ptr = make_shared[BlasFuncs](funcs) # <<<<<<<<<<<<<< + * + * cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil: +*/ + try { + __pyx_t_1 = std::make_shared(__pyx_v_funcs); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 23, __pyx_L1_error) + } + __pyx_v_self->ptr = __PYX_STD_MOVE_IF_SUPPORTED(__pyx_t_1); + + /* "thinc/backends/cblas.pyx":16 + * __slots__ = [] + * + * def __init__(self): # <<<<<<<<<<<<<< + * """Construct a CBlas instance set to use BLIS implementations of the + * supported BLAS functions.""" +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.backends.cblas.CBlas.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__, "CBlas.__reduce_cython__(self)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__ = {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__}; +static PyObject *__pyx_pw_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__(((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_self_ptr_cannot_be_converted_to, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.backends.cblas.CBlas.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__, "CBlas.__setstate_cython__(self, __pyx_state)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__ = {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__}; +static PyObject *__pyx_pw_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.cblas.CBlas.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__(((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_self_ptr_cannot_be_converted_to, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.backends.cblas.CBlas.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/cblas.pyx":25 + * self.ptr = make_shared[BlasFuncs](funcs) + * + * cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).daxpy + * +*/ + +static __pyx_t_5thinc_8backends_5cblas_daxpy_ptr __pyx_f_5thinc_8backends_5cblas_daxpy(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas) { + __pyx_t_5thinc_8backends_5cblas_daxpy_ptr __pyx_r; + + /* "thinc/backends/cblas.pyx":26 + * + * cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil: + * return deref(cblas.ptr).daxpy # <<<<<<<<<<<<<< + * + * cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil: +*/ + __pyx_r = (*__pyx_v_cblas->ptr).daxpy; + goto __pyx_L0; + + /* "thinc/backends/cblas.pyx":25 + * self.ptr = make_shared[BlasFuncs](funcs) + * + * cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).daxpy + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/cblas.pyx":28 + * return deref(cblas.ptr).daxpy + * + * cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).saxpy + * +*/ + +static __pyx_t_5thinc_8backends_5cblas_saxpy_ptr __pyx_f_5thinc_8backends_5cblas_saxpy(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas) { + __pyx_t_5thinc_8backends_5cblas_saxpy_ptr __pyx_r; + + /* "thinc/backends/cblas.pyx":29 + * + * cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil: + * return deref(cblas.ptr).saxpy # <<<<<<<<<<<<<< + * + * cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil: +*/ + __pyx_r = (*__pyx_v_cblas->ptr).saxpy; + goto __pyx_L0; + + /* "thinc/backends/cblas.pyx":28 + * return deref(cblas.ptr).daxpy + * + * cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).saxpy + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/cblas.pyx":31 + * return deref(cblas.ptr).saxpy + * + * cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).sgemm + * +*/ + +static __pyx_t_5thinc_8backends_5cblas_sgemm_ptr __pyx_f_5thinc_8backends_5cblas_sgemm(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas) { + __pyx_t_5thinc_8backends_5cblas_sgemm_ptr __pyx_r; + + /* "thinc/backends/cblas.pyx":32 + * + * cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil: + * return deref(cblas.ptr).sgemm # <<<<<<<<<<<<<< + * + * cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil: +*/ + __pyx_r = (*__pyx_v_cblas->ptr).sgemm; + goto __pyx_L0; + + /* "thinc/backends/cblas.pyx":31 + * return deref(cblas.ptr).saxpy + * + * cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil: # <<<<<<<<<<<<<< + * return deref(cblas.ptr).sgemm + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/cblas.pyx":34 + * return deref(cblas.ptr).sgemm + * + * cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).daxpy = daxpy + * +*/ + +static void __pyx_f_5thinc_8backends_5cblas_set_daxpy(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas, __pyx_t_5thinc_8backends_5cblas_daxpy_ptr __pyx_v_daxpy) { + + /* "thinc/backends/cblas.pyx":35 + * + * cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil: + * deref(cblas.ptr).daxpy = daxpy # <<<<<<<<<<<<<< + * + * cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil: +*/ + (*__pyx_v_cblas->ptr).daxpy = __pyx_v_daxpy; + + /* "thinc/backends/cblas.pyx":34 + * return deref(cblas.ptr).sgemm + * + * cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).daxpy = daxpy + * +*/ + + /* function exit code */ +} + +/* "thinc/backends/cblas.pyx":37 + * deref(cblas.ptr).daxpy = daxpy + * + * cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).saxpy = saxpy + * +*/ + +static void __pyx_f_5thinc_8backends_5cblas_set_saxpy(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas, __pyx_t_5thinc_8backends_5cblas_saxpy_ptr __pyx_v_saxpy) { + + /* "thinc/backends/cblas.pyx":38 + * + * cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil: + * deref(cblas.ptr).saxpy = saxpy # <<<<<<<<<<<<<< + * + * cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil: +*/ + (*__pyx_v_cblas->ptr).saxpy = __pyx_v_saxpy; + + /* "thinc/backends/cblas.pyx":37 + * deref(cblas.ptr).daxpy = daxpy + * + * cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).saxpy = saxpy + * +*/ + + /* function exit code */ +} + +/* "thinc/backends/cblas.pyx":40 + * deref(cblas.ptr).saxpy = saxpy + * + * cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).sgemm = sgemm +*/ + +static void __pyx_f_5thinc_8backends_5cblas_set_sgemm(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas, __pyx_t_5thinc_8backends_5cblas_sgemm_ptr __pyx_v_sgemm) { + + /* "thinc/backends/cblas.pyx":41 + * + * cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil: + * deref(cblas.ptr).sgemm = sgemm # <<<<<<<<<<<<<< +*/ + (*__pyx_v_cblas->ptr).sgemm = __pyx_v_sgemm; + + /* "thinc/backends/cblas.pyx":40 + * deref(cblas.ptr).saxpy = saxpy + * + * cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil: # <<<<<<<<<<<<<< + * deref(cblas.ptr).sgemm = sgemm +*/ + + /* function exit code */ +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_5thinc_8backends_5cblas_CBlas(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)o); + __Pyx_default_placement_construct(&(p->ptr)); + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_5cblas_CBlas(PyObject *o) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *p = (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_5cblas_CBlas) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + __Pyx_call_destructor(p->ptr); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static PyMethodDef __pyx_methods_5thinc_8backends_5cblas_CBlas[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_5cblas_5CBlas_2__reduce_cython__}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_5cblas_5CBlas_4__setstate_cython__}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_5cblas_CBlas_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_5cblas_CBlas}, + {Py_tp_doc, (void *)PyDoc_STR("CBlas()")}, + {Py_tp_methods, (void *)__pyx_methods_5thinc_8backends_5cblas_CBlas}, + {Py_tp_init, (void *)__pyx_pw_5thinc_8backends_5cblas_5CBlas_1__init__}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_5cblas_CBlas}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_5cblas_CBlas_spec = { + "thinc.backends.cblas.CBlas", + sizeof(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, + __pyx_type_5thinc_8backends_5cblas_CBlas_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_5cblas_CBlas = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.cblas.""CBlas", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_5cblas_CBlas, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + PyDoc_STR("CBlas()"), /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5thinc_8backends_5cblas_CBlas, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_pw_5thinc_8backends_5cblas_5CBlas_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_5cblas_CBlas, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_array __pyx_vtable_array; + +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_array_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_array_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_array; + p->mode = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_format = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (unlikely(__pyx_array___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_array(PyObject *o) { + struct __pyx_array_obj *p = (struct __pyx_array_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_array) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_array___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->mode); + Py_CLEAR(p->_format); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static PyObject *__pyx_sq_item_array(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_array(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_array___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_tp_getattro_array(PyObject *o, PyObject *n) { + PyObject *v = PyObject_GenericGetAttr(o, n); + if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + v = __pyx_array___getattr__(o, n); + } + return v; +} + +static PyObject *__pyx_getprop___pyx_array_memview(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(o); +} + +static PyMethodDef __pyx_methods_array[] = { + {"__getattr__", (PyCFunction)__pyx_array___getattr__, METH_O|METH_COEXIST, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_array[] = { + {"memview", __pyx_getprop___pyx_array_memview, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_array_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_array}, + {Py_sq_length, (void *)__pyx_array___len__}, + {Py_sq_item, (void *)__pyx_sq_item_array}, + {Py_mp_length, (void *)__pyx_array___len__}, + {Py_mp_subscript, (void *)__pyx_array___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_array}, + {Py_tp_getattro, (void *)__pyx_tp_getattro_array}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_array_getbuffer}, + #endif + {Py_tp_methods, (void *)__pyx_methods_array}, + {Py_tp_getset, (void *)__pyx_getsets_array}, + {Py_tp_new, (void *)__pyx_tp_new_array}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_array_spec = { + "thinc.backends.cblas.array", + sizeof(struct __pyx_array_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_array_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_array = { + __pyx_array___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_array, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_array = { + __pyx_array___len__, /*mp_length*/ + __pyx_array___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_array, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_array = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.cblas.""array", /*tp_name*/ + sizeof(struct __pyx_array_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_array, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_array, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_array, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + __pyx_tp_getattro_array, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_array, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_array, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_array, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_array, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_MemviewEnum_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_MemviewEnum_obj *)o); + p->name = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_Enum(PyObject *o) { + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_Enum) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->name); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_Enum(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->name) { + e = (*v)(p->name, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_Enum(PyObject *o) { + PyObject* tmp; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + tmp = ((PyObject*)p->name); + p->name = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_Enum[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_MemviewEnum_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_Enum}, + {Py_tp_repr, (void *)__pyx_MemviewEnum___repr__}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_Enum}, + {Py_tp_clear, (void *)__pyx_tp_clear_Enum}, + {Py_tp_methods, (void *)__pyx_methods_Enum}, + {Py_tp_init, (void *)__pyx_MemviewEnum___init__}, + {Py_tp_new, (void *)__pyx_tp_new_Enum}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_MemviewEnum_spec = { + "thinc.backends.cblas.Enum", + sizeof(struct __pyx_MemviewEnum_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_MemviewEnum_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_MemviewEnum = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.cblas.""Enum", /*tp_name*/ + sizeof(struct __pyx_MemviewEnum_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_Enum, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_MemviewEnum___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_Enum, /*tp_traverse*/ + __pyx_tp_clear_Enum, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_Enum, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_MemviewEnum___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_Enum, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_memoryview __pyx_vtable_memoryview; + +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryview_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryview_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_memoryview; + p->obj = Py_None; Py_INCREF(Py_None); + p->_size = Py_None; Py_INCREF(Py_None); + p->view.obj = NULL; + if (unlikely(__pyx_memoryview___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_memoryview(PyObject *o) { + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_memoryview) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryview___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->obj); + Py_CLEAR(p->_size); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_memoryview(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->obj) { + e = (*v)(p->obj, a); if (e) return e; + } + if (p->_size) { + e = (*v)(p->_size, a); if (e) return e; + } + if (p->view.obj) { + e = (*v)(p->view.obj, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_memoryview(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + tmp = ((PyObject*)p->obj); + p->obj = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_size); + p->_size = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + Py_CLEAR(p->view.obj); + return 0; +} + +static PyObject *__pyx_sq_item_memoryview(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_memoryview(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_memoryview___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_getprop___pyx_memoryview_T(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_base(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_shape(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_strides(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_suboffsets(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_ndim(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_itemsize(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_nbytes(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_size(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(o); +} + +static PyMethodDef __pyx_methods_memoryview[] = { + {"is_c_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_c_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"is_f_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_f_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy_fortran", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy_fortran, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_memoryview[] = { + {"T", __pyx_getprop___pyx_memoryview_T, 0, 0, 0}, + {"base", __pyx_getprop___pyx_memoryview_base, 0, 0, 0}, + {"shape", __pyx_getprop___pyx_memoryview_shape, 0, 0, 0}, + {"strides", __pyx_getprop___pyx_memoryview_strides, 0, 0, 0}, + {"suboffsets", __pyx_getprop___pyx_memoryview_suboffsets, 0, 0, 0}, + {"ndim", __pyx_getprop___pyx_memoryview_ndim, 0, 0, 0}, + {"itemsize", __pyx_getprop___pyx_memoryview_itemsize, 0, 0, 0}, + {"nbytes", __pyx_getprop___pyx_memoryview_nbytes, 0, 0, 0}, + {"size", __pyx_getprop___pyx_memoryview_size, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_memoryview_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_memoryview}, + {Py_tp_repr, (void *)__pyx_memoryview___repr__}, + {Py_sq_length, (void *)__pyx_memoryview___len__}, + {Py_sq_item, (void *)__pyx_sq_item_memoryview}, + {Py_mp_length, (void *)__pyx_memoryview___len__}, + {Py_mp_subscript, (void *)__pyx_memoryview___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_memoryview}, + {Py_tp_str, (void *)__pyx_memoryview___str__}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_memoryview_getbuffer}, + #endif + {Py_tp_traverse, (void *)__pyx_tp_traverse_memoryview}, + {Py_tp_clear, (void *)__pyx_tp_clear_memoryview}, + {Py_tp_methods, (void *)__pyx_methods_memoryview}, + {Py_tp_getset, (void *)__pyx_getsets_memoryview}, + {Py_tp_new, (void *)__pyx_tp_new_memoryview}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryview_spec = { + "thinc.backends.cblas.memoryview", + sizeof(struct __pyx_memoryview_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_memoryview_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_memoryview = { + __pyx_memoryview___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_memoryview, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_memoryview = { + __pyx_memoryview___len__, /*mp_length*/ + __pyx_memoryview___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_memoryview, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_memoryview = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.cblas.""memoryview", /*tp_name*/ + sizeof(struct __pyx_memoryview_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_memoryview, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_memoryview___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_memoryview, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_memoryview, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_memoryview___str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_memoryview, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_memoryview, /*tp_traverse*/ + __pyx_tp_clear_memoryview, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_memoryview, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_memoryview, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_memoryview, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct__memoryviewslice __pyx_vtable__memoryviewslice; + +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryviewslice_obj *p; + PyObject *o = __pyx_tp_new_memoryview(t, a, k); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryviewslice_obj *)o); + p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_memoryview*)__pyx_vtabptr__memoryviewslice; + __Pyx_default_placement_construct(&(p->from_slice)); + p->from_object = Py_None; Py_INCREF(Py_None); + p->from_slice.memview = NULL; + return o; +} + +static void __pyx_tp_dealloc__memoryviewslice(PyObject *o) { + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc__memoryviewslice) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryviewslice___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + __Pyx_call_destructor(p->from_slice); + Py_CLEAR(p->from_object); + PyObject_GC_Track(o); + __pyx_tp_dealloc_memoryview(o); +} + +static int __pyx_tp_traverse__memoryviewslice(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + e = __pyx_tp_traverse_memoryview(o, v, a); if (e) return e; + if (p->from_object) { + e = (*v)(p->from_object, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear__memoryviewslice(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + __pyx_tp_clear_memoryview(o); + tmp = ((PyObject*)p->from_object); + p->from_object = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + __PYX_XCLEAR_MEMVIEW(&p->from_slice, 1); + return 0; +} + +static PyMethodDef __pyx_methods__memoryviewslice[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_memoryviewslice_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc__memoryviewslice}, + {Py_tp_doc, (void *)PyDoc_STR("Internal class for passing memoryview slices to Python")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse__memoryviewslice}, + {Py_tp_clear, (void *)__pyx_tp_clear__memoryviewslice}, + {Py_tp_methods, (void *)__pyx_methods__memoryviewslice}, + {Py_tp_new, (void *)__pyx_tp_new__memoryviewslice}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryviewslice_spec = { + "thinc.backends.cblas._memoryviewslice", + sizeof(struct __pyx_memoryviewslice_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_memoryviewslice_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_memoryviewslice = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.cblas.""_memoryviewslice", /*tp_name*/ + sizeof(struct __pyx_memoryviewslice_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc__memoryviewslice, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___repr__, /*tp_repr*/ + #else + 0, /*tp_repr*/ + #endif + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___str__, /*tp_str*/ + #else + 0, /*tp_str*/ + #endif + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + PyDoc_STR("Internal class for passing memoryview slices to Python"), /*tp_doc*/ + __pyx_tp_traverse__memoryviewslice, /*tp_traverse*/ + __pyx_tp_clear__memoryviewslice, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods__memoryviewslice, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new__memoryviewslice, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_collections_abc_Sequence = Py_None; Py_INCREF(Py_None); + generic = Py_None; Py_INCREF(Py_None); + strided = Py_None; Py_INCREF(Py_None); + indirect = Py_None; Py_INCREF(Py_None); + contiguous = Py_None; Py_INCREF(Py_None); + indirect_contiguous = Py_None; Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + { + __pyx_t_1 = __Pyx_ApiExport_GetApiDict(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + const char * __pyx_export_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_pyx_t_5thinc_8backends_5cblas); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!__pyx_export_signature)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + const char * __pyx_export_name = __pyx_export_signature + 576; + void (*const __pyx_export_pointers[])(void) = {(void (*)(void))&__pyx_f_5thinc_8backends_5cblas_daxpy, (void (*)(void))&__pyx_f_5thinc_8backends_5cblas_saxpy, (void (*)(void))&__pyx_f_5thinc_8backends_5cblas_sgemm, (void (*)(void))&__pyx_f_5thinc_8backends_5cblas_set_daxpy, (void (*)(void))&__pyx_f_5thinc_8backends_5cblas_set_saxpy, (void (*)(void))&__pyx_f_5thinc_8backends_5cblas_set_sgemm, (void (*)(void)) NULL}; + void (*const *__pyx_export_pointer)(void) = __pyx_export_pointers; + const char *__pyx_export_current_signature = __pyx_export_signature; + while (*__pyx_export_pointer) { + if (__Pyx_ExportFunction(__pyx_t_1, __pyx_export_name, *__pyx_export_pointer, __pyx_export_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + ++__pyx_export_pointer; + __pyx_export_name = strchr(__pyx_export_name, '\0') + 1; + __pyx_export_signature = strchr(__pyx_export_signature, '\0') + 1; + if (*__pyx_export_signature != '\0') __pyx_export_current_signature = __pyx_export_signature; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_5cblas_CBlas_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas)) __PYX_ERR(0, 13, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_5cblas_CBlas_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas) < (0)) __PYX_ERR(0, 13, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas = &__pyx_type_5thinc_8backends_5cblas_CBlas; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas) < (0)) __PYX_ERR(0, 13, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_UPDATE_DESCRIPTOR_DOC + { + PyObject *wrapper = PyObject_GetAttrString((PyObject *)__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas, "__init__"); if (unlikely(!wrapper)) __PYX_ERR(0, 13, __pyx_L1_error) + if (__Pyx_IS_TYPE(wrapper, &PyWrapperDescr_Type)) { + __pyx_wrapperbase_5thinc_8backends_5cblas_5CBlas___init__ = *((PyWrapperDescrObject *)wrapper)->d_base; + __pyx_wrapperbase_5thinc_8backends_5cblas_5CBlas___init__.doc = __pyx_doc_5thinc_8backends_5cblas_5CBlas___init__; + ((PyWrapperDescrObject *)wrapper)->d_base = &__pyx_wrapperbase_5thinc_8backends_5cblas_5CBlas___init__; + } + } + #endif + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_CBlas, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas) < (0)) __PYX_ERR(0, 13, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas) < (0)) __PYX_ERR(0, 13, __pyx_L1_error) + __pyx_vtabptr_array = &__pyx_vtable_array; + __pyx_vtable_array.get_memview = (PyObject *(*)(struct __pyx_array_obj *))__pyx_array_get_memview; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_array_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_array_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_array_type)) __PYX_ERR(1, 110, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_array_type->tp_as_buffer = &__pyx_tp_as_buffer_array; + if (!__pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_array_spec, __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #else + __pyx_mstate->__pyx_array_type = &__pyx_type___pyx_array; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_array_type); + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_array_type, __pyx_vtabptr_array) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_MemviewEnum_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_MemviewEnum_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_MemviewEnum_type)) __PYX_ERR(1, 299, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_MemviewEnum_spec, __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #else + __pyx_mstate->__pyx_MemviewEnum_type = &__pyx_type___pyx_MemviewEnum; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_MemviewEnum_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_MemviewEnum_type->tp_dictoffset && __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + __pyx_vtabptr_memoryview = &__pyx_vtable_memoryview; + __pyx_vtable_memoryview.get_item_pointer = (char *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_get_item_pointer; + __pyx_vtable_memoryview.is_slice = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_is_slice; + __pyx_vtable_memoryview.setitem_slice_assignment = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_slice_assignment; + __pyx_vtable_memoryview.setitem_slice_assign_scalar = (PyObject *(*)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_setitem_slice_assign_scalar; + __pyx_vtable_memoryview.setitem_indexed = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_indexed; + __pyx_vtable_memoryview.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryview_convert_item_to_object; + __pyx_vtable_memoryview.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryview_assign_item_from_object; + __pyx_vtable_memoryview._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryview__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_memoryview_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryview_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_memoryview_type)) __PYX_ERR(1, 334, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer = &__pyx_tp_as_buffer_memoryview; + if (!__pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryview_spec, __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryview_type = &__pyx_type___pyx_memoryview; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryview_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryview_type->tp_dictoffset && __pyx_mstate->__pyx_memoryview_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryview_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryview_type, __pyx_vtabptr_memoryview) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + __pyx_vtabptr__memoryviewslice = &__pyx_vtable__memoryviewslice; + __pyx_vtable__memoryviewslice.__pyx_base = *__pyx_vtabptr_memoryview; + __pyx_vtable__memoryviewslice.__pyx_base.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryviewslice_convert_item_to_object; + __pyx_vtable__memoryviewslice.__pyx_base.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryviewslice_assign_item_from_object; + __pyx_vtable__memoryviewslice.__pyx_base._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryviewslice__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_memoryview_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_memoryviewslice_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryviewslice_spec, __pyx_t_1); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_mstate->__pyx_memoryviewslice_type)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryviewslice_spec, __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryviewslice_type = &__pyx_type___pyx_memoryviewslice; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate_global->__pyx_memoryviewslice_type->tp_base = __pyx_mstate_global->__pyx_memoryview_type; + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryviewslice_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryviewslice_type->tp_dictoffset && __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryviewslice_type, __pyx_vtabptr__memoryviewslice) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + { + __pyx_t_1 = PyImport_ImportModule("blis.cy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_void_int_double_double_const_int); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + const char * __pyx_import_name = __pyx_import_signature + 206; + void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_4blis_2cy_daxpy, (void (**)(void))&__pyx_f_4blis_2cy_saxpy, (void (**)(void))&__pyx_f_4blis_2cy_sgemm, (void (**)(void)) NULL}; + void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers; + const char *__pyx_import_current_signature = __pyx_import_signature; + while (*__pyx_import_pointer) { + if (__Pyx_ImportFunction_3_2_1(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + ++__pyx_import_pointer; + __pyx_import_name = strchr(__pyx_import_name, '\0') + 1; + __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1; + if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cblas(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cblas}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cblas", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cblas(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cblas(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cblas(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + static PyThread_type_lock __pyx_t_7[8]; + int __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cblas' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cblas" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_cblas", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__backends__cblas) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.backends.cblas")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.backends.cblas", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Execution code ---*/ + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":101 + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence # <<<<<<<<<<<<<< + * except: + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_collections_abc}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin___import__, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_abc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L7_try_end; + __pyx_L2_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":102 + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: # <<<<<<<<<<<<<< + * + * __pyx_collections_abc_Sequence = None +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":104 + * except: + * + * __pyx_collections_abc_Sequence = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, Py_None); + __Pyx_GIVEREF(Py_None); + goto __pyx_L3_exception_handled; + } + __pyx_L3_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L7_try_end:; + } + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":240 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":241 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L15_try_end; + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":242 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L11_exception_handled; + } + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L15_try_end:; + } + + /* "View.MemoryView":307 + * return self.name + * + * cdef generic = Enum("") # <<<<<<<<<<<<<< + * cdef strided = Enum("") # default + * cdef indirect = Enum("") +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct_or_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 307, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(generic); + __Pyx_DECREF_SET(generic, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":308 + * + * cdef generic = Enum("") + * cdef strided = Enum("") # default # <<<<<<<<<<<<<< + * cdef indirect = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 308, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(strided); + __Pyx_DECREF_SET(strided, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":309 + * cdef generic = Enum("") + * cdef strided = Enum("") # default + * cdef indirect = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 309, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect); + __Pyx_DECREF_SET(indirect, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":312 + * + * + * cdef contiguous = Enum("") # <<<<<<<<<<<<<< + * cdef indirect_contiguous = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 312, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(contiguous); + __Pyx_DECREF_SET(contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":313 + * + * cdef contiguous = Enum("") + * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 313, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect_contiguous); + __Pyx_DECREF_SET(indirect_contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":321 + * + * + * cdef int __pyx_memoryview_thread_locks_used = 0 # <<<<<<<<<<<<<< + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ + * PyThread_allocate_lock(), +*/ + __pyx_memoryview_thread_locks_used = 0; + + /* "View.MemoryView":322 + * + * cdef int __pyx_memoryview_thread_locks_used = 0 + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ # <<<<<<<<<<<<<< + * PyThread_allocate_lock(), + * PyThread_allocate_lock(), +*/ + __pyx_t_7[0] = PyThread_allocate_lock(); + __pyx_t_7[1] = PyThread_allocate_lock(); + __pyx_t_7[2] = PyThread_allocate_lock(); + __pyx_t_7[3] = PyThread_allocate_lock(); + __pyx_t_7[4] = PyThread_allocate_lock(); + __pyx_t_7[5] = PyThread_allocate_lock(); + __pyx_t_7[6] = PyThread_allocate_lock(); + __pyx_t_7[7] = PyThread_allocate_lock(); + memcpy(&(__pyx_memoryview_thread_locks[0]), __pyx_t_7, sizeof(__pyx_memoryview_thread_locks[0]) * (8)); + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":984 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":985 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L23_try_end; + __pyx_L18_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":986 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L19_exception_handled; + } + __pyx_L19_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L23_try_end:; + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_collections_abc_Sequence); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(1, 990, __pyx_L26_error) + if (__pyx_t_8) { + + /* "View.MemoryView":994 + * + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence.register(array) + * except: +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_memoryviewslice_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 994, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":995 + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) # <<<<<<<<<<<<<< + * except: + * pass # ignore failure, it's a minor issue +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_array_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 995, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L31_try_end; + __pyx_L26_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":996 + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) + * except: # <<<<<<<<<<<<<< + * pass # ignore failure, it's a minor issue + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L27_exception_handled; + } + __pyx_L27_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L31_try_end:; + } + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + __pyx_t_4 = PyCFunction_NewEx(&__pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum, NULL, __pyx_mstate_global->__pyx_n_u_View_MemoryView); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum, __pyx_t_4) < (0)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/cblas.pyx":14 + * + * cdef class CBlas: + * __slots__ = [] # <<<<<<<<<<<<<< + * + * def __init__(self): +*/ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas, __pyx_mstate_global->__pyx_n_u_slots, __pyx_t_4) < (0)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + __pyx_t_4 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_5cblas_5CBlas_3__reduce_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_CBlas___reduce_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_cblas, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_4); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_reduce_cython, __pyx_t_4) < (0)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self.ptr cannot be converted to a Python object for pickling" +*/ + __pyx_t_4 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_5cblas_5CBlas_5__setstate_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_CBlas___setstate_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_cblas, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_4); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_setstate_cython, __pyx_t_4) < (0)) __PYX_ERR(1, 3, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/cblas.pyx":1 + * # cython: profile=False # <<<<<<<<<<<<<< + * cimport blis.cy + * from cython.operator cimport dereference as deref +*/ + __pyx_t_4 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_4) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.backends.cblas", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.backends.cblas"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin___import__ = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_import); if (!__pyx_builtin___import__) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 409, __pyx_L1_error) + __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_id); if (!__pyx_builtin_id) __PYX_ERR(1, 619, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_self_name_is_not_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_New(1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_mstate_global->__pyx_tuple[1], 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(1, 583, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_mstate_global->__pyx_slice[0] = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[0])) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<2; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_slice; + for (Py_ssize_t i=0; i<1; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 10; } index[] = {{2},{68},{35},{54},{37},{60},{24},{52},{26},{34},{29},{33},{45},{22},{15},{179},{37},{30},{32},{1},{1},{1},{1},{1},{8},{5},{6},{15},{23},{25},{7},{6},{2},{6},{35},{9},{30},{50},{8},{21},{60},{20},{32},{22},{14},{30},{37},{5},{5},{23},{25},{8},{20},{8},{15},{3},{15},{18},{4},{1},{9},{17},{18},{5},{8},{15},{6},{9},{5},{5},{6},{7},{8},{12},{2},{10},{5},{13},{5},{8},{8},{7},{4},{10},{4},{8},{4},{7},{3},{4},{3},{12},{14},{11},{10},{19},{14},{12},{10},{17},{13},{8},{4},{12},{10},{12},{19},{5},{4},{9},{5},{4},{4},{6},{8},{20},{6},{6},{6},{1},{9},{623},{223},{1}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (1272 bytes) */ +const char* const cstring = "BZh91AY&SYh\367\327\202\000\000\306\337\342d\020B\377FW\277#\373\000\277\377\377\360@@@\000\000@P\005/q\210\253a\265\263l\264\r\250I F\223\nz\014\221\224\365=\252hh\321\220\323\023&\232\003\312\034\302`\023\000&\023\t\246\000\000\023&\232\006\022!\t= LI\246\232\236\204i\246\214!\2404\001\351\001\314&\0010\002a0\232`\000\0012i\240a\"\211\204h)\224z\206\364\211\262\200\000\003@\320\032\024\235\0140\277gs\016a\350\376\361\247\345\340y\235\203\374\337\344\335\260PL\236/\314\231\0318G\204k\315\003\246\337\032uH]\324\353\206P!\225\030\337\030\223\326\320X\220\306\030NJ\374\317]X\212\265|6F\367\271>\377\016+\265\326\220\006\357E^\242\265\"?^\032\227H\266\353\324O\363R\370\226?\276\255i\224\357\\7d\261n3\361\306\373\307\233:n\215\273\034N\n\322\\L\323\014F-\\g\032m\326\375\037\210:w{3H\017\240\327r\2109p,\205\302*Y\231\230\t='\007\205\004\364A\257\276oQ-v\233\315\227\037\037\206\260\026\346\005\341\022\254\014X\353\313@\257\350}=\2100\254Z\033\277\311\364bl\237,y\247kF\252\335E\226>\353)y\275\375\357\2037\032\373\266\3075zZ\221\027\202\376\nb\326\t\342\201(s\312\t\2379j\001\337\302\352\317Z\352\320\265\271\314K\n\361\372\264\024\377\021\357m\334\250\361p\347mkY\341D\\\342\252B\240\222\\2\035D\"\231\276\205i\344j2\001\027\242\001\221\n\3113Hl\324\254\360\213&\360\355\352\360\022\3527\236=wj\340L\303\253\316\026#\257}\305(\035K.xD!\035&\213P\214\371\334\021m=\244\354\204\332\355\351\250\362\241\230+\234\371\017\320\313\262\354M\267\2143g\017h+u\030\225-w$\\\205\002 \314\021\r\255\323\206\274Y\n\327FU/ WU\311~(\216v\317\002r\022\220\344\356\341\221\t\364f\0361\253\262\353\211\220\254\013\022\020!\251.\033&\204\302\021\373K\202\034\222n \256\314w\3606,s0\235\373\234\232\205\346\0323\004&\231\356\230\tIT\242q\236\377/\375\251Nz\334\307|\374\226o\345\225\323\025\212\347\216\322[\331\0273{X\255\"\323v\333\343o N\310\334\317Sm\r\33231\246\253\2729f\0360\215c\026\370\3016\263\332\252\353\314\216\007\200\311\351\256\225y\343\265\nBt\030\371\372\334\312\032\235\252Q\300\3071\217!\234v""\252.\220\263\266\310\357J\214\251\2535?\217\020\366r\234\215\233Q\230a\313T\214\374\032G=\002\251!\373\3379\036\334\251\202\232\2302\322\302F\254/\034\177p\277P\242^\364Ppf\306\357\256WI\313F\235J\334g\312\205P\321\266\305\366\357\250\341p\344\323`\341\222\253\033\235Dq\n\005\330\027G^\315\243~\351\327\320]8\256q\023y$\0333\327\255\004\201H\032\332\321\305\366\251\005xa\201ZO\333\257\276a\220\022U\215\215\256\322\253\365T\3474d<\307>\023C!\362\024\250D5E\351F\327(\264\352''67\025\331SBq\r\344\377nz\370\355XB\275\275\357b\017E\225\342\002\345\220\241\223-8\n\n\202\254\215\203lld\352L**@*%t\025\332<\010\2335F\014+\220\026\026)\220[\016\333\337 \340+\260\251wD\035\230\243\347\257\201!\204\036]\366\356\033\313\261b\r\311A\333\265\r\215\213^\035\234V\342\255\272\325\266\230\341\323de\233\2311\2238\214_\035\3724Y8\321\205\226s/C*\251\020\357K\000w\277j\207\001\n\022k\222\327S\231\277p\214\345\261\3339\026_\235\314\3716\311\321R\206\324Y(K'g+\nN\003/\020c\255\342-\212X*\3213 \244W\265\342I\210\007\273x\3152x\357\354\255\332l8\347,i\3672\024\247[\305\313\255f\367\322\365\264P\035Sn#knv{\223\037\244S'n\261\262\026\360g3\260] \247I\2241\2629\360*\214>(\273c\002\200\361\365\n8\324l\3265\310l\307\236\205\255\255\014m+\350A\204N\276\275%\355\27134G\324\013\234\r\026\234\205p\321\336\202\333R\266\205\327%\305\245jl\274)I\264\3669\322\343P\345\024P\223B\334\217s\020\270\246\323v[\353#c\267-\233\243\000\364\267\315\2267o\200\367\032\252\034Q\225\020Hm\222ll\003\357\r\240\306!\212\371E\214\030k\235\312x\301\304BO\252\204v\321\264T\247R \344Te\303\217\037\370\273\222)\302\204\203G\276\274\020"; + PyObject *data = __Pyx_DecompressString(cstring, 1272, 2); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (1117 bytes) */ +const char* const cstring = "x\332\275VMo$5\020\r\022'~E]V\223@\322\332\303\256\264B!h7\033\2449\220\rDp\265`\246\202P/]\365C\332@\\Jr\235\362@\311\265>\200\332\244\245w\215\014An\346%\001\237\023\370\026\026>;\035\341X\256\311\3133}2?\214U\301\317\303\340CB=w+i\r\261\3445\2362\360t\231H\232\251\031P\234\031\005\343Bf\247\320\221\325t\271\246CX\2270\347\337\227\002\177\346\002)\205kOP\026j.K\226\\\217Fk\026\030\010d\202\220i\245 \205?\0077W7g/^\275(\262\010\370\013\205\217\224\335BYb\216\344\3005ec\023\005cxc\003\363\0266>\203CJ\223h\035\350\336\276AZ\242\203\210\211\0270+\\\310Du\0132'm\316F\\\315\n\331\372;i#6\357\036\200G)\217\002\223Ja\234\320\274M8@/7\005?\322\363\007\014~\007\364O\256(\207\\\222\032W\030H\030\t{\376\366\013.\252\271\230\235|+\265\026\216\341\341b\201\020z\276V\336ZN\207\210i\344B\235\357i\231/\325d/\036oO\022\272\320&r\\,\321;\005\307LSW\324\303\025p3\342:\241K\2455v}ib5\321\234e4\037\020\316\277\201\347\007\332r\236\250ke\266\t\204\010\250\263B!@\347R\250\363\356\214\250\\\031i\351T\031g\022\035\326r/\"\332\266q\262\307I\316\327\336a\331\034\022\305\250=B(\216hU.%\334T\311T'%\235\301\250;Ku\234\327i\240\037\300r\270\307\252\335\242\363\340\374\341\256\353\242\317A\341E\336\022'\255\365\252\014\030.\035\264L\262y\342\264\212\237=\216\363\251y}{9\237_\276!\001\226G\263\205\252B)\304\264M\262\214\244\306\335\301\225\265f\210&\nq\263Y\323\357-5\206\270&\306~\304\366\026\337gt\n\271\257\232]\213\221H\246TD\225\251\214\033\247\214o\224""\0174\005\014\301\274\220\021\025Q\302-!\304\264\35001\333\374I\227\204\241\226\010R\341B\252;E\312'\3624\207\027\232\033MPR\225\005\312\201f\003\272\334\227\006\306\020|h\255\354\"\261\323\3134N\010!\332\354\024\007\353\246\"\205\321\364\353y\316\320\233\247\024;\335fY\2257\312O\210^RB\202f%\017J\236F\364\341u\266\364f\031m\237\244az\342\275\340\364\006\312}\360\203\020\003\301\247\344`8\201\262^\242\272\213\271\257_c>\274,\265\225UvEZ(\256\334toU\306?\373x\237\245\255\001w\272?\240u\273A\225\321\2623\221\246\032k\2740=&L\253\261\205\366\370\027Oh\241\350\252b\021\255OD\034\235\007\272\204CL\236~!3;\t#?\227\306\251\206\251C\032Y\215Z\020\303T\020}\347\201\204\2134\2473\306\365\257\237\375\361\371\027\277}\365\373\017c\361\342e1\024\257&K\361\262\230\n-\327\303Fpk\036\327@P-\010\343\277\264)\252\206/O\216>\346<~R\347\035\366\375\177u\276\362\364\327\354_\333\235\302?\206\363\323\205\210\377C\210\t\332\223\243R\320Q\254O\336>b\205\217\273\264\212\273\025\237\326\214\214K\247\364O]\246\236\232\336<\357\351\337A\n\276w8~N\205\224\223\326z9\275\036\031\325\275'l\236z|\324\317\023{\217C\034\024\377\356O\261\261\034\370"; + PyObject *data = __Pyx_DecompressString(cstring, 1117, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (2812 bytes) */ +const char* const bytes = ": All dimensions preceding dimension %d must be indexed and not slicedBuffer view does not expose stridesCan only create a buffer that is contiguous in memory.Cannot assign to read-only memoryviewCannot create writable memory view from read-only memoryviewCannot index with type 'Cannot transpose memoryview with indirect dimensionsDimension %d is not directEmpty shape tuple for cython.arrayIndex out of bounds (axis %d)Indirect dimensions not supportedInvalid mode, expected 'c' or 'fortran', got Invalid shape in axis ')?add_note and at 0xcollections.abcdisableenablegc (got got differing extents in dimension isenableditemsize <= 0 for cython.arrayno default __reduce__ due to non-trivial __cinit__ object>self.name is not Noneself.ptr cannot be converted to a Python object for picklingunable to allocate array data.unable to allocate shape and strides.ASCIICBlasCBlas.__reduce_cython__CBlas.__setstate_cython__Ellipsis__Pyx_PyDict_NextRefSequenceView.MemoryViewabcallocate_bufferasyncio.coroutinesbasec__class____class_getitem__cline_in_tracebackcount__dict__dtype_is_objectencodeenumerateerrorflagsformatfortran__func____getstate__id__import__index_is_coroutineitemsitemsize__main__memviewmode__module__name__name__ndim__new__objpackpop__pyx_capi____pyx_checksum__pyx_state__pyx_type__pyx_unpickle_Enum__pyx_vtable____qualname____reduce____reduce_cython____reduce_ex__registerself__set_name__setdefault__setstate____setstate_cython__shapesize__slots__startstepstopstruct__test__thinc.backends.cblasunpackupdatevaluesx\200\001\330\004\n\210+\220Q__pyx_t_5thinc_8""backends_5cblas_daxpy_ptr (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)\000__pyx_t_5thinc_8backends_5cblas_saxpy_ptr (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)\000__pyx_t_5thinc_8backends_5cblas_sgemm_ptr (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)\000void (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *, __pyx_t_5thinc_8backends_5cblas_daxpy_ptr)\000void (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *, __pyx_t_5thinc_8backends_5cblas_saxpy_ptr)\000void (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *, __pyx_t_5thinc_8backends_5cblas_sgemm_ptr)\000daxpy\000saxpy\000sgemm\000set_daxpy\000set_saxpy\000set_sgemmvoid (int, double, double const *, int, double *, int)\000void (int, float, float const *, int, float *, int)\000void (int, int, int, int, int, float, float const *, int, float const *, int, float, float *, int)\000daxpy\000saxpy\000sgemmO"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 120; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 47) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 120; i < 124; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 124; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 120; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 0; + int8_t const cint_constants_1[] = {0,-1,1}; + int32_t const cint_constants_4[] = {136983863L}; + for (int i = 0; i < 4; i++) { + numbertab[i] = PyLong_FromLong((i < 3 ? cint_constants_1[i - 0] : cint_constants_4[i - 3])); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 2; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 2; + unsigned int flags : 10; + unsigned int first_line : 2; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 1}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_reduce_cython, __pyx_mstate->__pyx_kp_b_iso88591_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 3}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_pyx_state}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_setstate_cython, __pyx_mstate->__pyx_kp_b_iso88591_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallOneArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallOneArg (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetAttrStr (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod0 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* PyObjectFastCallMethod */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) { + PyObject *result; + PyObject *attr = PyObject_GetAttr(args[0], name); + if (unlikely(!attr)) + return NULL; + result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1); + Py_DECREF(attr); + return result; +} +#endif + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* CIntToDigits (used by CIntToPyUnicode) */ +static const char DIGIT_PAIRS_10[2*10*10+1] = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; +static const char DIGIT_PAIRS_8[2*8*8+1] = { + "0001020304050607" + "1011121314151617" + "2021222324252627" + "3031323334353637" + "4041424344454647" + "5051525354555657" + "6061626364656667" + "7071727374757677" +}; +static const char DIGITS_HEX[2*16+1] = { + "0123456789abcdef" + "0123456789ABCDEF" +}; + +/* BuildPyUnicode (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char) { + PyObject *uval; + Py_ssize_t uoffset = ulength - clength; +#if CYTHON_USE_UNICODE_INTERNALS + Py_ssize_t i; + void *udata; + uval = PyUnicode_New(ulength, 127); + if (unlikely(!uval)) return NULL; + udata = PyUnicode_DATA(uval); + if (uoffset > 0) { + i = 0; + if (prepend_sign) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); + i++; + } + for (; i < uoffset; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); + } + } + for (i=0; i < clength; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); + } +#else + { + PyObject *sign = NULL, *padding = NULL; + uval = NULL; + if (uoffset > 0) { + prepend_sign = !!prepend_sign; + if (uoffset > prepend_sign) { + padding = PyUnicode_FromOrdinal(padding_char); + if (likely(padding) && uoffset > prepend_sign + 1) { + PyObject *tmp = PySequence_Repeat(padding, uoffset - prepend_sign); + Py_DECREF(padding); + padding = tmp; + } + if (unlikely(!padding)) goto done_or_error; + } + if (prepend_sign) { + sign = PyUnicode_FromOrdinal('-'); + if (unlikely(!sign)) goto done_or_error; + } + } + uval = PyUnicode_DecodeASCII(chars, clength, NULL); + if (likely(uval) && padding) { + PyObject *tmp = PyUnicode_Concat(padding, uval); + Py_DECREF(uval); + uval = tmp; + } + if (likely(uval) && sign) { + PyObject *tmp = PyUnicode_Concat(sign, uval); + Py_DECREF(uval); + uval = tmp; + } +done_or_error: + Py_XDECREF(padding); + Py_XDECREF(sign); + } +#endif + return uval; +} + +/* COrdinalToPyUnicode (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value) { + return value <= 1114111; +} +static PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t ulength, char padding_char) { + Py_ssize_t padding_length = ulength - 1; + if (likely((padding_length <= 250) && (value < 0xD800 || value > 0xDFFF))) { + char chars[256]; + if (value <= 255) { + memset(chars, padding_char, (size_t) padding_length); + chars[ulength-1] = (char) value; + return PyUnicode_DecodeLatin1(chars, ulength, NULL); + } + char *cpos = chars + sizeof(chars); + if (value < 0x800) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xc0 | (value & 0x1f)); + } else if (value < 0x10000) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xe0 | (value & 0x0f)); + } else { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xf0 | (value & 0x07)); + } + cpos -= padding_length; + memset(cpos, padding_char, (size_t) padding_length); + return PyUnicode_DecodeUTF8(cpos, chars + sizeof(chars) - cpos, NULL); + } + if (value <= 127 && CYTHON_USE_UNICODE_INTERNALS) { + const char chars[1] = {(char) value}; + return __Pyx_PyUnicode_BuildFromAscii(ulength, chars, 1, 0, padding_char); + } + { + PyObject *uchar, *padding_uchar, *padding, *result; + padding_uchar = PyUnicode_FromOrdinal(padding_char); + if (unlikely(!padding_uchar)) return NULL; + padding = PySequence_Repeat(padding_uchar, padding_length); + Py_DECREF(padding_uchar); + if (unlikely(!padding)) return NULL; + uchar = PyUnicode_FromOrdinal(value); + if (unlikely(!uchar)) { + Py_DECREF(padding); + return NULL; + } + result = PyUnicode_Concat(padding, uchar); + Py_DECREF(padding); + Py_DECREF(uchar); + return result; + } +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (int) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(int)*3+2]; + char *dpos, *end = digits + sizeof(int)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + int remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (int) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (int) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (int) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (Py_ssize_t) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(Py_ssize_t)*3+2]; + char *dpos, *end = digits + sizeof(Py_ssize_t)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + Py_ssize_t remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (Py_ssize_t) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (Py_ssize_t) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (Py_ssize_t) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* GetAttr */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) + return __Pyx_PyObject_GetAttrStr(o, n); +#endif + return PyObject_GetAttr(o, n); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* ObjectGetItem */ +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + PyObject *runerr = NULL; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1, 1); + } + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, __pyx_mstate_global->__pyx_n_u_class_getitem); + if (!meth) { + PyErr_Clear(); + } else { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + +/* RejectKeywords */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) { + PyObject *key = NULL; + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) { + key = __Pyx_PySequence_ITEM(kwds, 0); + } else { +#if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; +#else + Py_ssize_t pos = 0; +#endif +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return; +#endif + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); +#endif + } + if (likely(key)) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + Py_DECREF(key); + } +} + +/* DivInt[Py_ssize_t] */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t a, Py_ssize_t b, int b_is_constant) { + Py_ssize_t q = a / b; + Py_ssize_t r = a - q*b; + Py_ssize_t adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* GetAttr3 */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int res = PyObject_GetOptionalAttr(o, n, &r); + return (res != 0) ? r : __Pyx_NewRef(d); +#else + #if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } + #endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +#endif +} + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* RaiseTooManyValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* RaiseNoneIterError */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); +} + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); + return 0; +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* SwapException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; itp_as_sequence && type->tp_as_sequence->sq_repeat)) { + return type->tp_as_sequence->sq_repeat(seq, mul); + } else +#endif + { + return __Pyx_PySequence_Multiply_Generic(seq, mul); + } +} + +/* PyObjectFormatAndDecref */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f) { + if (unlikely(!s)) return NULL; + if (likely(PyUnicode_CheckExact(s))) return s; + return __Pyx_PyObject_FormatAndDecref(s, f); +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f) { + PyObject *result; + if (unlikely(!s)) return NULL; + result = PyObject_Format(s, f); + Py_DECREF(s); + return result; +} + +/* PyObjectFormat */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { + int ret; + _PyUnicodeWriter writer; + if (likely(PyFloat_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else if (likely(PyLong_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyLong_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else { + return PyObject_Format(obj, format_spec); + } + if (unlikely(ret == -1)) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} +#endif + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* DivInt[long] */ +static CYTHON_INLINE long __Pyx_div_long(long a, long b, int b_is_constant) { + long q = a / b; + long r = a - q*b; + long adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* FunctionExport */ +static int __Pyx_ExportFunction(PyObject *api_dict, const char *name, void (*f)(void), const char *sig) { + PyObject *cobj; + union { + void (*fp)(void); + void *p; + } tmp; + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(api_dict, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + return 0; +bad: + Py_XDECREF(cobj); + return -1; +} + +/* GetApiDict */ +static PyObject *__Pyx_ApiExport_GetApiDict(void) { + PyObject *d; + if (__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_capi, &d) == -1) + return NULL; + if (!d) { + d = PyDict_New(); + if (!d) + goto bad; + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_pyx_capi, d) < 0) + goto bad; + } + return d; +bad: + Py_XDECREF(d); + return NULL; +} + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* PyObjectCallNoArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* DelItemOnTypeDict (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_DelItem(tp_dict, k); + if (likely(!result)) PyType_Modified(tp); + return result; +} + +/* SetupReduce */ +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name_2); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_getstate = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + PyObject *getstate = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate); +#else + getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + if (!getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (getstate) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); +#else + object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + if (!object_getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (object_getstate != getstate) { + goto __PYX_GOOD; + } + } +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); + if (likely(reduce_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + goto __PYX_BAD; + } + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); + if (likely(setstate_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; +__PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; +__PYX_GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); + Py_XDECREF(object_getstate); + Py_XDECREF(getstate); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#endif + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* GetVTable (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* MergeVTables */ +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; +#if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; +#else + size = PyTuple_GET_SIZE(bases); +#endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; +#if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#else + basei = PyTuple_GET_ITEM(bases, i); +#endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; +bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); +#if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); +#endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +really_bad: // bad has failed! +#endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +other_failure: +#endif + PyMem_Free(base_vtables); + return -1; +} + +/* PxdImportShared (used by FunctionImport) */ +#ifndef __PYX_HAVE_RT_ImportFromPxd_3_2_1 +#define __PYX_HAVE_RT_ImportFromPxd_3_2_1 +static int __Pyx_ImportFromPxd_3_2_1(PyObject *module, const char *name, void **p, const char *sig, const char *what) { + PyObject *d = 0; + PyObject *cobj = 0; + d = PyObject_GetAttrString(module, "__pyx_capi__"); + if (!d) + goto bad; +#if (defined(Py_LIMITED_API) && Py_LIMITED_API >= 0x030d0000) || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030d0000) + PyDict_GetItemStringRef(d, name, &cobj); +#else + cobj = PyDict_GetItemString(d, name); + Py_XINCREF(cobj); +#endif + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C %.8s %.200s", + PyModule_GetName(module), what, name); + goto bad; + } + if (!PyCapsule_IsValid(cobj, sig)) { + PyErr_Format(PyExc_TypeError, + "C %.8s %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + what, PyModule_GetName(module), name, sig, PyCapsule_GetName(cobj)); + goto bad; + } + *p = PyCapsule_GetPointer(cobj, sig); + if (!(*p)) + goto bad; + Py_DECREF(d); + Py_DECREF(cobj); + return 0; +bad: + Py_XDECREF(d); + Py_XDECREF(cobj); + return -1; +} +#endif + +/* FunctionImport */ +#ifndef __PYX_HAVE_RT_ImportFunction_3_2_1 +#define __PYX_HAVE_RT_ImportFunction_3_2_1 +static int __Pyx_ImportFunction_3_2_1(PyObject *module, const char *funcname, void (**f)(void), const char *sig) { + union { + void (*fp)(void); + void *p; + } tmp; + int result = __Pyx_ImportFromPxd_3_2_1(module, funcname, &tmp.p, sig, "function"); + if (result == 0) { + *f = tmp.fp; + } + return result; +} +#endif + +/* dict_setdefault (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* AddModuleRef (used by FetchSharedCythonModule) */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* FetchCommonType (used by CommonTypesMetaclass) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +#if __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; +} +static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; +} +#endif +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* PyMethodNew (used by CythonFunctionShared) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared (used by CythonFunction) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +#endif +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, +#else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, +#endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#endif +#if CYTHON_METH_FASTCALL +#if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; +#endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); +#elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); +#else + PyObject_ClearManagedDict((PyObject*)m); +#endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); +#else + { + int e = +#if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict +#else + PyObject_VisitManagedDict +#endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } +#endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* MemviewRefcount */ +#include +#ifndef _Py_NO_RETURN +#define _Py_NO_RETURN +#endif +_Py_NO_RETURN +static void __pyx_fatalerror(const char *fmt, ...) { + va_list vargs; + char msg[200]; +#if PY_VERSION_HEX >= 0x030A0000 || defined(HAVE_STDARG_PROTOTYPES) + va_start(vargs, fmt); +#else + va_start(vargs); +#endif + vsnprintf(msg, 200, fmt, vargs); + va_end(vargs); + Py_FatalError(msg); +} +static CYTHON_INLINE int +__pyx_add_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)++; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE int +__pyx_sub_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)--; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE void +__Pyx_INC_MEMVIEW(__Pyx_memviewslice *memslice, int have_gil, int lineno) +{ + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + return; + } + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count+1, lineno); + } + } +} +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *memslice, + int have_gil, int lineno) { + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + memslice->memview = NULL; + return; + } + old_acquisition_count = __pyx_sub_acquisition_count(memview); + memslice->data = NULL; + if (likely(old_acquisition_count > 1)) { + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + if (have_gil) { + Py_CLEAR(memslice->memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_CLEAR(memslice->memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count-1, lineno); + } +} + +/* MemviewSliceIsContig */ +static int +__pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim) +{ + int i, index, step, start; + Py_ssize_t itemsize = mvs.memview->view.itemsize; + if (order == 'F') { + step = 1; + start = 0; + } else { + step = -1; + start = ndim - 1; + } + for (i = 0; i < ndim; i++) { + index = start + step * i; + if (mvs.suboffsets[index] >= 0 || mvs.strides[index] != itemsize) + return 0; + itemsize *= mvs.shape[index]; + } + return 1; +} + +/* OverlappingSlices */ +static void +__pyx_get_array_memory_extents(__Pyx_memviewslice *slice, + void **out_start, void **out_end, + int ndim, size_t itemsize) +{ + char *start, *end; + int i; + start = end = slice->data; + for (i = 0; i < ndim; i++) { + Py_ssize_t stride = slice->strides[i]; + Py_ssize_t extent = slice->shape[i]; + if (extent == 0) { + *out_start = *out_end = start; + return; + } else { + if (stride > 0) + end += stride * (extent - 1); + else + start += stride * (extent - 1); + } + } + *out_start = start; + *out_end = end + itemsize; +} +static int +__pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize) +{ + void *start1, *end1, *start2, *end2; + __pyx_get_array_memory_extents(slice1, &start1, &end1, ndim, itemsize); + __pyx_get_array_memory_extents(slice2, &start2, &end2, ndim, itemsize); + return (start1 < end2) && (start2 < end1); +} + +/* MemviewSliceInit */ +static int +__Pyx_init_memviewslice(struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference) +{ + __Pyx_RefNannyDeclarations + int i, retval=-1; + Py_buffer *buf = &memview->view; + __Pyx_RefNannySetupContext("init_memviewslice", 0); + if (unlikely(memviewslice->memview || memviewslice->data)) { + PyErr_SetString(PyExc_ValueError, + "memviewslice is already initialized!"); + goto fail; + } + if (buf->strides) { + for (i = 0; i < ndim; i++) { + memviewslice->strides[i] = buf->strides[i]; + } + } else { + Py_ssize_t stride = buf->itemsize; + for (i = ndim - 1; i >= 0; i--) { + memviewslice->strides[i] = stride; + stride *= buf->shape[i]; + } + } + for (i = 0; i < ndim; i++) { + memviewslice->shape[i] = buf->shape[i]; + if (buf->suboffsets) { + memviewslice->suboffsets[i] = buf->suboffsets[i]; + } else { + memviewslice->suboffsets[i] = -1; + } + } + memviewslice->memview = memview; + memviewslice->data = (char *)buf->buf; + if (__pyx_add_acquisition_count(memview) == 0 && !memview_is_new_reference) { + Py_INCREF((PyObject*)memview); + } + retval = 0; + goto no_fail; +fail: + memviewslice->memview = 0; + memviewslice->data = 0; + retval = -1; +no_fail: + __Pyx_RefNannyFinishContext(); + return retval; +} + +/* MemviewSliceCopy */ +static __Pyx_memviewslice +__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object) +{ + __Pyx_RefNannyDeclarations + int i; + __Pyx_memviewslice new_mvs = __Pyx_MEMSLICE_INIT; + struct __pyx_memoryview_obj *from_memview = from_mvs->memview; + Py_buffer *buf = &from_memview->view; + PyObject *shape_tuple = NULL; + PyObject *temp_int = NULL; + struct __pyx_array_obj *array_obj = NULL; + struct __pyx_memoryview_obj *memview_obj = NULL; + __Pyx_RefNannySetupContext("__pyx_memoryview_copy_new_contig", 0); + for (i = 0; i < ndim; i++) { + if (unlikely(from_mvs->suboffsets[i] >= 0)) { + PyErr_Format(PyExc_ValueError, "Cannot copy memoryview slice with " + "indirect dimensions (axis %d)", i); + goto fail; + } + } + shape_tuple = PyTuple_New(ndim); + if (unlikely(!shape_tuple)) { + goto fail; + } + __Pyx_GOTREF(shape_tuple); + for(i = 0; i < ndim; i++) { + temp_int = PyLong_FromSsize_t(from_mvs->shape[i]); + if(unlikely(!temp_int)) { + goto fail; + } else { +#if CYTHON_ASSUME_SAFE_MACROS + PyTuple_SET_ITEM(shape_tuple, i, temp_int); +#else + if (PyTuple_SetItem(shape_tuple, i, temp_int) < 0) { + goto fail; + } +#endif + temp_int = NULL; + } + } + array_obj = __pyx_array_new(shape_tuple, sizeof_dtype, buf->format, mode, NULL); + if (unlikely(!array_obj)) { + goto fail; + } + __Pyx_GOTREF(array_obj); + memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + (PyObject *) array_obj, contig_flag, + dtype_is_object, + from_mvs->memview->typeinfo); + if (unlikely(!memview_obj)) + goto fail; + if (unlikely(__Pyx_init_memviewslice(memview_obj, ndim, &new_mvs, 1) < 0)) + goto fail; + if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim, + dtype_is_object) < 0)) + goto fail; + goto no_fail; +fail: + __Pyx_XDECREF((PyObject *) new_mvs.memview); + new_mvs.memview = NULL; + new_mvs.data = NULL; +no_fail: + __Pyx_XDECREF(shape_tuple); + __Pyx_XDECREF(temp_int); + __Pyx_XDECREF((PyObject *) array_obj); + __Pyx_RefNannyFinishContext(); + return new_mvs; +} + +/* PyObjectCall2Args (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectCallMethod1 (used by UpdateUnpickledDict) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; +} +#endif +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); +#endif +} + +/* UpdateUnpickledDict */ +static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + PyObject *state_dict = __Pyx_PySequence_ITEM(state, index); + if (unlikely(!state_dict)) { + return -1; + } + int non_empty = PyObject_IsTrue(state_dict); + if (non_empty == 0) { + Py_DECREF(state_dict); + return 0; + } else if (unlikely(non_empty == -1)) { + return -1; + } + PyObject *dict; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = PyObject_GetAttrString(obj, "__dict__"); + #else + dict = PyObject_GenericGetDict(obj, NULL); + #endif + if (unlikely(!dict)) { + Py_DECREF(state_dict); + return -1; + } + int result; + if (likely(PyDict_CheckExact(dict))) { + result = PyDict_Update(dict, state_dict); + } else { + PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict); + if (likely(obj_result)) { + Py_DECREF(obj_result); + result = 0; + } else { + result = -1; + } + } + Py_DECREF(state_dict); + Py_DECREF(dict); + return result; +} +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(state_size == -1)) return -1; + #endif + if (state_size <= index) { + return 0; + } + return __Pyx__UpdateUnpickledDict(obj, state, index); +} + +/* CheckUnpickleChecksum */ +static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + PyObject *pickle_module = PyImport_ImportModule("pickle"); + if (unlikely(!pickle_module)) return; + PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (unlikely(!pickle_error)) return; + if (checksum2 == checksum1) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))", + checksum, checksum1, members); + } else if (checksum3 == checksum2) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, members); + } else { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, checksum3, members); + } + Py_DECREF(pickle_error); +} +static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + int found = 0; + found |= checksum1 == checksum; + found |= checksum2 == checksum; + found |= checksum3 == checksum; + if (likely(found)) + return 0; + __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members); + return -1; +} + +/* CIntFromPyVerify (used by CIntFromPy) */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* PyObjectVectorCallKwBuilder (used by CIntToPy) */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const char neg_one = (char) -1, const_zero = (char) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + char val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (char) -1; + val = __Pyx_PyLong_As_char(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 2 * PyLong_SHIFT)) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 3 * PyLong_SHIFT)) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 4 * PyLong_SHIFT)) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(char) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(char) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(char) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(char) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) + } else if ((sizeof(char) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + char val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (char) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (char) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (char) -1; + } else { + stepval = v; + } + v = NULL; + val = (char) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(char) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((char) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(char) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((char) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((char) 1) << (sizeof(char) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (char) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__6); + } + goto done; +} +#endif + +/* GetRuntimeVersion */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +void __Pyx_init_runtime_version(void) { + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } +} +#endif +static unsigned long __Pyx_get_runtime_version(void) { +#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* DecompressString */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; +import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); +bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pxd new file mode 100644 index 0000000000000000000000000000000000000000..aafdc1f260130fb202134d0a029f0d6d963aec04 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pxd @@ -0,0 +1,37 @@ +from libcpp.memory cimport shared_ptr + +ctypedef void (*sgemm_ptr)(bint transA, bint transB, int M, int N, int K, + float alpha, const float* A, int lda, const float *B, + int ldb, float beta, float* C, int ldc) noexcept nogil + + +ctypedef void (*saxpy_ptr)(int N, float alpha, const float* X, int incX, + float *Y, int incY) noexcept nogil + + +ctypedef void (*daxpy_ptr)(int N, double alpha, const double* X, int incX, + double *Y, int incY) noexcept nogil + + +# Forward-declaration of the BlasFuncs struct. This struct must be opaque, so +# that consumers of the CBlas class cannot become dependent on its size or +# ordering. +cdef struct BlasFuncs + + +cdef class CBlas: + cdef shared_ptr[BlasFuncs] ptr + + +# Note: the following functions are intentionally standalone. If we make them +# methods of CBlas, Cython will generate and use a vtable. This makes it +# impossible to add new BLAS functions later without breaking the ABI. +# +# See https://github.com/explosion/thinc/pull/700 for more information. + +cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil +cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil +cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil +cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil +cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil +cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pyx new file mode 100644 index 0000000000000000000000000000000000000000..5c0d2b9f43682022777d7a20f59344fc1a80cc12 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cblas.pyx @@ -0,0 +1,41 @@ +# cython: profile=False +cimport blis.cy +from cython.operator cimport dereference as deref +from libcpp.memory cimport make_shared + + +cdef struct BlasFuncs: + daxpy_ptr daxpy + saxpy_ptr saxpy + sgemm_ptr sgemm + + +cdef class CBlas: + __slots__ = [] + + def __init__(self): + """Construct a CBlas instance set to use BLIS implementations of the + supported BLAS functions.""" + cdef BlasFuncs funcs + funcs.daxpy = blis.cy.daxpy + funcs.saxpy = blis.cy.saxpy + funcs.sgemm = blis.cy.sgemm + self.ptr = make_shared[BlasFuncs](funcs) + +cdef daxpy_ptr daxpy(CBlas cblas) noexcept nogil: + return deref(cblas.ptr).daxpy + +cdef saxpy_ptr saxpy(CBlas cblas) noexcept nogil: + return deref(cblas.ptr).saxpy + +cdef sgemm_ptr sgemm(CBlas cblas) noexcept nogil: + return deref(cblas.ptr).sgemm + +cdef void set_daxpy(CBlas cblas, daxpy_ptr daxpy) noexcept nogil: + deref(cblas.ptr).daxpy = daxpy + +cdef void set_saxpy(CBlas cblas, saxpy_ptr saxpy) noexcept nogil: + deref(cblas.ptr).saxpy = saxpy + +cdef void set_sgemm(CBlas cblas, sgemm_ptr sgemm) noexcept nogil: + deref(cblas.ptr).sgemm = sgemm diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cpu_kernels.hh b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cpu_kernels.hh new file mode 100644 index 0000000000000000000000000000000000000000..16d9925965751ff8df09e3117f6be07925cb4f51 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cpu_kernels.hh @@ -0,0 +1,482 @@ +#ifndef CPU_KERNELS_HH +#define CPU_KERNELS_HH + +#include +#include +#include +#include +#include +#include + +// Ideally we'd use an alias declaration for a generic definition of +// *axpy. But Cython doesn't support alias declarations yet: +// +// https://github.com/cython/cython/issues/3272 +// +// template +// using axpy = void (*)(int N, T alpha, const T* X, int incX, +// T *Y, int incY); +// +// So, instead we'll do this the pre-C++11 way: + +template +struct axpy { + typedef void (*ptr)(int N, T alpha, const T* X, int incX, T *Y, int incY); +}; + + +// All elementwise functions, such as most activations, work in-place. + + +template +struct argmax_result { + T max; + L max_idx; +}; + +template +argmax_result argmax(T const *arr, L len) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + argmax_result r { arr[0], 0 }; + + for (L i = 1; i < len; ++i) { + if (arr[i] > r.max) { + r.max = arr[i]; + r.max_idx = i; + } + } + + return r; +} + +// The next two templates define argmax for a fixed number of elements. + +template +argmax_result argmax(T a) { + static_assert(std::is_floating_point::value, "Argument should be floating point"); + argmax_result acc { a, 0 }; + return acc; +} + +template +argmax_result argmax(T a, Args... args) { + static_assert(std::is_floating_point::value, "Arguments should be floating point"); + + auto acc = argmax(args...); + + if (acc.max > a) { + acc.max_idx += 1; + } else { + acc.max_idx = 0; + acc.max = a; + } + + return acc; +} + + +template +void vec_add(A* X, const A* Y, A scale, L N) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (L i = 0; i < N; ++i) + X[i] += scale * Y[i]; +} + +template +void cpu_maxout(A* best__bo, L* which__bo, const A* cands__bop, L B, L O, L P) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + // For small inputs, we use an unrolled argmax. + if (P == 2) { + for (int i = 0; i < B * O; ++i) { + A const *input = cands__bop + i * P; + auto r = argmax(input[0], input[1]); + which__bo[i] = r.max_idx; + best__bo[i] = r.max; + } + } else if (P == 3) { + for (int i = 0; i < B * O; ++i) { + A const *input = cands__bop + i * P; + auto r = argmax(input[0], input[1], input[2]); + which__bo[i] = r.max_idx; + best__bo[i] = r.max; + } + } else { + for (int i = 0; i < B * O; ++i) { + auto r = argmax(cands__bop + i * P, P); + which__bo[i] = r.max_idx; + best__bo[i] = r.max; + } + } +} + + +template +void cpu_backprop_maxout(A* dX__bop, const A* dX__bo, const L* which__bo, + L B, L O, L P) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (L b = 0; b < B; ++b) { + for (L o = 0; o < O; ++o) { + if (*which__bo >= P) { + throw std::out_of_range(std::string("index ") + std::to_string(*which__bo) + " is out of bounds for maxout with size " + std::to_string(P)); + } + + dX__bop[*which__bo] = *dX__bo; + dX__bop += P; + dX__bo += 1; + which__bo += 1; + } + } +} + +template +void cpu_reduce_max(A* maxes__bo, L* which__bo, const A* X__to, + const L* lengths__b, L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + if (*length <= 0) + throw std::invalid_argument(std::string("all sequence lengths must be > 0, was: ") + std::to_string(*length)); + else if (*length > T) { + throw std::out_of_range("lengths must sum up to the number of rows"); + } + + T -= *length; + + std::memcpy(maxes__bo, X__to, O * sizeof(*maxes__bo)); + X__to += O; + for (L i = 1; i < *length; ++i) { + for (L j = 0; j < O; ++j) { + if (X__to[j] > maxes__bo[j]) { + maxes__bo[j] = X__to[j]; + which__bo[j] = i; + } + } + X__to += O; + } + + maxes__bo += O; + which__bo += O; + } +} + +template +void cpu_backprop_reduce_max(A* dX__to, const A* d_maxes__bo, const L* which__bo, + const L* lengths__b, L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + for (L i = 0; i < O; ++i) { + L item = which__bo[i]; + if (item >= *length) { + throw std::out_of_range(std::string("index ") + std::to_string(item) + " is out of bounds for maxout with length " + std::to_string(*length)); + } + + dX__to[item * O + i] = d_maxes__bo[i]; + } + + dX__to += *length * O; + d_maxes__bo += O; + which__bo += O; + } +} + +template +void cpu_reduce_mean(A* means__bo, const A* X__to, const L* lengths__b, + L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + if (*length < 0) { + throw std::invalid_argument(std::string("all sequence lengths must be >= 0, was: ") + std::to_string(*length)); + } + else if (length == 0) { + means__bo += O; + continue; + } + else if (*length > T) { + throw std::out_of_range("lengths must sum up to the number of rows"); + } + + T -= *length; + + A scale = 1. / *length; + for (L i = 0; i < *length; ++i) { + vec_add(means__bo, X__to, scale, O); + X__to += O; + } + + means__bo += O; + } +} + +template +void cpu_backprop_reduce_mean(A* dX__to, const A* d_means__bo, const L* lengths__b, + L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + A scale = 1. / *length; + for (L i = 0; i < *length; ++i) { + vec_add(dX__to, d_means__bo, scale, O); + dX__to += O; + } + + d_means__bo += O; + } +} + +template +void cpu_mish(A* Y, L N, A threshold) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (L i = 0; i < N; ++i) { + if (Y[i] < threshold) { + Y[i] *= std::tanh(std::log(1.0 + std::exp(Y[i]))); + } + } +} + +template +void cpu_backprop_mish(A* dX, const A* X, L N, A threshold) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (L i = 0; i < N; ++i) { + A x = X[i]; + + if (x < threshold) { + A exp_x = std::exp(x); + A exp_2x = std::exp(2 * x); + A exp_3x = std::exp(3 * x); + A omega = (4. * (x + 1)) + (4 * exp_2x) + exp_3x + exp_x * (4. * x + 6); + A delta = 2. * exp_x + exp_2x + 2.; + dX[i] = dX[i] * ((exp_x * omega) / (delta * delta)); + } + } +} + +template +void cpu_reduce_sum(A* sums__bo, const A* X__to, const L* lengths__b, + L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + if (*length < 0) { + throw std::invalid_argument(std::string("all sequence lengths must be >= 0, was: ") + std::to_string(*length)); + } + else if (length == 0) { + sums__bo += O; + continue; + } + else if (*length > T) { + throw std::out_of_range("lengths must sum up to the number of rows"); + } + + T -= *length; + + for (L i = 0; i < *length; ++i) { + vec_add(sums__bo, X__to, static_cast(1.0), O); + X__to += O; + } + + sums__bo += O; + } +} + +template +void cpu_backprop_reduce_sum(A* dX__to, const A* d_sums__bo, const L* lengths__b, + L B, L T, L O) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (const L* length = lengths__b; length < lengths__b + B; ++length) { + for (L i = 0; i < *length; ++i) { + vec_add(dX__to, d_sums__bo, static_cast(1.0), O); + dX__to += O; + } + + d_sums__bo += O; + } +} + +template +void cpu_relu(A* X, L N) +{ + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + for (L i = 0; i < N; ++i) { + if (X[i] <= 0.0) { + X[i] = 0.0; + } + } +} + +template +void seq2col(A* output, const A* X, const L* lengths, L nW, L B, L I, L nL) +{ + // Let's say nW is 1 (it usually is). Then we want to take: + + // 1a 1b 1c + // 2a 2b 2c + // 3a 3b 3c + + // And make + + // __ __ __ 1a 1b 1c 2a 2b 2c + // 1a 1b 1c 2a 2b 2c 3a 3b 3c + // 2a 2b 2c 3a 3b 3c __ __ __ + + // Where __ is padding. + + // Now let's say nW is 2. Then we want to take: + + // 1a 1b 1c + // 2a 2b 2c + // 3a 3b 3c + + // And make + + // __ __ __ __ __ __ 1a 1b 1c 2a 2b 2c 3a 3b 3c + // __ __ __ 1a 1b 1c 2a 2b 2c 3a 3b 3c __ __ __ + // 1a 1b 1c 2a 2b 2c 3a 3b 3c __ __ __ __ __ __ + + // * x_start=-6, x_end=9 : (0-2) * 3, (0+2+1) * 3 + // * x_start=-3, x_end=13 : (1-2) * 3, (1+2+1) * 3 + // * x_start=0, x_end=16 : (2-2) * 3, (2+2+1) * 3 + + // If lengths > 1, then the sequence lengths dictate + // the boundaries/padding rather than the begin/end + // of X. + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + L nF = nW * 2 + 1; + + L seq_start = 0; + for (L i = 0; i < nL; ++i) { + // Calculate the bounds of the next sequence. + L seq_end = seq_start + lengths[i]; + + for (L j = seq_start; j < seq_end; ++j) { + // Find the unconstrained window around b, which + // may be out of the sequence bounds. + L window_start = j - nW; + L window_end = j + nW + 1; + + // Find the sequence-constrained window around b. + L x_start = std::max(seq_start, window_start); + L x_end = std::min(seq_end, window_end); + L n_elems = x_end - x_start; + + L out_offset = x_start - window_start; + + std::memcpy(output + (j * nF * I) + (out_offset * I), + X + (x_start * I), + n_elems * I * sizeof(*output)); + } + + seq_start += lengths[i]; + } +} + +template +void backprop_seq2col(A* d_seqs, const A* d_cols, const L* lengths, L B, L I, L nW, L nL) +{ + // here's what we're doing, if we had 2d indexing. + // for i in range(b): + // d_seq[i] += d_cols[i-2, 4] + // d_seq[i] += d_cols[i-1, 3] + // d_seq[i] += d_cols[i, 2] + // d_seq[i] += d_cols[i+1, 1] + // d_seq[i] += d_cols[i+2, 0] + static_assert(std::is_floating_point::value, + "Array should be floating point"); + static_assert(std::is_integral::value, "Array length should be integral"); + + L nF = nW * 2 + 1; + + L seq_start = 0; + for (L i = 0; i < nL; ++i) { + // Calculate the bounds of the next sequence. + L seq_end = seq_start + lengths[i]; + + for (L j = seq_start; j < seq_end; ++j) { + // Find the unconstrained window around b, which + // may be out of the sequence bounds. + L window_begin = j - nW; + L window_end = j + nW + 1; + + // Find the sequence-constrained window around b. + L d_seqs_begin = std::max(seq_start, window_begin); + L d_seqs_end = std::min(seq_end, window_end); + L n_elems = d_seqs_end - d_seqs_begin; + + // If the left window is cut short, we want to + // start by the same amount in the output. + L out_offset = d_seqs_begin - window_begin; + + vec_add(d_seqs + d_seqs_begin * I, + d_cols + (j * nF * I) + (out_offset * I), + static_cast(1.), n_elems * I); + } + + seq_start += lengths[i]; + } +} + +template +void cpu_gather_add(typename axpy::ptr axpy, F* out_bo, const F* table_to, const I* indices_bk, L T, L O, L B, L K) { + for (L b = 0; b < B; ++b) { + for (L k = 0; k < K; ++k) { + I idx = indices_bk[b * K + k]; + if (idx > T) { + throw std::out_of_range("Embedding index out-of-bounds"); + } + axpy(O, 1.0, table_to + idx * O, 1, out_bo + b * O, 1); + } + } +} + + +#endif // CPU_KERNELS_HH diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cupy_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cupy_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f01ef2181df813c0cc35c85f5cdafd3f2956f707 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/cupy_ops.py @@ -0,0 +1,366 @@ +import numpy + +from .. import registry +from ..compat import cublas, cupy, cupyx +from ..types import DeviceTypes +from ..util import ( + is_cupy_array, + is_mxnet_gpu_array, + is_tensorflow_gpu_array, + is_torch_cuda_array, + mxnet2xp, + tensorflow2xp, + torch2xp, +) +from . import _custom_kernels +from .numpy_ops import NumpyOps +from .ops import Ops + + +@registry.ops("CupyOps") +class CupyOps(Ops): + name = "cupy" + xp = cupy + _xp2 = cupyx + + def __init__( + self, device_type: DeviceTypes = "gpu", device_id: int = 0, **kwargs + ) -> None: + self.device_type = device_type + self.device_id = device_id + + def to_numpy(self, data, *, byte_order=None): + if not isinstance(data, numpy.ndarray): + data = data.get() + if byte_order: + dtype = data.dtype.newbyteorder(byte_order) + data = numpy.asarray(data, dtype=dtype) + return data + + def gather_add(self, table, indices): + if table.dtype in ("float32", "float64"): + return _custom_kernels.gather_add(table, indices) + else: + return super().gather_add(table, indices) + + def dish(self, X, inplace=False): + if X.dtype in ("float32", "float64"): + return _custom_kernels.dish(X, inplace=inplace) + else: + return super().dish(X, inplace=inplace) + + def backprop_dish(self, dY, X, inplace=False): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_dish(dY, X, inplace=inplace) + else: + return super().backprop_dish(dY, X, inplace=inplace) + + def gelu(self, X, inplace=False): + if X.dtype in ("float32", "float64"): + return _custom_kernels.gelu(X, inplace=inplace, threshold=6.0) + else: + return super().gelu(X, inplace=inplace) + + def backprop_gelu(self, dY, X, inplace=False): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_gelu(dY, X, inplace=inplace, threshold=6.0) + else: + return super().backprop_gelu(dY, X, inplace=inplace) + + def gemm(self, x, y, out=None, trans1=False, trans2=False): + if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): + raise ValueError( + "Encountered a numpy array when processing with cupy. " + "Did you call model.ops.asarray on your data?" + ) + if trans1: + x = x.T + if trans2: + y = y.T + if out is None: + return self.xp.dot(x, y) + else: + self.xp.dot(x, y, out=out) + return out + + def asarray(self, data, dtype=None): + # We'll try to perform a zero-copy conversion if possible. + if is_cupy_array(data): + array = self.xp.asarray(data, dtype=dtype) + elif is_torch_cuda_array(data): + array = torch2xp(data) + elif is_tensorflow_gpu_array(data): + array = tensorflow2xp(data) + elif is_mxnet_gpu_array(data): + array = mxnet2xp(data) + else: + array = self.xp.array(data, dtype=dtype) + + if dtype is not None: + array = array.astype(dtype=dtype, copy=False) + + return array + + def pad(self, seqs, round_to=1): + """Perform padding on a list of arrays so that they each have the same + length, by taking the maximum dimension across each axis. This only + works on non-empty sequences with the same `ndim` and `dtype`. + """ + # TODO: This should be generalized to handle different ranks + if not seqs: + raise ValueError("Cannot pad empty sequence") + if len(set(seq.ndim for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences with different ndims") + if len(set(seq.dtype for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences with different dtypes") + if len(set(seq.shape[1:] for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences that differ on other dimensions") + + # Our CUDA kernel can currently only handle C contiguous arrays. + if not all(seq.flags["C_CONTIGUOUS"] for seq in seqs) or seqs[0].dtype not in ( + "float32", + "float64", + "int32", + "int64", + ): + return super().pad(seqs, round_to) + + return _custom_kernels.pad(seqs, round_to) + + def maxout(self, X): + if X.dtype in ("float32", "float64"): + return _custom_kernels.maxout(X) + else: + return super().maxout(X) + + def backprop_maxout(self, dY, which, P): + if dY.dtype in ("float32", "float64") and which.dtype == "int32": + return _custom_kernels.backprop_maxout(dY, which, P) + else: + return super().backprop_maxout(dY, which, P) + + def relu(self, X, inplace=False): + if not inplace: + return X * (X > 0) + else: + X *= X > 0 + return X + + def backprop_relu(self, dY, Y, inplace=False): + if not inplace: + return dY * (Y > 0) + dY *= Y > 0 + return dY + + def clipped_linear( + self, + X, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, + inplace: bool = False, + ): + if X.dtype in ("float32", "float64"): + return _custom_kernels.clipped_linear( + X, + inplace=inplace, + slope=slope, + offset=offset, + min_val=min_val, + max_val=max_val, + ) + else: + return super().clipped_linear( + X, + inplace=inplace, + slope=slope, + offset=offset, + min_val=min_val, + max_val=max_val, + ) + + def backprop_clipped_linear( + self, + dY, + X, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, + inplace: bool = False, + ): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_clipped_linear( + dY, + X, + slope=slope, + offset=offset, + min_val=min_val, + max_val=max_val, + inplace=inplace, + ) + else: + return super().backprop_clipped_linear( + dY=dY, + X=X, + slope=slope, + offset=offset, + min_val=min_val, + max_val=max_val, + inplace=inplace, + ) + + def backprop_hard_swish(self, dY, X, inplace: bool = False): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_hard_swish(dY, X, inplace=inplace) + else: + return super().backprop_hard_swish(dY, X, inplace=inplace) + + def backprop_hard_swish_mobilenet(self, dY, X, inplace: bool = False): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_hard_swish_mobilenet(dY, X, inplace=inplace) + else: + return super().backprop_hard_swish_mobilenet(dY, X, inplace=inplace) + + def mish(self, X, threshold=20.0, inplace=False): + if X.dtype in ("float32", "float64"): + return _custom_kernels.mish(X, inplace=inplace, threshold=threshold) + else: + return super().mish(X, threshold, inplace) + + def backprop_mish(self, dY, X, threshold=20.0, inplace=False): + if X.dtype == dY.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_mish( + dY, X, inplace=inplace, threshold=threshold + ) + else: + return super().backprop_mish(dY, X, threshold, inplace) + + def swish(self, X, inplace=False): + if X.dtype in ("float32", "float64"): + return _custom_kernels.swish(X, inplace=inplace, threshold=17.0) + else: + return super().swish(X, inplace=inplace) + + def backprop_swish(self, dY, X, Y, inplace=False): + if X.dtype == dY.dtype == Y.dtype and X.dtype in ("float32", "float64"): + return _custom_kernels.backprop_swish( + dY, X, Y, inplace=inplace, threshold=17.0 + ) + else: + return super().backprop_swish(dY, X, Y, inplace=inplace) + + def clip_gradient(self, gradient, threshold): + # We do not use CuPy's linalg.norm, since it uses scalar reductions + # using one CUDA block. This is a lot slower than the cuBLAS + # implementation. + def frobenius_norm(X): + X_vec = X.reshape(-1) + return cublas.nrm2(X_vec) + + grad_norm = cupy.maximum(frobenius_norm(gradient), 1e-12) + gradient *= cupy.minimum(threshold, grad_norm) / grad_norm + return gradient + + def seq2col(self, seq, nW, *, lengths=None): + """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) sequence. + The new sequence is constructed by concatenating nW preceding and succeeding + vectors onto each column in the sequence, to extract a window of features. + """ + if seq.dtype in ("float32", "float64") and ( + lengths is None or lengths.dtype == "int32" + ): + return _custom_kernels.seq2col(seq, nW, lengths=lengths) + else: + return super().seq2col(seq, nW, lengths=lengths) + + def backprop_seq2col(self, dY, nW, *, lengths=None): + if dY.dtype in ("float32", "float64") and ( + lengths is None or lengths.dtype == "int32" + ): + return _custom_kernels.backprop_seq2col(dY, nW, lengths=lengths) + else: + return super().backprop_seq2col(dY, nW, lengths=lengths) + + def reduce_mean(self, X, lengths): + if X.dtype in ("float32", "float64") and lengths.dtype == "int32": + return _custom_kernels.reduce_mean(X, lengths=lengths) + else: + super().reduce_mean(X, lengths) + + def backprop_reduce_mean(self, d_means, lengths): + if d_means.dtype in ("float32", "float64") and lengths.dtype == "int32": + return _custom_kernels.backprop_reduce_mean(d_means, lengths) + else: + super().backprop_reduce_mean(d_means, lengths) + + def reduce_max(self, X, lengths): + if X.dtype in ("float32", "float64") and lengths.dtype == "int32": + return _custom_kernels.reduce_max(X, lengths) + else: + super().reduce_max(X, lengths) + + def backprop_reduce_max(self, d_maxes, which, lengths): + if ( + d_maxes.dtype in ("float32", "float64") + and which.dtype == "int32" + and lengths.dtype == "int32" + ): + return _custom_kernels.backprop_reduce_max(d_maxes, which, lengths) + else: + super().backprop_reduce_max(d_maxes, which, lengths) + + def reduce_sum(self, X, lengths): + if X.dtype in ("float32", "float64") and lengths.dtype == "int32": + return _custom_kernels.reduce_sum(X, lengths) + else: + return super().reduce_sum(X, lengths) + + def backprop_reduce_sum(self, d_sums, lengths): + if d_sums.dtype in ("float32", "float64") and lengths.dtype == "int32": + return _custom_kernels.backprop_reduce_sum(d_sums, lengths) + else: + return super().backprop_reduce_sum(d_sums, lengths) + + def hash(self, ids, seed): + return _custom_kernels.hash(ids, seed) + + def scatter_add(self, table, indices, values): + self._xp2.scatter_add(table, indices, values) + + def adam( + self, weights, gradient, mom1, mom2, beta1, beta2, eps, learn_rate, mod_rate=1.0 + ): + _check_compatible_shape(weights, gradient) + _check_compatible_shape(weights, mom1) + _check_compatible_shape(weights, mom2) + + adam_kernel( + gradient, learn_rate, 1 - beta1, 1 - beta2, eps, weights, mom1, mom2 + ) + gradient.fill(0) + return weights, gradient, mom1, mom2 + + def position_encode(self, N, D, period=10000, out=None): + positions = NumpyOps().position_encode(N, D, period=period, out=out) + return self.asarray(positions) + + +if cupy is not None: + adam_kernel = cupy.ElementwiseKernel( + "T grad, T lr, T one_minus_beta1, T one_minus_beta2, T eps", + "T param, T m, T v", + """m += one_minus_beta1 * (grad - m); + v += one_minus_beta2 * (grad * grad - v); + param -= lr * m / (sqrt(v) + eps);""", + "adam", + ) +else: + adam_kernel = None + + +def _check_compatible_shape(u, v): + if u.shape != v.shape: + msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + raise ValueError(msg) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cp312-win_amd64.pyd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cp312-win_amd64.pyd new file mode 100644 index 0000000000000000000000000000000000000000..5817e20a58a4d68824ddcdf70c23de6655711bc4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cp312-win_amd64.pyd differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cpp new file mode 100644 index 0000000000000000000000000000000000000000..85c5457d0798568918641afde7fe3ad9da0ec186 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.cpp @@ -0,0 +1,9407 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "include_dirs": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.backends.linalg", + "sources": [ + "thinc/backends/linalg.pyx" + ] + }, + "module_name": "thinc.backends.linalg" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__backends__linalg +#define __PYX_HAVE_API__thinc__backends__linalg +/* Early includes */ +#include +#include +#include "math.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* Header.proto */ +#if !defined(CYTHON_CCOMPLEX) + #if defined(__cplusplus) + #define CYTHON_CCOMPLEX 1 + #elif (defined(_Complex_I) && !defined(_MSC_VER)) || ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_COMPLEX__) && !defined(_MSC_VER)) + #define CYTHON_CCOMPLEX 1 + #else + #define CYTHON_CCOMPLEX 0 + #endif +#endif +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #include + #else + #include + #endif +#endif +#if CYTHON_CCOMPLEX && !defined(__cplusplus) && defined(__sun__) && defined(__GNUC__) + #undef _Complex_I + #define _Complex_I 1.0fj +#endif + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/backends/linalg.pyx", + "thinc/backends/linalg.pxd", + "cymem/cymem.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* Atomics.proto (used by CodeObjectCache) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "thinc/backends/linalg.pxd":9 + * from libc.string cimport memcpy, memset + * + * ctypedef float weight_t # <<<<<<<<<<<<<< + * + * DEF USE_BLAS = False +*/ +typedef float __pyx_t_5thinc_8backends_6linalg_weight_t; +/* #### Code section: complex_type_declarations ### */ +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< double > __pyx_t_double_complex; + #else + typedef double _Complex __pyx_t_double_complex; + #endif +#else + typedef struct { double real, imag; } __pyx_t_double_complex; +#endif +static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double, double); + +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc; +struct __pyx_obj_5cymem_5cymem_PyFree; +struct __pyx_obj_5cymem_5cymem_Pool; +struct __pyx_obj_5cymem_5cymem_Address; +struct __pyx_obj_5thinc_8backends_6linalg_Matrix; +struct __pyx_obj_5thinc_8backends_6linalg_Vec; +struct __pyx_obj_5thinc_8backends_6linalg_VecVec; +struct __pyx_obj_5thinc_8backends_6linalg_Mat; + +/* "cymem/cymem.pxd":1 + * ctypedef void* (*malloc_t)(size_t n) # <<<<<<<<<<<<<< + * ctypedef void (*free_t)(void *p) + * +*/ +typedef void *(*__pyx_t_5cymem_5cymem_malloc_t)(size_t); + +/* "cymem/cymem.pxd":2 + * ctypedef void* (*malloc_t)(size_t n) + * ctypedef void (*free_t)(void *p) # <<<<<<<<<<<<<< + * + * cdef class PyMalloc: +*/ +typedef void (*__pyx_t_5cymem_5cymem_free_t)(void *); + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtab; + __pyx_t_5cymem_5cymem_malloc_t malloc; +}; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ +struct __pyx_obj_5cymem_5cymem_PyFree { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtab; + __pyx_t_5cymem_5cymem_free_t free; +}; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ +struct __pyx_obj_5cymem_5cymem_Pool { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtab; + size_t size; + PyObject *addresses; + PyObject *refs; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "cymem/cymem.pxd":28 + * + * + * cdef class Address: # <<<<<<<<<<<<<< + * cdef void* ptr + * cdef readonly PyMalloc pymalloc +*/ +struct __pyx_obj_5cymem_5cymem_Address { + PyObject_HEAD + void *ptr; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "thinc/backends/linalg.pxd":23 + * + * + * cdef class Matrix: # <<<<<<<<<<<<<< + * cdef readonly Pool mem + * cdef weight_t* data +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Matrix { + PyObject_HEAD + struct __pyx_obj_5cymem_5cymem_Pool *mem; + __pyx_t_5thinc_8backends_6linalg_weight_t *data; + int32_t nr_row; + int32_t nr_col; +}; + + +/* "thinc/backends/linalg.pxd":30 + * + * + * cdef class Vec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Vec { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec *__pyx_vtab; +}; + + +/* "thinc/backends/linalg.pxd":159 + * + * + * cdef class VecVec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void add(weight_t* output, +*/ +struct __pyx_obj_5thinc_8backends_6linalg_VecVec { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec *__pyx_vtab; +}; + + +/* "thinc/backends/linalg.pxd":250 + * + * + * cdef class Mat: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void mean_row(weight_t* Ex, +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Mat { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat *__pyx_vtab; +}; + + + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyMalloc *, __pyx_t_5cymem_5cymem_malloc_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtabptr_5cymem_5cymem_PyMalloc; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyFree { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyFree *, __pyx_t_5cymem_5cymem_free_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtabptr_5cymem_5cymem_PyFree; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_Pool { + void *(*alloc)(struct __pyx_obj_5cymem_5cymem_Pool *, size_t, size_t); + void (*free)(struct __pyx_obj_5cymem_5cymem_Pool *, void *); + void *(*realloc)(struct __pyx_obj_5cymem_5cymem_Pool *, void *, size_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtabptr_5cymem_5cymem_Pool; + + +/* "thinc/backends/linalg.pxd":30 + * + * + * cdef class Vec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec { + int (*arg_max)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); + __pyx_t_5thinc_8backends_6linalg_weight_t (*max)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*sum)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*norm)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*add)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*pow)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*pow_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); + void (*div)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*div_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); + void (*exp)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*exp_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); + void (*reciprocal_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*mean)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*variance)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec *__pyx_vtabptr_5thinc_8backends_6linalg_Vec; +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + + +/* "thinc/backends/linalg.pxd":159 + * + * + * cdef class VecVec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void add(weight_t* output, +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec { + void (*add)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*batch_add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t, int32_t); + void (*add_pow)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_pow_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*mul_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*dot)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + int (*arg_max_if_true)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const *, int const ); + int (*arg_max_if_zero)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec *__pyx_vtabptr_5thinc_8backends_6linalg_VecVec; +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const *, int const ); +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); + + +/* "thinc/backends/linalg.pxd":250 + * + * + * cdef class Mat: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void mean_row(weight_t* Ex, +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat { + void (*mean_row)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t); + void (*var_row)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t, __pyx_t_5thinc_8backends_6linalg_weight_t); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat *__pyx_vtabptr_5thinc_8backends_6linalg_Mat; +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t, __pyx_t_5thinc_8backends_6linalg_weight_t); +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* RealImag.proto (used by SoftComplexToDouble) */ +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #define __Pyx_CREAL(z) ((z).real()) + #define __Pyx_CIMAG(z) ((z).imag()) + #else + #define __Pyx_CREAL(z) (__real__(z)) + #define __Pyx_CIMAG(z) (__imag__(z)) + #endif +#else + #define __Pyx_CREAL(z) ((z).real) + #define __Pyx_CIMAG(z) ((z).imag) +#endif +#if defined(__cplusplus) && CYTHON_CCOMPLEX\ + && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103) + #define __Pyx_SET_CREAL(z,x) ((z).real(x)) + #define __Pyx_SET_CIMAG(z,y) ((z).imag(y)) +#else + #define __Pyx_SET_CREAL(z,x) __Pyx_CREAL(z) = (x) + #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y) +#endif + +/* SoftComplexToDouble.proto */ +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_nogil); + +/* ErrOccurredWithGIL.proto */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by WriteUnraisableException) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* IncludeStringH.proto (used by FixUpExtensionType) */ +#include + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallNoArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* PyObjectCallOneArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectGetAttrStr.proto (used by PyObjectGetMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_1 +#define __PYX_HAVE_RT_ImportType_proto_3_2_1 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_2_1 { + __Pyx_ImportType_CheckSize_Error_3_2_1 = 0, + __Pyx_ImportType_CheckSize_Warn_3_2_1 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_2_1 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size); +#endif + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by HasAttr) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* TupleAndListFromArray.proto (used by ImportImpl) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* PyImportError_Check.proto */ +#define __Pyx_PyExc_ImportError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ImportError) + +/* dict_setdefault.proto (used by CLineInTraceback) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* PyDictVersioning.proto (used by CLineInTraceback) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_double(a, b) ((a)==(b)) + #define __Pyx_c_sum_double(a, b) ((a)+(b)) + #define __Pyx_c_diff_double(a, b) ((a)-(b)) + #define __Pyx_c_prod_double(a, b) ((a)*(b)) + #define __Pyx_c_quot_double(a, b) ((a)/(b)) + #define __Pyx_c_neg_double(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_double(z) ((z)==(double)0) + #define __Pyx_c_conj_double(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (::std::abs(z)) + #define __Pyx_c_pow_double(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_double(z) ((z)==0) + #define __Pyx_c_conj_double(z) (conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (cabs(z)) + #define __Pyx_c_pow_double(a, b) (cpow(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex); + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex, __pyx_t_double_complex); + #endif +#endif + +/* PyObjectVectorCallKwBuilder.proto (used by CIntToPy) */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* AddModuleRef.proto */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr, int32_t __pyx_v_nr_batch); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_costs, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Ex, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Vx, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_Ex, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_eps); /* proto*/ + +/* Module declarations from "cython" */ + +/* Module declarations from "cymem.cymem" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "thinc.backends.linalg" */ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.backends.linalg" +extern int __pyx_module_is_main_thinc__backends__linalg; +int __pyx_module_is_main_thinc__backends__linalg = 0; + +/* Implementation of "thinc.backends.linalg" */ +/* #### Code section: global_var ### */ +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Matrix(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Vec(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_VecVec(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Mat(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyMalloc; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyFree; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Pool; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Address; + PyObject *__pyx_type_5thinc_8backends_6linalg_Matrix; + PyObject *__pyx_type_5thinc_8backends_6linalg_Vec; + PyObject *__pyx_type_5thinc_8backends_6linalg_VecVec; + PyObject *__pyx_type_5thinc_8backends_6linalg_Mat; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Matrix; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Vec; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_VecVec; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Mat; + PyObject *__pyx_string_tab[21]; +/* #### Code section: module_state_contents ### */ +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_disable __pyx_string_tab[1] +#define __pyx_kp_u_enable __pyx_string_tab[2] +#define __pyx_kp_u_gc __pyx_string_tab[3] +#define __pyx_kp_u_isenabled __pyx_string_tab[4] +#define __pyx_kp_u_thinc_backends_blis_py __pyx_string_tab[5] +#define __pyx_n_u_Mat __pyx_string_tab[6] +#define __pyx_n_u_Matrix __pyx_string_tab[7] +#define __pyx_n_u_Vec __pyx_string_tab[8] +#define __pyx_n_u_VecVec __pyx_string_tab[9] +#define __pyx_n_u_blis __pyx_string_tab[10] +#define __pyx_n_u_blis_py __pyx_string_tab[11] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[12] +#define __pyx_n_u_main __pyx_string_tab[13] +#define __pyx_n_u_module __pyx_string_tab[14] +#define __pyx_n_u_name __pyx_string_tab[15] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[16] +#define __pyx_n_u_qualname __pyx_string_tab[17] +#define __pyx_n_u_set_name __pyx_string_tab[18] +#define __pyx_n_u_setdefault __pyx_string_tab[19] +#define __pyx_n_u_test __pyx_string_tab[20] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Matrix); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_6linalg_Matrix); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Vec); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_6linalg_Vec); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_VecVec); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_6linalg_VecVec); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Mat); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_6linalg_Mat); + for (int i=0; i<21; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Matrix); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_6linalg_Matrix); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Vec); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_6linalg_Vec); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_VecVec); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_6linalg_VecVec); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Mat); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_6linalg_Mat); + for (int i=0; i<21; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "thinc/backends/linalg.pxd":31 + * + * cdef class Vec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mode; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":33 + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: # <<<<<<<<<<<<<< + * return 0 if scores[0] > scores[1] else 1 + * cdef int i +*/ + __pyx_t_1 = (__pyx_v_n_classes == 2); + if (__pyx_t_1) { + + /* "thinc/backends/linalg.pxd":34 + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: + * return 0 if scores[0] > scores[1] else 1 # <<<<<<<<<<<<<< + * cdef int i + * cdef int best = 0 +*/ + __pyx_t_1 = ((__pyx_v_scores[0]) > (__pyx_v_scores[1])); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = 1; + } + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":33 + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: # <<<<<<<<<<<<<< + * return 0 if scores[0] > scores[1] else 1 + * cdef int i +*/ + } + + /* "thinc/backends/linalg.pxd":36 + * return 0 if scores[0] > scores[1] else 1 + * cdef int i + * cdef int best = 0 # <<<<<<<<<<<<<< + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): +*/ + __pyx_v_best = 0; + + /* "thinc/backends/linalg.pxd":37 + * cdef int i + * cdef int best = 0 + * cdef weight_t mode = scores[0] # <<<<<<<<<<<<<< + * for i in range(1, n_classes): + * if scores[i] > mode: +*/ + __pyx_v_mode = (__pyx_v_scores[0]); + + /* "thinc/backends/linalg.pxd":38 + * cdef int best = 0 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): # <<<<<<<<<<<<<< + * if scores[i] > mode: + * mode = scores[i] +*/ + __pyx_t_2 = __pyx_v_n_classes; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 1; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/backends/linalg.pxd":39 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): + * if scores[i] > mode: # <<<<<<<<<<<<<< + * mode = scores[i] + * best = i +*/ + __pyx_t_1 = ((__pyx_v_scores[__pyx_v_i]) > __pyx_v_mode); + if (__pyx_t_1) { + + /* "thinc/backends/linalg.pxd":40 + * for i in range(1, n_classes): + * if scores[i] > mode: + * mode = scores[i] # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_v_mode = (__pyx_v_scores[__pyx_v_i]); + + /* "thinc/backends/linalg.pxd":41 + * if scores[i] > mode: + * mode = scores[i] + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "thinc/backends/linalg.pxd":39 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): + * if scores[i] > mode: # <<<<<<<<<<<<<< + * mode = scores[i] + * best = i +*/ + } + } + + /* "thinc/backends/linalg.pxd":42 + * mode = scores[i] + * best = i + * return best # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":31 + * + * cdef class Vec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":44 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mode; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":46 + * @staticmethod + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: # <<<<<<<<<<<<<< + * return 0 + * cdef int i +*/ + __pyx_t_1 = (__pyx_v_nr == 0); + if (__pyx_t_1) { + + /* "thinc/backends/linalg.pxd":47 + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: + * return 0 # <<<<<<<<<<<<<< + * cdef int i + * cdef weight_t mode = x[0] +*/ + __pyx_r = 0.0; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":46 + * @staticmethod + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: # <<<<<<<<<<<<<< + * return 0 + * cdef int i +*/ + } + + /* "thinc/backends/linalg.pxd":49 + * return 0 + * cdef int i + * cdef weight_t mode = x[0] # <<<<<<<<<<<<<< + * for i in range(1, nr): + * if x[i] > mode: +*/ + __pyx_v_mode = (__pyx_v_x[0]); + + /* "thinc/backends/linalg.pxd":50 + * cdef int i + * cdef weight_t mode = x[0] + * for i in range(1, nr): # <<<<<<<<<<<<<< + * if x[i] > mode: + * mode = x[i] +*/ + __pyx_t_2 = __pyx_v_nr; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 1; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/backends/linalg.pxd":51 + * cdef weight_t mode = x[0] + * for i in range(1, nr): + * if x[i] > mode: # <<<<<<<<<<<<<< + * mode = x[i] + * return mode +*/ + __pyx_t_1 = ((__pyx_v_x[__pyx_v_i]) > __pyx_v_mode); + if (__pyx_t_1) { + + /* "thinc/backends/linalg.pxd":52 + * for i in range(1, nr): + * if x[i] > mode: + * mode = x[i] # <<<<<<<<<<<<<< + * return mode + * +*/ + __pyx_v_mode = (__pyx_v_x[__pyx_v_i]); + + /* "thinc/backends/linalg.pxd":51 + * cdef weight_t mode = x[0] + * for i in range(1, nr): + * if x[i] > mode: # <<<<<<<<<<<<<< + * mode = x[i] + * return mode +*/ + } + } + + /* "thinc/backends/linalg.pxd":53 + * if x[i] > mode: + * mode = x[i] + * return mode # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_mode; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":44 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":55 + * return mode + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/linalg.pxd":58 + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += vec[i] +*/ + __pyx_v_total = 0.0; + + /* "thinc/backends/linalg.pxd":59 + * cdef int i + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += vec[i] + * return total +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":60 + * cdef weight_t total = 0 + * for i in range(nr): + * total += vec[i] # <<<<<<<<<<<<<< + * return total + * +*/ + __pyx_v_total = (__pyx_v_total + (__pyx_v_vec[__pyx_v_i])); + } + + /* "thinc/backends/linalg.pxd":61 + * for i in range(nr): + * total += vec[i] + * return total # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_total; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":55 + * return mode + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":63 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + int32_t __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + + /* "thinc/backends/linalg.pxd":65 + * @staticmethod + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += vec[i] ** 2 +*/ + __pyx_v_total = 0.0; + + /* "thinc/backends/linalg.pxd":66 + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += vec[i] ** 2 + * return sqrt(total) +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":67 + * cdef weight_t total = 0 + * for i in range(nr): + * total += vec[i] ** 2 # <<<<<<<<<<<<<< + * return sqrt(total) + * +*/ + __pyx_v_total = (__pyx_v_total + powf(((__pyx_t_5thinc_8backends_6linalg_weight_t)(__pyx_v_vec[__pyx_v_i])), 2.0)); + } + + /* "thinc/backends/linalg.pxd":68 + * for i in range(nr): + * total += vec[i] ** 2 + * return sqrt(total) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = sqrt(__pyx_v_total); + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":63 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":70 + * return sqrt(total) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":73 + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.add_i(output, inc, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":74 + * weight_t inc, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * Vec.add_i(output, inc, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_v_output, __pyx_v_inc, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":70 + * return sqrt(total) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":76 + * Vec.add_i(output, inc, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":79 + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] += inc + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":80 + * cdef int i + * for i in range(nr): + * vec[i] += inc # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) + __pyx_v_inc); + } + + /* "thinc/backends/linalg.pxd":76 + * Vec.add_i(output, inc, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":82 + * vec[i] += inc + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":85 + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.mul_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":86 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.mul_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":82 + * vec[i] += inc + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":88 + * Vec.mul_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* vec, weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":94 + * blis.cy.scalv(BLIS_NO_CONJUGATE, nr, scal, vec, 1) + * ELSE: + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] *= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":95 + * ELSE: + * for i in range(nr): + * vec[i] *= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) * __pyx_v_scal); + } + + /* "thinc/backends/linalg.pxd":88 + * Vec.mul_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* vec, weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":97 + * vec[i] *= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":100 + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.pow_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":101 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.pow_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":97 + * vec[i] *= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":103 + * Vec.pow_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":106 + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] **= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":107 + * cdef int i + * for i in range(nr): + * vec[i] **= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = powf((__pyx_v_vec[__pyx_t_4]), ((__pyx_t_5thinc_8backends_6linalg_weight_t)__pyx_v_scal)); + } + + /* "thinc/backends/linalg.pxd":103 + * Vec.pow_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":109 + * vec[i] **= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":113 + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.div_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":114 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.div_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":109 + * vec[i] **= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":116 + * Vec.div_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":120 + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] /= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":121 + * cdef int i + * for i in range(nr): + * vec[i] /= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) / __pyx_v_scal); + } + + /* "thinc/backends/linalg.pxd":116 + * Vec.div_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":123 + * vec[i] /= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":125 + * @staticmethod + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.exp_i(output, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":126 + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.exp_i(output, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_v_output, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":123 + * vec[i] /= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":128 + * Vec.exp_i(output, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/linalg.pxd":131 + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] = exp(vec[i]) + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":132 + * cdef int i + * for i in range(nr): + * vec[i] = exp(vec[i]) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + (__pyx_v_vec[__pyx_v_i]) = exp((__pyx_v_vec[__pyx_v_i])); + } + + /* "thinc/backends/linalg.pxd":128 + * Vec.exp_i(output, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":134 + * vec[i] = exp(vec[i]) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/linalg.pxd":137 + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] = 1.0 / vec[i] + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":138 + * cdef int i + * for i in range(nr): + * vec[i] = 1.0 / vec[i] # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + (__pyx_v_vec[__pyx_v_i]) = (1.0 / (__pyx_v_vec[__pyx_v_i])); + } + + /* "thinc/backends/linalg.pxd":134 + * vec[i] = exp(vec[i]) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":140 + * vec[i] = 1.0 / vec[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mean; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_x; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_1; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_2; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_3; + + /* "thinc/backends/linalg.pxd":142 + * @staticmethod + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. # <<<<<<<<<<<<<< + * for x in X[:nr_dim]: + * mean += x +*/ + __pyx_v_mean = 0.; + + /* "thinc/backends/linalg.pxd":143 + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. + * for x in X[:nr_dim]: # <<<<<<<<<<<<<< + * mean += x + * return mean / nr_dim +*/ + __pyx_t_2 = (__pyx_v_X + __pyx_v_nr_dim); + for (__pyx_t_3 = __pyx_v_X; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_x = (__pyx_t_1[0]); + + /* "thinc/backends/linalg.pxd":144 + * cdef weight_t mean = 0. + * for x in X[:nr_dim]: + * mean += x # <<<<<<<<<<<<<< + * return mean / nr_dim + * +*/ + __pyx_v_mean = (__pyx_v_mean + __pyx_v_x); + } + + /* "thinc/backends/linalg.pxd":145 + * for x in X[:nr_dim]: + * mean += x + * return mean / nr_dim # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = (__pyx_v_mean / __pyx_v_nr_dim); + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":140 + * vec[i] = 1.0 / vec[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":147 + * return mean / nr_dim + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim) { + double __pyx_v_m; + double __pyx_v_v; + long __pyx_v_i; + double __pyx_v_diff; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + long __pyx_t_3; + + /* "thinc/backends/linalg.pxd":150 + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ + * cdef double m = X[0] # <<<<<<<<<<<<<< + * cdef double v = 0. + * for i in range(1, nr_dim): +*/ + __pyx_v_m = (__pyx_v_X[0]); + + /* "thinc/backends/linalg.pxd":151 + * # See https://www.johndcook.com/blog/standard_deviation/ + * cdef double m = X[0] + * cdef double v = 0. # <<<<<<<<<<<<<< + * for i in range(1, nr_dim): + * diff = X[i]-m +*/ + __pyx_v_v = 0.; + + /* "thinc/backends/linalg.pxd":152 + * cdef double m = X[0] + * cdef double v = 0. + * for i in range(1, nr_dim): # <<<<<<<<<<<<<< + * diff = X[i]-m + * m += diff / (i+1) +*/ + __pyx_t_1 = __pyx_v_nr_dim; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 1; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":153 + * cdef double v = 0. + * for i in range(1, nr_dim): + * diff = X[i]-m # <<<<<<<<<<<<<< + * m += diff / (i+1) + * v += diff * (X[i] - m) +*/ + __pyx_v_diff = ((__pyx_v_X[__pyx_v_i]) - __pyx_v_m); + + /* "thinc/backends/linalg.pxd":154 + * for i in range(1, nr_dim): + * diff = X[i]-m + * m += diff / (i+1) # <<<<<<<<<<<<<< + * v += diff * (X[i] - m) + * return v / nr_dim +*/ + __pyx_v_m = (__pyx_v_m + (__pyx_v_diff / (__pyx_v_i + 1))); + + /* "thinc/backends/linalg.pxd":155 + * diff = X[i]-m + * m += diff / (i+1) + * v += diff * (X[i] - m) # <<<<<<<<<<<<<< + * return v / nr_dim + * +*/ + __pyx_v_v = (__pyx_v_v + (__pyx_v_diff * ((__pyx_v_X[__pyx_v_i]) - __pyx_v_m))); + } + + /* "thinc/backends/linalg.pxd":156 + * m += diff / (i+1) + * v += diff * (X[i] - m) + * return v / nr_dim # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = (__pyx_v_v / __pyx_v_nr_dim); + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":147 + * return mean / nr_dim + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":160 + * + * cdef class VecVec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, + * const weight_t* x, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":166 + * weight_t scale, + * int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.add_i(output, y, scale, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":167 + * int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.add_i(output, y, scale, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_output, __pyx_v_y, __pyx_v_scale, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":160 + * + * cdef class VecVec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, + * const weight_t* x, +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":169 + * VecVec.add_i(output, y, scale, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* x, + * const weight_t* y, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":178 + * blis.cy.axpyv(BLIS_NO_CONJUGATE, nr, scale, y, 1, x, 1) + * ELSE: + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] += y[i] * scale + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":179 + * ELSE: + * for i in range(nr): + * x[i] += y[i] * scale # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_x[__pyx_t_4]) = ((__pyx_v_x[__pyx_t_4]) + ((__pyx_v_y[__pyx_v_i]) * __pyx_v_scale)); + } + + /* "thinc/backends/linalg.pxd":169 + * VecVec.add_i(output, y, scale, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* x, + * const weight_t* y, +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":181 + * x[i] += y[i] * scale + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void batch_add_i(weight_t* x, + * const weight_t* y, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr, int32_t __pyx_v_nr_batch) { + CYTHON_UNUSED int __pyx_v__; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/linalg.pxd":188 + * # For fixed x, matrix of y + * cdef int i, _ + * for _ in range(nr_batch): # <<<<<<<<<<<<<< + * VecVec.add_i(x, + * y, scale, nr) +*/ + __pyx_t_1 = __pyx_v_nr_batch; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v__ = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":189 + * cdef int i, _ + * for _ in range(nr_batch): + * VecVec.add_i(x, # <<<<<<<<<<<<<< + * y, scale, nr) + * y += nr +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_x, __pyx_v_y, __pyx_v_scale, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":191 + * VecVec.add_i(x, + * y, scale, nr) + * y += nr # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_v_y = (__pyx_v_y + __pyx_v_nr); + } + + /* "thinc/backends/linalg.pxd":181 + * x[i] += y[i] * scale + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void batch_add_i(weight_t* x, + * const weight_t* y, +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":193 + * y += nr + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":196 + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.add_pow_i(output, y, power, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":197 + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.add_pow_i(output, y, power, nr) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_v_output, __pyx_v_y, __pyx_v_power, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":193 + * y += nr + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":200 + * + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow_i(weight_t* x, + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + double __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "thinc/backends/linalg.pxd":204 + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] += y[i] ** power + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":205 + * cdef int i + * for i in range(nr): + * x[i] += y[i] ** power # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + __pyx_t_5 = __Pyx_SoftComplexToDouble(__Pyx_c_sum_double(__pyx_t_double_complex_from_parts((__pyx_v_x[__pyx_t_4]), 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts((__pyx_v_y[__pyx_v_i]), 0), __pyx_t_double_complex_from_parts(__pyx_v_power, 0))), 2); if (unlikely(__pyx_t_5 == ((double)-1) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 205, __pyx_L1_error) + (__pyx_v_x[__pyx_t_4]) = __pyx_t_5; + } + + /* "thinc/backends/linalg.pxd":200 + * + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow_i(weight_t* x, + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_WriteUnraisable("thinc.backends.linalg.VecVec.add_pow_i", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; +} + +/* "thinc/backends/linalg.pxd":207 + * x[i] += y[i] ** power + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + + /* "thinc/backends/linalg.pxd":210 + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.mul_i(output, y, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "thinc/backends/linalg.pxd":211 + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.mul_i(output, y, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_v_output, __pyx_v_y, __pyx_v_nr); + + /* "thinc/backends/linalg.pxd":207 + * x[i] += y[i] ** power + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":213 + * VecVec.mul_i(output, y, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* x, + * const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/linalg.pxd":217 + * const weight_t* y, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] *= y[i] + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":218 + * cdef int i + * for i in range(nr): + * x[i] *= y[i] # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_x[__pyx_t_4]) = ((__pyx_v_x[__pyx_t_4]) * (__pyx_v_y[__pyx_v_i])); + } + + /* "thinc/backends/linalg.pxd":213 + * VecVec.mul_i(output, y, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* x, + * const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":220 + * x[i] *= y[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t dot( + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/linalg.pxd":224 + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * cdef int i + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += x[i] * y[i] +*/ + __pyx_v_total = 0.0; + + /* "thinc/backends/linalg.pxd":225 + * cdef int i + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += x[i] * y[i] + * return total +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":226 + * cdef weight_t total = 0 + * for i in range(nr): + * total += x[i] * y[i] # <<<<<<<<<<<<<< + * return total + * +*/ + __pyx_v_total = (__pyx_v_total + ((__pyx_v_x[__pyx_v_i]) * (__pyx_v_y[__pyx_v_i]))); + } + + /* "thinc/backends/linalg.pxd":227 + * for i in range(nr): + * total += x[i] * y[i] + * return total # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_total; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":220 + * x[i] *= y[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t dot( + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":229 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_true( + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + + /* "thinc/backends/linalg.pxd":233 + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: + * cdef int i + * cdef int best = -1 # <<<<<<<<<<<<<< + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): +*/ + __pyx_v_best = -1; + + /* "thinc/backends/linalg.pxd":234 + * cdef int i + * cdef int best = -1 + * for i in range(n_classes): # <<<<<<<<<<<<<< + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i +*/ + __pyx_t_1 = __pyx_v_n_classes; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":235 + * cdef int best = -1 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_t_5 = ((__pyx_v_is_valid[__pyx_v_i]) != 0); + if (__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = (__pyx_v_best == -1L); + if (!__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = ((__pyx_v_scores[__pyx_v_i]) > (__pyx_v_scores[__pyx_v_best])); + __pyx_t_4 = __pyx_t_5; + __pyx_L6_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/linalg.pxd":236 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "thinc/backends/linalg.pxd":235 + * cdef int best = -1 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + } + } + + /* "thinc/backends/linalg.pxd":237 + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i + * return best # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":229 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_true( + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":239 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_zero( + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_costs, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + + /* "thinc/backends/linalg.pxd":243 + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: + * cdef int i + * cdef int best = -1 # <<<<<<<<<<<<<< + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): +*/ + __pyx_v_best = -1; + + /* "thinc/backends/linalg.pxd":244 + * cdef int i + * cdef int best = -1 + * for i in range(n_classes): # <<<<<<<<<<<<<< + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i +*/ + __pyx_t_1 = __pyx_v_n_classes; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":245 + * cdef int best = -1 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_t_5 = ((__pyx_v_costs[__pyx_v_i]) == 0.0); + if (__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = (__pyx_v_best == -1L); + if (!__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = ((__pyx_v_scores[__pyx_v_i]) > (__pyx_v_scores[__pyx_v_best])); + __pyx_t_4 = __pyx_t_5; + __pyx_L6_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/linalg.pxd":246 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "thinc/backends/linalg.pxd":245 + * cdef int best = -1 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + } + } + + /* "thinc/backends/linalg.pxd":247 + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i + * return best # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":239 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_zero( + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/linalg.pxd":251 + * + * cdef class Mat: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Ex, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col) { + int32_t __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + + /* "thinc/backends/linalg.pxd":254 + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) # <<<<<<<<<<<<<< + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) +*/ + (void)(memset(__pyx_v_Ex, 0, ((sizeof((__pyx_v_Ex[0]))) * __pyx_v_nr_col))); + + /* "thinc/backends/linalg.pxd":255 + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) + * for i in range(nr_row): # <<<<<<<<<<<<<< + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) +*/ + __pyx_t_1 = __pyx_v_nr_row; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/linalg.pxd":256 + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) # <<<<<<<<<<<<<< + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_Ex, (&(__pyx_v_mat[(__pyx_v_i * __pyx_v_nr_col)])), 1.0, __pyx_v_nr_col); + } + + /* "thinc/backends/linalg.pxd":257 + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_Ex, (1.0 / __pyx_v_nr_row), __pyx_v_nr_col); + + /* "thinc/backends/linalg.pxd":251 + * + * cdef class Mat: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/linalg.pxd":259 + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void var_row(weight_t* Vx, + * const weight_t* mat, const weight_t* Ex, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Vx, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_Ex, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_eps) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_sum_; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_sum2; + int32_t __pyx_v_i; + int32_t __pyx_v_j; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_x; + int __pyx_t_1; + int __pyx_t_2; + int32_t __pyx_t_3; + int32_t __pyx_t_4; + int32_t __pyx_t_5; + int32_t __pyx_t_6; + int32_t __pyx_t_7; + int32_t __pyx_t_8; + + /* "thinc/backends/linalg.pxd":264 + * int32_t nr_row, int32_t nr_col, weight_t eps) noexcept nogil: + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: # <<<<<<<<<<<<<< + * return + * cdef weight_t sum_, sum2 +*/ + __pyx_t_2 = (__pyx_v_nr_row == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_nr_col == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "thinc/backends/linalg.pxd":265 + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: + * return # <<<<<<<<<<<<<< + * cdef weight_t sum_, sum2 + * for i in range(nr_col): +*/ + goto __pyx_L0; + + /* "thinc/backends/linalg.pxd":264 + * int32_t nr_row, int32_t nr_col, weight_t eps) noexcept nogil: + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: # <<<<<<<<<<<<<< + * return + * cdef weight_t sum_, sum2 +*/ + } + + /* "thinc/backends/linalg.pxd":267 + * return + * cdef weight_t sum_, sum2 + * for i in range(nr_col): # <<<<<<<<<<<<<< + * sum_ = 0.0 + * sum2 = 0.0 +*/ + __pyx_t_3 = __pyx_v_nr_col; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "thinc/backends/linalg.pxd":268 + * cdef weight_t sum_, sum2 + * for i in range(nr_col): + * sum_ = 0.0 # <<<<<<<<<<<<<< + * sum2 = 0.0 + * for j in range(nr_row): +*/ + __pyx_v_sum_ = 0.0; + + /* "thinc/backends/linalg.pxd":269 + * for i in range(nr_col): + * sum_ = 0.0 + * sum2 = 0.0 # <<<<<<<<<<<<<< + * for j in range(nr_row): + * x = mat[j * nr_col + i] +*/ + __pyx_v_sum2 = 0.0; + + /* "thinc/backends/linalg.pxd":270 + * sum_ = 0.0 + * sum2 = 0.0 + * for j in range(nr_row): # <<<<<<<<<<<<<< + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 +*/ + __pyx_t_6 = __pyx_v_nr_row; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_j = __pyx_t_8; + + /* "thinc/backends/linalg.pxd":271 + * sum2 = 0.0 + * for j in range(nr_row): + * x = mat[j * nr_col + i] # <<<<<<<<<<<<<< + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] +*/ + __pyx_v_x = (__pyx_v_mat[((__pyx_v_j * __pyx_v_nr_col) + __pyx_v_i)]); + + /* "thinc/backends/linalg.pxd":272 + * for j in range(nr_row): + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 # <<<<<<<<<<<<<< + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row +*/ + __pyx_v_sum2 = (__pyx_v_sum2 + powf((__pyx_v_x - (__pyx_v_Ex[__pyx_v_i])), 2.0)); + + /* "thinc/backends/linalg.pxd":273 + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] # <<<<<<<<<<<<<< + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row + * Vx[i] += eps +*/ + __pyx_v_sum_ = (__pyx_v_sum_ + (__pyx_v_x - (__pyx_v_Ex[__pyx_v_i]))); + } + + /* "thinc/backends/linalg.pxd":274 + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row # <<<<<<<<<<<<<< + * Vx[i] += eps +*/ + (__pyx_v_Vx[__pyx_v_i]) = ((__pyx_v_sum2 - (powf(__pyx_v_sum_, 2.0) / __pyx_v_nr_row)) / __pyx_v_nr_row); + + /* "thinc/backends/linalg.pxd":275 + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row + * Vx[i] += eps # <<<<<<<<<<<<<< +*/ + __pyx_t_6 = __pyx_v_i; + (__pyx_v_Vx[__pyx_t_6]) = ((__pyx_v_Vx[__pyx_t_6]) + __pyx_v_eps); + } + + /* "thinc/backends/linalg.pxd":259 + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void var_row(weight_t* Vx, + * const weight_t* mat, const weight_t* Ex, +*/ + + /* function exit code */ + __pyx_L0:; +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Matrix(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_8backends_6linalg_Matrix *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_8backends_6linalg_Matrix *)o); + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_6linalg_Matrix(PyObject *o) { + struct __pyx_obj_5thinc_8backends_6linalg_Matrix *p = (struct __pyx_obj_5thinc_8backends_6linalg_Matrix *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_6linalg_Matrix) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->mem); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_5thinc_8backends_6linalg_Matrix(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_8backends_6linalg_Matrix *p = (struct __pyx_obj_5thinc_8backends_6linalg_Matrix *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->mem) { + e = (*v)(((PyObject *)p->mem), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_8backends_6linalg_Matrix(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5thinc_8backends_6linalg_Matrix *p = (struct __pyx_obj_5thinc_8backends_6linalg_Matrix *)o; + tmp = ((PyObject*)p->mem); + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_6linalg_Matrix_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_6linalg_Matrix}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_8backends_6linalg_Matrix}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_8backends_6linalg_Matrix}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_6linalg_Matrix}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_6linalg_Matrix_spec = { + "thinc.backends.linalg.Matrix", + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_8backends_6linalg_Matrix_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_6linalg_Matrix = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.linalg.""Matrix", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_6linalg_Matrix, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5thinc_8backends_6linalg_Matrix, /*tp_traverse*/ + __pyx_tp_clear_5thinc_8backends_6linalg_Matrix, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_6linalg_Matrix, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec __pyx_vtable_5thinc_8backends_6linalg_Vec; + +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Vec(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_8backends_6linalg_Vec *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_8backends_6linalg_Vec *)o); + p->__pyx_vtab = __pyx_vtabptr_5thinc_8backends_6linalg_Vec; + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_6linalg_Vec(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_6linalg_Vec) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_6linalg_Vec_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_6linalg_Vec}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_6linalg_Vec}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_6linalg_Vec_spec = { + "thinc.backends.linalg.Vec", + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Vec), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, + __pyx_type_5thinc_8backends_6linalg_Vec_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_6linalg_Vec = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.linalg.""Vec", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Vec), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_6linalg_Vec, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_6linalg_Vec, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec __pyx_vtable_5thinc_8backends_6linalg_VecVec; + +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_VecVec(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_8backends_6linalg_VecVec *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_8backends_6linalg_VecVec *)o); + p->__pyx_vtab = __pyx_vtabptr_5thinc_8backends_6linalg_VecVec; + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_6linalg_VecVec(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_6linalg_VecVec) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_6linalg_VecVec_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_6linalg_VecVec}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_6linalg_VecVec}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_6linalg_VecVec_spec = { + "thinc.backends.linalg.VecVec", + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, + __pyx_type_5thinc_8backends_6linalg_VecVec_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_6linalg_VecVec = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.linalg.""VecVec", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_6linalg_VecVec, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_6linalg_VecVec, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat __pyx_vtable_5thinc_8backends_6linalg_Mat; + +static PyObject *__pyx_tp_new_5thinc_8backends_6linalg_Mat(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_8backends_6linalg_Mat *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_8backends_6linalg_Mat *)o); + p->__pyx_vtab = __pyx_vtabptr_5thinc_8backends_6linalg_Mat; + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_6linalg_Mat(PyObject *o) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_6linalg_Mat) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_6linalg_Mat_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_6linalg_Mat}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_6linalg_Mat}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_6linalg_Mat_spec = { + "thinc.backends.linalg.Mat", + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Mat), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, + __pyx_type_5thinc_8backends_6linalg_Mat_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_6linalg_Mat = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.linalg.""Mat", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Mat), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_6linalg_Mat, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_6linalg_Mat, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_6linalg_Matrix_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix)) __PYX_ERR(1, 23, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_6linalg_Matrix_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix) < (0)) __PYX_ERR(1, 23, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix = &__pyx_type_5thinc_8backends_6linalg_Matrix; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix) < (0)) __PYX_ERR(1, 23, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_Matrix, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix) < (0)) __PYX_ERR(1, 23, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_Vec = &__pyx_vtable_5thinc_8backends_6linalg_Vec; + __pyx_vtable_5thinc_8backends_6linalg_Vec.arg_max = (int (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ))__pyx_f_5thinc_8backends_6linalg_3Vec_arg_max; + __pyx_vtable_5thinc_8backends_6linalg_Vec.max = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_max; + __pyx_vtable_5thinc_8backends_6linalg_Vec.sum = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_sum; + __pyx_vtable_5thinc_8backends_6linalg_Vec.norm = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_norm; + __pyx_vtable_5thinc_8backends_6linalg_Vec.add = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_add; + __pyx_vtable_5thinc_8backends_6linalg_Vec.add_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_add_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.mul = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_mul; + __pyx_vtable_5thinc_8backends_6linalg_Vec.mul_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_mul_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.pow = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_pow; + __pyx_vtable_5thinc_8backends_6linalg_Vec.pow_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_pow_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.div = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_div; + __pyx_vtable_5thinc_8backends_6linalg_Vec.div_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_div_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.exp = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_exp; + __pyx_vtable_5thinc_8backends_6linalg_Vec.exp_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_exp_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.reciprocal_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i; + __pyx_vtable_5thinc_8backends_6linalg_Vec.mean = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_mean; + __pyx_vtable_5thinc_8backends_6linalg_Vec.variance = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_3Vec_variance; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_6linalg_Vec_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec)) __PYX_ERR(1, 30, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_6linalg_Vec_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec) < (0)) __PYX_ERR(1, 30, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec = &__pyx_type_5thinc_8backends_6linalg_Vec; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec) < (0)) __PYX_ERR(1, 30, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec, __pyx_vtabptr_5thinc_8backends_6linalg_Vec) < (0)) __PYX_ERR(1, 30, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec) < (0)) __PYX_ERR(1, 30, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_Vec, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec) < (0)) __PYX_ERR(1, 30, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_VecVec = &__pyx_vtable_5thinc_8backends_6linalg_VecVec; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.add = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_add; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.add_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_add_i; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.batch_add_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.add_pow = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.add_pow_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.mul = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_mul; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.mul_i = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.dot = (__pyx_t_5thinc_8backends_6linalg_weight_t (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t))__pyx_f_5thinc_8backends_6linalg_6VecVec_dot; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.arg_max_if_true = (int (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const *, int const ))__pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true; + __pyx_vtable_5thinc_8backends_6linalg_VecVec.arg_max_if_zero = (int (*)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ))__pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_6linalg_VecVec_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec)) __PYX_ERR(1, 159, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_6linalg_VecVec_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec) < (0)) __PYX_ERR(1, 159, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec = &__pyx_type_5thinc_8backends_6linalg_VecVec; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec) < (0)) __PYX_ERR(1, 159, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec, __pyx_vtabptr_5thinc_8backends_6linalg_VecVec) < (0)) __PYX_ERR(1, 159, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec) < (0)) __PYX_ERR(1, 159, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_VecVec, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec) < (0)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_Mat = &__pyx_vtable_5thinc_8backends_6linalg_Mat; + __pyx_vtable_5thinc_8backends_6linalg_Mat.mean_row = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t))__pyx_f_5thinc_8backends_6linalg_3Mat_mean_row; + __pyx_vtable_5thinc_8backends_6linalg_Mat.var_row = (void (*)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t, __pyx_t_5thinc_8backends_6linalg_weight_t))__pyx_f_5thinc_8backends_6linalg_3Mat_var_row; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_6linalg_Mat_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat)) __PYX_ERR(1, 250, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_6linalg_Mat_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat) < (0)) __PYX_ERR(1, 250, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat = &__pyx_type_5thinc_8backends_6linalg_Mat; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat) < (0)) __PYX_ERR(1, 250, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat, __pyx_vtabptr_5thinc_8backends_6linalg_Mat) < (0)) __PYX_ERR(1, 250, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat) < (0)) __PYX_ERR(1, 250, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_Mat, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat) < (0)) __PYX_ERR(1, 250, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule("cymem.cymem"); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyMalloc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc) __PYX_ERR(2, 4, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyMalloc = (struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyMalloc)) __PYX_ERR(2, 4, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyFree", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree) __PYX_ERR(2, 10, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyFree = (struct __pyx_vtabstruct_5cymem_5cymem_PyFree*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyFree)) __PYX_ERR(2, 10, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Pool", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool) __PYX_ERR(2, 16, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_Pool = (struct __pyx_vtabstruct_5cymem_5cymem_Pool*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_Pool)) __PYX_ERR(2, 16, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Address = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Address", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Address) __PYX_ERR(2, 28, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_linalg(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_linalg}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "linalg", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_linalg(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_linalg(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_linalg(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'linalg' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "linalg" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_linalg", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__backends__linalg) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.backends.linalg")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.backends.linalg", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "thinc/backends/linalg.pyx":2 + * # cython: profile=False + * try: # <<<<<<<<<<<<<< + * import blis.py + * except ImportError: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "thinc/backends/linalg.pyx":3 + * # cython: profile=False + * try: + * import blis.py # <<<<<<<<<<<<<< + * except ImportError: + * pass +*/ + __pyx_t_5 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_blis_py, 0, 0, __pyx_mstate_global->__pyx_kp_u_thinc_backends_blis_py, -1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 3, __pyx_L2_error) + __pyx_t_4 = __pyx_t_5; + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_blis, __pyx_t_4) < (0)) __PYX_ERR(0, 3, __pyx_L2_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/linalg.pyx":2 + * # cython: profile=False + * try: # <<<<<<<<<<<<<< + * import blis.py + * except ImportError: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L7_try_end; + __pyx_L2_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/linalg.pyx":4 + * try: + * import blis.py + * except ImportError: # <<<<<<<<<<<<<< + * pass +*/ + __pyx_t_6 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_ImportError)))); + if (__pyx_t_6) { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L3_exception_handled; + } + goto __pyx_L4_except_error; + + /* "thinc/backends/linalg.pyx":2 + * # cython: profile=False + * try: # <<<<<<<<<<<<<< + * import blis.py + * except ImportError: +*/ + __pyx_L4_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L3_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L7_try_end:; + } + + /* "thinc/backends/linalg.pyx":1 + * # cython: profile=False # <<<<<<<<<<<<<< + * try: + * import blis.py +*/ + __pyx_t_4 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_4) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.backends.linalg", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.backends.linalg"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + __Pyx_RefNannyFinishContext(); + return 0; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 5; } index[] = {{1},{7},{6},{2},{9},{22},{3},{6},{3},{6},{4},{7},{18},{8},{10},{8},{14},{12},{12},{10},{8}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (154 bytes) */ +const char* const cstring = "BZh91AY&SY\267@\237\371\000\000\004\337\200@\000\000\001\000\000\200\002\001\000\277\357\377` \000\224\r4@(\364\324\323M\3525\036\240\310\232F\203G\250h\320i\022%R\266\362\321\014H\202\021\326\240Q\031\031P5\320Y'\256+\363\027\024\311\370\000\333\023\212G\2227\333\211K9\301\"\026t\200\350\336\217\341\203\005\273\3674\241\263\221\313\200\245(\324T\357#\310Z\306\020;\270\220\330^(\032s\256,\3321\241\226\231Z\275i?\213\271\"\234(H[\240O\374\200"; + PyObject *data = __Pyx_DecompressString(cstring, 154, 2); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (119 bytes) */ +const char* const cstring = "x\3325MA\016\302@\010\374\321~\301\027x\365J(\213Jd\261\026\326\264\277wi+\231\0143\231!\\\2528N\312l\311\017\022?T\215\247\030\225\t\351\305V\275L*^\346\355\2121\260\310zc:\220\301\031\222\2121\210A,H\234\247\000\r\207\037\323\336\265+\2472l\373\236\267\025\276\221\277\322}:\352?q\216\2635T\345;v\r\200`\037\374\003\3750E\365"; + PyObject *data = __Pyx_DecompressString(cstring, 119, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (176 bytes) */ +const char* const bytes = "?disableenablegcisenabledthinc.backends.blis.pyMatMatrixVecVecVecblisblis.pycline_in_traceback__main____module____name____pyx_vtable____qualname____set_name__setdefault__test__"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 21; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 6) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 21; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + } + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* SoftComplexToDouble */ +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_nogil) { + if (unlikely(__Pyx_CIMAG(value))) { + PyGILState_STATE gilstate; + if (have_nogil) + gilstate = PyGILState_Ensure(); + PyErr_SetString(PyExc_TypeError, + "Cannot convert 'complex' with non-zero imaginary component to 'double' " + "(this most likely comes from the '**' operator; " + "use 'cython.cpow(True)' to return 'nan' instead of a " + "complex number)."); + if (have_nogil) + PyGILState_Release(gilstate); + return -1.; + } + return __Pyx_CREAL(value); +} + +/* ErrOccurredWithGIL */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + PyGILState_STATE _save = PyGILState_Ensure(); + err = !!PyErr_Occurred(); + PyGILState_Release(_save); + return err; +} + +/* PyErrFetchRestore (used by WriteUnraisableException) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* WriteUnraisableException */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); + else state = (PyGILState_STATE)0; + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(0); + } + ctx = PyUnicode_FromString(name); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } + if (nogil) + PyGILState_Release(state); +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* PyObjectCall (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallNoArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallNoArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectCallOneArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetAttrStr (used by PyObjectGetMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#endif + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* GetVTable (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* MergeVTables */ +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; +#if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; +#else + size = PyTuple_GET_SIZE(bases); +#endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; +#if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#else + basei = PyTuple_GET_ITEM(bases, i); +#endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; +bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); +#if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); +#endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +really_bad: // bad has failed! +#endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +other_failure: +#endif + PyMem_Free(base_vtables); + return -1; +} + +/* TypeImport */ +#ifndef __PYX_HAVE_RT_ImportType_3_2_1 +#define __PYX_HAVE_RT_ImportType_3_2_1 +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size) +{ + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; +#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; +#endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; +#else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_1 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_1 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return NULL; +} +#endif + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyObjectGetAttrStrNoError (used by HasAttr) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* TupleAndListFromArray (used by ImportImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* dict_setdefault (used by CLineInTraceback) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* PyDictVersioning (used by CLineInTraceback) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* Declarations */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return ::std::complex< double >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return x + y*(__pyx_t_double_complex)_Complex_I; + } + #endif +#else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + __pyx_t_double_complex z; + z.real = x; + z.imag = y; + return z; + } +#endif + +/* Arithmetic */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) +#else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabs(b.real) >= fabs(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + double r = b.imag / b.real; + double s = (double)(1.0) / (b.real + b.imag * r); + return __pyx_t_double_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + double r = b.real / b.imag; + double s = (double)(1.0) / (b.imag + b.real * r); + return __pyx_t_double_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + double denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_double_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrt(z.real*z.real + z.imag*z.imag); + #else + return hypot(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + double r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + double denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_double(a, a); + case 3: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, a); + case 4: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = pow(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_double(a); + theta = atan2(a.imag, a.real); + } + lnr = log(r); + z_r = exp(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cos(z_theta); + z.imag = z_r * sin(z_theta); + return z; + } + #endif +#endif + +/* PyObjectVectorCallKwBuilder (used by CIntToPy) */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int32_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int32_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int32_t), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int32_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int32_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int32_t) -1; + val = __Pyx_PyLong_As_int32_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 2 * PyLong_SHIFT)) { + return (int32_t) (((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 3 * PyLong_SHIFT)) { + return (int32_t) (((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 4 * PyLong_SHIFT)) { + return (int32_t) (((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int32_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int32_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int32_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) ((((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) ((((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) ((((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int32_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, long, PyLong_AsLong(x)) + } else if ((sizeof(int32_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int32_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int32_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int32_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int32_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int32_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int32_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int32_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int32_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int32_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int32_t) 1) << (sizeof(int32_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int32_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int32_t"); + return (int32_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int32_t"); + return (int32_t) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u_); + } + goto done; +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } +} +#endif +static unsigned long __Pyx_get_runtime_version(void) { +#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + return __Pyx_cached_runtime_version; +#endif +} + +/* AddModuleRef */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* DecompressString */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; +import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); +bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pxd new file mode 100644 index 0000000000000000000000000000000000000000..8cceef60a36c4ac33341534158a29fc925a7eaed --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pxd @@ -0,0 +1,275 @@ +# cython: infer_types=True +# cython: cdivision=True + +cimport cython +from cymem.cymem cimport Pool +from libc.stdint cimport int32_t +from libc.string cimport memcpy, memset + +ctypedef float weight_t + +DEF USE_BLAS = False +DEF EPS = 1e-5 + + +IF USE_BLAS: + cimport blis.cy + +cdef extern from "math.h" nogil: + weight_t exp(weight_t x) + weight_t sqrt(weight_t x) + + +cdef class Matrix: + cdef readonly Pool mem + cdef weight_t* data + cdef readonly int32_t nr_row + cdef readonly int32_t nr_col + + +cdef class Vec: + @staticmethod + cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + if n_classes == 2: + return 0 if scores[0] > scores[1] else 1 + cdef int i + cdef int best = 0 + cdef weight_t mode = scores[0] + for i in range(1, n_classes): + if scores[i] > mode: + mode = scores[i] + best = i + return best + + @staticmethod + cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + if nr == 0: + return 0 + cdef int i + cdef weight_t mode = x[0] + for i in range(1, nr): + if x[i] > mode: + mode = x[i] + return mode + + @staticmethod + cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + cdef int i + cdef weight_t total = 0 + for i in range(nr): + total += vec[i] + return total + + @staticmethod + cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + cdef weight_t total = 0 + for i in range(nr): + total += vec[i] ** 2 + return sqrt(total) + + @staticmethod + cdef inline void add(weight_t* output, const weight_t* x, + weight_t inc, int32_t nr) noexcept nogil: + memcpy(output, x, sizeof(output[0]) * nr) + Vec.add_i(output, inc, nr) + + @staticmethod + cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + vec[i] += inc + + @staticmethod + cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + int32_t nr) noexcept nogil: + memcpy(output, vec, sizeof(output[0]) * nr) + Vec.mul_i(output, scal, nr) + + @staticmethod + cdef inline void mul_i(weight_t* vec, weight_t scal, int32_t nr) noexcept nogil: + cdef int i + IF USE_BLAS: + blis.cy.scalv(BLIS_NO_CONJUGATE, nr, scal, vec, 1) + ELSE: + for i in range(nr): + vec[i] *= scal + + @staticmethod + cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + int32_t nr) noexcept nogil: + memcpy(output, vec, sizeof(output[0]) * nr) + Vec.pow_i(output, scal, nr) + + @staticmethod + cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + vec[i] **= scal + + @staticmethod + @cython.cdivision(True) + cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, + int32_t nr) noexcept nogil: + memcpy(output, vec, sizeof(output[0]) * nr) + Vec.div_i(output, scal, nr) + + @staticmethod + @cython.cdivision(True) + cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + vec[i] /= scal + + @staticmethod + cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + memcpy(output, vec, sizeof(output[0]) * nr) + Vec.exp_i(output, nr) + + @staticmethod + cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + vec[i] = exp(vec[i]) + + @staticmethod + cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + vec[i] = 1.0 / vec[i] + + @staticmethod + cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + cdef weight_t mean = 0. + for x in X[:nr_dim]: + mean += x + return mean / nr_dim + + @staticmethod + cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + # See https://www.johndcook.com/blog/standard_deviation/ + cdef double m = X[0] + cdef double v = 0. + for i in range(1, nr_dim): + diff = X[i]-m + m += diff / (i+1) + v += diff * (X[i] - m) + return v / nr_dim + + +cdef class VecVec: + @staticmethod + cdef inline void add(weight_t* output, + const weight_t* x, + const weight_t* y, + weight_t scale, + int32_t nr) noexcept nogil: + memcpy(output, x, sizeof(output[0]) * nr) + VecVec.add_i(output, y, scale, nr) + + @staticmethod + cdef inline void add_i(weight_t* x, + const weight_t* y, + weight_t scale, + int32_t nr) noexcept nogil: + cdef int i + IF USE_BLAS: + blis.cy.axpyv(BLIS_NO_CONJUGATE, nr, scale, y, 1, x, 1) + ELSE: + for i in range(nr): + x[i] += y[i] * scale + + @staticmethod + cdef inline void batch_add_i(weight_t* x, + const weight_t* y, + weight_t scale, + int32_t nr, int32_t nr_batch) noexcept nogil: + # For fixed x, matrix of y + cdef int i, _ + for _ in range(nr_batch): + VecVec.add_i(x, + y, scale, nr) + y += nr + + @staticmethod + cdef inline void add_pow(weight_t* output, + const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + memcpy(output, x, sizeof(output[0]) * nr) + VecVec.add_pow_i(output, y, power, nr) + + + @staticmethod + cdef inline void add_pow_i(weight_t* x, + const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + x[i] += y[i] ** power + + @staticmethod + cdef inline void mul(weight_t* output, + const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + memcpy(output, x, sizeof(output[0]) * nr) + VecVec.mul_i(output, y, nr) + + @staticmethod + cdef inline void mul_i(weight_t* x, + const weight_t* y, int32_t nr) noexcept nogil: + cdef int i + for i in range(nr): + x[i] *= y[i] + + @staticmethod + cdef inline weight_t dot( + const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + cdef int i + cdef weight_t total = 0 + for i in range(nr): + total += x[i] * y[i] + return total + + @staticmethod + cdef inline int arg_max_if_true( + const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: + cdef int i + cdef int best = -1 + for i in range(n_classes): + if is_valid[i] and (best == -1 or scores[i] > scores[best]): + best = i + return best + + @staticmethod + cdef inline int arg_max_if_zero( + const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: + cdef int i + cdef int best = -1 + for i in range(n_classes): + if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + best = i + return best + + +cdef class Mat: + @staticmethod + cdef inline void mean_row(weight_t* Ex, + const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: + memset(Ex, 0, sizeof(Ex[0]) * nr_col) + for i in range(nr_row): + VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) + Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + + @staticmethod + cdef inline void var_row(weight_t* Vx, + const weight_t* mat, const weight_t* Ex, + int32_t nr_row, int32_t nr_col, weight_t eps) noexcept nogil: + # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + if nr_row == 0 or nr_col == 0: + return + cdef weight_t sum_, sum2 + for i in range(nr_col): + sum_ = 0.0 + sum2 = 0.0 + for j in range(nr_row): + x = mat[j * nr_col + i] + sum2 += (x - Ex[i]) ** 2 + sum_ += x - Ex[i] + Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row + Vx[i] += eps diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pyx new file mode 100644 index 0000000000000000000000000000000000000000..1592963f39b854c2ebad05421a961488e77b0e0b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/linalg.pyx @@ -0,0 +1,5 @@ +# cython: profile=False +try: + import blis.py +except ImportError: + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/mps_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/mps_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a59982a6da97b8bfdf76420ee08cfa1d957f7f68 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/mps_ops.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING + +import numpy + +from .. import registry +from .numpy_ops import NumpyOps +from .ops import Ops + +if TYPE_CHECKING: + # Type checking does not work with dynamic base classes, since MyPy cannot + # determine against which base class to check. So, always derive from Ops + # during type checking. + _Ops = Ops +else: + try: + from thinc_apple_ops import AppleOps + + _Ops = AppleOps + except ImportError: + _Ops = NumpyOps + + +@registry.ops("MPSOps") +class MPSOps(_Ops): + """Ops class for Metal Performance shaders.""" + + name = "mps" + xp = numpy diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1244e88a95a007194ad342614f91042cb3ef0bf9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.cpp @@ -0,0 +1,67580 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\arrayobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\arrayscalars.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ndarrayobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ndarraytypes.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ufuncobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include\\Python.h", + "thinc\\backends\\cpu_kernels.hh" + ], + "include_dirs": [ + "thinc/backends", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.backends.numpy_ops", + "sources": [ + "thinc/backends/numpy_ops.pyx" + ] + }, + "module_name": "thinc.backends.numpy_ops" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__backends__numpy_ops +#define __PYX_HAVE_API__thinc__backends__numpy_ops +/* Early includes */ +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include "cpu_kernels.hh" +#include +#include +#include + + /* Using NumPy API declarations from "numpy/__init__.cython-30.pxd" */ + +#include "numpy/arrayobject.h" +#include "numpy/ndarrayobject.h" +#include "numpy/ndarraytypes.h" +#include "numpy/arrayscalars.h" +#include "numpy/ufuncobject.h" +#include +#include +#include "math.h" +#include "pythread.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* Header.proto */ +#if !defined(CYTHON_CCOMPLEX) + #if defined(__cplusplus) + #define CYTHON_CCOMPLEX 1 + #elif (defined(_Complex_I) && !defined(_MSC_VER)) || ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_COMPLEX__) && !defined(_MSC_VER)) + #define CYTHON_CCOMPLEX 1 + #else + #define CYTHON_CCOMPLEX 0 + #endif +#endif +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #include + #else + #include + #endif +#endif +#if CYTHON_CCOMPLEX && !defined(__cplusplus) && defined(__sun__) && defined(__GNUC__) + #undef _Complex_I + #define _Complex_I 1.0fj +#endif + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/backends/numpy_ops.pyx", + "", + "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd", + "thinc/backends/linalg.pxd", + "thinc/backends/cblas.pxd", + "cpython/type.pxd", + "cymem/cymem.pxd", + "preshed/maps.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* BufferFormatStructs.proto (used by BufferFormatCheck) */ +struct __Pyx_StructField_; +#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0) +typedef struct { + const char* name; + const struct __Pyx_StructField_* fields; + size_t size; + size_t arraysize[8]; + int ndim; + char typegroup; + char is_unsigned; + int flags; +} __Pyx_TypeInfo; +typedef struct __Pyx_StructField_ { + const __Pyx_TypeInfo* type; + const char* name; + size_t offset; +} __Pyx_StructField; +typedef struct { + const __Pyx_StructField* field; + size_t parent_offset; +} __Pyx_BufFmt_StackElem; +typedef struct { + __Pyx_StructField root; + __Pyx_BufFmt_StackElem* head; + size_t fmt_offset; + size_t new_count, enc_count; + size_t struct_alignment; + int is_complex; + char enc_type; + char new_packmode; + char enc_packmode; + char is_valid_array; +} __Pyx_BufFmt_Context; + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* MemviewSliceStruct.proto */ +struct __pyx_memoryview_obj; +typedef struct { + struct __pyx_memoryview_obj *memview; + char *data; + Py_ssize_t shape[8]; + Py_ssize_t strides[8]; + Py_ssize_t suboffsets[8]; +} __Pyx_memviewslice; +#define __Pyx_MemoryView_Len(m) (m.shape[0]) +#define __Pyx_MEMVIEW_DIRECT 1 +#define __Pyx_MEMVIEW_PTR 2 +#define __Pyx_MEMVIEW_FULL 4 +#define __Pyx_MEMVIEW_CONTIG 8 +#define __Pyx_MEMVIEW_STRIDED 16 +#define __Pyx_MEMVIEW_FOLLOW 32 +#define __Pyx_IS_C_CONTIG 1 +#define __Pyx_IS_F_CONTIG 2 +#define __Pyx_MEMSLICE_INIT { 0, 0, { 0 }, { 0 }, { 0 } } +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "blis/cy.pxd":58 + * + * + * ctypedef int64_t dim_t # <<<<<<<<<<<<<< + * ctypedef int64_t inc_t + * ctypedef int64_t doff_t +*/ +typedef int64_t __pyx_t_4blis_2cy_dim_t; + +/* "blis/cy.pxd":59 + * + * ctypedef int64_t dim_t + * ctypedef int64_t inc_t # <<<<<<<<<<<<<< + * ctypedef int64_t doff_t + * +*/ +typedef int64_t __pyx_t_4blis_2cy_inc_t; + +/* "blis/cy.pxd":60 + * ctypedef int64_t dim_t + * ctypedef int64_t inc_t + * ctypedef int64_t doff_t # <<<<<<<<<<<<<< + * + * +*/ +typedef int64_t __pyx_t_4blis_2cy_doff_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":743 + * # in Cython to enable them only on the right systems. + * + * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t +*/ +typedef npy_int8 __pyx_t_5numpy_int8_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":744 + * + * ctypedef npy_int8 int8_t + * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< + * ctypedef npy_int32 int32_t + * ctypedef npy_int64 int64_t +*/ +typedef npy_int16 __pyx_t_5numpy_int16_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":745 + * ctypedef npy_int8 int8_t + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< + * ctypedef npy_int64 int64_t + * +*/ +typedef npy_int32 __pyx_t_5numpy_int32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":746 + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t + * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< + * + * ctypedef npy_uint8 uint8_t +*/ +typedef npy_int64 __pyx_t_5numpy_int64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":748 + * ctypedef npy_int64 int64_t + * + * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t +*/ +typedef npy_uint8 __pyx_t_5numpy_uint8_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":749 + * + * ctypedef npy_uint8 uint8_t + * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< + * ctypedef npy_uint32 uint32_t + * ctypedef npy_uint64 uint64_t +*/ +typedef npy_uint16 __pyx_t_5numpy_uint16_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":750 + * ctypedef npy_uint8 uint8_t + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< + * ctypedef npy_uint64 uint64_t + * +*/ +typedef npy_uint32 __pyx_t_5numpy_uint32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":751 + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t + * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< + * + * ctypedef npy_float32 float32_t +*/ +typedef npy_uint64 __pyx_t_5numpy_uint64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":753 + * ctypedef npy_uint64 uint64_t + * + * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< + * ctypedef npy_float64 float64_t + * #ctypedef npy_float80 float80_t +*/ +typedef npy_float32 __pyx_t_5numpy_float32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":754 + * + * ctypedef npy_float32 float32_t + * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< + * #ctypedef npy_float80 float80_t + * #ctypedef npy_float128 float128_t +*/ +typedef npy_float64 __pyx_t_5numpy_float64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":761 + * ctypedef double complex complex128_t + * + * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< + * ctypedef npy_ulonglong ulonglong_t + * +*/ +typedef npy_longlong __pyx_t_5numpy_longlong_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":762 + * + * ctypedef npy_longlong longlong_t + * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< + * + * ctypedef npy_intp intp_t +*/ +typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":764 + * ctypedef npy_ulonglong ulonglong_t + * + * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< + * ctypedef npy_uintp uintp_t + * +*/ +typedef npy_intp __pyx_t_5numpy_intp_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":765 + * + * ctypedef npy_intp intp_t + * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< + * + * ctypedef npy_double float_t +*/ +typedef npy_uintp __pyx_t_5numpy_uintp_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":767 + * ctypedef npy_uintp uintp_t + * + * ctypedef npy_double float_t # <<<<<<<<<<<<<< + * ctypedef npy_double double_t + * ctypedef npy_longdouble longdouble_t +*/ +typedef npy_double __pyx_t_5numpy_float_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":768 + * + * ctypedef npy_double float_t + * ctypedef npy_double double_t # <<<<<<<<<<<<<< + * ctypedef npy_longdouble longdouble_t + * +*/ +typedef npy_double __pyx_t_5numpy_double_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":769 + * ctypedef npy_double float_t + * ctypedef npy_double double_t + * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< + * + * ctypedef float complex cfloat_t +*/ +typedef npy_longdouble __pyx_t_5numpy_longdouble_t; + +/* "preshed/maps.pxd":5 + * + * + * ctypedef uint64_t key_t # <<<<<<<<<<<<<< + * + * +*/ +typedef uint64_t __pyx_t_7preshed_4maps_key_t; + +/* "linalg.pxd":9 + * from libc.string cimport memcpy, memset + * + * ctypedef float weight_t # <<<<<<<<<<<<<< + * + * DEF USE_BLAS = False +*/ +typedef float __pyx_t_5thinc_8backends_6linalg_weight_t; + +/* "thinc/backends/numpy_ops.pyx":35 + * + * + * ctypedef float weight_t # <<<<<<<<<<<<<< + * + * +*/ +typedef float __pyx_t_5thinc_8backends_9numpy_ops_weight_t; +/* #### Code section: complex_type_declarations ### */ +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< double > __pyx_t_double_complex; + #else + typedef double _Complex __pyx_t_double_complex; + #endif +#else + typedef struct { double real, imag; } __pyx_t_double_complex; +#endif +static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double, double); + +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< float > __pyx_t_float_complex; + #else + typedef float _Complex __pyx_t_float_complex; + #endif +#else + typedef struct { float real, imag; } __pyx_t_float_complex; +#endif +static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float, float); + +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< long double > __pyx_t_long_double_complex; + #else + typedef long double _Complex __pyx_t_long_double_complex; + #endif +#else + typedef struct { long double real, imag; } __pyx_t_long_double_complex; +#endif +static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double, long double); + +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5thinc_8backends_5cblas_CBlas; +struct __pyx_obj_5cymem_5cymem_PyMalloc; +struct __pyx_obj_5cymem_5cymem_PyFree; +struct __pyx_obj_5cymem_5cymem_Pool; +struct __pyx_obj_5cymem_5cymem_Address; +struct __pyx_obj_7preshed_4maps_PreshMap; +struct __pyx_obj_7preshed_4maps_PreshMapArray; +struct __pyx_obj_5thinc_8backends_6linalg_Matrix; +struct __pyx_obj_5thinc_8backends_6linalg_Vec; +struct __pyx_obj_5thinc_8backends_6linalg_VecVec; +struct __pyx_obj_5thinc_8backends_6linalg_Mat; +struct __pyx_defaults; +struct __pyx_defaults1; +struct __pyx_array_obj; +struct __pyx_MemviewEnum_obj; +struct __pyx_memoryview_obj; +struct __pyx_memoryviewslice_obj; + +/* "cblas.pxd":3 + * from libcpp.memory cimport shared_ptr + * + * ctypedef void (*sgemm_ptr)(bint transA, bint transB, int M, int N, int K, # <<<<<<<<<<<<<< + * float alpha, const float* A, int lda, const float *B, + * int ldb, float beta, float* C, int ldc) noexcept nogil +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_sgemm_ptr)(int, int, int, int, int, float, float const *, int, float const *, int, float, float *, int); + +/* "cblas.pxd":8 + * + * + * ctypedef void (*saxpy_ptr)(int N, float alpha, const float* X, int incX, # <<<<<<<<<<<<<< + * float *Y, int incY) noexcept nogil + * +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_saxpy_ptr)(int, float, float const *, int, float *, int); + +/* "cblas.pxd":12 + * + * + * ctypedef void (*daxpy_ptr)(int N, double alpha, const double* X, int incX, # <<<<<<<<<<<<<< + * double *Y, int incY) noexcept nogil + * +*/ +typedef void (*__pyx_t_5thinc_8backends_5cblas_daxpy_ptr)(int, double, double const *, int, double *, int); + +/* "blis/cy.pxd":66 + * # the header into our header. + * # We get some piece of mind from checking the values on init. + * cpdef enum trans_t: # <<<<<<<<<<<<<< + * NO_TRANSPOSE = 0 + * TRANSPOSE = 8 +*/ +enum __pyx_t_4blis_2cy_trans_t { + __pyx_e_4blis_2cy_NO_TRANSPOSE = 0, + __pyx_e_4blis_2cy_TRANSPOSE = 8, + __pyx_e_4blis_2cy_CONJ_NO_TRANSPOSE = 16, + __pyx_e_4blis_2cy_CONJ_TRANSPOSE = 24 +}; + +/* "blis/cy.pxd":73 + * + * + * cpdef enum conj_t: # <<<<<<<<<<<<<< + * NO_CONJUGATE = 0 + * CONJUGATE = 16 +*/ +enum __pyx_t_4blis_2cy_conj_t { + __pyx_e_4blis_2cy_NO_CONJUGATE = 0, + __pyx_e_4blis_2cy_CONJUGATE = 16 +}; + +/* "blis/cy.pxd":78 + * + * + * cpdef enum side_t: # <<<<<<<<<<<<<< + * LEFT = 0 + * RIGHT = 1 +*/ +enum __pyx_t_4blis_2cy_side_t { + __pyx_e_4blis_2cy_LEFT = 0, + __pyx_e_4blis_2cy_RIGHT = 1 +}; + +/* "blis/cy.pxd":83 + * + * + * cpdef enum uplo_t: # <<<<<<<<<<<<<< + * LOWER = 192 + * UPPER = 96 +*/ +enum __pyx_t_4blis_2cy_uplo_t { + __pyx_e_4blis_2cy_LOWER = 0xC0, + __pyx_e_4blis_2cy_UPPER = 96, + __pyx_e_4blis_2cy_DENSE = 0xE0 +}; + +/* "blis/cy.pxd":89 + * + * + * cpdef enum diag_t: # <<<<<<<<<<<<<< + * NONUNIT_DIAG = 0 + * UNIT_DIAG = 256 +*/ +enum __pyx_t_4blis_2cy_diag_t { + __pyx_e_4blis_2cy_NONUNIT_DIAG = 0, + __pyx_e_4blis_2cy_UNIT_DIAG = 0x100 +}; + +/* "blis/cy.pxd":6 + * + * + * ctypedef float[::1] float1d_t # <<<<<<<<<<<<<< + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_float1d_t; + +/* "blis/cy.pxd":7 + * + * ctypedef float[::1] float1d_t + * ctypedef double[::1] double1d_t # <<<<<<<<<<<<<< + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_double1d_t; + +/* "blis/cy.pxd":8 + * ctypedef float[::1] float1d_t + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t # <<<<<<<<<<<<<< + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_float2d_t; + +/* "blis/cy.pxd":9 + * ctypedef double[::1] double1d_t + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t # <<<<<<<<<<<<<< + * ctypedef float* floats_t + * ctypedef double* doubles_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_double2d_t; + +/* "blis/cy.pxd":10 + * ctypedef float[:, ::1] float2d_t + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t # <<<<<<<<<<<<<< + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t +*/ +typedef float *__pyx_t_4blis_2cy_floats_t; + +/* "blis/cy.pxd":11 + * ctypedef double[:, ::1] double2d_t + * ctypedef float* floats_t + * ctypedef double* doubles_t # <<<<<<<<<<<<<< + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t +*/ +typedef double *__pyx_t_4blis_2cy_doubles_t; + +/* "blis/cy.pxd":12 + * ctypedef float* floats_t + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t # <<<<<<<<<<<<<< + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_float1d_t; + +/* "blis/cy.pxd":13 + * ctypedef double* doubles_t + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t # <<<<<<<<<<<<<< + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_double1d_t; + +/* "blis/cy.pxd":14 + * ctypedef const float[::1] const_float1d_t + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t # <<<<<<<<<<<<<< + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_float2d_t; + +/* "blis/cy.pxd":15 + * ctypedef const double[::1] const_double1d_t + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t # <<<<<<<<<<<<<< + * ctypedef const float* const_floats_t + * ctypedef const double* const_doubles_t +*/ +typedef __Pyx_memviewslice __pyx_t_4blis_2cy_const_double2d_t; + +/* "blis/cy.pxd":16 + * ctypedef const float[:, ::1] const_float2d_t + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t # <<<<<<<<<<<<<< + * ctypedef const double* const_doubles_t + * +*/ +typedef float const *__pyx_t_4blis_2cy_const_floats_t; + +/* "blis/cy.pxd":17 + * ctypedef const double[:, ::1] const_double2d_t + * ctypedef const float* const_floats_t + * ctypedef const double* const_doubles_t # <<<<<<<<<<<<<< + * + * +*/ +typedef double const *__pyx_t_4blis_2cy_const_doubles_t; + +/* "cymem/cymem.pxd":1 + * ctypedef void* (*malloc_t)(size_t n) # <<<<<<<<<<<<<< + * ctypedef void (*free_t)(void *p) + * +*/ +typedef void *(*__pyx_t_5cymem_5cymem_malloc_t)(size_t); + +/* "cymem/cymem.pxd":2 + * ctypedef void* (*malloc_t)(size_t n) + * ctypedef void (*free_t)(void *p) # <<<<<<<<<<<<<< + * + * cdef class PyMalloc: +*/ +typedef void (*__pyx_t_5cymem_5cymem_free_t)(void *); +struct __pyx_t_7preshed_4maps_Cell; +struct __pyx_t_7preshed_4maps_Result; +struct __pyx_t_7preshed_4maps_MapStruct; + +/* "preshed/maps.pxd":8 + * + * + * cdef struct Cell: # <<<<<<<<<<<<<< + * key_t key + * void* value +*/ +struct __pyx_t_7preshed_4maps_Cell { + __pyx_t_7preshed_4maps_key_t key; + void *value; +}; + +/* "preshed/maps.pxd":13 + * + * + * cdef struct Result: # <<<<<<<<<<<<<< + * int found + * void* value +*/ +struct __pyx_t_7preshed_4maps_Result { + int found; + void *value; +}; + +/* "preshed/maps.pxd":18 + * + * + * cdef struct MapStruct: # <<<<<<<<<<<<<< + * Cell* cells + * void* value_for_empty_key +*/ +struct __pyx_t_7preshed_4maps_MapStruct { + struct __pyx_t_7preshed_4maps_Cell *cells; + void *value_for_empty_key; + void *value_for_del_key; + __pyx_t_7preshed_4maps_key_t length; + __pyx_t_7preshed_4maps_key_t filled; + int is_empty_key_set; + int is_del_key_set; +}; + +/* "thinc/backends/numpy_ops.pxd":3 + * from .cblas cimport saxpy_ptr + * + * ctypedef double[:, ::1] double2d_t # <<<<<<<<<<<<<< + * ctypedef double[:, :, ::1] double3d_t + * ctypedef float[:, ::1] float2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_double2d_t; + +/* "thinc/backends/numpy_ops.pxd":4 + * + * ctypedef double[:, ::1] double2d_t + * ctypedef double[:, :, ::1] double3d_t # <<<<<<<<<<<<<< + * ctypedef float[:, ::1] float2d_t + * ctypedef float[:, :, ::1] float3d_t +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_double3d_t; + +/* "thinc/backends/numpy_ops.pxd":5 + * ctypedef double[:, ::1] double2d_t + * ctypedef double[:, :, ::1] double3d_t + * ctypedef float[:, ::1] float2d_t # <<<<<<<<<<<<<< + * ctypedef float[:, :, ::1] float3d_t + * ctypedef int[:, ::1] int2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_float2d_t; + +/* "thinc/backends/numpy_ops.pxd":6 + * ctypedef double[:, :, ::1] double3d_t + * ctypedef float[:, ::1] float2d_t + * ctypedef float[:, :, ::1] float3d_t # <<<<<<<<<<<<<< + * ctypedef int[:, ::1] int2d_t + * ctypedef unsigned int[:, ::1] uint2d_t +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_float3d_t; + +/* "thinc/backends/numpy_ops.pxd":7 + * ctypedef float[:, ::1] float2d_t + * ctypedef float[:, :, ::1] float3d_t + * ctypedef int[:, ::1] int2d_t # <<<<<<<<<<<<<< + * ctypedef unsigned int[:, ::1] uint2d_t + * +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_int2d_t; + +/* "thinc/backends/numpy_ops.pxd":8 + * ctypedef float[:, :, ::1] float3d_t + * ctypedef int[:, ::1] int2d_t + * ctypedef unsigned int[:, ::1] uint2d_t # <<<<<<<<<<<<<< + * + * cdef fused ints2d_ft: +*/ +typedef __Pyx_memviewslice __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t; + +/* "cblas.pxd":22 + * + * + * cdef class CBlas: # <<<<<<<<<<<<<< + * cdef shared_ptr[BlasFuncs] ptr + * +*/ +struct __pyx_obj_5thinc_8backends_5cblas_CBlas { + PyObject_HEAD + std::shared_ptr ptr; +}; + + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtab; + __pyx_t_5cymem_5cymem_malloc_t malloc; +}; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ +struct __pyx_obj_5cymem_5cymem_PyFree { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtab; + __pyx_t_5cymem_5cymem_free_t free; +}; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ +struct __pyx_obj_5cymem_5cymem_Pool { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtab; + size_t size; + PyObject *addresses; + PyObject *refs; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "cymem/cymem.pxd":28 + * + * + * cdef class Address: # <<<<<<<<<<<<<< + * cdef void* ptr + * cdef readonly PyMalloc pymalloc +*/ +struct __pyx_obj_5cymem_5cymem_Address { + PyObject_HEAD + void *ptr; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ +struct __pyx_obj_7preshed_4maps_PreshMap { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtab; + struct __pyx_t_7preshed_4maps_MapStruct *c_map; + struct __pyx_obj_5cymem_5cymem_Pool *mem; +}; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ +struct __pyx_obj_7preshed_4maps_PreshMapArray { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtab; + struct __pyx_obj_5cymem_5cymem_Pool *mem; + struct __pyx_t_7preshed_4maps_MapStruct *maps; + size_t length; +}; + + +/* "linalg.pxd":23 + * + * + * cdef class Matrix: # <<<<<<<<<<<<<< + * cdef readonly Pool mem + * cdef weight_t* data +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Matrix { + PyObject_HEAD + struct __pyx_obj_5cymem_5cymem_Pool *mem; + __pyx_t_5thinc_8backends_6linalg_weight_t *data; + int32_t nr_row; + int32_t nr_col; +}; + + +/* "linalg.pxd":30 + * + * + * cdef class Vec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Vec { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec *__pyx_vtab; +}; + + +/* "linalg.pxd":159 + * + * + * cdef class VecVec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void add(weight_t* output, +*/ +struct __pyx_obj_5thinc_8backends_6linalg_VecVec { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec *__pyx_vtab; +}; + + +/* "linalg.pxd":250 + * + * + * cdef class Mat: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void mean_row(weight_t* Ex, +*/ +struct __pyx_obj_5thinc_8backends_6linalg_Mat { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat *__pyx_vtab; +}; + + +/* "thinc/backends/numpy_ops.pyx":165 + * return dX, d_params + * + * def maxout(self, reals3d_ft X): # <<<<<<<<<<<<<< + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] +*/ +struct __pyx_defaults { + PyObject_HEAD + PyObject *arg0; +}; + + +/* "thinc/backends/numpy_ops.pyx":228 + * return super().backprop_mish(dY, X, threshold, inplace) + * + * def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + * sequence. The new sequence is constructed by concatenating nW preceding +*/ +struct __pyx_defaults1 { + PyObject_HEAD + __Pyx_memviewslice arg0; +}; + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ +struct __pyx_array_obj { + PyObject_HEAD + struct __pyx_vtabstruct_array *__pyx_vtab; + char *data; + Py_ssize_t len; + char *format; + int ndim; + Py_ssize_t *_shape; + Py_ssize_t *_strides; + Py_ssize_t itemsize; + PyObject *mode; + PyObject *_format; + void (*callback_free_data)(void *); + int free_data; + int dtype_is_object; +}; + + +/* "View.MemoryView":299 + * + * + * @cname('__pyx_MemviewEnum') # <<<<<<<<<<<<<< + * cdef class Enum(object): + * cdef object name +*/ +struct __pyx_MemviewEnum_obj { + PyObject_HEAD + PyObject *name; +}; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ +struct __pyx_memoryview_obj { + PyObject_HEAD + struct __pyx_vtabstruct_memoryview *__pyx_vtab; + PyObject *obj; + PyObject *_size; + void *_unused; + PyThread_type_lock lock; + __pyx_atomic_int_type acquisition_count; + Py_buffer view; + int flags; + int dtype_is_object; + __Pyx_TypeInfo const *typeinfo; +}; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ +struct __pyx_memoryviewslice_obj { + struct __pyx_memoryview_obj __pyx_base; + __Pyx_memviewslice from_slice; + PyObject *from_object; + PyObject *(*to_object_func)(char *); + int (*to_dtype_func)(char *, PyObject *); +}; + + + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyMalloc *, __pyx_t_5cymem_5cymem_malloc_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtabptr_5cymem_5cymem_PyMalloc; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyFree { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyFree *, __pyx_t_5cymem_5cymem_free_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtabptr_5cymem_5cymem_PyFree; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_Pool { + void *(*alloc)(struct __pyx_obj_5cymem_5cymem_Pool *, size_t, size_t); + void (*free)(struct __pyx_obj_5cymem_5cymem_Pool *, void *); + void *(*realloc)(struct __pyx_obj_5cymem_5cymem_Pool *, void *, size_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtabptr_5cymem_5cymem_Pool; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMap { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtabptr_7preshed_4maps_PreshMap; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtabptr_7preshed_4maps_PreshMapArray; + + +/* "linalg.pxd":30 + * + * + * cdef class Vec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec { + int (*arg_max)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); + __pyx_t_5thinc_8backends_6linalg_weight_t (*max)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*sum)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*norm)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*add)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*pow)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*pow_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); + void (*div)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*div_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); + void (*exp)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*exp_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); + void (*reciprocal_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*mean)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*variance)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec *__pyx_vtabptr_5thinc_8backends_6linalg_Vec; +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const , int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + + +/* "linalg.pxd":159 + * + * + * cdef class VecVec: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void add(weight_t* output, +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec { + void (*add)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*batch_add_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t, int32_t); + void (*add_pow)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*add_pow_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); + void (*mul)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + void (*mul_i)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + __pyx_t_5thinc_8backends_6linalg_weight_t (*dot)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); + int (*arg_max_if_true)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const *, int const ); + int (*arg_max_if_zero)(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec *__pyx_vtabptr_5thinc_8backends_6linalg_VecVec; +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t); +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *, int const *, int const ); +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int const ); + + +/* "linalg.pxd":250 + * + * + * cdef class Mat: # <<<<<<<<<<<<<< + * @staticmethod + * cdef inline void mean_row(weight_t* Ex, +*/ + +struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat { + void (*mean_row)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t); + void (*var_row)(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t, __pyx_t_5thinc_8backends_6linalg_weight_t); +}; +static struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat *__pyx_vtabptr_5thinc_8backends_6linalg_Mat; +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t); +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, __pyx_t_5thinc_8backends_6linalg_weight_t const *, int32_t, int32_t, __pyx_t_5thinc_8backends_6linalg_weight_t); + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ + +struct __pyx_vtabstruct_array { + PyObject *(*get_memview)(struct __pyx_array_obj *); +}; +static struct __pyx_vtabstruct_array *__pyx_vtabptr_array; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ + +struct __pyx_vtabstruct_memoryview { + char *(*get_item_pointer)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*is_slice)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_slice_assignment)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*setitem_slice_assign_scalar)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_indexed)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*convert_item_to_object)(struct __pyx_memoryview_obj *, char *); + PyObject *(*assign_item_from_object)(struct __pyx_memoryview_obj *, char *, PyObject *); + PyObject *(*_get_base)(struct __pyx_memoryview_obj *); +}; +static struct __pyx_vtabstruct_memoryview *__pyx_vtabptr_memoryview; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ + +struct __pyx_vtabstruct__memoryviewslice { + struct __pyx_vtabstruct_memoryview __pyx_base; +}; +static struct __pyx_vtabstruct__memoryviewslice *__pyx_vtabptr__memoryviewslice; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* UnicodeAsUCS4.proto (used by object_ord) */ +static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject*); + +/* object_ord.proto */ +#define __Pyx_PyObject_Ord(c)\ + (likely(PyUnicode_Check(c)) ? (long)__Pyx_PyUnicode_AsPy_UCS4(c) : __Pyx__PyObject_Ord(c)) +static long __Pyx__PyObject_Ord(PyObject* c); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* memoryview_get_from_buffer.proto */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_PyMemoryView_Get_itemsize(o) PyMemoryView_GET_BUFFER(o)->itemsize +#else + // can't get format like this unfortunately. It's unicode via getattr +static Py_ssize_t __Pyx_PyMemoryView_Get_itemsize(PyObject *obj); +#endif + +/* memoryview_get_from_buffer.proto */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_PyMemoryView_Get_ndim(o) PyMemoryView_GET_BUFFER(o)->ndim +#else + // can't get format like this unfortunately. It's unicode via getattr +static int __Pyx_PyMemoryView_Get_ndim(PyObject *obj); +#endif + +/* PyValueError_Check.proto */ +#define __Pyx_PyExc_ValueError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ValueError) + +/* PyTypeError_Check.proto */ +#define __Pyx_PyExc_TypeError_Check(obj) __Pyx_TypeCheck(obj, PyExc_TypeError) + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* dict_getitem_default.proto */ +static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value); + +/* CallCFunction.proto (used by CallUnboundCMethod1) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCall2Args) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCall2Args.proto (used by CallUnboundCMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod1) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod1.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg); +#else +#define __Pyx_CallUnboundCMethod1(cfunc, self, arg) __Pyx__CallUnboundCMethod1(cfunc, self, arg) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* IterFinish.proto (used by dict_iter) */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* PyObjectCallOneArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by dict_iter) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* RaiseNeedMoreValuesToUnpack.proto (used by UnpackTuple2) */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* RaiseTooManyValuesToUnpack.proto (used by UnpackItemEndCheck) */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* UnpackItemEndCheck.proto (used by UnpackTuple2) */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* RaiseNoneIterError.proto (used by UnpackTupleError) */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); + +/* UnpackTupleError.proto (used by UnpackTuple2) */ +static void __Pyx_UnpackTupleError(PyObject *, Py_ssize_t index); + +/* UnpackTuple2.proto (used by dict_iter) */ +static CYTHON_INLINE int __Pyx_unpack_tuple2( + PyObject* tuple, PyObject** value1, PyObject** value2, int is_tuple, int has_known_size, int decref_tuple); +static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( + PyObject* tuple, PyObject** value1, PyObject** value2, int decref_tuple); +static int __Pyx_unpack_tuple2_generic( + PyObject* tuple, PyObject** value1, PyObject** value2, int has_known_size, int decref_tuple); + +/* dict_iter.proto */ +static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* dict, int is_dict, PyObject* method_name, + Py_ssize_t* p_orig_length, int* p_is_dict); +static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t orig_length, Py_ssize_t* ppos, + PyObject** pkey, PyObject** pvalue, PyObject** pitem, int is_dict); + +/* PyObjectFastCallMethod.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL) +#else +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf); +#endif + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* PyMemoryError_Check.proto */ +#define __Pyx_PyExc_MemoryError_Check(obj) __Pyx_TypeCheck(obj, PyExc_MemoryError) + +/* BuildPyUnicode.proto (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char); + +/* COrdinalToPyUnicode.proto (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value); +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t width, char padding_char); + +/* GCCDiagnostics.proto (used by CIntToPyUnicode) */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* IncludeStdlibH.proto (used by CIntToPyUnicode) */ +#include + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_int(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_int(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_int(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char); + +/* JoinPyUnicode.export */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyObjectFormatSimple.proto */ +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#elif CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) :\ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) :\ + PyObject_Format(s, f)) +#else + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#endif + +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *); /*proto*/ +/* GetAttr.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ObjectGetItem.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key); +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/* RejectKeywords.export */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds); + +/* DivInt[Py_ssize_t].proto */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t, Py_ssize_t, int b_is_constant); + +/* UnaryNegOverflows.proto */ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x)\ + (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* ExtTypeTest.proto */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* PySequenceMultiply.proto */ +#define __Pyx_PySequence_Multiply_Left(mul, seq) __Pyx_PySequence_Multiply(seq, mul) +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul); + +/* PyObjectFormatAndDecref.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f); +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f); + +/* PyObjectFormat.proto */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* s, PyObject* f); +#else +#define __Pyx_PyObject_Format(s, f) PyObject_Format(s, f) +#endif + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* PyIndexError_Check.proto */ +#define __Pyx_PyExc_IndexError_Check(obj) __Pyx_TypeCheck(obj, PyExc_IndexError) + +/* DivInt[long].proto */ +static CYTHON_INLINE long __Pyx_div_long(long, long, int b_is_constant); + +/* PyImportError_Check.proto */ +#define __Pyx_PyExc_ImportError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ImportError) + +/* RealImag.proto (used by SoftComplexToDouble) */ +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #define __Pyx_CREAL(z) ((z).real()) + #define __Pyx_CIMAG(z) ((z).imag()) + #else + #define __Pyx_CREAL(z) (__real__(z)) + #define __Pyx_CIMAG(z) (__imag__(z)) + #endif +#else + #define __Pyx_CREAL(z) ((z).real) + #define __Pyx_CIMAG(z) ((z).imag) +#endif +#if defined(__cplusplus) && CYTHON_CCOMPLEX\ + && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103) + #define __Pyx_SET_CREAL(z,x) ((z).real(x)) + #define __Pyx_SET_CIMAG(z,y) ((z).imag(y)) +#else + #define __Pyx_SET_CREAL(z,x) __Pyx_CREAL(z) = (x) + #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y) +#endif + +/* SoftComplexToDouble.proto */ +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_nogil); + +/* ErrOccurredWithGIL.proto */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + +/* PyObjectDelAttr.proto (used by PyObjectSetAttrStr) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +#define __Pyx_PyObject_DelAttr(o, n) PyObject_SetAttr(o, n, NULL) +#else +#define __Pyx_PyObject_DelAttr(o, n) PyObject_DelAttr(o, n) +#endif + +/* PyObjectSetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value); +#else +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_DelAttr(o,n) +#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) +#endif + +/* PyObjectVectorCallKwBuilder.proto (used by PyObjectVectorCallMethodKwBuilder) */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* PyObjectVectorCallMethodKwBuilder.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_VectorcallMethod_CallFromBuilder PyObject_VectorcallMethod +#else +static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#endif + +/* PyDictContains.proto */ +static CYTHON_INLINE int __Pyx_PyDict_ContainsTF(PyObject* item, PyObject* dict, int eq) { + int result = PyDict_Contains(dict, item); + return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); +} + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_long(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_long(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_long(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_long(long value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_long(long value, Py_ssize_t width, char padding_char, char format_char); + +/* BufferIndexError.proto */ +static void __Pyx_RaiseBufferIndexError(int axis); + +/* PyLongCompare.proto */ +static CYTHON_INLINE int __Pyx_PyLong_BoolNeObjC(PyObject *op1, PyObject *op2, long intval, long inplace); + +/* IsLittleEndian.proto (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void); + +/* BufferFormatCheck.proto (used by BufferGetAndValidate) */ +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts); +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type); + +/* BufferGetAndValidate.proto */ +#define __Pyx_GetBufferAndValidate(buf, obj, dtype, flags, nd, cast, stack)\ + ((obj == Py_None || obj == NULL) ?\ + (__Pyx_ZeroBuffer(buf), 0) :\ + __Pyx__GetBufferAndValidate(buf, obj, dtype, flags, nd, cast, stack)) +static int __Pyx__GetBufferAndValidate(Py_buffer* buf, PyObject* obj, + const __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack); +static void __Pyx_ZeroBuffer(Py_buffer* buf); +static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info); +static Py_ssize_t __Pyx_minusones[] = { -1, -1, -1, -1, -1, -1, -1, -1 }; +static Py_ssize_t __Pyx_zeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +/* SliceObject.proto */ +#define __Pyx_PyObject_DelSlice(obj, cstart, cstop, py_start, py_stop, py_slice, has_cstart, has_cstop, wraparound)\ + __Pyx_PyObject_SetSlice(obj, (PyObject*)NULL, cstart, cstop, py_start, py_stop, py_slice, has_cstart, has_cstop, wraparound) +static CYTHON_INLINE int __Pyx_PyObject_SetSlice( + PyObject* obj, PyObject* value, Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** py_start, PyObject** py_stop, PyObject** py_slice, + int has_cstart, int has_cstop, int wraparound); + +/* SliceObject.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice( + PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** py_start, PyObject** py_stop, PyObject** py_slice, + int has_cstart, int has_cstop, int wraparound); + +/* ListAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_PyList_Append(L,x) PyList_Append(L,x) +#endif + +/* PyObjectCallMethod1.proto (used by append) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* append.proto */ +static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x); + +/* PyLongCompare.proto */ +static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace); + +/* PyRange_Check.proto */ +#if CYTHON_COMPILING_IN_PYPY && !defined(PyRange_Check) + #define PyRange_Check(obj) __Pyx_TypeCheck((obj), &PyRange_Type) +#endif + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* DefaultPlacementNew.proto */ +#include +template +void __Pyx_default_placement_construct(T* x) { + new (static_cast(x)) T(); +} + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* DelItemOnTypeDict.proto (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k); +#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k) + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_1 +#define __PYX_HAVE_RT_ImportType_proto_3_2_1 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_2_1 { + __Pyx_ImportType_CheckSize_Error_3_2_1 = 0, + __Pyx_ImportType_CheckSize_Warn_3_2_1 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_2_1 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size); +#endif + +/* FunctionImport.proto */ +static int __Pyx_ImportFunction_3_2_1(PyObject *module, const char *funcname, void (**f)(void), const char *sig); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* Py3UpdateBases.proto */ +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); + +/* CalculateMetaclass.proto */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases); + +/* SetNameInClass.proto */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? _PyDict_SetItem_KnownHash(ns, name, value, ((PyASCIIObject *) name)->hash) : PyObject_SetItem(ns, name, value)) +#elif CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? PyDict_SetItem(ns, name, value) : PyObject_SetItem(ns, name, value)) +#else +#define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) +#endif + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* FusedFunction.proto */ +typedef struct { + __pyx_CyFunctionObject func; + PyObject *__signatures__; + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyMethodDef *ml; +#endif +} __pyx_FusedFunctionObject; +static PyObject *__pyx_FusedFunction_New(PyMethodDef *ml, int flags, + PyObject *qualname, PyObject *closure, + PyObject *module, PyObject *globals, + PyObject *code); +static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self); +static int __pyx_FusedFunction_init(PyObject *module); +#define __Pyx_FusedFunction_USED + +/* PyObjectLookupSpecial.proto (used by Py3ClassCreate) */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* Py3ClassCreate.proto */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, + PyObject *mkw, PyObject *modname, PyObject *doc); +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, + PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); + +/* CyFunctionClassCell.proto */ +static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* BufferStructDeclare.proto */ +typedef struct { + Py_ssize_t shape, strides, suboffsets; +} __Pyx_Buf_DimInfo; +typedef struct { + size_t refcount; + Py_buffer pybuffer; +} __Pyx_Buffer; +typedef struct { + __Pyx_Buffer *rcbuffer; + char *data; + __Pyx_Buf_DimInfo diminfo[8]; +} __Pyx_LocalBuf_ND; + +/* MemviewRefcount.proto */ +static CYTHON_INLINE int __pyx_add_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +#define __pyx_get_slice_count_pointer(memview) (&memview->acquisition_count) +#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) +static CYTHON_INLINE void __Pyx_INC_MEMVIEW(__Pyx_memviewslice *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *, int, int); + +/* MemviewSliceIsContig.proto */ +static int __pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim); + +/* OverlappingSlices.proto */ +static int __pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize); + +/* MemviewSliceInit.proto */ +static int __Pyx_init_memviewslice( + struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference); + +/* TypeInfoCompare.proto (used by MemviewSliceValidateAndInit) */ +static int __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b); + +/* MemviewSliceValidateAndInit.export */ +static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_int(PyObject *, int writable_flag); + +/* MemviewDtypeToObject.proto */ +static CYTHON_INLINE PyObject *__pyx_memview_get_int(const char *itemp); +static CYTHON_INLINE int __pyx_memview_set_int(char *itemp, PyObject *obj); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_nn_uint64_t__const__(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_nn___pyx_t_5thinc_8backends_9numpy_ops_weight_t(PyObject *, int writable_flag); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_float(PyObject *, int writable_flag); + +/* MemviewDtypeToObject.proto */ +static CYTHON_INLINE PyObject *__pyx_memview_get_float(const char *itemp); +static CYTHON_INLINE int __pyx_memview_set_float(char *itemp, PyObject *obj); + +/* MemviewDtypeToObject.proto */ +static CYTHON_INLINE PyObject *__pyx_memview_get_double(const char *itemp); +static CYTHON_INLINE int __pyx_memview_set_double(char *itemp, PyObject *obj); + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* MemviewDtypeToObject.proto */ +static CYTHON_INLINE PyObject *__pyx_memview_get_nn_uint64_t__const__(const char *itemp); + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_double(a, b) ((a)==(b)) + #define __Pyx_c_sum_double(a, b) ((a)+(b)) + #define __Pyx_c_diff_double(a, b) ((a)-(b)) + #define __Pyx_c_prod_double(a, b) ((a)*(b)) + #define __Pyx_c_quot_double(a, b) ((a)/(b)) + #define __Pyx_c_neg_double(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_double(z) ((z)==(double)0) + #define __Pyx_c_conj_double(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (::std::abs(z)) + #define __Pyx_c_pow_double(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_double(z) ((z)==0) + #define __Pyx_c_conj_double(z) (conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (cabs(z)) + #define __Pyx_c_pow_double(a, b) (cpow(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex); + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex, __pyx_t_double_complex); + #endif +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_float(a, b) ((a)==(b)) + #define __Pyx_c_sum_float(a, b) ((a)+(b)) + #define __Pyx_c_diff_float(a, b) ((a)-(b)) + #define __Pyx_c_prod_float(a, b) ((a)*(b)) + #define __Pyx_c_quot_float(a, b) ((a)/(b)) + #define __Pyx_c_neg_float(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_float(z) ((z)==(float)0) + #define __Pyx_c_conj_float(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_float(z) (::std::abs(z)) + #define __Pyx_c_pow_float(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_float(z) ((z)==0) + #define __Pyx_c_conj_float(z) (conjf(z)) + #if 1 + #define __Pyx_c_abs_float(z) (cabsf(z)) + #define __Pyx_c_pow_float(a, b) (cpowf(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sum_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_diff_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prod_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_neg_float(__pyx_t_float_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_float(__pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conj_float(__pyx_t_float_complex); + #if 1 + static CYTHON_INLINE float __Pyx_c_abs_float(__pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_pow_float(__pyx_t_float_complex, __pyx_t_float_complex); + #endif +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_long__double(a, b) ((a)==(b)) + #define __Pyx_c_sum_long__double(a, b) ((a)+(b)) + #define __Pyx_c_diff_long__double(a, b) ((a)-(b)) + #define __Pyx_c_prod_long__double(a, b) ((a)*(b)) + #define __Pyx_c_quot_long__double(a, b) ((a)/(b)) + #define __Pyx_c_neg_long__double(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_long__double(z) ((z)==(long double)0) + #define __Pyx_c_conj_long__double(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_long__double(z) (::std::abs(z)) + #define __Pyx_c_pow_long__double(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_long__double(z) ((z)==0) + #define __Pyx_c_conj_long__double(z) (conjl(z)) + #if 1 + #define __Pyx_c_abs_long__double(z) (cabsl(z)) + #define __Pyx_c_pow_long__double(a, b) (cpowl(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_sum_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_diff_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_prod_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_neg_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_conj_long__double(__pyx_t_long_double_complex); + #if 1 + static CYTHON_INLINE long double __Pyx_c_abs_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_pow_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + #endif +#endif + +/* MemviewSliceCopy.proto */ +static __Pyx_memviewslice +__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object); + +/* ImportNumPyArray.proto */ +static PyObject* __Pyx_ImportNumPyArrayTypeIfAvailable(void); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE uint32_t __Pyx_PyLong_As_uint32_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_uint64_t(uint64_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_npy_intp(npy_intp value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE npy_intp __Pyx_PyLong_As_npy_intp(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *); + +/* PyUCS4InUnicode.proto */ +static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character); + +/* UpdateUnpickledDict.proto */ +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index); + +/* CheckUnpickleChecksum.proto */ +static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members); + +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self); /* proto*/ +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto*/ +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self); /* proto*/ +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_8itemsize_itemsize(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_9alignment_alignment(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_6fields_fields(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_5names_names(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyArray_ArrayDescr *__pyx_f_5numpy_5dtype_8subarray_subarray(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_uint64 __pyx_f_5numpy_5dtype_5flags_flags(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_7numiter_numiter(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_4size_size(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_5index_index(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_2nd_nd(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_9broadcast_10dimensions_dimensions(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE void **__pyx_f_5numpy_9broadcast_5iters_iters(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_7ndarray_4base_base(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyArray_Descr *__pyx_f_5numpy_7ndarray_5descr_descr(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_7ndarray_4ndim_ndim(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_5shape_shape(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_7strides_strides(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_7ndarray_4size_size(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE char *__pyx_f_5numpy_7ndarray_4data_data(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr, int32_t __pyx_v_nr_batch); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr); /* proto*/ +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_costs, int const __pyx_v_n_classes); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Ex, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Vx, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_Ex, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_eps); /* proto*/ + +/* Module declarations from "libcpp" */ + +/* Module declarations from "libcpp.memory" */ + +/* Module declarations from "thinc.backends.cblas" */ +static __pyx_t_5thinc_8backends_5cblas_daxpy_ptr (*__pyx_f_5thinc_8backends_5cblas_daxpy)(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *); /*proto*/ +static __pyx_t_5thinc_8backends_5cblas_saxpy_ptr (*__pyx_f_5thinc_8backends_5cblas_saxpy)(struct __pyx_obj_5thinc_8backends_5cblas_CBlas *); /*proto*/ + +/* Module declarations from "cython.view" */ + +/* Module declarations from "cython.dataclasses" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "blis.cy" */ +static void (*__pyx_fuse_0__pyx_f_4blis_2cy_gemm)(enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t); /*proto*/ +static void (*__pyx_fuse_1__pyx_f_4blis_2cy_gemm)(enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t); /*proto*/ +static void (*__pyx_fuse_2__pyx_f_4blis_2cy_gemm)(enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t); /*proto*/ +static void (*__pyx_fuse_3__pyx_f_4blis_2cy_gemm)(enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t); /*proto*/ + +/* Module declarations from "blis" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "libc.stdio" */ + +/* Module declarations from "__builtin__" */ + +/* Module declarations from "cpython.type" */ + +/* Module declarations from "cpython" */ + +/* Module declarations from "cpython.object" */ + +/* Module declarations from "cpython.ref" */ + +/* Module declarations from "numpy" */ + +/* Module declarations from "numpy" */ + +/* Module declarations from "cymem.cymem" */ + +/* Module declarations from "libc.math" */ + +/* Module declarations from "libc.stdlib" */ + +/* Module declarations from "murmurhash.mrmr" */ +static uint64_t (*__pyx_f_10murmurhash_4mrmr_hash64)(void *, int, uint64_t); /*proto*/ + +/* Module declarations from "preshed.maps" */ + +/* Module declarations from "thinc.backends.linalg" */ + +/* Module declarations from "thinc.backends.numpy_ops" */ +static PyObject *__pyx_collections_abc_Sequence = 0; +static PyObject *generic = 0; +static PyObject *strided = 0; +static PyObject *indirect = 0; +static PyObject *contiguous = 0; +static PyObject *indirect_contiguous = 0; +static int __pyx_memoryview_thread_locks_used; +static PyThread_type_lock __pyx_memoryview_thread_locks[8]; +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_position_encode(float *, float, int, int); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_scatter_add(float *, int const *, float const *, int, int); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops__adam_momentum(__pyx_t_5thinc_8backends_9numpy_ops_weight_t *, __pyx_t_5thinc_8backends_9numpy_ops_weight_t *, __pyx_t_5thinc_8backends_9numpy_ops_weight_t *, int, __pyx_t_5thinc_8backends_9numpy_ops_weight_t, __pyx_t_5thinc_8backends_9numpy_ops_weight_t, __pyx_t_5thinc_8backends_9numpy_ops_weight_t, __pyx_t_5thinc_8backends_9numpy_ops_weight_t); /*proto*/ +static int __pyx_f_5thinc_8backends_9numpy_ops__lstm_forward_training(int, int, int, int, int, PyArrayObject *, float *, float *, float *, float *, float *, PyArrayObject *, int *, float *, float *); /*proto*/ +static int __pyx_f_5thinc_8backends_9numpy_ops__lstm_backward_training(int, int, int, int, int, float *, float *, float *, float *, float *, float *, float *, float const *, float const *, float const *, float const *, float const *, float const *, PyObject *); /*proto*/ +static CYTHON_INLINE float __pyx_f_5thinc_8backends_9numpy_ops_dsigmoid(float); /*proto*/ +static CYTHON_INLINE float __pyx_f_5thinc_8backends_9numpy_ops_dtanh(float); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_activate_fwd(float *, int, int); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_fwd(float *, float *, float const *, float const *, int, int); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_bwd(float *, float *, float const *, float const *, float const *, float const *, float const *, int); /*proto*/ +static void __pyx_f_5thinc_8backends_9numpy_ops_MurmurHash3_x86_128_uint64(uint64_t const , uint32_t const , uint32_t *); /*proto*/ +static CYTHON_INLINE PyArrayObject *__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(PyArrayObject *, PyObject *); /*proto*/ +static PyObject *__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc(PyObject *, PyTypeObject *); /*proto*/ +static PyObject *__pyx_ff_match_signatures_single(PyObject *, PyObject *); /*proto*/ +static PyObject *__pyx_ff_index_signature(PyObject *, PyObject *, PyObject *); /*proto*/ +static PyObject *__pyx_ff_match_signatures(PyObject *, PyObject *, PyObject *); /*proto*/ +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *); /*proto*/ +static struct __pyx_array_obj *__pyx_array_new(PyObject *, Py_ssize_t, char *, char const *, char *); /*proto*/ +static PyObject *__pyx_memoryview_new(PyObject *, int, int, __Pyx_TypeInfo const *); /*proto*/ +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *); /*proto*/ +static PyObject *_unellipsify(PyObject *, int); /*proto*/ +static int assert_direct_dimensions(Py_ssize_t *, int); /*proto*/ +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *, PyObject *); /*proto*/ +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int, int); /*proto*/ +static char *__pyx_pybuffer_index(Py_buffer *, char *, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memslice_transpose(__Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice, int, PyObject *(*)(char *), int (*)(char *, PyObject *), int); /*proto*/ +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static Py_ssize_t abs_py_ssize_t(Py_ssize_t); /*proto*/ +static char __pyx_get_best_slice_order(__Pyx_memviewslice *, int); /*proto*/ +static void _copy_strided_to_strided(char *, Py_ssize_t *, char *, Py_ssize_t *, Py_ssize_t *, Py_ssize_t *, int, size_t); /*proto*/ +static void copy_strided_to_strided(__Pyx_memviewslice *, __Pyx_memviewslice *, int, size_t); /*proto*/ +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *, int); /*proto*/ +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *, Py_ssize_t *, Py_ssize_t, int, char); /*proto*/ +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *, __Pyx_memviewslice *, char, int); /*proto*/ +static int __pyx_memoryview_err_extents(int, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memoryview_err_dim(PyObject *, PyObject *, int); /*proto*/ +static int __pyx_memoryview_err(PyObject *, PyObject *); /*proto*/ +static int __pyx_memoryview_err_no_memory(void); /*proto*/ +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice, __Pyx_memviewslice, int, int, int); /*proto*/ +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *, int, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *, int, size_t, void *, int); /*proto*/ +static void __pyx_memoryview__slice_assign_scalar(char *, Py_ssize_t *, Py_ssize_t *, int, size_t, void *); /*proto*/ +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +static const __Pyx_TypeInfo __Pyx_TypeInfo_nn_uint32_t = { "uint32_t", NULL, sizeof(uint32_t), { 0 }, 0, __PYX_IS_UNSIGNED(uint32_t) ? 'U' : 'I', __PYX_IS_UNSIGNED(uint32_t), 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t = { "float32_t", NULL, sizeof(__pyx_t_5numpy_float32_t), { 0 }, 0, 'R', 0, 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_float = { "float", NULL, sizeof(float), { 0 }, 0, 'R', 0, 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_double = { "double", NULL, sizeof(double), { 0 }, 0, 'R', 0, 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_int = { "int", NULL, sizeof(int), { 0 }, 0, __PYX_IS_UNSIGNED(int) ? 'U' : 'I', __PYX_IS_UNSIGNED(int), 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_unsigned_int = { "unsigned int", NULL, sizeof(unsigned int), { 0 }, 0, __PYX_IS_UNSIGNED(unsigned int) ? 'U' : 'I', __PYX_IS_UNSIGNED(unsigned int), 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_nn_uint64_t__const__ = { "const uint64_t", NULL, sizeof(uint64_t const ), { 0 }, 0, __PYX_IS_UNSIGNED(uint64_t const ) ? 'U' : 'I', __PYX_IS_UNSIGNED(uint64_t const ), 0 }; +static const __Pyx_TypeInfo __Pyx_TypeInfo_nn___pyx_t_5thinc_8backends_9numpy_ops_weight_t = { "weight_t", NULL, sizeof(__pyx_t_5thinc_8backends_9numpy_ops_weight_t), { 0 }, 0, 'R', 0, 0 }; +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.backends.numpy_ops" +extern int __pyx_module_is_main_thinc__backends__numpy_ops; +int __pyx_module_is_main_thinc__backends__numpy_ops = 0; + +/* Implementation of "thinc.backends.numpy_ops" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_super; +static PyObject *__pyx_builtin_reversed; +static PyObject *__pyx_builtin_zip; +static PyObject *__pyx_builtin___import__; +static PyObject *__pyx_builtin_enumerate; +static PyObject *__pyx_builtin_Ellipsis; +static PyObject *__pyx_builtin_id; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object); /* proto */ +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_device_type, PyObject *__pyx_v_device_id, PyObject *__pyx_v_use_blis); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_2asarray(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_data, PyObject *__pyx_v_dtype); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_4alloc(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_shape, PyObject *__pyx_v_dtype, PyObject *__pyx_v_zeros); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_6cblas(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_8gemm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_x, PyArrayObject *__pyx_v_y, PyArrayObject *__pyx_v_out, PyObject *__pyx_v_trans1, PyObject *__pyx_v_trans2); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_10relu(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_inplace); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_12backprop_relu(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_Y, PyObject *__pyx_v_inplace); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_14lstm_forward_training(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_H0, PyArrayObject *__pyx_v_C0, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_size_at_t); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_16lstm_forward_inference(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_H0, PyArrayObject *__pyx_v_C0, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_size_at_t); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_18backprop_lstm(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_lengths, PyArrayObject *__pyx_v_params, PyObject *__pyx_v_fwd_state); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_56maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float3d_t __pyx_v_X); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_58maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double3d_t __pyx_v_X); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_62backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_dY, __Pyx_memviewslice __pyx_v_which, int __pyx_v_P); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_64backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_dY, __Pyx_memviewslice __pyx_v_which, int __pyx_v_P); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_24mish(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_threshold, PyObject *__pyx_v_inplace); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_26backprop_mish(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_threshold, PyObject *__pyx_v_inplace); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_20__defaults__(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_28seq2col(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_seq, int __pyx_v_nW, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_22__defaults__(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_30backprop_seq2col(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, int __pyx_v_nW, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_32hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __Pyx_memviewslice __pyx_v_ids, uint32_t __pyx_v_seed); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_68reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_70reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_74backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_means, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_76backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_means, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_80reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_82reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_86backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_sums, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_88backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_sums, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_92reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_94reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_98backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_maxes, __Pyx_memviewslice __pyx_v_which, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_100backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_maxes, __Pyx_memviewslice __pyx_v_which, __Pyx_memviewslice __pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, PyObject *__pyx_v__fused_sigindex); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_104gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_106gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_108gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_110gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_48scatter_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_table, PyArrayObject *__pyx_v_indices, PyArrayObject *__pyx_v_values); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_50adam(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_weights, PyArrayObject *__pyx_v_gradient, PyArrayObject *__pyx_v_mom1, PyArrayObject *__pyx_v_mom2, float __pyx_v_beta1, float __pyx_v_beta2, float __pyx_v_eps, float __pyx_v_learn_rate, CYTHON_UNUSED float __pyx_v_mod_rate); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_52ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, int __pyx_v_n, __Pyx_memviewslice __pyx_v_keys); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_54position_encode(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, int __pyx_v_N, int __pyx_v_D, int __pyx_v_period, PyObject *__pyx_v_out); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_check_seq2col_lengths(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_ops, PyObject *__pyx_v_lengths, PyObject *__pyx_v_B); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_2cpu_clip_gradient(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_memviewslice __pyx_v_gradient, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_threshold); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_4add_gradient_noise(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_memviewslice __pyx_v_gradient, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_noise_level, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_timestep); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_6lstm_forward_training(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_c_init, PyArrayObject *__pyx_v_h_init, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8backprop_lstm(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_lengths, PyArrayObject *__pyx_v_params, PyObject *__pyx_v_fwd_state); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_10_split_directions(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_X, PyObject *__pyx_v_dirs); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_12_split_weights(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_params, CYTHON_UNUSED int __pyx_v_i, int __pyx_v_nO, int __pyx_v_nI, int __pyx_v_params_i); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_14_transpose_weights(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_params); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_16_untranspose_unsplit_weights(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_params); /* proto */ +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_18_check_compatible_shape(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_u, PyObject *__pyx_v_v); /* proto */ +static PyObject *__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults1(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyTypeObject *__pyx_ptype_5thinc_8backends_5cblas_CBlas; + PyTypeObject *__pyx_ptype_7cpython_4type_type; + PyTypeObject *__pyx_ptype_5numpy_dtype; + PyTypeObject *__pyx_ptype_5numpy_flatiter; + PyTypeObject *__pyx_ptype_5numpy_broadcast; + PyTypeObject *__pyx_ptype_5numpy_ndarray; + PyTypeObject *__pyx_ptype_5numpy_generic; + PyTypeObject *__pyx_ptype_5numpy_number; + PyTypeObject *__pyx_ptype_5numpy_integer; + PyTypeObject *__pyx_ptype_5numpy_signedinteger; + PyTypeObject *__pyx_ptype_5numpy_unsignedinteger; + PyTypeObject *__pyx_ptype_5numpy_inexact; + PyTypeObject *__pyx_ptype_5numpy_floating; + PyTypeObject *__pyx_ptype_5numpy_complexfloating; + PyTypeObject *__pyx_ptype_5numpy_flexible; + PyTypeObject *__pyx_ptype_5numpy_character; + PyTypeObject *__pyx_ptype_5numpy_ufunc; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyMalloc; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyFree; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Pool; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Address; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMap; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMapArray; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Matrix; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Vec; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_VecVec; + PyTypeObject *__pyx_ptype_5thinc_8backends_6linalg_Mat; + PyObject *__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults; + PyObject *__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1; + PyObject *__pyx_type___pyx_array; + PyObject *__pyx_type___pyx_MemviewEnum; + PyObject *__pyx_type___pyx_memoryview; + PyObject *__pyx_type___pyx_memoryviewslice; + PyTypeObject *__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults; + PyTypeObject *__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1; + PyTypeObject *__pyx_array_type; + PyTypeObject *__pyx_MemviewEnum_type; + PyTypeObject *__pyx_memoryview_type; + PyTypeObject *__pyx_memoryviewslice_type; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_get; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_slice[3]; + PyObject *__pyx_tuple[11]; + PyObject *__pyx_codeobj_tab[58]; + PyObject *__pyx_string_tab[509]; + PyObject *__pyx_number_tab[8]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + +/* FusedFunction.module_state_decls */ +PyTypeObject *__pyx_FusedFunctionType; + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* ImportNumPyArray.module_state_decls */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS +__pyx_atomic_ptr_type __pyx_numpy_ndarray; +#else +PyObject *__pyx_numpy_ndarray; +#endif + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_All_dimensions_preceding_dimensi __pyx_string_tab[1] +#define __pyx_kp_u_All_sequence_lengths_must_be_0 __pyx_string_tab[2] +#define __pyx_kp_u_BLIS_support_requires_blis_pip_i __pyx_string_tab[3] +#define __pyx_kp_u_Buffer_view_does_not_expose_stri __pyx_string_tab[4] +#define __pyx_kp_u_Can_only_create_a_buffer_that_is __pyx_string_tab[5] +#define __pyx_kp_u_Cannot_assign_to_read_only_memor __pyx_string_tab[6] +#define __pyx_kp_u_Cannot_create_writable_memory_vi __pyx_string_tab[7] +#define __pyx_kp_u_Cannot_index_with_type __pyx_string_tab[8] +#define __pyx_kp_u_Cannot_transpose_memoryview_with __pyx_string_tab[9] +#define __pyx_kp_u_Dimension_d_is_not_direct __pyx_string_tab[10] +#define __pyx_kp_u_Empty_shape_tuple_for_cython_arr __pyx_string_tab[11] +#define __pyx_kp_u_Expected_at_least __pyx_string_tab[12] +#define __pyx_kp_u_Function_call_with_ambiguous_arg __pyx_string_tab[13] +#define __pyx_kp_u_Index_out_of_bounds_axis_d __pyx_string_tab[14] +#define __pyx_kp_u_Indirect_dimensions_not_supporte __pyx_string_tab[15] +#define __pyx_kp_u_Invalid_mode_expected_c_or_fortr __pyx_string_tab[16] +#define __pyx_kp_u_Invalid_shape_in_axis __pyx_string_tab[17] +#define __pyx_kp_u_MemoryView_of __pyx_string_tab[18] +#define __pyx_kp_u_No_matching_signature_found __pyx_string_tab[19] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[20] +#define __pyx_kp_u_Optional_DTypes __pyx_string_tab[21] +#define __pyx_kp_u_Out_of_bounds_on_buffer_access_a __pyx_string_tab[22] +#define __pyx_kp_u_Provided_x_array_should_be_2_dim __pyx_string_tab[23] +#define __pyx_kp_u_Provided_y_array_should_be_2_dim __pyx_string_tab[24] +#define __pyx_kp_u_Step_may_not_be_zero_axis_d __pyx_string_tab[25] +#define __pyx_kp_u_The_lengths_must_sum_up_to_the_b __pyx_string_tab[26] +#define __pyx_kp_u_Unable_to_convert_item_to_object __pyx_string_tab[27] +#define __pyx_kp_u__2 __pyx_string_tab[28] +#define __pyx_kp_u__3 __pyx_string_tab[29] +#define __pyx_kp_u__4 __pyx_string_tab[30] +#define __pyx_kp_u__5 __pyx_string_tab[31] +#define __pyx_kp_u__6 __pyx_string_tab[32] +#define __pyx_kp_u__7 __pyx_string_tab[33] +#define __pyx_kp_u__9 __pyx_string_tab[34] +#define __pyx_kp_u_add_note __pyx_string_tab[35] +#define __pyx_kp_u_all_sequence_lengths_must_be_0_g __pyx_string_tab[36] +#define __pyx_kp_u_all_sequence_lengths_must_be_0_g_2 __pyx_string_tab[37] +#define __pyx_kp_u_and __pyx_string_tab[38] +#define __pyx_kp_u_arguments_got __pyx_string_tab[39] +#define __pyx_kp_u_arrays_have_incompatible_shapes __pyx_string_tab[40] +#define __pyx_kp_u_at_0x __pyx_string_tab[41] +#define __pyx_kp_u_backprop_seq2col_requires_gradie __pyx_string_tab[42] +#define __pyx_kp_u_backprop_seq2col_requires_gradie_2 __pyx_string_tab[43] +#define __pyx_kp_u_collections_abc __pyx_string_tab[44] +#define __pyx_kp_u_contiguous_and_direct __pyx_string_tab[45] +#define __pyx_kp_u_contiguous_and_indirect __pyx_string_tab[46] +#define __pyx_kp_u_dimension_s __pyx_string_tab[47] +#define __pyx_kp_u_disable __pyx_string_tab[48] +#define __pyx_kp_u_double2d_t_int2d_t __pyx_string_tab[49] +#define __pyx_kp_u_double2d_t_uint2d_t __pyx_string_tab[50] +#define __pyx_kp_u_enable __pyx_string_tab[51] +#define __pyx_kp_u_float2d_t_int2d_t __pyx_string_tab[52] +#define __pyx_kp_u_float2d_t_uint2d_t __pyx_string_tab[53] +#define __pyx_kp_u_gc __pyx_string_tab[54] +#define __pyx_kp_u_got __pyx_string_tab[55] +#define __pyx_kp_u_got_differing_extents_in_dimensi __pyx_string_tab[56] +#define __pyx_kp_u_isenabled __pyx_string_tab[57] +#define __pyx_kp_u_itemsize_0_for_cython_array __pyx_string_tab[58] +#define __pyx_kp_u_no_default___reduce___due_to_non __pyx_string_tab[59] +#define __pyx_kp_u_np_ndarray __pyx_string_tab[60] +#define __pyx_kp_u_numpy__core_multiarray_failed_to __pyx_string_tab[61] +#define __pyx_kp_u_numpy__core_umath_failed_to_impo __pyx_string_tab[62] +#define __pyx_kp_u_object __pyx_string_tab[63] +#define __pyx_kp_u_self_name_is_not_None __pyx_string_tab[64] +#define __pyx_kp_u_seq2col_requires_sequence_array __pyx_string_tab[65] +#define __pyx_kp_u_seq2col_requires_sequence_array_2 __pyx_string_tab[66] +#define __pyx_kp_u_strided_and_direct __pyx_string_tab[67] +#define __pyx_kp_u_strided_and_direct_or_indirect __pyx_string_tab[68] +#define __pyx_kp_u_strided_and_indirect __pyx_string_tab[69] +#define __pyx_kp_u_thinc __pyx_string_tab[70] +#define __pyx_kp_u_thinc_backends_blis_py __pyx_string_tab[71] +#define __pyx_kp_u_thinc_backends_collections_abc __pyx_string_tab[72] +#define __pyx_kp_u_thinc_backends_numpy __pyx_string_tab[73] +#define __pyx_kp_u_thinc_backends_numpy_ops_pyx __pyx_string_tab[74] +#define __pyx_kp_u_thinc_backends_ops __pyx_string_tab[75] +#define __pyx_kp_u_thinc_backends_typing __pyx_string_tab[76] +#define __pyx_kp_u_thinc_types __pyx_string_tab[77] +#define __pyx_kp_u_thinc_util __pyx_string_tab[78] +#define __pyx_kp_u_unable_to_allocate_array_data __pyx_string_tab[79] +#define __pyx_kp_u_unable_to_allocate_shape_and_str __pyx_string_tab[80] +#define __pyx_n_u_ASCII __pyx_string_tab[81] +#define __pyx_n_u_ArrayXd __pyx_string_tab[82] +#define __pyx_n_u_B __pyx_string_tab[83] +#define __pyx_n_u_C __pyx_string_tab[84] +#define __pyx_n_u_C0 __pyx_string_tab[85] +#define __pyx_n_u_CBlas __pyx_string_tab[86] +#define __pyx_n_u_Cid __pyx_string_tab[87] +#define __pyx_n_u_Ct2 __pyx_string_tab[88] +#define __pyx_n_u_D __pyx_string_tab[89] +#define __pyx_n_u_DTypes __pyx_string_tab[90] +#define __pyx_n_u_DeviceTypes __pyx_string_tab[91] +#define __pyx_n_u_Ellipsis __pyx_string_tab[92] +#define __pyx_n_u_G __pyx_string_tab[93] +#define __pyx_n_u_Gid __pyx_string_tab[94] +#define __pyx_n_u_H __pyx_string_tab[95] +#define __pyx_n_u_H0 __pyx_string_tab[96] +#define __pyx_n_u_I __pyx_string_tab[97] +#define __pyx_n_u_N __pyx_string_tab[98] +#define __pyx_n_u_None __pyx_string_tab[99] +#define __pyx_n_u_NumpyOps __pyx_string_tab[100] +#define __pyx_n_u_NumpyOps___init __pyx_string_tab[101] +#define __pyx_n_u_NumpyOps_adam __pyx_string_tab[102] +#define __pyx_n_u_NumpyOps_alloc __pyx_string_tab[103] +#define __pyx_n_u_NumpyOps_asarray __pyx_string_tab[104] +#define __pyx_n_u_NumpyOps_backprop_lstm __pyx_string_tab[105] +#define __pyx_n_u_NumpyOps_backprop_maxout __pyx_string_tab[106] +#define __pyx_n_u_NumpyOps_backprop_maxout_double2 __pyx_string_tab[107] +#define __pyx_n_u_NumpyOps_backprop_maxout_float2d __pyx_string_tab[108] +#define __pyx_n_u_NumpyOps_backprop_mish __pyx_string_tab[109] +#define __pyx_n_u_NumpyOps_backprop_reduce_max __pyx_string_tab[110] +#define __pyx_n_u_NumpyOps_backprop_reduce_max_dou __pyx_string_tab[111] +#define __pyx_n_u_NumpyOps_backprop_reduce_max_flo __pyx_string_tab[112] +#define __pyx_n_u_NumpyOps_backprop_reduce_mean __pyx_string_tab[113] +#define __pyx_n_u_NumpyOps_backprop_reduce_mean_do __pyx_string_tab[114] +#define __pyx_n_u_NumpyOps_backprop_reduce_mean_fl __pyx_string_tab[115] +#define __pyx_n_u_NumpyOps_backprop_reduce_sum __pyx_string_tab[116] +#define __pyx_n_u_NumpyOps_backprop_reduce_sum_dou __pyx_string_tab[117] +#define __pyx_n_u_NumpyOps_backprop_reduce_sum_flo __pyx_string_tab[118] +#define __pyx_n_u_NumpyOps_backprop_relu __pyx_string_tab[119] +#define __pyx_n_u_NumpyOps_backprop_seq2col __pyx_string_tab[120] +#define __pyx_n_u_NumpyOps_cblas __pyx_string_tab[121] +#define __pyx_n_u_NumpyOps_gather_add __pyx_string_tab[122] +#define __pyx_n_u_NumpyOps_gather_add_double2d_t_i __pyx_string_tab[123] +#define __pyx_n_u_NumpyOps_gather_add_double2d_t_u __pyx_string_tab[124] +#define __pyx_n_u_NumpyOps_gather_add_float2d_t_in __pyx_string_tab[125] +#define __pyx_n_u_NumpyOps_gather_add_float2d_t_ui __pyx_string_tab[126] +#define __pyx_n_u_NumpyOps_gemm __pyx_string_tab[127] +#define __pyx_n_u_NumpyOps_hash __pyx_string_tab[128] +#define __pyx_n_u_NumpyOps_lstm_forward_inference __pyx_string_tab[129] +#define __pyx_n_u_NumpyOps_lstm_forward_training __pyx_string_tab[130] +#define __pyx_n_u_NumpyOps_maxout __pyx_string_tab[131] +#define __pyx_n_u_NumpyOps_maxout_double3d_t __pyx_string_tab[132] +#define __pyx_n_u_NumpyOps_maxout_float3d_t __pyx_string_tab[133] +#define __pyx_n_u_NumpyOps_mish __pyx_string_tab[134] +#define __pyx_n_u_NumpyOps_ngrams __pyx_string_tab[135] +#define __pyx_n_u_NumpyOps_position_encode __pyx_string_tab[136] +#define __pyx_n_u_NumpyOps_reduce_max __pyx_string_tab[137] +#define __pyx_n_u_NumpyOps_reduce_max_double2d_t __pyx_string_tab[138] +#define __pyx_n_u_NumpyOps_reduce_max_float2d_t __pyx_string_tab[139] +#define __pyx_n_u_NumpyOps_reduce_mean __pyx_string_tab[140] +#define __pyx_n_u_NumpyOps_reduce_mean_double2d_t __pyx_string_tab[141] +#define __pyx_n_u_NumpyOps_reduce_mean_float2d_t __pyx_string_tab[142] +#define __pyx_n_u_NumpyOps_reduce_sum __pyx_string_tab[143] +#define __pyx_n_u_NumpyOps_reduce_sum_double2d_t __pyx_string_tab[144] +#define __pyx_n_u_NumpyOps_reduce_sum_float2d_t __pyx_string_tab[145] +#define __pyx_n_u_NumpyOps_relu __pyx_string_tab[146] +#define __pyx_n_u_NumpyOps_scatter_add __pyx_string_tab[147] +#define __pyx_n_u_NumpyOps_seq2col __pyx_string_tab[148] +#define __pyx_n_u_O __pyx_string_tab[149] +#define __pyx_n_u_Ops __pyx_string_tab[150] +#define __pyx_n_u_Optional __pyx_string_tab[151] +#define __pyx_n_u_P __pyx_string_tab[152] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[153] +#define __pyx_n_u_Sequence __pyx_string_tab[154] +#define __pyx_n_u_Shape __pyx_string_tab[155] +#define __pyx_n_u_Sized __pyx_string_tab[156] +#define __pyx_n_u_T __pyx_string_tab[157] +#define __pyx_n_u_View_MemoryView __pyx_string_tab[158] +#define __pyx_n_u_Wh __pyx_string_tab[159] +#define __pyx_n_u_Wh_size __pyx_string_tab[160] +#define __pyx_n_u_Wx __pyx_string_tab[161] +#define __pyx_n_u_Wx_size __pyx_string_tab[162] +#define __pyx_n_u_X __pyx_string_tab[163] +#define __pyx_n_u_X_2 __pyx_string_tab[164] +#define __pyx_n_u_Xs __pyx_string_tab[165] +#define __pyx_n_u_Y __pyx_string_tab[166] +#define __pyx_n_u_Y_ptr __pyx_string_tab[167] +#define __pyx_n_u_Yid __pyx_string_tab[168] +#define __pyx_n_u_Yt2 __pyx_string_tab[169] +#define __pyx_n_u__10 __pyx_string_tab[170] +#define __pyx_n_u__8 __pyx_string_tab[171] +#define __pyx_n_u_abc __pyx_string_tab[172] +#define __pyx_n_u_adam __pyx_string_tab[173] +#define __pyx_n_u_add __pyx_string_tab[174] +#define __pyx_n_u_add_gradient_noise __pyx_string_tab[175] +#define __pyx_n_u_all __pyx_string_tab[176] +#define __pyx_n_u_all_layer_grads __pyx_string_tab[177] +#define __pyx_n_u_all_layer_params __pyx_string_tab[178] +#define __pyx_n_u_alloc __pyx_string_tab[179] +#define __pyx_n_u_allocate_buffer __pyx_string_tab[180] +#define __pyx_n_u_append __pyx_string_tab[181] +#define __pyx_n_u_args __pyx_string_tab[182] +#define __pyx_n_u_array __pyx_string_tab[183] +#define __pyx_n_u_as_contig __pyx_string_tab[184] +#define __pyx_n_u_asarray __pyx_string_tab[185] +#define __pyx_n_u_asarray1i __pyx_string_tab[186] +#define __pyx_n_u_ascontig __pyx_string_tab[187] +#define __pyx_n_u_ascontiguousarray __pyx_string_tab[188] +#define __pyx_n_u_astype __pyx_string_tab[189] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[190] +#define __pyx_n_u_at __pyx_string_tab[191] +#define __pyx_n_u_backprop_lstm __pyx_string_tab[192] +#define __pyx_n_u_backprop_maxout __pyx_string_tab[193] +#define __pyx_n_u_backprop_maxout_double2d_t __pyx_string_tab[194] +#define __pyx_n_u_backprop_maxout_float2d_t __pyx_string_tab[195] +#define __pyx_n_u_backprop_mish __pyx_string_tab[196] +#define __pyx_n_u_backprop_reduce_max __pyx_string_tab[197] +#define __pyx_n_u_backprop_reduce_max_double2d_t __pyx_string_tab[198] +#define __pyx_n_u_backprop_reduce_max_float2d_t __pyx_string_tab[199] +#define __pyx_n_u_backprop_reduce_mean __pyx_string_tab[200] +#define __pyx_n_u_backprop_reduce_mean_double2d_t __pyx_string_tab[201] +#define __pyx_n_u_backprop_reduce_mean_float2d_t __pyx_string_tab[202] +#define __pyx_n_u_backprop_reduce_sum __pyx_string_tab[203] +#define __pyx_n_u_backprop_reduce_sum_double2d_t __pyx_string_tab[204] +#define __pyx_n_u_backprop_reduce_sum_float2d_t __pyx_string_tab[205] +#define __pyx_n_u_backprop_relu __pyx_string_tab[206] +#define __pyx_n_u_backprop_seq2col __pyx_string_tab[207] +#define __pyx_n_u_base __pyx_string_tab[208] +#define __pyx_n_u_batch_size __pyx_string_tab[209] +#define __pyx_n_u_best __pyx_string_tab[210] +#define __pyx_n_u_beta __pyx_string_tab[211] +#define __pyx_n_u_beta1 __pyx_string_tab[212] +#define __pyx_n_u_beta2 __pyx_string_tab[213] +#define __pyx_n_u_bh __pyx_string_tab[214] +#define __pyx_n_u_bh_size __pyx_string_tab[215] +#define __pyx_n_u_bias __pyx_string_tab[216] +#define __pyx_n_u_blis __pyx_string_tab[217] +#define __pyx_n_u_blis_py __pyx_string_tab[218] +#define __pyx_n_u_bool __pyx_string_tab[219] +#define __pyx_n_u_bx __pyx_string_tab[220] +#define __pyx_n_u_bx_size __pyx_string_tab[221] +#define __pyx_n_u_c __pyx_string_tab[222] +#define __pyx_n_u_c_contiguous __pyx_string_tab[223] +#define __pyx_n_u_c_init __pyx_string_tab[224] +#define __pyx_n_u_cblas __pyx_string_tab[225] +#define __pyx_n_u_check_compatible_shape __pyx_string_tab[226] +#define __pyx_n_u_check_seq2col_lengths __pyx_string_tab[227] +#define __pyx_n_u_class __pyx_string_tab[228] +#define __pyx_n_u_class_getitem __pyx_string_tab[229] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[230] +#define __pyx_n_u_collections_abc __pyx_string_tab[231] +#define __pyx_n_u_cols __pyx_string_tab[232] +#define __pyx_n_u_concatenate __pyx_string_tab[233] +#define __pyx_n_u_copy __pyx_string_tab[234] +#define __pyx_n_u_copy_array __pyx_string_tab[235] +#define __pyx_n_u_count __pyx_string_tab[236] +#define __pyx_n_u_cpu __pyx_string_tab[237] +#define __pyx_n_u_cpu_clip_gradient __pyx_string_tab[238] +#define __pyx_n_u_d __pyx_string_tab[239] +#define __pyx_n_u_dC __pyx_string_tab[240] +#define __pyx_n_u_dG __pyx_string_tab[241] +#define __pyx_n_u_dWh __pyx_string_tab[242] +#define __pyx_n_u_dWx __pyx_string_tab[243] +#define __pyx_n_u_dX __pyx_string_tab[244] +#define __pyx_n_u_dX_ptr __pyx_string_tab[245] +#define __pyx_n_u_dXs __pyx_string_tab[246] +#define __pyx_n_u_dY __pyx_string_tab[247] +#define __pyx_n_u_dYid __pyx_string_tab[248] +#define __pyx_n_u_d_bias __pyx_string_tab[249] +#define __pyx_n_u_d_maxes __pyx_string_tab[250] +#define __pyx_n_u_d_means __pyx_string_tab[251] +#define __pyx_n_u_d_params __pyx_string_tab[252] +#define __pyx_n_u_d_sums __pyx_string_tab[253] +#define __pyx_n_u_data __pyx_string_tab[254] +#define __pyx_n_u_defaults __pyx_string_tab[255] +#define __pyx_n_u_depth __pyx_string_tab[256] +#define __pyx_n_u_dest __pyx_string_tab[257] +#define __pyx_n_u_device_id __pyx_string_tab[258] +#define __pyx_n_u_device_type __pyx_string_tab[259] +#define __pyx_n_u_dict __pyx_string_tab[260] +#define __pyx_n_u_dims __pyx_string_tab[261] +#define __pyx_n_u_dir_grads __pyx_string_tab[262] +#define __pyx_n_u_dirs __pyx_string_tab[263] +#define __pyx_n_u_doc __pyx_string_tab[264] +#define __pyx_n_u_double2d_t __pyx_string_tab[265] +#define __pyx_n_u_double3d_t __pyx_string_tab[266] +#define __pyx_n_u_dtype __pyx_string_tab[267] +#define __pyx_n_u_dtype_is_object __pyx_string_tab[268] +#define __pyx_n_u_empty __pyx_string_tab[269] +#define __pyx_n_u_encode __pyx_string_tab[270] +#define __pyx_n_u_enumerate __pyx_string_tab[271] +#define __pyx_n_u_eps __pyx_string_tab[272] +#define __pyx_n_u_error __pyx_string_tab[273] +#define __pyx_n_u_f __pyx_string_tab[274] +#define __pyx_n_u_fill __pyx_string_tab[275] +#define __pyx_n_u_flags __pyx_string_tab[276] +#define __pyx_n_u_float2d_t __pyx_string_tab[277] +#define __pyx_n_u_float32 __pyx_string_tab[278] +#define __pyx_n_u_float3d_t __pyx_string_tab[279] +#define __pyx_n_u_float64 __pyx_string_tab[280] +#define __pyx_n_u_format __pyx_string_tab[281] +#define __pyx_n_u_fortran __pyx_string_tab[282] +#define __pyx_n_u_func __pyx_string_tab[283] +#define __pyx_n_u_fused_sigindex __pyx_string_tab[284] +#define __pyx_n_u_fwd_state __pyx_string_tab[285] +#define __pyx_n_u_gather_add __pyx_string_tab[286] +#define __pyx_n_u_gather_add_double2d_t_int2d_t __pyx_string_tab[287] +#define __pyx_n_u_gather_add_double2d_t_uint2d_t __pyx_string_tab[288] +#define __pyx_n_u_gather_add_float2d_t_int2d_t __pyx_string_tab[289] +#define __pyx_n_u_gather_add_float2d_t_uint2d_t __pyx_string_tab[290] +#define __pyx_n_u_gemm __pyx_string_tab[291] +#define __pyx_n_u_get __pyx_string_tab[292] +#define __pyx_n_u_get_array_module __pyx_string_tab[293] +#define __pyx_n_u_getstate __pyx_string_tab[294] +#define __pyx_n_u_grad_norm __pyx_string_tab[295] +#define __pyx_n_u_grad_parts __pyx_string_tab[296] +#define __pyx_n_u_gradient __pyx_string_tab[297] +#define __pyx_n_u_h_init __pyx_string_tab[298] +#define __pyx_n_u_has_blis __pyx_string_tab[299] +#define __pyx_n_u_hash __pyx_string_tab[300] +#define __pyx_n_u_i __pyx_string_tab[301] +#define __pyx_n_u_id __pyx_string_tab[302] +#define __pyx_n_u_ids __pyx_string_tab[303] +#define __pyx_n_u_import __pyx_string_tab[304] +#define __pyx_n_u_index __pyx_string_tab[305] +#define __pyx_n_u_indices __pyx_string_tab[306] +#define __pyx_n_u_init __pyx_string_tab[307] +#define __pyx_n_u_inplace __pyx_string_tab[308] +#define __pyx_n_u_int __pyx_string_tab[309] +#define __pyx_n_u_int2d_t __pyx_string_tab[310] +#define __pyx_n_u_int32 __pyx_string_tab[311] +#define __pyx_n_u_is_coroutine __pyx_string_tab[312] +#define __pyx_n_u_items __pyx_string_tab[313] +#define __pyx_n_u_itemsize __pyx_string_tab[314] +#define __pyx_n_u_keys __pyx_string_tab[315] +#define __pyx_n_u_keys_2 __pyx_string_tab[316] +#define __pyx_n_u_kind __pyx_string_tab[317] +#define __pyx_n_u_kwargs __pyx_string_tab[318] +#define __pyx_n_u_layer_grads __pyx_string_tab[319] +#define __pyx_n_u_layer_params __pyx_string_tab[320] +#define __pyx_n_u_learn_rate __pyx_string_tab[321] +#define __pyx_n_u_length __pyx_string_tab[322] +#define __pyx_n_u_lengths __pyx_string_tab[323] +#define __pyx_n_u_loc __pyx_string_tab[324] +#define __pyx_n_u_lstm_forward_inference __pyx_string_tab[325] +#define __pyx_n_u_lstm_forward_training __pyx_string_tab[326] +#define __pyx_n_u_main __pyx_string_tab[327] +#define __pyx_n_u_maxes __pyx_string_tab[328] +#define __pyx_n_u_maxout __pyx_string_tab[329] +#define __pyx_n_u_maxout_double3d_t __pyx_string_tab[330] +#define __pyx_n_u_maxout_float3d_t __pyx_string_tab[331] +#define __pyx_n_u_means __pyx_string_tab[332] +#define __pyx_n_u_memview __pyx_string_tab[333] +#define __pyx_n_u_metaclass __pyx_string_tab[334] +#define __pyx_n_u_mish __pyx_string_tab[335] +#define __pyx_n_u_mod_rate __pyx_string_tab[336] +#define __pyx_n_u_mode __pyx_string_tab[337] +#define __pyx_n_u_module __pyx_string_tab[338] +#define __pyx_n_u_mom1 __pyx_string_tab[339] +#define __pyx_n_u_mom2 __pyx_string_tab[340] +#define __pyx_n_u_mro_entries __pyx_string_tab[341] +#define __pyx_n_u_msg __pyx_string_tab[342] +#define __pyx_n_u_n __pyx_string_tab[343] +#define __pyx_n_u_nF __pyx_string_tab[344] +#define __pyx_n_u_nI __pyx_string_tab[345] +#define __pyx_n_u_nL __pyx_string_tab[346] +#define __pyx_n_u_nO __pyx_string_tab[347] +#define __pyx_n_u_nT __pyx_string_tab[348] +#define __pyx_n_u_nW __pyx_string_tab[349] +#define __pyx_n_u_n_inputs __pyx_string_tab[350] +#define __pyx_n_u_name __pyx_string_tab[351] +#define __pyx_n_u_name_2 __pyx_string_tab[352] +#define __pyx_n_u_ndarray __pyx_string_tab[353] +#define __pyx_n_u_ndim __pyx_string_tab[354] +#define __pyx_n_u_new __pyx_string_tab[355] +#define __pyx_n_u_ngrams __pyx_string_tab[356] +#define __pyx_n_u_noise_level __pyx_string_tab[357] +#define __pyx_n_u_normal __pyx_string_tab[358] +#define __pyx_n_u_numpy __pyx_string_tab[359] +#define __pyx_n_u_obj __pyx_string_tab[360] +#define __pyx_n_u_ops __pyx_string_tab[361] +#define __pyx_n_u_orig_X __pyx_string_tab[362] +#define __pyx_n_u_out __pyx_string_tab[363] +#define __pyx_n_u_out_2 __pyx_string_tab[364] +#define __pyx_n_u_output __pyx_string_tab[365] +#define __pyx_n_u_output_2 __pyx_string_tab[366] +#define __pyx_n_u_pack __pyx_string_tab[367] +#define __pyx_n_u_params __pyx_string_tab[368] +#define __pyx_n_u_params_i __pyx_string_tab[369] +#define __pyx_n_u_period __pyx_string_tab[370] +#define __pyx_n_u_pop __pyx_string_tab[371] +#define __pyx_n_u_position_encode __pyx_string_tab[372] +#define __pyx_n_u_prepare __pyx_string_tab[373] +#define __pyx_n_u_py __pyx_string_tab[374] +#define __pyx_n_u_pyx_checksum __pyx_string_tab[375] +#define __pyx_n_u_pyx_state __pyx_string_tab[376] +#define __pyx_n_u_pyx_type __pyx_string_tab[377] +#define __pyx_n_u_pyx_unpickle_Enum __pyx_string_tab[378] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[379] +#define __pyx_n_u_qualname __pyx_string_tab[380] +#define __pyx_n_u_random __pyx_string_tab[381] +#define __pyx_n_u_ravel __pyx_string_tab[382] +#define __pyx_n_u_reduce __pyx_string_tab[383] +#define __pyx_n_u_reduce_cython __pyx_string_tab[384] +#define __pyx_n_u_reduce_ex __pyx_string_tab[385] +#define __pyx_n_u_reduce_max __pyx_string_tab[386] +#define __pyx_n_u_reduce_max_double2d_t __pyx_string_tab[387] +#define __pyx_n_u_reduce_max_float2d_t __pyx_string_tab[388] +#define __pyx_n_u_reduce_mean __pyx_string_tab[389] +#define __pyx_n_u_reduce_mean_double2d_t __pyx_string_tab[390] +#define __pyx_n_u_reduce_mean_float2d_t __pyx_string_tab[391] +#define __pyx_n_u_reduce_sum __pyx_string_tab[392] +#define __pyx_n_u_reduce_sum_double2d_t __pyx_string_tab[393] +#define __pyx_n_u_reduce_sum_float2d_t __pyx_string_tab[394] +#define __pyx_n_u_register __pyx_string_tab[395] +#define __pyx_n_u_registry __pyx_string_tab[396] +#define __pyx_n_u_relu __pyx_string_tab[397] +#define __pyx_n_u_reshape __pyx_string_tab[398] +#define __pyx_n_u_return __pyx_string_tab[399] +#define __pyx_n_u_reverse __pyx_string_tab[400] +#define __pyx_n_u_reversed __pyx_string_tab[401] +#define __pyx_n_u_rows __pyx_string_tab[402] +#define __pyx_n_u_scale __pyx_string_tab[403] +#define __pyx_n_u_scatter_add __pyx_string_tab[404] +#define __pyx_n_u_seed __pyx_string_tab[405] +#define __pyx_n_u_self __pyx_string_tab[406] +#define __pyx_n_u_seq __pyx_string_tab[407] +#define __pyx_n_u_seq2col __pyx_string_tab[408] +#define __pyx_n_u_seq_i __pyx_string_tab[409] +#define __pyx_n_u_set_name __pyx_string_tab[410] +#define __pyx_n_u_setdefault __pyx_string_tab[411] +#define __pyx_n_u_setstate __pyx_string_tab[412] +#define __pyx_n_u_setstate_cython __pyx_string_tab[413] +#define __pyx_n_u_shape __pyx_string_tab[414] +#define __pyx_n_u_signatures __pyx_string_tab[415] +#define __pyx_n_u_size __pyx_string_tab[416] +#define __pyx_n_u_size_at_t __pyx_string_tab[417] +#define __pyx_n_u_split_directions __pyx_string_tab[418] +#define __pyx_n_u_split_weights __pyx_string_tab[419] +#define __pyx_n_u_start __pyx_string_tab[420] +#define __pyx_n_u_step __pyx_string_tab[421] +#define __pyx_n_u_stop __pyx_string_tab[422] +#define __pyx_n_u_strip __pyx_string_tab[423] +#define __pyx_n_u_struct __pyx_string_tab[424] +#define __pyx_n_u_sum __pyx_string_tab[425] +#define __pyx_n_u_sums __pyx_string_tab[426] +#define __pyx_n_u_super __pyx_string_tab[427] +#define __pyx_n_u_table __pyx_string_tab[428] +#define __pyx_n_u_test __pyx_string_tab[429] +#define __pyx_n_u_thinc_backends_numpy_ops __pyx_string_tab[430] +#define __pyx_n_u_threshold __pyx_string_tab[431] +#define __pyx_n_u_timestep __pyx_string_tab[432] +#define __pyx_n_u_trans1 __pyx_string_tab[433] +#define __pyx_n_u_trans2 __pyx_string_tab[434] +#define __pyx_n_u_transpose __pyx_string_tab[435] +#define __pyx_n_u_transpose_weights __pyx_string_tab[436] +#define __pyx_n_u_types __pyx_string_tab[437] +#define __pyx_n_u_typing __pyx_string_tab[438] +#define __pyx_n_u_u __pyx_string_tab[439] +#define __pyx_n_u_uint2d_t __pyx_string_tab[440] +#define __pyx_n_u_uint32 __pyx_string_tab[441] +#define __pyx_n_u_uint64 __pyx_string_tab[442] +#define __pyx_n_u_unpack __pyx_string_tab[443] +#define __pyx_n_u_untranspose_unsplit_weights __pyx_string_tab[444] +#define __pyx_n_u_update __pyx_string_tab[445] +#define __pyx_n_u_use_blis __pyx_string_tab[446] +#define __pyx_n_u_util __pyx_string_tab[447] +#define __pyx_n_u_v __pyx_string_tab[448] +#define __pyx_n_u_values __pyx_string_tab[449] +#define __pyx_n_u_variance __pyx_string_tab[450] +#define __pyx_n_u_weights __pyx_string_tab[451] +#define __pyx_n_u_which __pyx_string_tab[452] +#define __pyx_n_u_x __pyx_string_tab[453] +#define __pyx_n_u_xp __pyx_string_tab[454] +#define __pyx_n_u_y __pyx_string_tab[455] +#define __pyx_n_u_zeros __pyx_string_tab[456] +#define __pyx_n_u_zip __pyx_string_tab[457] +#define __pyx_kp_b_iso88591_1G3a_A_AYawe1A_1_gS_A_AZq_uAQ_1 __pyx_string_tab[458] +#define __pyx_kp_b_iso88591_1_1G3a_A_AYawe1G7_1_gS_A_AZq_uA __pyx_string_tab[459] +#define __pyx_kp_b_iso88591_22G_UV_1F_Q_A_UUVVWWX_1F_Q_A_UU __pyx_string_tab[460] +#define __pyx_kp_b_iso88591_5_4xvQa_z_A_81D_KxvQa __pyx_string_tab[461] +#define __pyx_kp_b_iso88591_99I_q_A_2WCz_QgS_a_Qhb_xq_uAWGS __pyx_string_tab[462] +#define __pyx_kp_b_iso88591_9_q_A_1A_6aq_2WCz_QgS_a_U_1_5_C __pyx_string_tab[463] +#define __pyx_kp_b_iso88591_A_2Rq_4vRuF_1D_AS_F_2Rq_fBj_a_G1 __pyx_string_tab[464] +#define __pyx_kp_b_iso88591_A_2_V2S_at4vUV_k_Q_E_as_1_as_4vQ __pyx_string_tab[465] +#define __pyx_kp_b_iso88591_A_4_AXT_S_q __pyx_string_tab[466] +#define __pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_6_as_a __pyx_string_tab[467] +#define __pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_q_Qc_Q __pyx_string_tab[468] +#define __pyx_kp_b_iso88591_A_5_s_G7_Q_F_A_E_q_G6_F_G6_A_E_s __pyx_string_tab[469] +#define __pyx_kp_b_iso88591_A_K_AT_t1 __pyx_string_tab[470] +#define __pyx_kp_b_iso88591_A_Q_1_O1_M_L_4z_T_AQ __pyx_string_tab[471] +#define __pyx_kp_b_iso88591_A_QfAQ_QfAQ_QfAQ_F_7_T_y_a_4vQgS __pyx_string_tab[472] +#define __pyx_kp_b_iso88591_A_QfAQ_QfAQ_QfAQ_F_7_T_y_a_4vQgS_2 __pyx_string_tab[473] +#define __pyx_kp_b_iso88591_A_QfD_1_A_AV1_D_a_AV1_D_A_D_6_a __pyx_string_tab[474] +#define __pyx_kp_b_iso88591_A_RvQa_RvQa_fBc_D_a_q_7_1Bas_auA __pyx_string_tab[475] +#define __pyx_kp_b_iso88591_A_RvQa_RvQa_fBc_D_a_q_AS_AU_3dRU __pyx_string_tab[476] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q __pyx_string_tab[477] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_2 __pyx_string_tab[478] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_3 __pyx_string_tab[479] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_4 __pyx_string_tab[480] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_F_7_T_uF_2S_Rs __pyx_string_tab[481] +#define __pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_F_7_T_uF_2S_Rs_2 __pyx_string_tab[482] +#define __pyx_kp_b_iso88591_A_WF_1_V6_Q_JgRq_wb_j_J_1_2S_Rs __pyx_string_tab[483] +#define __pyx_kp_b_iso88591_A_WF_1_V6_Q_JgRq_wb_j_J_1_2S_Rs_2 __pyx_string_tab[484] +#define __pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wb_j_J_1_2S_R __pyx_string_tab[485] +#define __pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wb_j_J_1_2S_R_2 __pyx_string_tab[486] +#define __pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wc_j_I_2S_Rs __pyx_string_tab[487] +#define __pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wc_j_I_2S_Rs_2 __pyx_string_tab[488] +#define __pyx_kp_b_iso88591_A_q_q_q_axxwhd_Q_6_gWE_AXWA_HG1L __pyx_string_tab[489] +#define __pyx_kp_b_iso88591_A_r_q_3b_aq_r_q_3b_aq_AXT_S_s __pyx_string_tab[490] +#define __pyx_kp_b_iso88591_B_3fCq_MQcQR_AQ_4s_AQ_S_aq_S_aq __pyx_string_tab[491] +#define __pyx_kp_b_iso88591_F_6_q_vQa_q_aq_q_at3b_F_q_at5_a __pyx_string_tab[492] +#define __pyx_kp_b_iso88591_G1_6_6_82Qc_D_Cs_XRq_1_82Qc_D_C __pyx_string_tab[493] +#define __pyx_kp_b_iso88591_J_2V3a_VVWWYYZ_AQ_4r_q_AQ_RvQa __pyx_string_tab[494] +#define __pyx_kp_b_iso88591_Rr_3a_y_1_E_G7_6_4t5_1A_Q __pyx_string_tab[495] +#define __pyx_kp_b_iso88591_T_t6_82T_BfBa_Rs_S_3b_b_82T_Jb __pyx_string_tab[496] +#define __pyx_kp_b_iso88591_W_J_Qaab_1_4s_a_4s_a __pyx_string_tab[497] +#define __pyx_kp_b_iso88591_a_4s_4vRs_fA_1_t6_S_t6_S_1HD_xs __pyx_string_tab[498] +#define __pyx_kp_b_iso88591_a_q_s_1_1_q_j __pyx_string_tab[499] +#define __pyx_kp_b_iso88591_b_Rq_b_b_Rq_b_q_9Bhhb_D_q_9Bhhb __pyx_string_tab[500] +#define __pyx_kp_b_iso88591_q_uA __pyx_string_tab[501] +#define __pyx_kp_b_iso88591_s_T_QfAQ_AV1A_6_6_q_6_3e6_6_3b __pyx_string_tab[502] +#define __pyx_kp_b_iso88591_uCq_q_Qhb_q_S_Q_E_aq_gQe_Qb_A_q __pyx_string_tab[503] +#define __pyx_kp_b_iso88591_xs_Zq_s_T_S_ay_4q_s_T_3c_1 __pyx_string_tab[504] +#define __pyx_kp_b_pyx_t_5thinc_8backends_5cblas __pyx_string_tab[505] +#define __pyx_kp_b_uint64_t_void_int_uint64_t_hash6 __pyx_string_tab[506] +#define __pyx_kp_b_void_enum___pyx_t_4blis_2cy_tran __pyx_string_tab[507] +#define __pyx_n_b_O __pyx_string_tab[508] +#define __pyx_float_0_ __pyx_number_tab[0] +#define __pyx_float_20_0 __pyx_number_tab[1] +#define __pyx_int_0 __pyx_number_tab[2] +#define __pyx_int_neg_1 __pyx_number_tab[3] +#define __pyx_int_1 __pyx_number_tab[4] +#define __pyx_int_2 __pyx_number_tab[5] +#define __pyx_int_4 __pyx_number_tab[6] +#define __pyx_int_136983863 __pyx_number_tab[7] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_5cblas_CBlas); + Py_CLEAR(clear_module_state->__pyx_ptype_7cpython_4type_type); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_dtype); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_flatiter); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_broadcast); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_ndarray); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_generic); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_number); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_integer); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_signedinteger); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_unsignedinteger); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_inexact); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_floating); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_complexfloating); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_flexible); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_character); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_ufunc); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Matrix); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Vec); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_VecVec); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_6linalg_Mat); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1); + Py_CLEAR(clear_module_state->__pyx_array_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_array); + Py_CLEAR(clear_module_state->__pyx_MemviewEnum_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_MemviewEnum); + Py_CLEAR(clear_module_state->__pyx_memoryview_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryview); + Py_CLEAR(clear_module_state->__pyx_memoryviewslice_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<3; ++i) { Py_CLEAR(clear_module_state->__pyx_slice[i]); } + for (int i=0; i<11; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<58; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<509; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<8; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* FusedFunction.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_5cblas_CBlas); + Py_VISIT(traverse_module_state->__pyx_ptype_7cpython_4type_type); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_dtype); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_flatiter); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_broadcast); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_ndarray); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_generic); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_number); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_integer); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_signedinteger); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_unsignedinteger); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_inexact); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_floating); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_complexfloating); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_flexible); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_character); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_ufunc); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Matrix); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Vec); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_VecVec); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_6linalg_Mat); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1); + Py_VISIT(traverse_module_state->__pyx_array_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_array); + Py_VISIT(traverse_module_state->__pyx_MemviewEnum_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_MemviewEnum); + Py_VISIT(traverse_module_state->__pyx_memoryview_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryview); + Py_VISIT(traverse_module_state->__pyx_memoryviewslice_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<3; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_slice[i]); } + for (int i=0; i<11; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<58; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<509; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<8; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* FusedFunction.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(object, int) + * + * @cname('__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + +static PyObject *__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc(PyObject *__pyx_v_arg, PyTypeObject *__pyx_v_ndarray) { + __Pyx_memviewslice __pyx_v_memslice; + Py_ssize_t __pyx_v_itemsize; + CYTHON_UNUSED int __pyx_v_dtype_signed; + Py_UCS4 __pyx_v_kind; + PyObject *__pyx_v_arg_as_memoryview = 0; + PyObject *__pyx_v_dtype = NULL; + PyObject *__pyx_v_arg_base = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + long __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("map_fused_type", 0); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":22 + * cdef Py_UCS4 kind + * + * itemsize = -1 # <<<<<<<<<<<<<< + * + * cdef memoryview arg_as_memoryview +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":27 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + __pyx_t_1 = (__pyx_v_ndarray != ((PyTypeObject*)Py_None)); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":28 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":29 + * if ndarray is not None: + * if isinstance(arg, ndarray): + * dtype = arg.dtype # <<<<<<<<<<<<<< + * + * elif __pyx_memoryview_check(arg): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":28 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":31 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + __pyx_t_1 = __pyx_memoryview_check(__pyx_v_arg); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":32 + * + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base # <<<<<<<<<<<<<< + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_base = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg_base, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":34 + * arg_base = arg.base + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg_base, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + goto __pyx_L5; + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":36 + * dtype = arg_base.dtype + * else: + * dtype = None # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L5:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":31 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":38 + * dtype = None + * else: + * dtype = None # <<<<<<<<<<<<<< + * + * itemsize = -1 +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L4:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":40 + * dtype = None + * + * itemsize = -1 # <<<<<<<<<<<<<< + * if dtype is not None: + * itemsize = dtype.itemsize +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":41 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + __pyx_t_1 = (__pyx_v_dtype != Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":42 + * itemsize = -1 + * if dtype is not None: + * itemsize = dtype.itemsize # <<<<<<<<<<<<<< + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_itemsize = __pyx_t_3; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":43 + * if dtype is not None: + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) # <<<<<<<<<<<<<< + * dtype_signed = kind == u'i' + * if kind in u'iu': +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_kind); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_Ord(__pyx_t_2); if (unlikely(__pyx_t_4 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_kind = __pyx_t_4; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":44 + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' # <<<<<<<<<<<<<< + * if kind in u'iu': + * pass +*/ + __pyx_v_dtype_signed = (__pyx_v_kind == 0x69); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":45 + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' + * if kind in u'iu': # <<<<<<<<<<<<<< + * pass + * elif kind == u'f': +*/ + switch (__pyx_v_kind) { + case 0x69: + case 0x75: + break; + case 0x66: + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":49 + * elif kind == u'f': + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 3: # <<<<<<<<<<<<<< + * return 'float3d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 3: +*/ + __pyx_t_5 = ((sizeof(float)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 3); + __pyx_t_1 = __pyx_t_5; + __pyx_L8_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":50 + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 3: + * return 'float3d_t' # <<<<<<<<<<<<<< + * if sizeof(double) == itemsize and (arg.ndim) == 3: + * return 'double3d_t' +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float3d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float3d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":49 + * elif kind == u'f': + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 3: # <<<<<<<<<<<<<< + * return 'float3d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 3: +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * if sizeof(float) == itemsize and (arg.ndim) == 3: + * return 'float3d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 3: # <<<<<<<<<<<<<< + * return 'double3d_t' + * elif kind == u'c': +*/ + __pyx_t_5 = ((sizeof(double)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 51, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 51, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 3); + __pyx_t_1 = __pyx_t_5; + __pyx_L11_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":52 + * return 'float3d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 3: + * return 'double3d_t' # <<<<<<<<<<<<<< + * elif kind == u'c': + * pass +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_double3d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_double3d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * if sizeof(float) == itemsize and (arg.ndim) == 3: + * return 'float3d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 3: # <<<<<<<<<<<<<< + * return 'double3d_t' + * elif kind == u'c': +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":47 + * if kind in u'iu': + * pass + * elif kind == u'f': # <<<<<<<<<<<<<< + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 3: +*/ + break; + case 99: + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":53 + * if sizeof(double) == itemsize and (arg.ndim) == 3: + * return 'double3d_t' + * elif kind == u'c': # <<<<<<<<<<<<<< + * pass + * +*/ + break; + default: break; + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":41 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":27 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":56 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'float3d_t' + * +*/ + __pyx_t_1 = (__pyx_v_arg == Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":57 + * + * if arg is None: + * return 'float3d_t' # <<<<<<<<<<<<<< + * + * try: +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float3d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float3d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":56 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'float3d_t' + * +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float3d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":60 + * + * try: + * arg_as_memoryview = memoryview(arg) # <<<<<<<<<<<<<< + * except (ValueError, TypeError): + * pass +*/ + __pyx_t_2 = PyMemoryView_FromObject(__pyx_v_arg); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 60, __pyx_L14_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_as_memoryview = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float3d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): +*/ + /*else:*/ { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":67 + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L23_next_or; + } else { + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): +*/ + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(float))); + if (!__pyx_t_5) { + } else { + goto __pyx_L22_next_and; + } + __pyx_L23_next_or:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":67 + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(float))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L21_bool_binop_done; + } + __pyx_L22_next_and:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":68 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 68, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_9 == 3); + __pyx_t_1 = __pyx_t_5; + __pyx_L21_bool_binop_done:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":69 + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":70 + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":71 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'float3d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":73 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'float3d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float3d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float3d_t; + goto __pyx_L17_except_return; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":70 + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":75 + * return 'float3d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * + * # try double +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 3): +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 3): +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L29_next_or; + } else { + } + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 78, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(double))); + if (!__pyx_t_5) { + } else { + goto __pyx_L28_next_and; + } + __pyx_L29_next_or:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":79 + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) + * or itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(double))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L27_bool_binop_done; + } + __pyx_L28_next_and:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":80 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 3): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 80, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_9 == 3); + __pyx_t_1 = __pyx_t_5; + __pyx_L27_bool_binop_done:; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 3): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":81 + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":82 + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":83 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'double3d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":85 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'double3d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_double3d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_double3d_t; + goto __pyx_L17_except_return; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":82 + * and arg_as_memoryview.ndim == 3): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":87 + * return 'double3d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * return None +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 3): +*/ + } + } + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L19_try_end; + __pyx_L14_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":61 + * try: + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): # <<<<<<<<<<<<<< + * pass + * else: +*/ + __pyx_t_9 = __Pyx_PyErr_ExceptionMatches2(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), ((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_9) { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L15_exception_handled; + } + goto __pyx_L16_except_error; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float3d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + __pyx_L16_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L1_error; + __pyx_L17_except_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L0; + __pyx_L15_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + __pyx_L19_try_end:; + } + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":88 + * else: + * __pyx_PyErr_Clear() + * return None # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(object, int) + * + * @cname('__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("__pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc.map_fused_type", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_arg_as_memoryview); + __Pyx_XDECREF(__pyx_v_dtype); + __Pyx_XDECREF(__pyx_v_arg_base); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "match_signatures_single":3 + * + * + * @cname("__pyx_ff_match_signatures_single") # <<<<<<<<<<<<<< + * cdef object match_signatures_single(signatures: dict, dest_type): + * found_match = signatures.get(dest_type) +*/ + +static PyObject *__pyx_ff_match_signatures_single(PyObject *__pyx_v_signatures, PyObject *__pyx_v_dest_type) { + PyObject *__pyx_v_found_match = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("match_signatures_single", 0); + + /* "match_signatures_single":5 + * @cname("__pyx_ff_match_signatures_single") + * cdef object match_signatures_single(signatures: dict, dest_type): + * found_match = signatures.get(dest_type) # <<<<<<<<<<<<<< + * if found_match is None: + * raise TypeError("No matching signature found") +*/ + __pyx_t_1 = __Pyx_PyDict_GetItemDefault(__pyx_v_signatures, __pyx_v_dest_type, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_found_match = __pyx_t_1; + __pyx_t_1 = 0; + + /* "match_signatures_single":6 + * cdef object match_signatures_single(signatures: dict, dest_type): + * found_match = signatures.get(dest_type) + * if found_match is None: # <<<<<<<<<<<<<< + * raise TypeError("No matching signature found") + * return found_match +*/ + __pyx_t_2 = (__pyx_v_found_match == Py_None); + if (unlikely(__pyx_t_2)) { + + /* "match_signatures_single":7 + * found_match = signatures.get(dest_type) + * if found_match is None: + * raise TypeError("No matching signature found") # <<<<<<<<<<<<<< + * return found_match + * +*/ + __pyx_t_3 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_kp_u_No_matching_signature_found}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 7, __pyx_L1_error) + + /* "match_signatures_single":6 + * cdef object match_signatures_single(signatures: dict, dest_type): + * found_match = signatures.get(dest_type) + * if found_match is None: # <<<<<<<<<<<<<< + * raise TypeError("No matching signature found") + * return found_match +*/ + } + + /* "match_signatures_single":8 + * if found_match is None: + * raise TypeError("No matching signature found") + * return found_match # <<<<<<<<<<<<<< + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_found_match); + __pyx_r = __pyx_v_found_match; + goto __pyx_L0; + + /* "match_signatures_single":3 + * + * + * @cname("__pyx_ff_match_signatures_single") # <<<<<<<<<<<<<< + * cdef object match_signatures_single(signatures: dict, dest_type): + * found_match = signatures.get(dest_type) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("match_signatures_single.match_signatures_single", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_found_match); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(object, int) + * + * @cname('__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + +static PyObject *__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(PyObject *__pyx_v_arg, PyTypeObject *__pyx_v_ndarray) { + __Pyx_memviewslice __pyx_v_memslice; + Py_ssize_t __pyx_v_itemsize; + CYTHON_UNUSED int __pyx_v_dtype_signed; + Py_UCS4 __pyx_v_kind; + PyObject *__pyx_v_arg_as_memoryview = 0; + PyObject *__pyx_v_dtype = NULL; + PyObject *__pyx_v_arg_base = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + long __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("map_fused_type", 0); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":22 + * cdef Py_UCS4 kind + * + * itemsize = -1 # <<<<<<<<<<<<<< + * + * cdef memoryview arg_as_memoryview +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":27 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + __pyx_t_1 = (__pyx_v_ndarray != ((PyTypeObject*)Py_None)); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":28 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":29 + * if ndarray is not None: + * if isinstance(arg, ndarray): + * dtype = arg.dtype # <<<<<<<<<<<<<< + * + * elif __pyx_memoryview_check(arg): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":28 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":31 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + __pyx_t_1 = __pyx_memoryview_check(__pyx_v_arg); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":32 + * + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base # <<<<<<<<<<<<<< + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_base = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg_base, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":34 + * arg_base = arg.base + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg_base, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + goto __pyx_L5; + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":36 + * dtype = arg_base.dtype + * else: + * dtype = None # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L5:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":31 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":38 + * dtype = None + * else: + * dtype = None # <<<<<<<<<<<<<< + * + * itemsize = -1 +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L4:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":40 + * dtype = None + * + * itemsize = -1 # <<<<<<<<<<<<<< + * if dtype is not None: + * itemsize = dtype.itemsize +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":41 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + __pyx_t_1 = (__pyx_v_dtype != Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":42 + * itemsize = -1 + * if dtype is not None: + * itemsize = dtype.itemsize # <<<<<<<<<<<<<< + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_itemsize = __pyx_t_3; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":43 + * if dtype is not None: + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) # <<<<<<<<<<<<<< + * dtype_signed = kind == u'i' + * if kind in u'iu': +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_kind); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_Ord(__pyx_t_2); if (unlikely(__pyx_t_4 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_kind = __pyx_t_4; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":44 + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' # <<<<<<<<<<<<<< + * if kind in u'iu': + * pass +*/ + __pyx_v_dtype_signed = (__pyx_v_kind == 0x69); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":45 + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' + * if kind in u'iu': # <<<<<<<<<<<<<< + * pass + * elif kind == u'f': +*/ + switch (__pyx_v_kind) { + case 0x69: + case 0x75: + break; + case 0x66: + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":49 + * elif kind == u'f': + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 2: # <<<<<<<<<<<<<< + * return 'float2d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 2: +*/ + __pyx_t_5 = ((sizeof(float)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L8_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":50 + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 2: + * return 'float2d_t' # <<<<<<<<<<<<<< + * if sizeof(double) == itemsize and (arg.ndim) == 2: + * return 'double2d_t' +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":49 + * elif kind == u'f': + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 2: # <<<<<<<<<<<<<< + * return 'float2d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 2: +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * if sizeof(float) == itemsize and (arg.ndim) == 2: + * return 'float2d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 2: # <<<<<<<<<<<<<< + * return 'double2d_t' + * elif kind == u'c': +*/ + __pyx_t_5 = ((sizeof(double)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 51, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 51, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L11_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":52 + * return 'float2d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 2: + * return 'double2d_t' # <<<<<<<<<<<<<< + * elif kind == u'c': + * pass +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_double2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_double2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * if sizeof(float) == itemsize and (arg.ndim) == 2: + * return 'float2d_t' + * if sizeof(double) == itemsize and (arg.ndim) == 2: # <<<<<<<<<<<<<< + * return 'double2d_t' + * elif kind == u'c': +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":47 + * if kind in u'iu': + * pass + * elif kind == u'f': # <<<<<<<<<<<<<< + * pass + * if sizeof(float) == itemsize and (arg.ndim) == 2: +*/ + break; + case 99: + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":53 + * if sizeof(double) == itemsize and (arg.ndim) == 2: + * return 'double2d_t' + * elif kind == u'c': # <<<<<<<<<<<<<< + * pass + * +*/ + break; + default: break; + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":41 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":27 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":56 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'float2d_t' + * +*/ + __pyx_t_1 = (__pyx_v_arg == Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":57 + * + * if arg is None: + * return 'float2d_t' # <<<<<<<<<<<<<< + * + * try: +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":56 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'float2d_t' + * +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":60 + * + * try: + * arg_as_memoryview = memoryview(arg) # <<<<<<<<<<<<<< + * except (ValueError, TypeError): + * pass +*/ + __pyx_t_2 = PyMemoryView_FromObject(__pyx_v_arg); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 60, __pyx_L14_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_as_memoryview = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): +*/ + /*else:*/ { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":67 + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L23_next_or; + } else { + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): +*/ + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(float))); + if (!__pyx_t_5) { + } else { + goto __pyx_L22_next_and; + } + __pyx_L23_next_or:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":67 + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(float))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L21_bool_binop_done; + } + __pyx_L22_next_and:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":68 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 68, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_9 == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L21_bool_binop_done:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":69 + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":70 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":71 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'float2d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":73 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'float2d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_float2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_float2d_t; + goto __pyx_L17_except_return; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":70 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":75 + * return 'float2d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * + * # try double +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * # try float + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(float)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(float)) + * and arg_as_memoryview.ndim == 2): +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 2): +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L29_next_or; + } else { + } + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 78, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(double))); + if (!__pyx_t_5) { + } else { + goto __pyx_L28_next_and; + } + __pyx_L29_next_or:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":79 + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) + * or itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(double))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L27_bool_binop_done; + } + __pyx_L28_next_and:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":80 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 2): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 80, __pyx_L16_except_error) + __pyx_t_5 = (__pyx_t_9 == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L27_bool_binop_done:; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 2): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":81 + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":82 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":83 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'double2d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":85 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'double2d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_double2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_double2d_t; + goto __pyx_L17_except_return; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":82 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":87 + * return 'double2d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * return None +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":78 + * + * # try double + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(double)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(double)) + * and arg_as_memoryview.ndim == 2): +*/ + } + } + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L19_try_end; + __pyx_L14_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":61 + * try: + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): # <<<<<<<<<<<<<< + * pass + * else: +*/ + __pyx_t_9 = __Pyx_PyErr_ExceptionMatches2(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), ((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_9) { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L15_exception_handled; + } + goto __pyx_L16_except_error; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * return 'float2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + __pyx_L16_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L1_error; + __pyx_L17_except_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L0; + __pyx_L15_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + __pyx_L19_try_end:; + } + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":88 + * else: + * __pyx_PyErr_Clear() + * return None # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(object, int) + * + * @cname('__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("__pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc.map_fused_type", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_arg_as_memoryview); + __Pyx_XDECREF(__pyx_v_dtype); + __Pyx_XDECREF(__pyx_v_arg_base); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(object, int) + * + * @cname('__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + +static PyObject *__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc(PyObject *__pyx_v_arg, PyTypeObject *__pyx_v_ndarray) { + __Pyx_memviewslice __pyx_v_memslice; + Py_ssize_t __pyx_v_itemsize; + int __pyx_v_dtype_signed; + Py_UCS4 __pyx_v_kind; + PyObject *__pyx_v_arg_as_memoryview = 0; + int __pyx_v_int_is_signed; + int __pyx_v_unsigned_int_is_signed; + PyObject *__pyx_v_dtype = NULL; + PyObject *__pyx_v_arg_base = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + long __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("map_fused_type", 0); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":22 + * cdef Py_UCS4 kind + * + * itemsize = -1 # <<<<<<<<<<<<<< + * + * cdef memoryview arg_as_memoryview +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":27 + * + * cdef bint int_is_signed + * int_is_signed = not ( -1 > 0) # <<<<<<<<<<<<<< + * + * cdef bint unsigned_int_is_signed +*/ + __pyx_v_int_is_signed = (!(((int)-1L) > 0)); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":30 + * + * cdef bint unsigned_int_is_signed + * unsigned_int_is_signed = not ( -1 > 0) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_unsigned_int_is_signed = (!(((unsigned int)-1L) > 0)); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + __pyx_t_1 = (__pyx_v_ndarray != ((PyTypeObject*)Py_None)); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":34 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":35 + * if ndarray is not None: + * if isinstance(arg, ndarray): + * dtype = arg.dtype # <<<<<<<<<<<<<< + * + * elif __pyx_memoryview_check(arg): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 35, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":34 + * + * if ndarray is not None: + * if isinstance(arg, ndarray): # <<<<<<<<<<<<<< + * dtype = arg.dtype + * +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":37 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + __pyx_t_1 = __pyx_memoryview_check(__pyx_v_arg); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":38 + * + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base # <<<<<<<<<<<<<< + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_base = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":39 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_arg_base, __pyx_v_ndarray); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":40 + * arg_base = arg.base + * if isinstance(arg_base, ndarray): + * dtype = arg_base.dtype # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg_base, __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_dtype = __pyx_t_2; + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":39 + * elif __pyx_memoryview_check(arg): + * arg_base = arg.base + * if isinstance(arg_base, ndarray): # <<<<<<<<<<<<<< + * dtype = arg_base.dtype + * else: +*/ + goto __pyx_L5; + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":42 + * dtype = arg_base.dtype + * else: + * dtype = None # <<<<<<<<<<<<<< + * else: + * dtype = None +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L5:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":37 + * dtype = arg.dtype + * + * elif __pyx_memoryview_check(arg): # <<<<<<<<<<<<<< + * arg_base = arg.base + * if isinstance(arg_base, ndarray): +*/ + goto __pyx_L4; + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":44 + * dtype = None + * else: + * dtype = None # <<<<<<<<<<<<<< + * + * itemsize = -1 +*/ + /*else*/ { + __Pyx_INCREF(Py_None); + __pyx_v_dtype = Py_None; + } + __pyx_L4:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":46 + * dtype = None + * + * itemsize = -1 # <<<<<<<<<<<<<< + * if dtype is not None: + * itemsize = dtype.itemsize +*/ + __pyx_v_itemsize = -1L; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":47 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + __pyx_t_1 = (__pyx_v_dtype != Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":48 + * itemsize = -1 + * if dtype is not None: + * itemsize = dtype.itemsize # <<<<<<<<<<<<<< + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 48, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_itemsize = __pyx_t_3; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":49 + * if dtype is not None: + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) # <<<<<<<<<<<<<< + * dtype_signed = kind == u'i' + * if kind in u'iu': +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dtype, __pyx_mstate_global->__pyx_n_u_kind); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_Ord(__pyx_t_2); if (unlikely(__pyx_t_4 == ((long)(long)(Py_UCS4)-1))) __PYX_ERR(1, 49, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_kind = __pyx_t_4; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":50 + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' # <<<<<<<<<<<<<< + * if kind in u'iu': + * pass +*/ + __pyx_v_dtype_signed = (__pyx_v_kind == 0x69); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' + * if kind in u'iu': # <<<<<<<<<<<<<< + * pass + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): +*/ + switch (__pyx_v_kind) { + case 0x69: + case 0x75: + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":53 + * if kind in u'iu': + * pass + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): # <<<<<<<<<<<<<< + * return 'int2d_t' + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): +*/ + __pyx_t_5 = ((sizeof(int)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 2); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_5 = (!(__pyx_v_int_is_signed ^ __pyx_v_dtype_signed)); + __pyx_t_1 = __pyx_t_5; + __pyx_L8_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":54 + * pass + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): + * return 'int2d_t' # <<<<<<<<<<<<<< + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): + * return 'uint2d_t' +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_int2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_int2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":53 + * if kind in u'iu': + * pass + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): # <<<<<<<<<<<<<< + * return 'int2d_t' + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":55 + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): + * return 'int2d_t' + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): # <<<<<<<<<<<<<< + * return 'uint2d_t' + * elif kind == u'f': +*/ + __pyx_t_5 = ((sizeof(unsigned int)) == __pyx_v_itemsize); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_arg, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 55, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyIndex_AsSsize_t(__pyx_t_2); if (unlikely((__pyx_t_3 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 55, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (((Py_ssize_t)__pyx_t_3) == 2); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_5 = (!(__pyx_v_unsigned_int_is_signed ^ __pyx_v_dtype_signed)); + __pyx_t_1 = __pyx_t_5; + __pyx_L12_bool_binop_done:; + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":56 + * return 'int2d_t' + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): + * return 'uint2d_t' # <<<<<<<<<<<<<< + * elif kind == u'f': + * pass +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_uint2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_uint2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":55 + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): + * return 'int2d_t' + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): # <<<<<<<<<<<<<< + * return 'uint2d_t' + * elif kind == u'f': +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":51 + * kind = ord(dtype.kind) + * dtype_signed = kind == u'i' + * if kind in u'iu': # <<<<<<<<<<<<<< + * pass + * if sizeof(int) == itemsize and (arg.ndim) == 2 and not (int_is_signed ^ dtype_signed): +*/ + break; + case 0x66: + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":57 + * if sizeof(unsigned int) == itemsize and (arg.ndim) == 2 and not (unsigned_int_is_signed ^ dtype_signed): + * return 'uint2d_t' + * elif kind == u'f': # <<<<<<<<<<<<<< + * pass + * elif kind == u'c': +*/ + break; + case 99: + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":59 + * elif kind == u'f': + * pass + * elif kind == u'c': # <<<<<<<<<<<<<< + * pass + * +*/ + break; + default: break; + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":47 + * + * itemsize = -1 + * if dtype is not None: # <<<<<<<<<<<<<< + * itemsize = dtype.itemsize + * kind = ord(dtype.kind) +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":33 + * + * + * if ndarray is not None: # <<<<<<<<<<<<<< + * if isinstance(arg, ndarray): + * dtype = arg.dtype +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":62 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'int2d_t' + * +*/ + __pyx_t_1 = (__pyx_v_arg == Py_None); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":63 + * + * if arg is None: + * return 'int2d_t' # <<<<<<<<<<<<<< + * + * try: +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_int2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_int2d_t; + goto __pyx_L0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":62 + * pass + * + * if arg is None: # <<<<<<<<<<<<<< + * return 'int2d_t' + * +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":65 + * return 'int2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":66 + * + * try: + * arg_as_memoryview = memoryview(arg) # <<<<<<<<<<<<<< + * except (ValueError, TypeError): + * pass +*/ + __pyx_t_2 = PyMemoryView_FromObject(__pyx_v_arg); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 66, __pyx_L16_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_arg_as_memoryview = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":65 + * return 'int2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":72 + * + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): +*/ + /*else:*/ { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":73 + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) + * or itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L25_next_or; + } else { + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":72 + * + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): +*/ + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 72, __pyx_L18_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(int))); + if (!__pyx_t_5) { + } else { + goto __pyx_L24_next_and; + } + __pyx_L25_next_or:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":73 + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) + * or itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(int))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L23_bool_binop_done; + } + __pyx_L24_next_and:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":74 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 74, __pyx_L18_except_error) + __pyx_t_5 = (__pyx_t_9 == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L23_bool_binop_done:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":72 + * + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":75 + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":76 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":77 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'int2d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":79 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'int2d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_int2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_int2d_t; + goto __pyx_L19_except_return; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":76 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":81 + * return 'int2d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * + * # try unsigned int +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":72 + * + * # try int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(int)) + * and arg_as_memoryview.ndim == 2): +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":84 + * + * # try unsigned int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(unsigned int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(unsigned int)) + * and arg_as_memoryview.ndim == 2): +*/ + __pyx_t_5 = (__pyx_v_itemsize == -1L); + if (!__pyx_t_5) { + goto __pyx_L31_next_or; + } else { + } + __pyx_t_3 = __Pyx_PyMemoryView_Get_itemsize(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 84, __pyx_L18_except_error) + __pyx_t_5 = (__pyx_t_3 == (sizeof(unsigned int))); + if (!__pyx_t_5) { + } else { + goto __pyx_L30_next_and; + } + __pyx_L31_next_or:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":85 + * # try unsigned int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(unsigned int)) + * or itemsize == sizeof(unsigned int)) # <<<<<<<<<<<<<< + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) +*/ + __pyx_t_5 = (__pyx_v_itemsize == (sizeof(unsigned int))); + if (__pyx_t_5) { + } else { + __pyx_t_1 = __pyx_t_5; + goto __pyx_L29_bool_binop_done; + } + __pyx_L30_next_and:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":86 + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(unsigned int)) + * or itemsize == sizeof(unsigned int)) + * and arg_as_memoryview.ndim == 2): # <<<<<<<<<<<<<< + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) + * if memslice.memview: +*/ + __pyx_t_9 = __Pyx_PyMemoryView_Get_ndim(__pyx_v_arg_as_memoryview); if (unlikely(__pyx_t_9 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(1, 86, __pyx_L18_except_error) + __pyx_t_5 = (__pyx_t_9 == 2); + __pyx_t_1 = __pyx_t_5; + __pyx_L29_bool_binop_done:; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":84 + * + * # try unsigned int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(unsigned int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(unsigned int)) + * and arg_as_memoryview.ndim == 2): +*/ + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":87 + * or itemsize == sizeof(unsigned int)) + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) # <<<<<<<<<<<<<< + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) +*/ + __pyx_v_memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(__pyx_v_arg_as_memoryview, 0); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":88 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + __pyx_t_1 = (__pyx_v_memslice.memview != 0); + if (__pyx_t_1) { + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":89 + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) + * if memslice.memview: + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) # <<<<<<<<<<<<<< + * # print 'found a match for the buffer through format parsing' + * return 'uint2d_t' +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_memslice), 1); + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":91 + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' + * return 'uint2d_t' # <<<<<<<<<<<<<< + * else: + * __pyx_PyErr_Clear() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_uint2d_t); + __pyx_r = __pyx_mstate_global->__pyx_n_u_uint2d_t; + goto __pyx_L19_except_return; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":88 + * and arg_as_memoryview.ndim == 2): + * memslice = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(arg_as_memoryview, 0) + * if memslice.memview: # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&memslice, 1) + * # print 'found a match for the buffer through format parsing' +*/ + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":93 + * return 'uint2d_t' + * else: + * __pyx_PyErr_Clear() # <<<<<<<<<<<<<< + * return None +*/ + /*else*/ { + PyErr_Clear(); + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":84 + * + * # try unsigned int + * if (((itemsize == -1 and arg_as_memoryview.itemsize == sizeof(unsigned int)) # <<<<<<<<<<<<<< + * or itemsize == sizeof(unsigned int)) + * and arg_as_memoryview.ndim == 2): +*/ + } + } + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L21_try_end; + __pyx_L16_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":67 + * try: + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): # <<<<<<<<<<<<<< + * pass + * else: +*/ + __pyx_t_9 = __Pyx_PyErr_ExceptionMatches2(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), ((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_9) { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L17_exception_handled; + } + goto __pyx_L18_except_error; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":65 + * return 'int2d_t' + * + * try: # <<<<<<<<<<<<<< + * arg_as_memoryview = memoryview(arg) + * except (ValueError, TypeError): +*/ + __pyx_L18_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L1_error; + __pyx_L19_except_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L0; + __pyx_L17_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + __pyx_L21_try_end:; + } + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":94 + * else: + * __pyx_PyErr_Clear() + * return None # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc":14 + * __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(object, int) + * + * @cname('__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc') # <<<<<<<<<<<<<< + * cdef str map_fused_type(object arg, type ndarray): + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("__pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc.map_fused_type", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_arg_as_memoryview); + __Pyx_XDECREF(__pyx_v_dtype); + __Pyx_XDECREF(__pyx_v_arg_base); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "match_signatures":15 + * cimport cython + * + * @cname("__pyx_ff_index_signature") # <<<<<<<<<<<<<< + * cdef object index_signature(dest_sig: tuple, signatures: dict, sigindex: dict): + * matched_function = None +*/ + +static PyObject *__pyx_ff_index_signature(PyObject *__pyx_v_dest_sig, PyObject *__pyx_v_signatures, PyObject *__pyx_v_sigindex) { + PyObject *__pyx_v_matched_function = NULL; + PyObject *__pyx_v_sig = NULL; + PyObject *__pyx_v_function = NULL; + PyObject *__pyx_v_types = NULL; + PyObject *__pyx_v_type_name = NULL; + PyObject *__pyx_v_dest_type = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + size_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *(*__pyx_t_10)(PyObject *); + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + PyObject *(*__pyx_t_14)(PyObject *); + int __pyx_t_15; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("index_signature", 0); + + /* "match_signatures":17 + * @cname("__pyx_ff_index_signature") + * cdef object index_signature(dest_sig: tuple, signatures: dict, sigindex: dict): + * matched_function = None # <<<<<<<<<<<<<< + * for sig, function in signatures.items(): + * types = ( sig).strip('()').split('|') +*/ + __Pyx_INCREF(Py_None); + __pyx_v_matched_function = Py_None; + + /* "match_signatures":18 + * cdef object index_signature(dest_sig: tuple, signatures: dict, sigindex: dict): + * matched_function = None + * for sig, function in signatures.items(): # <<<<<<<<<<<<<< + * types = ( sig).strip('()').split('|') + * +*/ + __pyx_t_2 = 0; + __pyx_t_5 = __Pyx_dict_iterator(__pyx_v_signatures, 1, __pyx_mstate_global->__pyx_n_u_items, (&__pyx_t_3), (&__pyx_t_4)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_1); + __pyx_t_1 = __pyx_t_5; + __pyx_t_5 = 0; + while (1) { + __pyx_t_7 = __Pyx_dict_iter_next(__pyx_t_1, __pyx_t_3, &__pyx_t_2, &__pyx_t_5, &__pyx_t_6, NULL, __pyx_t_4); + if (unlikely(__pyx_t_7 == 0)) break; + if (unlikely(__pyx_t_7 == -1)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GOTREF(__pyx_t_6); + __Pyx_XDECREF_SET(__pyx_v_sig, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XDECREF_SET(__pyx_v_function, __pyx_t_6); + __pyx_t_6 = 0; + + /* "match_signatures":19 + * matched_function = None + * for sig, function in signatures.items(): + * types = ( sig).strip('()').split('|') # <<<<<<<<<<<<<< + * + * for type_name, dest_type in zip(types, dest_sig): +*/ + __pyx_t_5 = __pyx_v_sig; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_strip, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_5 = PyUnicode_Split(((PyObject*)__pyx_t_6), __pyx_mstate_global->__pyx_kp_u__2, -1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF_SET(__pyx_v_types, ((PyObject*)__pyx_t_5)); + __pyx_t_5 = 0; + + /* "match_signatures":21 + * types = ( sig).strip('()').split('|') + * + * for type_name, dest_type in zip(types, dest_sig): # <<<<<<<<<<<<<< + * if dest_type is None: + * continue +*/ + __pyx_t_6 = NULL; + __pyx_t_8 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_v_types, __pyx_v_dest_sig}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_zip, __pyx_callargs+__pyx_t_8, (3-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + if (likely(PyList_CheckExact(__pyx_t_5)) || PyTuple_CheckExact(__pyx_t_5)) { + __pyx_t_6 = __pyx_t_5; __Pyx_INCREF(__pyx_t_6); + __pyx_t_9 = 0; + __pyx_t_10 = NULL; + } else { + __pyx_t_9 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_10 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 21, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + for (;;) { + if (likely(!__pyx_t_10)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 21, __pyx_L1_error) + #endif + if (__pyx_t_9 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_6, __pyx_t_9, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_9; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 21, __pyx_L1_error) + #endif + if (__pyx_t_9 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_9)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_9); + #endif + ++__pyx_t_9; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 21, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_10(__pyx_t_6); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 21, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 21, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_11 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_11); + __pyx_t_12 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_12); + } else { + __pyx_t_11 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_12); + } + #else + __pyx_t_11 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_13 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_13)) __PYX_ERR(1, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_14 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_13); + index = 0; __pyx_t_11 = __pyx_t_14(__pyx_t_13); if (unlikely(!__pyx_t_11)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_11); + index = 1; __pyx_t_12 = __pyx_t_14(__pyx_t_13); if (unlikely(!__pyx_t_12)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_12); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_14(__pyx_t_13), 2) < (0)) __PYX_ERR(1, 21, __pyx_L1_error) + __pyx_t_14 = NULL; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + goto __pyx_L8_unpacking_done; + __pyx_L7_unpacking_failed:; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_14 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(1, 21, __pyx_L1_error) + __pyx_L8_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_type_name, __pyx_t_11); + __pyx_t_11 = 0; + __Pyx_XDECREF_SET(__pyx_v_dest_type, __pyx_t_12); + __pyx_t_12 = 0; + + /* "match_signatures":22 + * + * for type_name, dest_type in zip(types, dest_sig): + * if dest_type is None: # <<<<<<<<<<<<<< + * continue + * if dest_type != type_name: +*/ + __pyx_t_15 = (__pyx_v_dest_type == Py_None); + if (__pyx_t_15) { + + /* "match_signatures":23 + * for type_name, dest_type in zip(types, dest_sig): + * if dest_type is None: + * continue # <<<<<<<<<<<<<< + * if dest_type != type_name: + * break +*/ + goto __pyx_L5_continue; + + /* "match_signatures":22 + * + * for type_name, dest_type in zip(types, dest_sig): + * if dest_type is None: # <<<<<<<<<<<<<< + * continue + * if dest_type != type_name: +*/ + } + + /* "match_signatures":24 + * if dest_type is None: + * continue + * if dest_type != type_name: # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_t_5 = PyObject_RichCompare(__pyx_v_dest_type, __pyx_v_type_name, Py_NE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_15 < 0))) __PYX_ERR(1, 24, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_15) { + + /* "match_signatures":25 + * continue + * if dest_type != type_name: + * break # <<<<<<<<<<<<<< + * else: + * if matched_function is not None: +*/ + goto __pyx_L6_break; + + /* "match_signatures":24 + * if dest_type is None: + * continue + * if dest_type != type_name: # <<<<<<<<<<<<<< + * break + * else: +*/ + } + + /* "match_signatures":21 + * types = ( sig).strip('()').split('|') + * + * for type_name, dest_type in zip(types, dest_sig): # <<<<<<<<<<<<<< + * if dest_type is None: + * continue +*/ + __pyx_L5_continue:; + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L11_for_else; + __pyx_L6_break:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L12_for_end; + /*else*/ { + __pyx_L11_for_else:; + + /* "match_signatures":27 + * break + * else: + * if matched_function is not None: # <<<<<<<<<<<<<< + * raise TypeError("Function call with ambiguous argument types") + * matched_function = function +*/ + __pyx_t_15 = (__pyx_v_matched_function != Py_None); + if (unlikely(__pyx_t_15)) { + + /* "match_signatures":28 + * else: + * if matched_function is not None: + * raise TypeError("Function call with ambiguous argument types") # <<<<<<<<<<<<<< + * matched_function = function + * +*/ + __pyx_t_5 = NULL; + __pyx_t_8 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_Function_call_with_ambiguous_arg}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 28, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_Raise(__pyx_t_6, 0, 0, 0); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __PYX_ERR(1, 28, __pyx_L1_error) + + /* "match_signatures":27 + * break + * else: + * if matched_function is not None: # <<<<<<<<<<<<<< + * raise TypeError("Function call with ambiguous argument types") + * matched_function = function +*/ + } + + /* "match_signatures":29 + * if matched_function is not None: + * raise TypeError("Function call with ambiguous argument types") + * matched_function = function # <<<<<<<<<<<<<< + * + * if matched_function is None: +*/ + __Pyx_INCREF(__pyx_v_function); + __Pyx_DECREF_SET(__pyx_v_matched_function, __pyx_v_function); + } + __pyx_L12_for_end:; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "match_signatures":31 + * matched_function = function + * + * if matched_function is None: # <<<<<<<<<<<<<< + * raise TypeError("No matching signature found") + * +*/ + __pyx_t_15 = (__pyx_v_matched_function == Py_None); + if (unlikely(__pyx_t_15)) { + + /* "match_signatures":32 + * + * if matched_function is None: + * raise TypeError("No matching signature found") # <<<<<<<<<<<<<< + * + * sigindex[dest_sig] = matched_function +*/ + __pyx_t_6 = NULL; + __pyx_t_8 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_mstate_global->__pyx_kp_u_No_matching_signature_found}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 32, __pyx_L1_error) + + /* "match_signatures":31 + * matched_function = function + * + * if matched_function is None: # <<<<<<<<<<<<<< + * raise TypeError("No matching signature found") + * +*/ + } + + /* "match_signatures":34 + * raise TypeError("No matching signature found") + * + * sigindex[dest_sig] = matched_function # <<<<<<<<<<<<<< + * return matched_function + * +*/ + if (unlikely((PyDict_SetItem(__pyx_v_sigindex, __pyx_v_dest_sig, __pyx_v_matched_function) < 0))) __PYX_ERR(1, 34, __pyx_L1_error) + + /* "match_signatures":35 + * + * sigindex[dest_sig] = matched_function + * return matched_function # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_matched_function); + __pyx_r = __pyx_v_matched_function; + goto __pyx_L0; + + /* "match_signatures":15 + * cimport cython + * + * @cname("__pyx_ff_index_signature") # <<<<<<<<<<<<<< + * cdef object index_signature(dest_sig: tuple, signatures: dict, sigindex: dict): + * matched_function = None +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_AddTraceback("match_signatures.index_signature", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_matched_function); + __Pyx_XDECREF(__pyx_v_sig); + __Pyx_XDECREF(__pyx_v_function); + __Pyx_XDECREF(__pyx_v_types); + __Pyx_XDECREF(__pyx_v_type_name); + __Pyx_XDECREF(__pyx_v_dest_type); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "match_signatures":38 + * + * + * @cname("__pyx_ff_match_signatures") # <<<<<<<<<<<<<< + * cdef object match_signatures(signatures: dict, dest_sig: tuple, sigindex: dict): + * found_match = sigindex.get(dest_sig) +*/ + +static PyObject *__pyx_ff_match_signatures(PyObject *__pyx_v_signatures, PyObject *__pyx_v_dest_sig, PyObject *__pyx_v_sigindex) { + PyObject *__pyx_v_found_match = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("match_signatures", 0); + + /* "match_signatures":40 + * @cname("__pyx_ff_match_signatures") + * cdef object match_signatures(signatures: dict, dest_sig: tuple, sigindex: dict): + * found_match = sigindex.get(dest_sig) # <<<<<<<<<<<<<< + * if found_match is None: + * +*/ + __pyx_t_1 = __Pyx_PyDict_GetItemDefault(__pyx_v_sigindex, __pyx_v_dest_sig, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_found_match = __pyx_t_1; + __pyx_t_1 = 0; + + /* "match_signatures":41 + * cdef object match_signatures(signatures: dict, dest_sig: tuple, sigindex: dict): + * found_match = sigindex.get(dest_sig) + * if found_match is None: # <<<<<<<<<<<<<< + * + * found_match = index_signature(dest_sig, signatures, sigindex) +*/ + __pyx_t_2 = (__pyx_v_found_match == Py_None); + if (__pyx_t_2) { + + /* "match_signatures":43 + * if found_match is None: + * + * found_match = index_signature(dest_sig, signatures, sigindex) # <<<<<<<<<<<<<< + * return found_match +*/ + __pyx_t_1 = __pyx_ff_index_signature(__pyx_v_dest_sig, __pyx_v_signatures, __pyx_v_sigindex); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF_SET(__pyx_v_found_match, __pyx_t_1); + __pyx_t_1 = 0; + + /* "match_signatures":41 + * cdef object match_signatures(signatures: dict, dest_sig: tuple, sigindex: dict): + * found_match = sigindex.get(dest_sig) + * if found_match is None: # <<<<<<<<<<<<<< + * + * found_match = index_signature(dest_sig, signatures, sigindex) +*/ + } + + /* "match_signatures":44 + * + * found_match = index_signature(dest_sig, signatures, sigindex) + * return found_match # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_found_match); + __pyx_r = __pyx_v_found_match; + goto __pyx_L0; + + /* "match_signatures":38 + * + * + * @cname("__pyx_ff_match_signatures") # <<<<<<<<<<<<<< + * cdef object match_signatures(signatures: dict, dest_sig: tuple, sigindex: dict): + * found_match = sigindex.get(dest_sig) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("match_signatures.match_signatures", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_found_match); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + +/* Python wrapper */ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_shape = 0; + Py_ssize_t __pyx_v_itemsize; + PyObject *__pyx_v_format = 0; + PyObject *__pyx_v_mode = 0; + int __pyx_v_allocate_buffer; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_shape,&__pyx_mstate_global->__pyx_n_u_itemsize,&__pyx_mstate_global->__pyx_n_u_format,&__pyx_mstate_global->__pyx_n_u_mode,&__pyx_mstate_global->__pyx_n_u_allocate_buffer,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 129, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 129, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, i); __PYX_ERR(1, 129, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + } + __pyx_v_shape = ((PyObject*)values[0]); + __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_v_format = values[2]; + __pyx_v_mode = values[3]; + if (values[4]) { + __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) + } else { + + /* "View.MemoryView":130 + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, + * mode="c", bint allocate_buffer=True): # <<<<<<<<<<<<<< + * + * cdef int idx +*/ + __pyx_v_allocate_buffer = ((int)1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, __pyx_nargs); __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 129, __pyx_L1_error) + if (unlikely(((PyObject *)__pyx_v_format) == Py_None)) { + PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 129, __pyx_L1_error) + } + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v_shape, __pyx_v_itemsize, __pyx_v_format, __pyx_v_mode, __pyx_v_allocate_buffer); + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer) { + int __pyx_v_idx; + Py_ssize_t __pyx_v_dim; + char __pyx_v_order; + int __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11[5]; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + __Pyx_INCREF(__pyx_v_format); + + /* "View.MemoryView":135 + * cdef Py_ssize_t dim + * + * self.ndim = len(shape) # <<<<<<<<<<<<<< + * self.itemsize = itemsize + * +*/ + if (unlikely(__pyx_v_shape == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 135, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 135, __pyx_L1_error) + __pyx_v_self->ndim = ((int)__pyx_t_1); + + /* "View.MemoryView":136 + * + * self.ndim = len(shape) + * self.itemsize = itemsize # <<<<<<<<<<<<<< + * + * if not self.ndim: +*/ + __pyx_v_self->itemsize = __pyx_v_itemsize; + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + __pyx_t_2 = (!(__pyx_v_self->ndim != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":139 + * + * if not self.ndim: + * raise ValueError, "Empty shape tuple for cython.array" # <<<<<<<<<<<<<< + * + * if itemsize <= 0: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Empty_shape_tuple_for_cython_arr, 0, 0); + __PYX_ERR(1, 139, __pyx_L1_error) + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + } + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + __pyx_t_2 = (__pyx_v_itemsize <= 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":142 + * + * if itemsize <= 0: + * raise ValueError, "itemsize <= 0 for cython.array" # <<<<<<<<<<<<<< + * + * if not isinstance(format, bytes): +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_itemsize_0_for_cython_array, 0, 0); + __PYX_ERR(1, 142, __pyx_L1_error) + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + } + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + __pyx_t_2 = PyBytes_Check(__pyx_v_format); + __pyx_t_3 = (!__pyx_t_2); + if (__pyx_t_3) { + + /* "View.MemoryView":145 + * + * if not isinstance(format, bytes): + * format = format.encode('ASCII') # <<<<<<<<<<<<<< + * self._format = format # keep a reference to the byte string + * self.format = self._format +*/ + __pyx_t_5 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ASCII}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_encode, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF_SET(__pyx_v_format, __pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + } + + /* "View.MemoryView":146 + * if not isinstance(format, bytes): + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string # <<<<<<<<<<<<<< + * self.format = self._format + * +*/ + __pyx_t_4 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_4); + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_v_self->_format); + __Pyx_DECREF(__pyx_v_self->_format); + __pyx_v_self->_format = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":147 + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string + * self.format = self._format # <<<<<<<<<<<<<< + * + * +*/ + if (unlikely(__pyx_v_self->_format == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(1, 147, __pyx_L1_error) + } + __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) + __pyx_v_self->format = __pyx_t_7; + + /* "View.MemoryView":150 + * + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) # <<<<<<<<<<<<<< + * self._strides = self._shape + self.ndim + * +*/ + __pyx_v_self->_shape = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * __pyx_v_self->ndim) * 2))); + + /* "View.MemoryView":151 + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) + * self._strides = self._shape + self.ndim # <<<<<<<<<<<<<< + * + * if not self._shape: +*/ + __pyx_v_self->_strides = (__pyx_v_self->_shape + __pyx_v_self->ndim); + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + __pyx_t_3 = (!(__pyx_v_self->_shape != 0)); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":154 + * + * if not self._shape: + * raise MemoryError, "unable to allocate shape and strides." # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_shape_and_str, 0, 0); + __PYX_ERR(1, 154, __pyx_L1_error) + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + } + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + __pyx_t_8 = 0; + __pyx_t_4 = __pyx_v_shape; __Pyx_INCREF(__pyx_t_4); + __pyx_t_1 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 157, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_4, __pyx_t_1); + #endif + ++__pyx_t_1; + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_9; + __pyx_v_idx = __pyx_t_8; + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + __pyx_t_3 = (__pyx_v_dim <= 0); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":159 + * for idx, dim in enumerate(shape): + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." # <<<<<<<<<<<<<< + * self._shape[idx] = dim + * +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_int(__pyx_v_idx, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11[0] = __pyx_mstate_global->__pyx_kp_u_Invalid_shape_in_axis; + __pyx_t_11[1] = __pyx_t_5; + __pyx_t_11[2] = __pyx_mstate_global->__pyx_kp_u__3; + __pyx_t_11[3] = __pyx_t_10; + __pyx_t_11[4] = __pyx_mstate_global->__pyx_kp_u__4; + __pyx_t_12 = __Pyx_PyUnicode_Join(__pyx_t_11, 5, 22 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_10) + 1, 127); + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 159, __pyx_L1_error) + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + } + + /* "View.MemoryView":160 + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim # <<<<<<<<<<<<<< + * + * cdef char order +*/ + (__pyx_v_self->_shape[__pyx_v_idx]) = __pyx_v_dim; + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 163, __pyx_L1_error) + if (__pyx_t_3) { + + /* "View.MemoryView":164 + * cdef char order + * if mode == 'c': + * order = b'C' # <<<<<<<<<<<<<< + * self.mode = u'c' + * elif mode == 'fortran': +*/ + __pyx_v_order = 'C'; + + /* "View.MemoryView":165 + * if mode == 'c': + * order = b'C' + * self.mode = u'c' # <<<<<<<<<<<<<< + * elif mode == 'fortran': + * order = b'F' +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_c; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 166, __pyx_L1_error) + if (likely(__pyx_t_3)) { + + /* "View.MemoryView":167 + * self.mode = u'c' + * elif mode == 'fortran': + * order = b'F' # <<<<<<<<<<<<<< + * self.mode = u'fortran' + * else: +*/ + __pyx_v_order = 'F'; + + /* "View.MemoryView":168 + * elif mode == 'fortran': + * order = b'F' + * self.mode = u'fortran' # <<<<<<<<<<<<<< + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_fortran; + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":170 + * self.mode = u'fortran' + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" # <<<<<<<<<<<<<< + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) +*/ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_mode, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_12 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_Invalid_mode_expected_c_or_fortr, __pyx_t_4); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 170, __pyx_L1_error) + } + __pyx_L11:; + + /* "View.MemoryView":172 + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) # <<<<<<<<<<<<<< + * + * self.free_data = allocate_buffer +*/ + __pyx_v_self->len = __pyx_fill_contig_strides_array(__pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_itemsize, __pyx_v_self->ndim, __pyx_v_order); + + /* "View.MemoryView":174 + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) + * + * self.free_data = allocate_buffer # <<<<<<<<<<<<<< + * self.dtype_is_object = format == b'O' + * +*/ + __pyx_v_self->free_data = __pyx_v_allocate_buffer; + + /* "View.MemoryView":175 + * + * self.free_data = allocate_buffer + * self.dtype_is_object = format == b'O' # <<<<<<<<<<<<<< + * + * if allocate_buffer: +*/ + __pyx_t_12 = PyObject_RichCompare(__pyx_v_format, __pyx_mstate_global->__pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_12); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_12); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 175, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_v_self->dtype_is_object = __pyx_t_3; + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + if (__pyx_v_allocate_buffer) { + + /* "View.MemoryView":178 + * + * if allocate_buffer: + * _allocate_buffer(self) # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_t_8 = __pyx_array_allocate_buffer(__pyx_v_self); if (unlikely(__pyx_t_8 == ((int)-1))) __PYX_ERR(1, 178, __pyx_L1_error) + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + } + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_format); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(((struct __pyx_array_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_bufmode; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + char *__pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":182 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 # <<<<<<<<<<<<<< + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": +*/ + __pyx_v_bufmode = -1; + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_t_1 = ((__pyx_v_flags & ((PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS) | PyBUF_ANY_CONTIGUOUS)) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 184, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":185 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_v_bufmode = (PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 186, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":187 + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." +*/ + __pyx_v_bufmode = (PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + } + __pyx_L4:; + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + __pyx_t_1 = (!((__pyx_v_flags & __pyx_v_bufmode) != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":189 + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." # <<<<<<<<<<<<<< + * info.buf = self.data + * info.len = self.len +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Can_only_create_a_buffer_that_is, 0, 0); + __PYX_ERR(1, 189, __pyx_L1_error) + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + } + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + } + + /* "View.MemoryView":190 + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data # <<<<<<<<<<<<<< + * info.len = self.len + * +*/ + __pyx_t_2 = __pyx_v_self->data; + __pyx_v_info->buf = __pyx_t_2; + + /* "View.MemoryView":191 + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data + * info.len = self.len # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + __pyx_t_3 = __pyx_v_self->len; + __pyx_v_info->len = __pyx_t_3; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":194 + * + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim # <<<<<<<<<<<<<< + * info.shape = self._shape + * info.strides = self._strides +*/ + __pyx_t_4 = __pyx_v_self->ndim; + __pyx_v_info->ndim = __pyx_t_4; + + /* "View.MemoryView":195 + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim + * info.shape = self._shape # <<<<<<<<<<<<<< + * info.strides = self._strides + * else: +*/ + __pyx_t_5 = __pyx_v_self->_shape; + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":196 + * info.ndim = self.ndim + * info.shape = self._shape + * info.strides = self._strides # <<<<<<<<<<<<<< + * else: + * info.ndim = 1 +*/ + __pyx_t_5 = __pyx_v_self->_strides; + __pyx_v_info->strides = __pyx_t_5; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":198 + * info.strides = self._strides + * else: + * info.ndim = 1 # <<<<<<<<<<<<<< + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL +*/ + /*else*/ { + __pyx_v_info->ndim = 1; + + /* "View.MemoryView":199 + * else: + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL # <<<<<<<<<<<<<< + * info.strides = NULL + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + __pyx_t_5 = (&__pyx_v_self->len); + } else { + __pyx_t_5 = NULL; + } + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":200 + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL # <<<<<<<<<<<<<< + * + * info.suboffsets = NULL +*/ + __pyx_v_info->strides = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":202 + * info.strides = NULL + * + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * info.itemsize = self.itemsize + * info.readonly = 0 +*/ + __pyx_v_info->suboffsets = NULL; + + /* "View.MemoryView":203 + * + * info.suboffsets = NULL + * info.itemsize = self.itemsize # <<<<<<<<<<<<<< + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL +*/ + __pyx_t_3 = __pyx_v_self->itemsize; + __pyx_v_info->itemsize = __pyx_t_3; + + /* "View.MemoryView":204 + * info.suboffsets = NULL + * info.itemsize = self.itemsize + * info.readonly = 0 # <<<<<<<<<<<<<< + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self +*/ + __pyx_v_info->readonly = 0; + + /* "View.MemoryView":205 + * info.itemsize = self.itemsize + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + __pyx_t_2 = __pyx_v_self->format; + } else { + __pyx_t_2 = NULL; + } + __pyx_v_info->format = __pyx_t_2; + + /* "View.MemoryView":206 + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self # <<<<<<<<<<<<<< + * + * def __dealloc__(array self): +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + +/* Python wrapper */ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + __pyx_t_1 = (__pyx_v_self->callback_free_data != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":210 + * def __dealloc__(array self): + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) # <<<<<<<<<<<<<< + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: +*/ + __pyx_v_self->callback_free_data(__pyx_v_self->data); + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + if (__pyx_v_self->free_data) { + } else { + __pyx_t_1 = __pyx_v_self->free_data; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_self->data != NULL); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":213 + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) # <<<<<<<<<<<<<< + * free(self.data) + * PyObject_Free(self._shape) +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_self->data, __pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_self->ndim, 0); + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + } + + /* "View.MemoryView":214 + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) # <<<<<<<<<<<<<< + * PyObject_Free(self._shape) + * +*/ + free(__pyx_v_self->data); + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + } + __pyx_L3:; + + /* "View.MemoryView":215 + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) + * PyObject_Free(self._shape) # <<<<<<<<<<<<<< + * + * @property +*/ + PyObject_Free(__pyx_v_self->_shape); + + /* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_5array_7memview___get__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":219 + * @property + * def memview(self): + * return self.get_memview() # <<<<<<<<<<<<<< + * + * @cname('get_memview') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.memview.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_memview", 0); + + /* "View.MemoryView":223 + * @cname('get_memview') + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE # <<<<<<<<<<<<<< + * return memoryview(self, flags, self.dtype_is_object) + * +*/ + __pyx_v_flags = ((PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT) | PyBUF_WRITABLE); + + /* "View.MemoryView":224 + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE + * return memoryview(self, flags, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, ((PyObject *)__pyx_v_self), __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_r = ((PyObject *)__pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array.get_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + + /* "View.MemoryView":227 + * + * def __len__(self): + * return self._shape[0] # <<<<<<<<<<<<<< + * + * def __getattr__(self, attr): +*/ + __pyx_r = (__pyx_v_self->_shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr); /*proto*/ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getattr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_attr)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getattr__", 0); + + /* "View.MemoryView":230 + * + * def __getattr__(self, attr): + * return getattr(self.memview, attr) # <<<<<<<<<<<<<< + * + * def __getitem__(self, item): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getattr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item); /*proto*/ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":233 + * + * def __getitem__(self, item): + * return self.memview[item] # <<<<<<<<<<<<<< + * + * def __setitem__(self, item, value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + +/* Python wrapper */ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + + /* "View.MemoryView":236 + * + * def __setitem__(self, item, value): + * self.memview[item] = value # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely((PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0))) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_array___reduce_cython__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_array_2__setstate_cython__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_v_i; + PyObject **__pyx_v_p; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":252 + * cdef PyObject **p + * + * self.free_data = True # <<<<<<<<<<<<<< + * self.data = malloc(self.len) + * if not self.data: +*/ + __pyx_v_self->free_data = 1; + + /* "View.MemoryView":253 + * + * self.free_data = True + * self.data = malloc(self.len) # <<<<<<<<<<<<<< + * if not self.data: + * raise MemoryError, "unable to allocate array data." +*/ + __pyx_v_self->data = ((char *)malloc(__pyx_v_self->len)); + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + __pyx_t_1 = (!(__pyx_v_self->data != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":255 + * self.data = malloc(self.len) + * if not self.data: + * raise MemoryError, "unable to allocate array data." # <<<<<<<<<<<<<< + * + * if self.dtype_is_object: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_array_data, 0, 0); + __PYX_ERR(1, 255, __pyx_L1_error) + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":258 + * + * if self.dtype_is_object: + * p = self.data # <<<<<<<<<<<<<< + * for i in range(self.len // self.itemsize): + * p[i] = Py_None +*/ + __pyx_v_p = ((PyObject **)__pyx_v_self->data); + + /* "View.MemoryView":259 + * if self.dtype_is_object: + * p = self.data + * for i in range(self.len // self.itemsize): # <<<<<<<<<<<<<< + * p[i] = Py_None + * Py_INCREF(Py_None) +*/ + if (unlikely(__pyx_v_self->itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_self->itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_self->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_div_Py_ssize_t(__pyx_v_self->len, __pyx_v_self->itemsize, 0); + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":260 + * p = self.data + * for i in range(self.len // self.itemsize): + * p[i] = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * return 0 +*/ + (__pyx_v_p[__pyx_v_i]) = Py_None; + + /* "View.MemoryView":261 + * for i in range(self.len // self.itemsize): + * p[i] = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * return 0 + * +*/ + Py_INCREF(Py_None); + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + } + + /* "View.MemoryView":262 + * p[i] = Py_None + * Py_INCREF(Py_None) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._allocate_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + +static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, char *__pyx_v_format, char const *__pyx_v_c_mode, char *__pyx_v_buf) { + struct __pyx_array_obj *__pyx_v_result = 0; + PyObject *__pyx_v_mode = 0; + struct __pyx_array_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("array_cwrapper", 0); + + /* "View.MemoryView":268 + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. # <<<<<<<<<<<<<< + * + * if buf is NULL: +*/ + __pyx_t_2 = ((__pyx_v_c_mode[0]) == 'f'); + if (__pyx_t_2) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_fortran; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_c; + } + __pyx_v_mode = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + __pyx_t_2 = (__pyx_v_buf == NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":271 + * + * if buf is NULL: + * result = array.__new__(array, shape, itemsize, format, mode) # <<<<<<<<<<<<<< + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) +*/ + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_3) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_4, NULL)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":273 + * result = array.__new__(array, shape, itemsize, format, mode) + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) # <<<<<<<<<<<<<< + * result.data = buf + * +*/ + /*else*/ { + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_3) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_allocate_buffer, Py_False) < (0)) __PYX_ERR(1, 273, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_1, __pyx_t_4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":274 + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) + * result.data = buf # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->data = __pyx_v_buf; + } + __pyx_L3:; + + /* "View.MemoryView":276 + * result.data = buf + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_mode); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + +/* Python wrapper */ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_name = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 302, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(1, 302, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, i); __PYX_ERR(1, 302, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + } + __pyx_v_name = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 302, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v_name); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__", 0); + + /* "View.MemoryView":303 + * cdef object name + * def __init__(self, name): + * self.name = name # <<<<<<<<<<<<<< + * def __repr__(self): + * return self.name +*/ + __Pyx_INCREF(__pyx_v_name); + __Pyx_GIVEREF(__pyx_v_name); + __Pyx_GOTREF(__pyx_v_self->name); + __Pyx_DECREF(__pyx_v_self->name); + __pyx_v_self->name = __pyx_v_name; + + /* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":305 + * self.name = name + * def __repr__(self): + * return self.name # <<<<<<<<<<<<<< + * + * cdef generic = Enum("") +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->name); + __pyx_r = __pyx_v_self->name; + goto __pyx_L0; + + /* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_MemviewEnum___reduce_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_v_state = 0; + PyObject *__pyx_v__dict = 0; + int __pyx_v_use_setstate; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":5 + * cdef object _dict + * cdef bint use_setstate + * state = (self.name,) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_self->name); + __Pyx_GIVEREF(__pyx_v_self->name); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->name) != (0)) __PYX_ERR(1, 5, __pyx_L1_error); + __pyx_v_state = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "(tree fragment)":6 + * cdef bint use_setstate + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None and _dict: + * state += (_dict,) +*/ + __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v__dict = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + __pyx_t_3 = (__pyx_v__dict != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_2) { + + /* "(tree fragment)":8 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(1, 8, __pyx_L1_error); + __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "(tree fragment)":9 + * if _dict is not None and _dict: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = ('self.name is not None',) +*/ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + goto __pyx_L3; + } + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + /*else*/ { + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_mstate_global->__pyx_tuple[0]); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_v_use_setstate = __pyx_t_2; + } + __pyx_L3:; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + if (__pyx_v_use_setstate) { + + /* "(tree fragment)":13 + * use_setstate = ('self.name is not None',) + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + } + + /* "(tree fragment)":15 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.Enum.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 16, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 16, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 16, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 16, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_MemviewEnum_2__setstate_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":17 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) # <<<<<<<<<<<<<< +*/ + __pyx_t_1 = __pyx_v___pyx_state; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(1, 17, __pyx_L1_error) + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 17, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + +/* Python wrapper */ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + int __pyx_v_flags; + int __pyx_v_dtype_is_object; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_obj,&__pyx_mstate_global->__pyx_n_u_flags,&__pyx_mstate_global->__pyx_n_u_dtype_is_object,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 348, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 348, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, i); __PYX_ERR(1, 348, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_obj = values[0]; + __pyx_v_flags = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + if (values[2]) { + __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + } else { + __pyx_v_dtype_is_object = ((int)0); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, __pyx_nargs); __PYX_ERR(1, 348, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_obj, __pyx_v_flags, __pyx_v_dtype_is_object); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + Py_intptr_t __pyx_t_4; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "View.MemoryView":349 + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj # <<<<<<<<<<<<<< + * self.flags = flags + * if type(self) is memoryview or obj is not None: +*/ + __Pyx_INCREF(__pyx_v_obj); + __Pyx_GIVEREF(__pyx_v_obj); + __Pyx_GOTREF(__pyx_v_self->obj); + __Pyx_DECREF(__pyx_v_self->obj); + __pyx_v_self->obj = __pyx_v_obj; + + /* "View.MemoryView":350 + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj + * self.flags = flags # <<<<<<<<<<<<<< + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) +*/ + __pyx_v_self->flags = __pyx_v_flags; + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + __pyx_t_2 = (((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))) == ((PyObject *)__pyx_mstate_global->__pyx_memoryview_type)); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_obj != Py_None); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":352 + * self.flags = flags + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) # <<<<<<<<<<<<<< + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None +*/ + __pyx_t_3 = PyObject_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 352, __pyx_L1_error) + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_self->view.obj) == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":354 + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = Py_None; + + /* "View.MemoryView":355 + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + } + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + __pyx_t_1 = (!__PYX_CYTHON_ATOMICS_ENABLED()); + if (__pyx_t_1) { + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + __pyx_t_2 = (__pyx_memoryview_thread_locks_used < 8); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L9_bool_binop_done; + } + + /* "View.MemoryView":361 + * if (__pyx_memoryview_thread_locks_used < 8 and + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 +*/ + __pyx_t_2 = (!__PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()); + __pyx_t_1 = __pyx_t_2; + __pyx_L9_bool_binop_done:; + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":362 + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: +*/ + __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + + /* "View.MemoryView":363 + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":365 + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< + * if self.lock is NULL: + * raise MemoryError +*/ + __pyx_v_self->lock = PyThread_allocate_lock(); + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":367 + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 367, __pyx_L1_error) + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + } + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":370 + * + * if flags & PyBUF_FORMAT: + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') # <<<<<<<<<<<<<< + * else: + * self.dtype_is_object = dtype_is_object +*/ + __pyx_t_2 = ((__pyx_v_self->view.format[0]) == 'O'); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_self->view.format[1]) == '\x00'); + __pyx_t_1 = __pyx_t_2; + __pyx_L14_bool_binop_done:; + __pyx_v_self->dtype_is_object = __pyx_t_1; + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":372 + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: + * self.dtype_is_object = dtype_is_object # <<<<<<<<<<<<<< + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 +*/ + /*else*/ { + __pyx_v_self->dtype_is_object = __pyx_v_dtype_is_object; + } + __pyx_L13:; + + /* "View.MemoryView":374 + * self.dtype_is_object = dtype_is_object + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 # <<<<<<<<<<<<<< + * self.typeinfo = NULL + * +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_4 = ((Py_intptr_t)((void *)(&__pyx_v_self->acquisition_count))); + __pyx_t_5 = (sizeof(__pyx_atomic_int_type)); + if (unlikely(__pyx_t_5 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 374, __pyx_L1_error) + } + __pyx_t_1 = ((__pyx_t_4 % __pyx_t_5) == 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 374, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 374, __pyx_L1_error) + #endif + + /* "View.MemoryView":375 + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 + * self.typeinfo = NULL # <<<<<<<<<<<<<< + * + * def __dealloc__(memoryview self): +*/ + __pyx_v_self->typeinfo = NULL; + + /* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + +/* Python wrapper */ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self) { + int __pyx_v_i; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + PyThread_type_lock __pyx_t_5; + PyThread_type_lock __pyx_t_6; + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + __pyx_t_1 = (__pyx_v_self->obj != Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":379 + * def __dealloc__(memoryview self): + * if self.obj is not None: + * PyBuffer_Release(&self.view) # <<<<<<<<<<<<<< + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * +*/ + PyBuffer_Release((&__pyx_v_self->view)); + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + __pyx_t_1 = (((Py_buffer *)(&__pyx_v_self->view))->obj == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":382 + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * + * (<__pyx_buffer *> &self.view).obj = NULL # <<<<<<<<<<<<<< + * Py_DECREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = NULL; + + /* "View.MemoryView":383 + * + * (<__pyx_buffer *> &self.view).obj = NULL + * Py_DECREF(Py_None) # <<<<<<<<<<<<<< + * + * cdef int i +*/ + Py_DECREF(Py_None); + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + } + __pyx_L3:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + __pyx_t_1 = (__pyx_v_self->lock != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":388 + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): # <<<<<<<<<<<<<< + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 +*/ + __pyx_t_1 = __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING(); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = __pyx_memoryview_thread_locks_used; + } + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + __pyx_t_1 = ((__pyx_memoryview_thread_locks[__pyx_v_i]) == __pyx_v_self->lock); + if (__pyx_t_1) { + + /* "View.MemoryView":390 + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 # <<<<<<<<<<<<<< + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used - 1); + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + __pyx_t_1 = (__pyx_v_i != __pyx_memoryview_thread_locks_used); + if (__pyx_t_1) { + + /* "View.MemoryView":393 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_t_5 = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + __pyx_t_6 = (__pyx_memoryview_thread_locks[__pyx_v_i]); + + /* "View.MemoryView":392 + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break +*/ + (__pyx_memoryview_thread_locks[__pyx_v_i]) = __pyx_t_5; + (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]) = __pyx_t_6; + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + } + + /* "View.MemoryView":394 + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break # <<<<<<<<<<<<<< + * else: + * PyThread_free_lock(self.lock) +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + } + } + /*else*/ { + + /* "View.MemoryView":396 + * break + * else: + * PyThread_free_lock(self.lock) # <<<<<<<<<<<<<< + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: +*/ + PyThread_free_lock(__pyx_v_self->lock); + } + __pyx_L6_break:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + } + + /* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + Py_ssize_t __pyx_v_dim; + char *__pyx_v_itemp; + PyObject *__pyx_v_idx = NULL; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_item_pointer", 0); + + /* "View.MemoryView":400 + * cdef char *get_item_pointer(memoryview self, object index) except NULL: + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf # <<<<<<<<<<<<<< + * + * for dim, idx in enumerate(index): +*/ + __pyx_v_itemp = ((char *)__pyx_v_self->view.buf); + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + __pyx_t_1 = 0; + if (likely(PyList_CheckExact(__pyx_v_index)) || PyTuple_CheckExact(__pyx_v_index)) { + __pyx_t_2 = __pyx_v_index; __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 402, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 402, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_3, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 402, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 402, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_1; + __pyx_t_1 = (__pyx_t_1 + 1); + + /* "View.MemoryView":403 + * + * for dim, idx in enumerate(index): + * itemp = pybuffer_index(&self.view, itemp, idx, dim) # <<<<<<<<<<<<<< + * + * return itemp +*/ + __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_7; + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":405 + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * + * return itemp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_itemp; + goto __pyx_L0; + + /* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.get_item_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index); /*proto*/ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_indices = NULL; + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + char *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + __pyx_t_1 = (__pyx_v_index == __pyx_builtin_Ellipsis); + if (__pyx_t_1) { + + /* "View.MemoryView":410 + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: + * return self # <<<<<<<<<<<<<< + * + * have_slices, indices = _unellipsify(index, self.view.ndim) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_self); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + } + + /* "View.MemoryView":412 + * return self + * + * have_slices, indices = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * cdef char *itemp +*/ + __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (likely(__pyx_t_2 != Py_None)) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 412, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 412, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_indices = __pyx_t_4; + __pyx_t_4 = 0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 415, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":416 + * cdef char *itemp + * if have_slices: + * return memview_slice(self, indices) # <<<<<<<<<<<<<< + * else: + * itemp = self.get_item_pointer(indices) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + } + + /* "View.MemoryView":418 + * return memview_slice(self, indices) + * else: + * itemp = self.get_item_pointer(indices) # <<<<<<<<<<<<<< + * return self.convert_item_to_object(itemp) + * +*/ + /*else*/ { + __pyx_t_5 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_5; + + /* "View.MemoryView":419 + * else: + * itemp = self.get_item_pointer(indices) + * return self.convert_item_to_object(itemp) # <<<<<<<<<<<<<< + * + * def __setitem__(memoryview self, object index, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + +/* Python wrapper */ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_obj = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + __Pyx_INCREF(__pyx_v_index); + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + if (unlikely(__pyx_v_self->view.readonly)) { + + /* "View.MemoryView":423 + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" # <<<<<<<<<<<<<< + * + * have_slices, index = _unellipsify(index, self.view.ndim) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_Cannot_assign_to_read_only_memor, 0, 0); + __PYX_ERR(1, 423, __pyx_L1_error) + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + } + + /* "View.MemoryView":425 + * raise TypeError, "Cannot assign to read-only memoryview" + * + * have_slices, index = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * if have_slices: +*/ + __pyx_t_1 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (likely(__pyx_t_1 != Py_None)) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 425, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 425, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_DECREF_SET(__pyx_v_index, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(1, 427, __pyx_L1_error) + if (__pyx_t_4) { + + /* "View.MemoryView":428 + * + * if have_slices: + * obj = self.is_slice(value) # <<<<<<<<<<<<<< + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 428, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_obj = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + __pyx_t_4 = (__pyx_v_obj != Py_None); + if (__pyx_t_4) { + + /* "View.MemoryView":430 + * obj = self.is_slice(value) + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) # <<<<<<<<<<<<<< + * else: + * self.setitem_slice_assign_scalar(self[index], value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_1, __pyx_v_obj); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":432 + * self.setitem_slice_assignment(self[index], obj) + * else: + * self.setitem_slice_assign_scalar(self[index], value) # <<<<<<<<<<<<<< + * else: + * self.setitem_indexed(index, value) +*/ + /*else*/ { + __pyx_t_3 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 432, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_3), __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L5:; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":434 + * self.setitem_slice_assign_scalar(self[index], value) + * else: + * self.setitem_indexed(index, value) # <<<<<<<<<<<<<< + * + * cdef is_slice(self, obj): +*/ + /*else*/ { + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L4:; + + /* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_slice", 0); + __Pyx_INCREF(__pyx_v_obj); + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_obj, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + /*try:*/ { + + /* "View.MemoryView":439 + * if not isinstance(obj, memoryview): + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< + * self.dtype_is_object) + * except TypeError: +*/ + __pyx_t_7 = NULL; + __pyx_t_8 = __Pyx_PyLong_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + + /* "View.MemoryView":440 + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) # <<<<<<<<<<<<<< + * except TypeError: + * return None +*/ + __pyx_t_9 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 440, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_7, __pyx_v_obj, __pyx_t_8, __pyx_t_9}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_10, (4-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF((PyObject *)__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_obj, ((PyObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L9_try_end; + __pyx_L4_error:; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":441 + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) + * except TypeError: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_11) { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":442 + * self.dtype_is_object) + * except TypeError: + * return None # <<<<<<<<<<<<<< + * + * return obj +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L7_except_return; + } + goto __pyx_L6_except_error; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + __pyx_L6_except_error:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L1_error; + __pyx_L7_except_return:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L0; + __pyx_L9_try_end:; + } + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + } + + /* "View.MemoryView":444 + * return None + * + * return obj # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assignment(self, dst, src): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_obj); + __pyx_r = __pyx_v_obj; + goto __pyx_L0; + + /* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src) { + __Pyx_memviewslice __pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_src_slice; + __Pyx_memviewslice __pyx_v_msrc; + __Pyx_memviewslice __pyx_v_mdst; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assignment", 0); + + /* "View.MemoryView":449 + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * +*/ + if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_v_msrc = (__pyx_t_1[0]); + + /* "View.MemoryView":450 + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] # <<<<<<<<<<<<<< + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) +*/ + if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_v_mdst = (__pyx_t_1[0]); + + /* "View.MemoryView":452 + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __pyx_memoryview_copy_contents(__pyx_v_msrc, __pyx_v_mdst, __pyx_t_3, __pyx_t_4, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 452, __pyx_L1_error) + + /* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assignment", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value) { + int __pyx_v_array[128]; + void *__pyx_v_tmp; + void *__pyx_v_item; + __Pyx_memviewslice *__pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_tmp_slice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + char const *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assign_scalar", 0); + + /* "View.MemoryView":456 + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): + * cdef int array[128] + * cdef void *tmp = NULL # <<<<<<<<<<<<<< + * cdef void *item + * +*/ + __pyx_v_tmp = NULL; + + /* "View.MemoryView":461 + * cdef __Pyx_memviewslice *dst_slice + * cdef __Pyx_memviewslice tmp_slice + * dst_slice = get_slice_from_memview(dst, &tmp_slice) # <<<<<<<<<<<<<< + * + * if self.view.itemsize > sizeof(array): +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 461, __pyx_L1_error) + __pyx_v_dst_slice = __pyx_t_1; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + __pyx_t_2 = (((size_t)__pyx_v_self->view.itemsize) > (sizeof(__pyx_v_array))); + if (__pyx_t_2) { + + /* "View.MemoryView":464 + * + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) # <<<<<<<<<<<<<< + * if tmp == NULL: + * raise MemoryError +*/ + __pyx_v_tmp = PyMem_Malloc(__pyx_v_self->view.itemsize); + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + __pyx_t_2 = (__pyx_v_tmp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":466 + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * item = tmp + * else: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 466, __pyx_L1_error) + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + } + + /* "View.MemoryView":467 + * if tmp == NULL: + * raise MemoryError + * item = tmp # <<<<<<<<<<<<<< + * else: + * item = array +*/ + __pyx_v_item = __pyx_v_tmp; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":469 + * item = tmp + * else: + * item = array # <<<<<<<<<<<<<< + * + * try: +*/ + /*else*/ { + __pyx_v_item = ((void *)__pyx_v_array); + } + __pyx_L3:; + + /* "View.MemoryView":471 + * item = array + * + * try: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * ( item)[0] = value +*/ + /*try:*/ { + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":473 + * try: + * if self.dtype_is_object: + * ( item)[0] = value # <<<<<<<<<<<<<< + * else: + * self.assign_item_from_object( item, value) +*/ + (((PyObject **)__pyx_v_item)[0]) = ((PyObject *)__pyx_v_value); + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":475 + * ( item)[0] = value + * else: + * self.assign_item_from_object( item, value) # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 475, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L8:; + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + __pyx_t_2 = (__pyx_v_self->view.suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":480 + * + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) # <<<<<<<<<<<<<< + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, + * item, self.dtype_is_object) +*/ + __pyx_t_4 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 480, __pyx_L6_error) + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + } + + /* "View.MemoryView":481 + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, # <<<<<<<<<<<<<< + * item, self.dtype_is_object) + * finally: +*/ + __pyx_memoryview_slice_assign_scalar(__pyx_v_dst_slice, __pyx_v_dst->view.ndim, __pyx_v_self->view.itemsize, __pyx_v_item, __pyx_v_self->dtype_is_object); + } + + /* "View.MemoryView":484 + * item, self.dtype_is_object) + * finally: + * PyMem_Free(tmp) # <<<<<<<<<<<<<< + * + * cdef setitem_indexed(self, index, value): +*/ + /*finally:*/ { + /*normal exit:*/{ + PyMem_Free(__pyx_v_tmp); + goto __pyx_L7; + } + __pyx_L6_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_ExceptionSwap(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); + if ( unlikely(__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0)) __Pyx_ErrFetch(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_XGOTREF(__pyx_t_12); + __pyx_t_4 = __pyx_lineno; __pyx_t_5 = __pyx_clineno; __pyx_t_6 = __pyx_filename; + { + PyMem_Free(__pyx_v_tmp); + } + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ErrRestore(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_5; __pyx_filename = __pyx_t_6; + goto __pyx_L1_error; + } + __pyx_L7:; + } + + /* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assign_scalar", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_indexed", 0); + + /* "View.MemoryView":487 + * + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) # <<<<<<<<<<<<<< + * self.assign_item_from_object(itemp, value) + * +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 487, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_1; + + /* "View.MemoryView":488 + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 488, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_indexed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_v_struct = NULL; + PyObject *__pyx_v_bytesitem = 0; + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":493 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef bytes bytesitem + * +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 493, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":496 + * cdef bytes bytesitem + * + * bytesitem = itemp[:self.view.itemsize] # <<<<<<<<<<<<<< + * try: + * result = struct.unpack(self.view.format, bytesitem) +*/ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 496, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_bytesitem = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_2, &__pyx_t_3, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + /*try:*/ { + + /* "View.MemoryView":498 + * bytesitem = itemp[:self.view.itemsize] + * try: + * result = struct.unpack(self.view.format, bytesitem) # <<<<<<<<<<<<<< + * except struct.error: + * raise ValueError, "Unable to convert item to object" +*/ + __pyx_t_5 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_t_6, __pyx_v_bytesitem}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_unpack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_result = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + } + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + /*else:*/ { + __pyx_t_8 = __Pyx_ssize_strlen(__pyx_v_self->view.format); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 502, __pyx_L5_except_error) + __pyx_t_9 = (__pyx_t_8 == 1); + if (__pyx_t_9) { + + /* "View.MemoryView":503 + * else: + * if len(self.view.format) == 1: + * return result[0] # <<<<<<<<<<<<<< + * return result + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 503, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L6_except_return; + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + } + + /* "View.MemoryView":504 + * if len(self.view.format) == 1: + * return result[0] + * return result # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L6_except_return; + } + __pyx_L3_error:; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "View.MemoryView":499 + * try: + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: # <<<<<<<<<<<<<< + * raise ValueError, "Unable to convert item to object" + * else: +*/ + __Pyx_ErrFetch(&__pyx_t_1, &__pyx_t_6, &__pyx_t_5); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_error); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_1, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_ErrRestore(__pyx_t_1, __pyx_t_6, __pyx_t_5); + __pyx_t_1 = 0; __pyx_t_6 = 0; __pyx_t_5 = 0; + if (__pyx_t_11) { + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_1) < 0) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_1); + + /* "View.MemoryView":500 + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: + * raise ValueError, "Unable to convert item to object" # <<<<<<<<<<<<<< + * else: + * if len(self.view.format) == 1: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Unable_to_convert_item_to_object, 0, 0); + __PYX_ERR(1, 500, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L1_error; + __pyx_L6_except_return:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L0; + } + + /* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesitem); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_v_struct = NULL; + char __pyx_v_c; + PyObject *__pyx_v_bytesvalue = 0; + Py_ssize_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + char *__pyx_t_10; + char *__pyx_t_11; + Py_ssize_t __pyx_t_12; + char *__pyx_t_13; + char *__pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":509 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef char c + * cdef bytes bytesvalue +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 509, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + __pyx_t_3 = PyTuple_Check(__pyx_v_value); + if (__pyx_t_3) { + + /* "View.MemoryView":515 + * + * if isinstance(value, tuple): + * bytesvalue = struct.pack(self.view.format, *value) # <<<<<<<<<<<<<< + * else: + * bytesvalue = struct.pack(self.view.format, value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 515, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 515, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":517 + * bytesvalue = struct.pack(self.view.format, *value) + * else: + * bytesvalue = struct.pack(self.view.format, value) # <<<<<<<<<<<<<< + * + * for i, c in enumerate(bytesvalue): +*/ + /*else*/ { + __pyx_t_6 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_t_1, __pyx_v_value}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_pack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 517, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = 0; + if (unlikely(__pyx_v_bytesvalue == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(1, 519, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytesvalue); + __pyx_t_9 = __pyx_v_bytesvalue; + __pyx_t_11 = __Pyx_PyBytes_AsWritableString(__pyx_t_9); if (unlikely(__pyx_t_11 == ((char *)NULL))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyBytes_GET_SIZE(__pyx_t_9); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_13 = (__pyx_t_11 + __pyx_t_12); + for (__pyx_t_14 = __pyx_t_11; __pyx_t_14 < __pyx_t_13; __pyx_t_14++) { + __pyx_t_10 = __pyx_t_14; + __pyx_v_c = (__pyx_t_10[0]); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_v_i = __pyx_t_8; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + (__pyx_v_itemp[__pyx_v_i]) = __pyx_v_c; + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t *__pyx_t_3; + char *__pyx_t_4; + void *__pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + __pyx_t_2 = ((__pyx_v_flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_L4_bool_binop_done:; + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":525 + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: + * raise ValueError, "Cannot create writable memory view from read-only memoryview" # <<<<<<<<<<<<<< + * + * if flags & PyBUF_ND: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Cannot_create_writable_memory_vi, 0, 0); + __PYX_ERR(1, 525, __pyx_L1_error) + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + } + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":528 + * + * if flags & PyBUF_ND: + * info.shape = self.view.shape # <<<<<<<<<<<<<< + * else: + * info.shape = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.shape; + __pyx_v_info->shape = __pyx_t_3; + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":530 + * info.shape = self.view.shape + * else: + * info.shape = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + /*else*/ { + __pyx_v_info->shape = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":533 + * + * if flags & PyBUF_STRIDES: + * info.strides = self.view.strides # <<<<<<<<<<<<<< + * else: + * info.strides = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.strides; + __pyx_v_info->strides = __pyx_t_3; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":535 + * info.strides = self.view.strides + * else: + * info.strides = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_INDIRECT: +*/ + /*else*/ { + __pyx_v_info->strides = NULL; + } + __pyx_L7:; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_INDIRECT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":538 + * + * if flags & PyBUF_INDIRECT: + * info.suboffsets = self.view.suboffsets # <<<<<<<<<<<<<< + * else: + * info.suboffsets = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.suboffsets; + __pyx_v_info->suboffsets = __pyx_t_3; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":540 + * info.suboffsets = self.view.suboffsets + * else: + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + /*else*/ { + __pyx_v_info->suboffsets = NULL; + } + __pyx_L8:; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":543 + * + * if flags & PyBUF_FORMAT: + * info.format = self.view.format # <<<<<<<<<<<<<< + * else: + * info.format = NULL +*/ + __pyx_t_4 = __pyx_v_self->view.format; + __pyx_v_info->format = __pyx_t_4; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":545 + * info.format = self.view.format + * else: + * info.format = NULL # <<<<<<<<<<<<<< + * + * info.buf = self.view.buf +*/ + /*else*/ { + __pyx_v_info->format = NULL; + } + __pyx_L9:; + + /* "View.MemoryView":547 + * info.format = NULL + * + * info.buf = self.view.buf # <<<<<<<<<<<<<< + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize +*/ + __pyx_t_5 = __pyx_v_self->view.buf; + __pyx_v_info->buf = __pyx_t_5; + + /* "View.MemoryView":548 + * + * info.buf = self.view.buf + * info.ndim = self.view.ndim # <<<<<<<<<<<<<< + * info.itemsize = self.view.itemsize + * info.len = self.view.len +*/ + __pyx_t_6 = __pyx_v_self->view.ndim; + __pyx_v_info->ndim = __pyx_t_6; + + /* "View.MemoryView":549 + * info.buf = self.view.buf + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize # <<<<<<<<<<<<<< + * info.len = self.view.len + * info.readonly = self.view.readonly +*/ + __pyx_t_7 = __pyx_v_self->view.itemsize; + __pyx_v_info->itemsize = __pyx_t_7; + + /* "View.MemoryView":550 + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize + * info.len = self.view.len # <<<<<<<<<<<<<< + * info.readonly = self.view.readonly + * info.obj = self +*/ + __pyx_t_7 = __pyx_v_self->view.len; + __pyx_v_info->len = __pyx_t_7; + + /* "View.MemoryView":551 + * info.itemsize = self.view.itemsize + * info.len = self.view.len + * info.readonly = self.view.readonly # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_v_info->readonly = __pyx_t_1; + + /* "View.MemoryView":552 + * info.len = self.view.len + * info.readonly = self.view.readonly + * info.obj = self # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":557 + * @property + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) # <<<<<<<<<<<<<< + * transpose_memslice(&result.from_slice) + * return result +*/ + __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 557, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 557, __pyx_L1_error) + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":558 + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 558, __pyx_L1_error) + + /* "View.MemoryView":559 + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) + * return result # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.T.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":563 + * @property + * def base(self): + * return self._get_base() # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->_get_base(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 563, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.base.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":566 + * + * cdef _get_base(self): + * return self.obj # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->obj); + __pyx_r = __pyx_v_self->obj; + goto __pyx_L0; + + /* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_7genexpr__pyx_v_length; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":570 + * @property + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_7genexpr__pyx_v_length = (__pyx_t_2[0]); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_7genexpr__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } /* exit inner scope */ + __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.shape.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr1__pyx_v_stride; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + __pyx_t_1 = (__pyx_v_self->view.strides == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":576 + * if self.view.strides == NULL: + * + * raise ValueError, "Buffer view does not expose strides" # <<<<<<<<<<<<<< + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Buffer_view_does_not_expose_stri, 0, 0); + __PYX_ERR(1, 576, __pyx_L1_error) + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + } + + /* "View.MemoryView":578 + * raise ValueError, "Buffer view does not expose strides" + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.strides + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.strides; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr1__pyx_v_stride = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr1__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.strides.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr2__pyx_v_suboffset; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + __pyx_t_1 = (__pyx_v_self->view.suboffsets == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PySequence_Multiply(__pyx_mstate_global->__pyx_tuple[1], __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + } + + /* "View.MemoryView":585 + * return (-1,) * self.view.ndim + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.suboffsets + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.suboffsets; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr2__pyx_v_suboffset = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr2__pyx_v_suboffset); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.suboffsets.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":589 + * @property + * def ndim(self): + * return self.view.ndim # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 589, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.ndim.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":593 + * @property + * def itemsize(self): + * return self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 593, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.itemsize.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":597 + * @property + * def nbytes(self): + * return self.size * self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.nbytes.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + __pyx_t_1 = (__pyx_v_self->_size == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":602 + * def size(self): + * if self._size is None: + * result = 1 # <<<<<<<<<<<<<< + * + * for length in self.view.shape[:self.view.ndim]: +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __pyx_v_result = __pyx_mstate_global->__pyx_int_1; + + /* "View.MemoryView":604 + * result = 1 + * + * for length in self.view.shape[:self.view.ndim]: # <<<<<<<<<<<<<< + * result *= length + * +*/ + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_t_5 = PyLong_FromSsize_t((__pyx_t_2[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 604, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_5); + __pyx_t_5 = 0; + + /* "View.MemoryView":605 + * + * for length in self.view.shape[:self.view.ndim]: + * result *= length # <<<<<<<<<<<<<< + * + * self._size = result +*/ + __pyx_t_5 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 605, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_result, __pyx_t_5); + __pyx_t_5 = 0; + } + + /* "View.MemoryView":607 + * result *= length + * + * self._size = result # <<<<<<<<<<<<<< + * + * return self._size +*/ + __Pyx_INCREF(__pyx_v_result); + __Pyx_GIVEREF(__pyx_v_result); + __Pyx_GOTREF(__pyx_v_self->_size); + __Pyx_DECREF(__pyx_v_self->_size); + __pyx_v_self->_size = __pyx_v_result; + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + } + + /* "View.MemoryView":609 + * self._size = result + * + * return self._size # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->_size); + __pyx_r = __pyx_v_self->_size; + goto __pyx_L0; + + /* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.size.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + __pyx_t_1 = (__pyx_v_self->view.ndim >= 1); + if (__pyx_t_1) { + + /* "View.MemoryView":613 + * def __len__(self): + * if self.view.ndim >= 1: + * return self.view.shape[0] # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_r = (__pyx_v_self->view.shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + } + + /* "View.MemoryView":615 + * return self.view.shape[0] + * + * return 0 # <<<<<<<<<<<<<< + * + * def __repr__(self): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[5]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":619 + * def __repr__(self): + * return "" % (self.base.__class__.__name__, + * id(self)) # <<<<<<<<<<<<<< + * + * def __str__(self): +*/ + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_Format(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_x); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_4[1] = __pyx_t_2; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_at_0x; + __pyx_t_4[3] = __pyx_t_3; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u__5; + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_4, 5, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3[3]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__str__", 0); + + /* "View.MemoryView":622 + * + * def __str__(self): + * return "" % (self.base.__class__.__name__,) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_3[1] = __pyx_t_2; + __pyx_t_3[2] = __pyx_mstate_global->__pyx_kp_u_object; + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_3, 3, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 8, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_c_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_c_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_c_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_c_contig", 0); + + /* "View.MemoryView":628 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 628, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":629 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'C', self.view.ndim) # <<<<<<<<<<<<<< + * + * def is_f_contig(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 629, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_c_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_f_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_f_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_f_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_f_contig", 0); + + /* "View.MemoryView":634 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 634, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":635 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'F', self.view.ndim) # <<<<<<<<<<<<<< + * + * def copy(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 635, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_f_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_mslice; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy", 0); + + /* "View.MemoryView":639 + * def copy(self): + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &mslice) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_F_CONTIGUOUS)); + + /* "View.MemoryView":641 + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS + * + * slice_copy(self, &mslice) # <<<<<<<<<<<<<< + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_mslice)); + + /* "View.MemoryView":642 + * + * slice_copy(self, &mslice) + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_C_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char const *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 642, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":647 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &mslice) # <<<<<<<<<<<<<< + * + * def copy_fortran(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 647, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy_fortran (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy_fortran", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy_fortran", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy_fortran", 0); + + /* "View.MemoryView":651 + * def copy_fortran(self): + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &src) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_C_CONTIGUOUS)); + + /* "View.MemoryView":653 + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS + * + * slice_copy(self, &src) # <<<<<<<<<<<<<< + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_src)); + + /* "View.MemoryView":654 + * + * slice_copy(self, &src) + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_F_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char const *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 654, __pyx_L1_error) + __pyx_v_dst = __pyx_t_1; + + /* "View.MemoryView":659 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &dst) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 659, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy_fortran", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryview___reduce_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryview_2__setstate_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + +static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, int __pyx_v_dtype_is_object, __Pyx_TypeInfo const *__pyx_v_typeinfo) { + struct __pyx_memoryview_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_cwrapper", 0); + + /* "View.MemoryView":664 + * @cname('__pyx_memoryview_new') + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) # <<<<<<<<<<<<<< + * result.typeinfo = typeinfo + * return result +*/ + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, __pyx_v_o, __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_v_result = ((struct __pyx_memoryview_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":665 + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_v_result->typeinfo = __pyx_v_typeinfo; + + /* "View.MemoryView":666 + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_check') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { + int __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":670 + * @cname('__pyx_memoryview_check') + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) # <<<<<<<<<<<<<< + * + * cdef tuple _unellipsify(object index, int ndim): +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_o, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + +static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_idx; + PyObject *__pyx_v_tup = NULL; + PyObject *__pyx_v_result = NULL; + int __pyx_v_have_slices; + int __pyx_v_seen_ellipsis; + PyObject *__pyx_v_item = NULL; + Py_ssize_t __pyx_v_nslices; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6[3]; + PyObject *__pyx_t_7 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_unellipsify", 0); + + /* "View.MemoryView":678 + * """ + * cdef Py_ssize_t idx + * tup = index if isinstance(index, tuple) else (index,) # <<<<<<<<<<<<<< + * + * result = [slice(None)] * ndim +*/ + __pyx_t_2 = PyTuple_Check(__pyx_v_index); + if (__pyx_t_2) { + __Pyx_INCREF(((PyObject*)__pyx_v_index)); + __pyx_t_1 = __pyx_v_index; + } else { + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_index); + __Pyx_GIVEREF(__pyx_v_index); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_index) != (0)) __PYX_ERR(1, 678, __pyx_L1_error); + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_v_tup = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_t_1 = PyList_New(1 * ((__pyx_v_ndim<0) ? 0:__pyx_v_ndim)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_ndim; __pyx_temp++) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, __pyx_temp, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(1, 680, __pyx_L1_error); + } + } + __pyx_v_result = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":681 + * + * result = [slice(None)] * ndim + * have_slices = False # <<<<<<<<<<<<<< + * seen_ellipsis = False + * idx = 0 +*/ + __pyx_v_have_slices = 0; + + /* "View.MemoryView":682 + * result = [slice(None)] * ndim + * have_slices = False + * seen_ellipsis = False # <<<<<<<<<<<<<< + * idx = 0 + * for item in tup: +*/ + __pyx_v_seen_ellipsis = 0; + + /* "View.MemoryView":683 + * have_slices = False + * seen_ellipsis = False + * idx = 0 # <<<<<<<<<<<<<< + * for item in tup: + * if item is Ellipsis: +*/ + __pyx_v_idx = 0; + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(1, 684, __pyx_L1_error) + } + __pyx_t_1 = __pyx_v_tup; __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 684, __pyx_L1_error) + #endif + if (__pyx_t_4 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_4); + #endif + ++__pyx_t_4; + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 684, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + __pyx_t_2 = (__pyx_v_item == __pyx_builtin_Ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + __pyx_t_2 = (!__pyx_v_seen_ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":687 + * if item is Ellipsis: + * if not seen_ellipsis: + * idx += ndim - len(tup) # <<<<<<<<<<<<<< + * seen_ellipsis = True + * have_slices = True +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 687, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(__pyx_v_tup); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 687, __pyx_L1_error) + __pyx_v_idx = (__pyx_v_idx + (__pyx_v_ndim - __pyx_t_5)); + + /* "View.MemoryView":688 + * if not seen_ellipsis: + * idx += ndim - len(tup) + * seen_ellipsis = True # <<<<<<<<<<<<<< + * have_slices = True + * else: +*/ + __pyx_v_seen_ellipsis = 1; + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + } + + /* "View.MemoryView":689 + * idx += ndim - len(tup) + * seen_ellipsis = True + * have_slices = True # <<<<<<<<<<<<<< + * else: + * if isinstance(item, slice): +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + /*else*/ { + __pyx_t_2 = PySlice_Check(__pyx_v_item); + if (__pyx_t_2) { + + /* "View.MemoryView":692 + * else: + * if isinstance(item, slice): + * have_slices = True # <<<<<<<<<<<<<< + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + __pyx_t_2 = (!(PyIndex_Check(__pyx_v_item) != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":694 + * have_slices = True + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" # <<<<<<<<<<<<<< + * result[idx] = item + * idx += 1 +*/ + __pyx_t_3 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_item)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6[0] = __pyx_mstate_global->__pyx_kp_u_Cannot_index_with_type; + __pyx_t_6[1] = __pyx_t_3; + __pyx_t_6[2] = __pyx_mstate_global->__pyx_kp_u__6; + __pyx_t_7 = __Pyx_PyUnicode_Join(__pyx_t_6, 3, 24 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_t_7, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 694, __pyx_L1_error) + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + } + __pyx_L7:; + + /* "View.MemoryView":695 + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item # <<<<<<<<<<<<<< + * idx += 1 + * +*/ + if (unlikely((__Pyx_SetItemInt(__pyx_v_result, __pyx_v_idx, __pyx_v_item, Py_ssize_t, 1, PyLong_FromSsize_t, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(1, 695, __pyx_L1_error) + } + __pyx_L5:; + + /* "View.MemoryView":696 + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item + * idx += 1 # <<<<<<<<<<<<<< + * + * nslices = ndim - idx +*/ + __pyx_v_idx = (__pyx_v_idx + 1); + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":698 + * idx += 1 + * + * nslices = ndim - idx # <<<<<<<<<<<<<< + * return have_slices or nslices, tuple(result) + * +*/ + __pyx_v_nslices = (__pyx_v_ndim - __pyx_v_idx); + + /* "View.MemoryView":699 + * + * nslices = ndim - idx + * return have_slices or nslices, tuple(result) # <<<<<<<<<<<<<< + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: +*/ + __Pyx_XDECREF(__pyx_r); + if (!__pyx_v_have_slices) { + } else { + __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_7 = PyLong_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_L9_bool_binop_done:; + __pyx_t_7 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_7 = 0; + __pyx_r = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("View.MemoryView._unellipsify", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_tup); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_item); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + +static int assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_suboffset; + int __pyx_r; + Py_ssize_t *__pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":702 + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" +*/ + __pyx_t_2 = (__pyx_v_suboffsets + __pyx_v_ndim); + for (__pyx_t_3 = __pyx_v_suboffsets; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_suboffset = (__pyx_t_1[0]); + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + __pyx_t_4 = (__pyx_v_suboffset >= 0); + if (unlikely(__pyx_t_4)) { + + /* "View.MemoryView":704 + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" # <<<<<<<<<<<<<< + * return 0 # return type just used as an error flag + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Indirect_dimensions_not_supporte, 0, 0); + __PYX_ERR(1, 704, __pyx_L1_error) + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + } + } + + /* "View.MemoryView":705 + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.assert_direct_dimensions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *__pyx_v_memview, PyObject *__pyx_v_indices) { + int __pyx_v_new_ndim; + int __pyx_v_suboffset_dim; + int __pyx_v_dim; + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + __Pyx_memviewslice *__pyx_v_p_src; + struct __pyx_memoryviewslice_obj *__pyx_v_memviewsliceobj = 0; + __Pyx_memviewslice *__pyx_v_p_dst; + int *__pyx_v_p_suboffset_dim; + Py_ssize_t __pyx_v_start; + Py_ssize_t __pyx_v_stop; + Py_ssize_t __pyx_v_step; + Py_ssize_t __pyx_v_cindex; + int __pyx_v_have_start; + int __pyx_v_have_stop; + int __pyx_v_have_step; + PyObject *__pyx_v_index = NULL; + struct __pyx_memoryview_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + struct __pyx_memoryview_obj *__pyx_t_3; + char *__pyx_t_4; + int __pyx_t_5; + Py_ssize_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memview_slice", 0); + + /* "View.MemoryView":713 + * @cname('__pyx_memview_slice') + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim # <<<<<<<<<<<<<< + * cdef bint negative_step + * cdef __Pyx_memviewslice src, dst +*/ + __pyx_v_new_ndim = 0; + __pyx_v_suboffset_dim = -1; + + /* "View.MemoryView":720 + * + * + * memset(&dst, 0, sizeof(dst)) # <<<<<<<<<<<<<< + * + * cdef _memoryviewslice memviewsliceobj +*/ + (void)(memset((&__pyx_v_dst), 0, (sizeof(__pyx_v_dst)))); + + /* "View.MemoryView":724 + * cdef _memoryviewslice memviewsliceobj + * + * assert memview.view.ndim > 0 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_memview->view.ndim > 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 724, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 724, __pyx_L1_error) + #endif + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":727 + * + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview # <<<<<<<<<<<<<< + * p_src = &memviewsliceobj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 727, __pyx_L1_error) + __pyx_v_memviewsliceobj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":728 + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, &src) +*/ + __pyx_v_p_src = (&__pyx_v_memviewsliceobj->from_slice); + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":730 + * p_src = &memviewsliceobj.from_slice + * else: + * slice_copy(memview, &src) # <<<<<<<<<<<<<< + * p_src = &src + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_src)); + + /* "View.MemoryView":731 + * else: + * slice_copy(memview, &src) + * p_src = &src # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_p_src = (&__pyx_v_src); + } + __pyx_L3:; + + /* "View.MemoryView":737 + * + * + * dst.memview = p_src.memview # <<<<<<<<<<<<<< + * dst.data = p_src.data + * +*/ + __pyx_t_3 = __pyx_v_p_src->memview; + __pyx_v_dst.memview = __pyx_t_3; + + /* "View.MemoryView":738 + * + * dst.memview = p_src.memview + * dst.data = p_src.data # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_v_p_src->data; + __pyx_v_dst.data = __pyx_t_4; + + /* "View.MemoryView":743 + * + * + * cdef __Pyx_memviewslice *p_dst = &dst # <<<<<<<<<<<<<< + * cdef int *p_suboffset_dim = &suboffset_dim + * cdef Py_ssize_t start, stop, step, cindex +*/ + __pyx_v_p_dst = (&__pyx_v_dst); + + /* "View.MemoryView":744 + * + * cdef __Pyx_memviewslice *p_dst = &dst + * cdef int *p_suboffset_dim = &suboffset_dim # <<<<<<<<<<<<<< + * cdef Py_ssize_t start, stop, step, cindex + * cdef bint have_start, have_stop, have_step +*/ + __pyx_v_p_suboffset_dim = (&__pyx_v_suboffset_dim); + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_indices)) || PyTuple_CheckExact(__pyx_v_indices)) { + __pyx_t_2 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 748, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 748, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_6)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 748, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_2); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 748, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_index, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_dim = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + __pyx_t_1 = (PyIndex_Check(__pyx_v_index) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":750 + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): + * cindex = index # <<<<<<<<<<<<<< + * slice_memviewslice( + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], +*/ + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 750, __pyx_L1_error) + __pyx_v_cindex = __pyx_t_9; + + /* "View.MemoryView":751 + * if PyIndex_Check(index): + * cindex = index + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_cindex, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 751, __pyx_L1_error) + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + __pyx_t_1 = (__pyx_v_index == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":758 + * False) + * elif index is None: + * p_dst.shape[new_ndim] = 1 # <<<<<<<<<<<<<< + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 +*/ + (__pyx_v_p_dst->shape[__pyx_v_new_ndim]) = 1; + + /* "View.MemoryView":759 + * elif index is None: + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 # <<<<<<<<<<<<<< + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 +*/ + (__pyx_v_p_dst->strides[__pyx_v_new_ndim]) = 0; + + /* "View.MemoryView":760 + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 # <<<<<<<<<<<<<< + * new_ndim += 1 + * else: +*/ + (__pyx_v_p_dst->suboffsets[__pyx_v_new_ndim]) = -1L; + + /* "View.MemoryView":761 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 # <<<<<<<<<<<<<< + * else: + * start = index.start or 0 +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":763 + * new_ndim += 1 + * else: + * start = index.start or 0 # <<<<<<<<<<<<<< + * stop = index.stop or 0 + * step = index.step or 0 +*/ + /*else*/ { + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 763, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 763, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 763, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L7_bool_binop_done:; + __pyx_v_start = __pyx_t_9; + + /* "View.MemoryView":764 + * else: + * start = index.start or 0 + * stop = index.stop or 0 # <<<<<<<<<<<<<< + * step = index.step or 0 + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 764, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 764, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 764, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L9_bool_binop_done:; + __pyx_v_stop = __pyx_t_9; + + /* "View.MemoryView":765 + * start = index.start or 0 + * stop = index.stop or 0 + * step = index.step or 0 # <<<<<<<<<<<<<< + * + * have_start = index.start is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 765, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 765, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 765, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L11_bool_binop_done:; + __pyx_v_step = __pyx_t_9; + + /* "View.MemoryView":767 + * step = index.step or 0 + * + * have_start = index.start is not None # <<<<<<<<<<<<<< + * have_stop = index.stop is not None + * have_step = index.step is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 767, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_start = __pyx_t_1; + + /* "View.MemoryView":768 + * + * have_start = index.start is not None + * have_stop = index.stop is not None # <<<<<<<<<<<<<< + * have_step = index.step is not None + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 768, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_stop = __pyx_t_1; + + /* "View.MemoryView":769 + * have_start = index.start is not None + * have_stop = index.stop is not None + * have_step = index.step is not None # <<<<<<<<<<<<<< + * + * slice_memviewslice( +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_step = __pyx_t_1; + + /* "View.MemoryView":771 + * have_step = index.step is not None + * + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 771, __pyx_L1_error) + + /* "View.MemoryView":777 + * have_start, have_stop, have_step, + * True) + * new_ndim += 1 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + } + __pyx_L6:; + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":781 + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, # <<<<<<<<<<<<<< + * memviewsliceobj.to_dtype_func, + * memview.dtype_is_object) +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 781, __pyx_L1_error) } + + /* "View.MemoryView":782 + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * else: +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 782, __pyx_L1_error) } + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 780, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + } + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + /*else*/ { + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":786 + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 785, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 785, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("View.MemoryView.memview_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_memviewsliceobj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, Py_ssize_t __pyx_v_shape, Py_ssize_t __pyx_v_stride, Py_ssize_t __pyx_v_suboffset, int __pyx_v_dim, int __pyx_v_new_ndim, int *__pyx_v_suboffset_dim, Py_ssize_t __pyx_v_start, Py_ssize_t __pyx_v_stop, Py_ssize_t __pyx_v_step, int __pyx_v_have_start, int __pyx_v_have_stop, int __pyx_v_have_step, int __pyx_v_is_slice) { + Py_ssize_t __pyx_v_new_shape; + int __pyx_v_negative_step; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + __pyx_t_1 = (!__pyx_v_is_slice); + if (__pyx_t_1) { + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + __pyx_t_1 = (__pyx_v_start < 0); + if (__pyx_t_1) { + + /* "View.MemoryView":817 + * + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + } + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + __pyx_t_1 = (0 <= __pyx_v_start); + if (__pyx_t_1) { + __pyx_t_1 = (__pyx_v_start < __pyx_v_shape); + } + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":819 + * start += shape + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_Index_out_of_bounds_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 819, __pyx_L1_error) + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + /*else*/ { + __pyx_t_2 = (__pyx_v_have_step != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":823 + * + * if have_step: + * negative_step = step < 0 # <<<<<<<<<<<<<< + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) +*/ + __pyx_v_negative_step = (__pyx_v_step < 0); + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + __pyx_t_2 = (__pyx_v_step == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":825 + * negative_step = step < 0 + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * negative_step = False +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Step_may_not_be_zero_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 825, __pyx_L1_error) + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":827 + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: + * negative_step = False # <<<<<<<<<<<<<< + * step = 1 + * +*/ + /*else*/ { + __pyx_v_negative_step = 0; + + /* "View.MemoryView":828 + * else: + * negative_step = False + * step = 1 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_step = 1; + } + __pyx_L6:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + __pyx_t_2 = (__pyx_v_have_start != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":833 + * if have_start: + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if start < 0: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":835 + * start += shape + * if start < 0: + * start = 0 # <<<<<<<<<<<<<< + * elif start >= shape: + * if negative_step: +*/ + __pyx_v_start = 0; + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + } + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + __pyx_t_2 = (__pyx_v_start >= __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + if (__pyx_v_negative_step) { + + /* "View.MemoryView":838 + * elif start >= shape: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = shape +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":840 + * start = shape - 1 + * else: + * start = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + /*else*/ { + __pyx_v_start = __pyx_v_shape; + } + __pyx_L11:; + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + } + __pyx_L9:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":843 + * else: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":845 + * start = shape - 1 + * else: + * start = 0 # <<<<<<<<<<<<<< + * + * if have_stop: +*/ + /*else*/ { + __pyx_v_start = 0; + } + __pyx_L12:; + } + __pyx_L8:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + __pyx_t_2 = (__pyx_v_have_stop != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":849 + * if have_stop: + * if stop < 0: + * stop += shape # <<<<<<<<<<<<<< + * if stop < 0: + * stop = 0 +*/ + __pyx_v_stop = (__pyx_v_stop + __pyx_v_shape); + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":851 + * stop += shape + * if stop < 0: + * stop = 0 # <<<<<<<<<<<<<< + * elif stop > shape: + * stop = shape +*/ + __pyx_v_stop = 0; + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + } + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + goto __pyx_L14; + } + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + __pyx_t_2 = (__pyx_v_stop > __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":853 + * stop = 0 + * elif stop > shape: + * stop = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + __pyx_v_stop = __pyx_v_shape; + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + } + __pyx_L14:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":856 + * else: + * if negative_step: + * stop = -1 # <<<<<<<<<<<<<< + * else: + * stop = shape +*/ + __pyx_v_stop = -1L; + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + goto __pyx_L16; + } + + /* "View.MemoryView":858 + * stop = -1 + * else: + * stop = shape # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_v_stop = __pyx_v_shape; + } + __pyx_L16:; + } + __pyx_L13:; + + /* "View.MemoryView":862 + * + * with cython.cdivision(True): + * new_shape = (stop - start) // step # <<<<<<<<<<<<<< + * + * if (stop - start) - step * new_shape: +*/ + __pyx_v_new_shape = ((__pyx_v_stop - __pyx_v_start) / __pyx_v_step); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + __pyx_t_2 = (((__pyx_v_stop - __pyx_v_start) - (__pyx_v_step * __pyx_v_new_shape)) != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":865 + * + * if (stop - start) - step * new_shape: + * new_shape += 1 # <<<<<<<<<<<<<< + * + * if new_shape < 0: +*/ + __pyx_v_new_shape = (__pyx_v_new_shape + 1); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + } + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + __pyx_t_2 = (__pyx_v_new_shape < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":868 + * + * if new_shape < 0: + * new_shape = 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_new_shape = 0; + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + } + + /* "View.MemoryView":871 + * + * + * dst.strides[new_ndim] = stride * step # <<<<<<<<<<<<<< + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset +*/ + (__pyx_v_dst->strides[__pyx_v_new_ndim]) = (__pyx_v_stride * __pyx_v_step); + + /* "View.MemoryView":872 + * + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape # <<<<<<<<<<<<<< + * dst.suboffsets[new_ndim] = suboffset + * +*/ + (__pyx_v_dst->shape[__pyx_v_new_ndim]) = __pyx_v_new_shape; + + /* "View.MemoryView":873 + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_dst->suboffsets[__pyx_v_new_ndim]) = __pyx_v_suboffset; + } + __pyx_L3:; + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + __pyx_t_2 = ((__pyx_v_suboffset_dim[0]) < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":877 + * + * if suboffset_dim[0] < 0: + * dst.data += start * stride # <<<<<<<<<<<<<< + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride +*/ + __pyx_v_dst->data = (__pyx_v_dst->data + (__pyx_v_start * __pyx_v_stride)); + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + goto __pyx_L19; + } + + /* "View.MemoryView":879 + * dst.data += start * stride + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride # <<<<<<<<<<<<<< + * + * if suboffset >= 0: +*/ + /*else*/ { + __pyx_t_3 = (__pyx_v_suboffset_dim[0]); + (__pyx_v_dst->suboffsets[__pyx_t_3]) = ((__pyx_v_dst->suboffsets[__pyx_t_3]) + (__pyx_v_start * __pyx_v_stride)); + } + __pyx_L19:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + __pyx_t_2 = (!__pyx_v_is_slice); + if (__pyx_t_2) { + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + __pyx_t_2 = (__pyx_v_new_ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":884 + * if not is_slice: + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset # <<<<<<<<<<<<<< + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " +*/ + __pyx_v_dst->data = ((((char **)__pyx_v_dst->data)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + goto __pyx_L22; + } + + /* "View.MemoryView":886 + * dst.data = ( dst.data)[0] + suboffset + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " # <<<<<<<<<<<<<< + * "must be indexed and not sliced", dim) + * else: +*/ + /*else*/ { + + /* "View.MemoryView":887 + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " + * "must be indexed and not sliced", dim) # <<<<<<<<<<<<<< + * else: + * suboffset_dim[0] = new_ndim +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_All_dimensions_preceding_dimensi, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 886, __pyx_L1_error) + } + __pyx_L22:; + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + goto __pyx_L21; + } + + /* "View.MemoryView":889 + * "must be indexed and not sliced", dim) + * else: + * suboffset_dim[0] = new_ndim # <<<<<<<<<<<<<< + * + * return 0 +*/ + /*else*/ { + (__pyx_v_suboffset_dim[0]) = __pyx_v_new_ndim; + } + __pyx_L21:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + } + + /* "View.MemoryView":891 + * suboffset_dim[0] = new_ndim + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.slice_memviewslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + +static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, Py_ssize_t __pyx_v_index, Py_ssize_t __pyx_v_dim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_suboffset; + Py_ssize_t __pyx_v_itemsize; + char *__pyx_v_resultp; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[3]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("pybuffer_index", 0); + + /* "View.MemoryView":899 + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 # <<<<<<<<<<<<<< + * cdef Py_ssize_t itemsize = view.itemsize + * cdef char *resultp +*/ + __pyx_v_suboffset = -1L; + + /* "View.MemoryView":900 + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 + * cdef Py_ssize_t itemsize = view.itemsize # <<<<<<<<<<<<<< + * cdef char *resultp + * +*/ + __pyx_t_1 = __pyx_v_view->itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + __pyx_t_2 = (__pyx_v_view->ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":904 + * + * if view.ndim == 0: + * shape = view.len // itemsize # <<<<<<<<<<<<<< + * stride = itemsize + * else: +*/ + if (unlikely(__pyx_v_itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_view->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + __pyx_v_shape = __Pyx_div_Py_ssize_t(__pyx_v_view->len, __pyx_v_itemsize, 0); + + /* "View.MemoryView":905 + * if view.ndim == 0: + * shape = view.len // itemsize + * stride = itemsize # <<<<<<<<<<<<<< + * else: + * shape = view.shape[dim] +*/ + __pyx_v_stride = __pyx_v_itemsize; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":907 + * stride = itemsize + * else: + * shape = view.shape[dim] # <<<<<<<<<<<<<< + * stride = view.strides[dim] + * if view.suboffsets != NULL: +*/ + /*else*/ { + __pyx_v_shape = (__pyx_v_view->shape[__pyx_v_dim]); + + /* "View.MemoryView":908 + * else: + * shape = view.shape[dim] + * stride = view.strides[dim] # <<<<<<<<<<<<<< + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] +*/ + __pyx_v_stride = (__pyx_v_view->strides[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + __pyx_t_2 = (__pyx_v_view->suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":910 + * stride = view.strides[dim] + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] # <<<<<<<<<<<<<< + * + * if index < 0: +*/ + __pyx_v_suboffset = (__pyx_v_view->suboffsets[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + } + } + __pyx_L3:; + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":913 + * + * if index < 0: + * index += view.shape[dim] # <<<<<<<<<<<<<< + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" +*/ + __pyx_v_index = (__pyx_v_index + (__pyx_v_view->shape[__pyx_v_dim])); + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":915 + * index += view.shape[dim] + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * if index >= shape: +*/ + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_3; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__7; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 915, __pyx_L1_error) + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + } + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index >= __pyx_v_shape); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":918 + * + * if index >= shape: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * resultp = bufp + index * stride +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_5; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__7; + __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 1, 127); + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 918, __pyx_L1_error) + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":920 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * resultp = bufp + index * stride # <<<<<<<<<<<<<< + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset +*/ + __pyx_v_resultp = (__pyx_v_bufp + (__pyx_v_index * __pyx_v_stride)); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":922 + * resultp = bufp + index * stride + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset # <<<<<<<<<<<<<< + * + * return resultp +*/ + __pyx_v_resultp = ((((char **)__pyx_v_resultp)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + } + + /* "View.MemoryView":924 + * resultp = ( resultp)[0] + suboffset + * + * return resultp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_resultp; + goto __pyx_L0; + + /* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.pybuffer_index", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + +static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { + int __pyx_v_ndim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + int __pyx_v_i; + int __pyx_v_j; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + long __pyx_t_3; + long __pyx_t_4; + Py_ssize_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":931 + * @cname('__pyx_memslice_transpose') + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim # <<<<<<<<<<<<<< + * + * cdef Py_ssize_t *shape = memslice.shape +*/ + __pyx_t_1 = __pyx_v_memslice->memview->view.ndim; + __pyx_v_ndim = __pyx_t_1; + + /* "View.MemoryView":933 + * cdef int ndim = memslice.memview.view.ndim + * + * cdef Py_ssize_t *shape = memslice.shape # <<<<<<<<<<<<<< + * cdef Py_ssize_t *strides = memslice.strides + * +*/ + __pyx_t_2 = __pyx_v_memslice->shape; + __pyx_v_shape = __pyx_t_2; + + /* "View.MemoryView":934 + * + * cdef Py_ssize_t *shape = memslice.shape + * cdef Py_ssize_t *strides = memslice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_v_memslice->strides; + __pyx_v_strides = __pyx_t_2; + + /* "View.MemoryView":938 + * + * cdef int i, j + * for i in range(ndim // 2): # <<<<<<<<<<<<<< + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] +*/ + __pyx_t_3 = __Pyx_div_long(__pyx_v_ndim, 2, 1); + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_4; __pyx_t_1+=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":939 + * cdef int i, j + * for i in range(ndim // 2): + * j = ndim - 1 - i # <<<<<<<<<<<<<< + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] +*/ + __pyx_v_j = ((__pyx_v_ndim - 1) - __pyx_v_i); + + /* "View.MemoryView":940 + * for i in range(ndim // 2): + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] # <<<<<<<<<<<<<< + * shape[i], shape[j] = shape[j], shape[i] + * +*/ + __pyx_t_5 = (__pyx_v_strides[__pyx_v_j]); + __pyx_t_6 = (__pyx_v_strides[__pyx_v_i]); + (__pyx_v_strides[__pyx_v_i]) = __pyx_t_5; + (__pyx_v_strides[__pyx_v_j]) = __pyx_t_6; + + /* "View.MemoryView":941 + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] # <<<<<<<<<<<<<< + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: +*/ + __pyx_t_6 = (__pyx_v_shape[__pyx_v_j]); + __pyx_t_5 = (__pyx_v_shape[__pyx_v_i]); + (__pyx_v_shape[__pyx_v_i]) = __pyx_t_6; + (__pyx_v_shape[__pyx_v_j]) = __pyx_t_5; + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_i]) >= 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_j]) >= 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L6_bool_binop_done:; + if (__pyx_t_7) { + + /* "View.MemoryView":944 + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_t_9 = __pyx_memoryview_err(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Cannot_transpose_memoryview_with); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 944, __pyx_L1_error) + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + } + } + + /* "View.MemoryView":946 + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.transpose_memslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + +/* Python wrapper */ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + + /* "View.MemoryView":965 + * + * def __dealloc__(self): + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_self->from_slice), 1); + + /* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_object_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":969 + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) # <<<<<<<<<<<<<< + * else: + * return memoryview.convert_item_to_object(self, itemp) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 969, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + } + + /* "View.MemoryView":971 + * return self.to_object_func(itemp) + * else: + * return memoryview.convert_item_to_object(self, itemp) # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 971, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_dtype_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":975 + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) # <<<<<<<<<<<<<< + * else: + * memoryview.assign_item_from_object(self, itemp, value) +*/ + __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 975, __pyx_L1_error) + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":977 + * self.to_dtype_func(itemp, value) + * else: + * memoryview.assign_item_from_object(self, itemp, value) # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + /*else*/ { + __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 977, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":980 + * + * cdef _get_base(self): + * return self.from_object # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->from_object); + __pyx_r = __pyx_v_self->from_object; + goto __pyx_L0; + + /* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryviewslice___reduce_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryviewslice_2__setstate_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewslice, int __pyx_v_ndim, PyObject *(*__pyx_v_to_object_func)(char *), int (*__pyx_v_to_dtype_func)(char *, PyObject *), int __pyx_v_dtype_is_object) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + Py_ssize_t __pyx_v_suboffset; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_TypeInfo const *__pyx_t_4; + Py_buffer __pyx_t_5; + Py_ssize_t *__pyx_t_6; + Py_ssize_t *__pyx_t_7; + Py_ssize_t *__pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_fromslice", 0); + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_memviewslice.memview) == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":1009 + * + * if memviewslice.memview == Py_None: + * return None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + } + + /* "View.MemoryView":1014 + * + * + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) # <<<<<<<<<<<<<< + * + * result.from_slice = memviewslice +*/ + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, Py_None) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_0) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_2 = ((PyObject *)__pyx_tp_new__memoryviewslice(((PyTypeObject *)__pyx_mstate_global->__pyx_memoryviewslice_type), __pyx_t_3, NULL)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1016 + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) + * + * result.from_slice = memviewslice # <<<<<<<<<<<<<< + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * +*/ + __pyx_v_result->from_slice = __pyx_v_memviewslice; + + /* "View.MemoryView":1017 + * + * result.from_slice = memviewslice + * __PYX_INC_MEMVIEW(&memviewslice, 1) # <<<<<<<<<<<<<< + * + * result.from_object = ( memviewslice.memview)._get_base() +*/ + __PYX_INC_MEMVIEW((&__pyx_v_memviewslice), 1); + + /* "View.MemoryView":1019 + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * + * result.from_object = ( memviewslice.memview)._get_base() # <<<<<<<<<<<<<< + * result.typeinfo = memviewslice.memview.typeinfo + * +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->__pyx_vtab)->_get_base(((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1019, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_result->from_object); + __Pyx_DECREF(__pyx_v_result->from_object); + __pyx_v_result->from_object = __pyx_t_2; + __pyx_t_2 = 0; + + /* "View.MemoryView":1020 + * + * result.from_object = ( memviewslice.memview)._get_base() + * result.typeinfo = memviewslice.memview.typeinfo # <<<<<<<<<<<<<< + * + * result.view = memviewslice.memview.view +*/ + __pyx_t_4 = __pyx_v_memviewslice.memview->typeinfo; + __pyx_v_result->__pyx_base.typeinfo = __pyx_t_4; + + /* "View.MemoryView":1022 + * result.typeinfo = memviewslice.memview.typeinfo + * + * result.view = memviewslice.memview.view # <<<<<<<<<<<<<< + * result.view.buf = memviewslice.data + * result.view.ndim = ndim +*/ + __pyx_t_5 = __pyx_v_memviewslice.memview->view; + __pyx_v_result->__pyx_base.view = __pyx_t_5; + + /* "View.MemoryView":1023 + * + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data # <<<<<<<<<<<<<< + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None +*/ + __pyx_v_result->__pyx_base.view.buf = ((void *)__pyx_v_memviewslice.data); + + /* "View.MemoryView":1024 + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data + * result.view.ndim = ndim # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_v_result->__pyx_base.view.ndim = __pyx_v_ndim; + + /* "View.MemoryView":1025 + * result.view.buf = memviewslice.data + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_result->__pyx_base.view))->obj = Py_None; + + /* "View.MemoryView":1026 + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + __pyx_t_1 = ((((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1029 + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: + * result.flags = PyBUF_RECORDS # <<<<<<<<<<<<<< + * else: + * result.flags = PyBUF_RECORDS_RO +*/ + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS; + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1031 + * result.flags = PyBUF_RECORDS + * else: + * result.flags = PyBUF_RECORDS_RO # <<<<<<<<<<<<<< + * + * result.view.shape = result.from_slice.shape +*/ + /*else*/ { + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS_RO; + } + __pyx_L4:; + + /* "View.MemoryView":1033 + * result.flags = PyBUF_RECORDS_RO + * + * result.view.shape = result.from_slice.shape # <<<<<<<<<<<<<< + * result.view.strides = result.from_slice.strides + * +*/ + __pyx_v_result->__pyx_base.view.shape = ((Py_ssize_t *)__pyx_v_result->from_slice.shape); + + /* "View.MemoryView":1034 + * + * result.view.shape = result.from_slice.shape + * result.view.strides = result.from_slice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_result->__pyx_base.view.strides = ((Py_ssize_t *)__pyx_v_result->from_slice.strides); + + /* "View.MemoryView":1037 + * + * + * result.view.suboffsets = NULL # <<<<<<<<<<<<<< + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: +*/ + __pyx_v_result->__pyx_base.view.suboffsets = NULL; + + /* "View.MemoryView":1038 + * + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets +*/ + __pyx_t_7 = (__pyx_v_result->from_slice.suboffsets + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->from_slice.suboffsets; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_v_suboffset = (__pyx_t_6[0]); + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + __pyx_t_1 = (__pyx_v_suboffset >= 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1040 + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_result->__pyx_base.view.suboffsets = ((Py_ssize_t *)__pyx_v_result->from_slice.suboffsets); + + /* "View.MemoryView":1041 + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets + * break # <<<<<<<<<<<<<< + * + * result.view.len = result.view.itemsize +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + } + } + __pyx_L6_break:; + + /* "View.MemoryView":1043 + * break + * + * result.view.len = result.view.itemsize # <<<<<<<<<<<<<< + * for length in result.view.shape[:ndim]: + * result.view.len *= length +*/ + __pyx_t_9 = __pyx_v_result->__pyx_base.view.itemsize; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + + /* "View.MemoryView":1044 + * + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: # <<<<<<<<<<<<<< + * result.view.len *= length + * +*/ + __pyx_t_7 = (__pyx_v_result->__pyx_base.view.shape + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->__pyx_base.view.shape; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_t_2 = PyLong_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1044, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1045 + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: + * result.view.len *= length # <<<<<<<<<<<<<< + * + * result.to_object_func = to_object_func +*/ + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + } + + /* "View.MemoryView":1047 + * result.view.len *= length + * + * result.to_object_func = to_object_func # <<<<<<<<<<<<<< + * result.to_dtype_func = to_dtype_func + * +*/ + __pyx_v_result->to_object_func = __pyx_v_to_object_func; + + /* "View.MemoryView":1048 + * + * result.to_object_func = to_object_func + * result.to_dtype_func = to_dtype_func # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->to_dtype_func = __pyx_v_to_dtype_func; + + /* "View.MemoryView":1050 + * result.to_dtype_func = to_dtype_func + * + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview_fromslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_mslice) { + struct __pyx_memoryviewslice_obj *__pyx_v_obj = 0; + __Pyx_memviewslice *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_slice_from_memview", 0); + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1057 + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): + * obj = memview # <<<<<<<<<<<<<< + * return &obj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 1057, __pyx_L1_error) + __pyx_v_obj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1058 + * if isinstance(memview, _memoryviewslice): + * obj = memview + * return &obj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, mslice) +*/ + __pyx_r = (&__pyx_v_obj->from_slice); + goto __pyx_L0; + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + } + + /* "View.MemoryView":1060 + * return &obj.from_slice + * else: + * slice_copy(memview, mslice) # <<<<<<<<<<<<<< + * return mslice + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, __pyx_v_mslice); + + /* "View.MemoryView":1061 + * else: + * slice_copy(memview, mslice) + * return mslice # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_slice_copy') +*/ + __pyx_r = __pyx_v_mslice; + goto __pyx_L0; + } + + /* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.get_slice_from_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_obj); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_dst) { + int __pyx_v_dim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + Py_ssize_t *__pyx_v_suboffsets; + Py_ssize_t *__pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + + /* "View.MemoryView":1068 + * cdef (Py_ssize_t*) shape, strides, suboffsets + * + * shape = memview.view.shape # <<<<<<<<<<<<<< + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets +*/ + __pyx_t_1 = __pyx_v_memview->view.shape; + __pyx_v_shape = __pyx_t_1; + + /* "View.MemoryView":1069 + * + * shape = memview.view.shape + * strides = memview.view.strides # <<<<<<<<<<<<<< + * suboffsets = memview.view.suboffsets + * +*/ + __pyx_t_1 = __pyx_v_memview->view.strides; + __pyx_v_strides = __pyx_t_1; + + /* "View.MemoryView":1070 + * shape = memview.view.shape + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets # <<<<<<<<<<<<<< + * + * dst.memview = <__pyx_memoryview *> memview +*/ + __pyx_t_1 = __pyx_v_memview->view.suboffsets; + __pyx_v_suboffsets = __pyx_t_1; + + /* "View.MemoryView":1072 + * suboffsets = memview.view.suboffsets + * + * dst.memview = <__pyx_memoryview *> memview # <<<<<<<<<<<<<< + * dst.data = memview.view.buf + * +*/ + __pyx_v_dst->memview = ((struct __pyx_memoryview_obj *)__pyx_v_memview); + + /* "View.MemoryView":1073 + * + * dst.memview = <__pyx_memoryview *> memview + * dst.data = memview.view.buf # <<<<<<<<<<<<<< + * + * for dim in range(memview.view.ndim): +*/ + __pyx_v_dst->data = ((char *)__pyx_v_memview->view.buf); + + /* "View.MemoryView":1075 + * dst.data = memview.view.buf + * + * for dim in range(memview.view.ndim): # <<<<<<<<<<<<<< + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] +*/ + __pyx_t_2 = __pyx_v_memview->view.ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_dim = __pyx_t_4; + + /* "View.MemoryView":1076 + * + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] # <<<<<<<<<<<<<< + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 +*/ + (__pyx_v_dst->shape[__pyx_v_dim]) = (__pyx_v_shape[__pyx_v_dim]); + + /* "View.MemoryView":1077 + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] # <<<<<<<<<<<<<< + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * +*/ + (__pyx_v_dst->strides[__pyx_v_dim]) = (__pyx_v_strides[__pyx_v_dim]); + + /* "View.MemoryView":1078 + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object') +*/ + __pyx_t_6 = (__pyx_v_suboffsets != 0); + if (__pyx_t_6) { + __pyx_t_5 = (__pyx_v_suboffsets[__pyx_v_dim]); + } else { + __pyx_t_5 = -1L; + } + (__pyx_v_dst->suboffsets[__pyx_v_dim]) = __pyx_t_5; + } + + /* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx_v_memview) { + __Pyx_memviewslice __pyx_v_memviewslice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy", 0); + + /* "View.MemoryView":1084 + * "Create a new memoryview object" + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) # <<<<<<<<<<<<<< + * return memoryview_copy_from_slice(memview, &memviewslice) + * +*/ + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_memviewslice)); + + /* "View.MemoryView":1085 + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) + * return memoryview_copy_from_slice(memview, &memviewslice) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object_from_slice') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1085, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_memviewslice) { + PyObject *(*__pyx_v_to_object_func)(char *); + int (*__pyx_v_to_dtype_func)(char *, PyObject *); + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *(*__pyx_t_2)(char *); + int (*__pyx_t_3)(char *, PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy_from_slice", 0); + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1096 + * + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func # <<<<<<<<<<<<<< + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: +*/ + __pyx_t_2 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_object_func; + __pyx_v_to_object_func = __pyx_t_2; + + /* "View.MemoryView":1097 + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func # <<<<<<<<<<<<<< + * else: + * to_object_func = NULL +*/ + __pyx_t_3 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_dtype_func; + __pyx_v_to_dtype_func = __pyx_t_3; + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1099 + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: + * to_object_func = NULL # <<<<<<<<<<<<<< + * to_dtype_func = NULL + * +*/ + /*else*/ { + __pyx_v_to_object_func = NULL; + + /* "View.MemoryView":1100 + * else: + * to_object_func = NULL + * to_dtype_func = NULL # <<<<<<<<<<<<<< + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, +*/ + __pyx_v_to_dtype_func = NULL; + } + __pyx_L3:; + + /* "View.MemoryView":1102 + * to_dtype_func = NULL + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, # <<<<<<<<<<<<<< + * to_object_func, to_dtype_func, + * memview.dtype_is_object) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "View.MemoryView":1104 + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, + * to_object_func, to_dtype_func, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_from_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + +static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":1111 + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: + * return -arg if arg < 0 else arg # <<<<<<<<<<<<<< + * + * @cname('__pyx_get_best_slice_order') +*/ + __pyx_t_2 = (__pyx_v_arg < 0); + if (__pyx_t_2) { + __pyx_t_1 = (-__pyx_v_arg); + } else { + __pyx_t_1 = __pyx_v_arg; + } + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + +static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim) { + int __pyx_v_i; + Py_ssize_t __pyx_v_c_stride; + Py_ssize_t __pyx_v_f_stride; + char __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1119 + * """ + * cdef int i + * cdef Py_ssize_t c_stride = 0 # <<<<<<<<<<<<<< + * cdef Py_ssize_t f_stride = 0 + * +*/ + __pyx_v_c_stride = 0; + + /* "View.MemoryView":1120 + * cdef int i + * cdef Py_ssize_t c_stride = 0 + * cdef Py_ssize_t f_stride = 0 # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_f_stride = 0; + + /* "View.MemoryView":1122 + * cdef Py_ssize_t f_stride = 0 + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1124 + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_c_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1125 + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + goto __pyx_L4_break; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L4_break:; + + /* "View.MemoryView":1127 + * break + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] +*/ + __pyx_t_1 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_1; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1129 + * for i in range(ndim): + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_f_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1130 + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): +*/ + goto __pyx_L7_break; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L7_break:; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + __pyx_t_2 = (abs_py_ssize_t(__pyx_v_c_stride) <= abs_py_ssize_t(__pyx_v_f_stride)); + if (__pyx_t_2) { + + /* "View.MemoryView":1133 + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): + * return 'C' # <<<<<<<<<<<<<< + * else: + * return 'F' +*/ + __pyx_r = 'C'; + goto __pyx_L0; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + } + + /* "View.MemoryView":1135 + * return 'C' + * else: + * return 'F' # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + /*else*/ { + __pyx_r = 'F'; + goto __pyx_L0; + } + + /* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + +static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v_src_strides, char *__pyx_v_dst_data, Py_ssize_t *__pyx_v_dst_strides, Py_ssize_t *__pyx_v_src_shape, Py_ssize_t *__pyx_v_dst_shape, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + CYTHON_UNUSED Py_ssize_t __pyx_v_src_extent; + Py_ssize_t __pyx_v_dst_extent; + Py_ssize_t __pyx_v_src_stride; + Py_ssize_t __pyx_v_dst_stride; + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + + /* "View.MemoryView":1145 + * + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] +*/ + __pyx_v_src_extent = (__pyx_v_src_shape[0]); + + /* "View.MemoryView":1146 + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] +*/ + __pyx_v_dst_extent = (__pyx_v_dst_shape[0]); + + /* "View.MemoryView":1147 + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_stride = dst_strides[0] + * +*/ + __pyx_v_src_stride = (__pyx_v_src_strides[0]); + + /* "View.MemoryView":1148 + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_dst_stride = (__pyx_v_dst_strides[0]); + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + __pyx_t_2 = (__pyx_v_src_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_dst_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + + /* "View.MemoryView":1152 + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: +*/ + __pyx_t_2 = (((size_t)__pyx_v_src_stride) == __pyx_v_itemsize); + if (__pyx_t_2) { + __pyx_t_2 = (__pyx_v_itemsize == ((size_t)__pyx_v_dst_stride)); + } + __pyx_t_1 = __pyx_t_2; + __pyx_L5_bool_binop_done:; + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":1153 + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, (__pyx_v_itemsize * __pyx_v_dst_extent))); + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1155 + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1156 + * else: + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) # <<<<<<<<<<<<<< + * src_data += src_stride + * dst_data += dst_stride +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, __pyx_v_itemsize)); + + /* "View.MemoryView":1157 + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * else: +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1158 + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L4:; + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1160 + * dst_data += dst_stride + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * _copy_strided_to_strided(src_data, src_strides + 1, + * dst_data, dst_strides + 1, +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1161 + * else: + * for i in range(dst_extent): + * _copy_strided_to_strided(src_data, src_strides + 1, # <<<<<<<<<<<<<< + * dst_data, dst_strides + 1, + * src_shape + 1, dst_shape + 1, +*/ + _copy_strided_to_strided(__pyx_v_src_data, (__pyx_v_src_strides + 1), __pyx_v_dst_data, (__pyx_v_dst_strides + 1), (__pyx_v_src_shape + 1), (__pyx_v_dst_shape + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize); + + /* "View.MemoryView":1165 + * src_shape + 1, dst_shape + 1, + * ndim - 1, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1166 + * ndim - 1, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + +static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + + /* "View.MemoryView":1171 + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: + * _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, # <<<<<<<<<<<<<< + * src.shape, dst.shape, ndim, itemsize) + * +*/ + _copy_strided_to_strided(__pyx_v_src->data, __pyx_v_src->strides, __pyx_v_dst->data, __pyx_v_dst->strides, __pyx_v_src->shape, __pyx_v_dst->shape, __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_src, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_size; + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + + /* "View.MemoryView":1177 + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" + * cdef Py_ssize_t shape, size = src.memview.view.itemsize # <<<<<<<<<<<<<< + * + * for shape in src.shape[:ndim]: +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_size = __pyx_t_1; + + /* "View.MemoryView":1179 + * cdef Py_ssize_t shape, size = src.memview.view.itemsize + * + * for shape in src.shape[:ndim]: # <<<<<<<<<<<<<< + * size *= shape + * +*/ + __pyx_t_3 = (__pyx_v_src->shape + __pyx_v_ndim); + for (__pyx_t_4 = __pyx_v_src->shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_v_shape = (__pyx_t_2[0]); + + /* "View.MemoryView":1180 + * + * for shape in src.shape[:ndim]: + * size *= shape # <<<<<<<<<<<<<< + * + * return size +*/ + __pyx_v_size = (__pyx_v_size * __pyx_v_shape); + } + + /* "View.MemoryView":1182 + * size *= shape + * + * return size # <<<<<<<<<<<<<< + * + * @cname('__pyx_fill_contig_strides_array') +*/ + __pyx_r = __pyx_v_size; + goto __pyx_L0; + + /* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, Py_ssize_t __pyx_v_stride, int __pyx_v_ndim, char __pyx_v_order) { + int __pyx_v_idx; + Py_ssize_t __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + __pyx_t_1 = (__pyx_v_order == 'F'); + if (__pyx_t_1) { + + /* "View.MemoryView":1195 + * + * if order == 'F': + * for idx in range(ndim): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + __pyx_t_2 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_idx = __pyx_t_4; + + /* "View.MemoryView":1196 + * if order == 'F': + * for idx in range(ndim): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * else: +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1197 + * for idx in range(ndim): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * else: + * for idx in range(ndim - 1, -1, -1): +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1199 + * stride *= shape[idx] + * else: + * for idx in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + /*else*/ { + for (__pyx_t_2 = (__pyx_v_ndim - 1); __pyx_t_2 > -1; __pyx_t_2-=1) { + __pyx_v_idx = __pyx_t_2; + + /* "View.MemoryView":1200 + * else: + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1201 + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * + * return stride +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + } + __pyx_L3:; + + /* "View.MemoryView":1203 + * stride *= shape[idx] + * + * return stride # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_data_to_temp') +*/ + __pyx_r = __pyx_v_stride; + goto __pyx_L0; + + /* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_tmpslice, char __pyx_v_order, int __pyx_v_ndim) { + int __pyx_v_i; + void *__pyx_v_result; + size_t __pyx_v_itemsize; + size_t __pyx_v_size; + void *__pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + struct __pyx_memoryview_obj *__pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1217 + * cdef void *result + * + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef size_t size = slice_get_size(src, ndim) + * +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1218 + * + * cdef size_t itemsize = src.memview.view.itemsize + * cdef size_t size = slice_get_size(src, ndim) # <<<<<<<<<<<<<< + * + * result = malloc(size) +*/ + __pyx_v_size = __pyx_memoryview_slice_get_size(__pyx_v_src, __pyx_v_ndim); + + /* "View.MemoryView":1220 + * cdef size_t size = slice_get_size(src, ndim) + * + * result = malloc(size) # <<<<<<<<<<<<<< + * if not result: + * _err_no_memory() +*/ + __pyx_v_result = malloc(__pyx_v_size); + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + __pyx_t_2 = (!(__pyx_v_result != 0)); + if (__pyx_t_2) { + + /* "View.MemoryView":1222 + * result = malloc(size) + * if not result: + * _err_no_memory() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __pyx_memoryview_err_no_memory(); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1222, __pyx_L1_error) + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + } + + /* "View.MemoryView":1225 + * + * + * tmpslice.data = result # <<<<<<<<<<<<<< + * tmpslice.memview = src.memview + * for i in range(ndim): +*/ + __pyx_v_tmpslice->data = ((char *)__pyx_v_result); + + /* "View.MemoryView":1226 + * + * tmpslice.data = result + * tmpslice.memview = src.memview # <<<<<<<<<<<<<< + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] +*/ + __pyx_t_4 = __pyx_v_src->memview; + __pyx_v_tmpslice->memview = __pyx_t_4; + + /* "View.MemoryView":1227 + * tmpslice.data = result + * tmpslice.memview = src.memview + * for i in range(ndim): # <<<<<<<<<<<<<< + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1228 + * tmpslice.memview = src.memview + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] # <<<<<<<<<<<<<< + * tmpslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_tmpslice->shape[__pyx_v_i]) = (__pyx_v_src->shape[__pyx_v_i]); + + /* "View.MemoryView":1229 + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) +*/ + (__pyx_v_tmpslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1231 + * tmpslice.suboffsets[i] = -1 + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # <<<<<<<<<<<<<< + * + * +*/ + (void)(__pyx_fill_contig_strides_array((&(__pyx_v_tmpslice->shape[0])), (&(__pyx_v_tmpslice->strides[0])), __pyx_v_itemsize, __pyx_v_ndim, __pyx_v_order)); + + /* "View.MemoryView":1234 + * + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + __pyx_t_2 = ((__pyx_v_tmpslice->shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1236 + * for i in range(ndim): + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 # <<<<<<<<<<<<<< + * + * if slice_is_contig(src[0], order, ndim): +*/ + (__pyx_v_tmpslice->strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + } + } + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig((__pyx_v_src[0]), __pyx_v_order, __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1239 + * + * if slice_is_contig(src[0], order, ndim): + * memcpy(result, src.data, size) # <<<<<<<<<<<<<< + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) +*/ + (void)(memcpy(__pyx_v_result, __pyx_v_src->data, __pyx_v_size)); + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":1241 + * memcpy(result, src.data, size) + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) # <<<<<<<<<<<<<< + * + * return result +*/ + /*else*/ { + copy_strided_to_strided(__pyx_v_src, __pyx_v_tmpslice, __pyx_v_ndim, __pyx_v_itemsize); + } + __pyx_L9:; + + /* "View.MemoryView":1243 + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.copy_data_to_temp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + +static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent1, Py_ssize_t __pyx_v_extent2) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[7]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_extents", 0); + + /* "View.MemoryView":1250 + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_dim') +*/ + __pyx_t_1 = __Pyx_PyUnicode_From_int(__pyx_v_i, 0, ' ', 'd'); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent1, 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent2, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_got_differing_extents_in_dimensi; + __pyx_t_4[1] = __pyx_t_1; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_got; + __pyx_t_4[3] = __pyx_t_2; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_4[5] = __pyx_t_3; + __pyx_t_4[6] = __pyx_mstate_global->__pyx_kp_u__7; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 7, 35 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_1) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 1250, __pyx_L1_error) + + /* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView._err_extents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + +static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, PyObject *__pyx_v_msg, int __pyx_v_dim) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_dim", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1254 + * @cname('__pyx_memoryview_err_dim') + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err') +*/ + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_FormatSafe(__pyx_v_msg, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_t_2, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 1254, __pyx_L1_error) + + /* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._err_dim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + +static int __pyx_memoryview_err(PyObject *__pyx_v_error, PyObject *__pyx_v_msg) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1258 + * @cname('__pyx_memoryview_err') + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_no_memory') +*/ + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_v_msg, 0, 0); + __PYX_ERR(1, 1258, __pyx_L1_error) + + /* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + +static int __pyx_memoryview_err_no_memory(void) { + int __pyx_r; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1262 + * @cname('__pyx_memoryview_err_no_memory') + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError # <<<<<<<<<<<<<< + * + * +*/ + PyErr_NoMemory(); __PYX_ERR(1, 1262, __pyx_L1_error) + + /* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err_no_memory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_memviewslice __pyx_v_dst, int __pyx_v_src_ndim, int __pyx_v_dst_ndim, int __pyx_v_dtype_is_object) { + void *__pyx_v_tmpdata; + size_t __pyx_v_itemsize; + int __pyx_v_i; + char __pyx_v_order; + int __pyx_v_broadcasting; + int __pyx_v_direct_copy; + __Pyx_memviewslice __pyx_v_tmp; + int __pyx_v_ndim; + int __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + void *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1274 + * Check for overlapping memory and verify the shapes. + * """ + * cdef void *tmpdata = NULL # <<<<<<<<<<<<<< + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i +*/ + __pyx_v_tmpdata = NULL; + + /* "View.MemoryView":1275 + * """ + * cdef void *tmpdata = NULL + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) +*/ + __pyx_t_1 = __pyx_v_src.memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1277 + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) # <<<<<<<<<<<<<< + * cdef bint broadcasting = False + * cdef bint direct_copy = False +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_src), __pyx_v_src_ndim); + + /* "View.MemoryView":1278 + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False # <<<<<<<<<<<<<< + * cdef bint direct_copy = False + * cdef __Pyx_memviewslice tmp +*/ + __pyx_v_broadcasting = 0; + + /* "View.MemoryView":1279 + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False + * cdef bint direct_copy = False # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice tmp + * +*/ + __pyx_v_direct_copy = 0; + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + __pyx_t_2 = (__pyx_v_src_ndim < __pyx_v_dst_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1283 + * + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_src), __pyx_v_src_ndim, __pyx_v_dst_ndim); + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + __pyx_t_2 = (__pyx_v_dst_ndim < __pyx_v_src_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1285 + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) # <<<<<<<<<<<<<< + * + * cdef int ndim = max(src_ndim, dst_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_dst), __pyx_v_dst_ndim, __pyx_v_src_ndim); + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + } + __pyx_L3:; + + /* "View.MemoryView":1287 + * broadcast_leading(&dst, dst_ndim, src_ndim) + * + * cdef int ndim = max(src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + __pyx_t_3 = __pyx_v_dst_ndim; + __pyx_t_4 = __pyx_v_src_ndim; + __pyx_t_2 = (__pyx_t_3 > __pyx_t_4); + if (__pyx_t_2) { + __pyx_t_5 = __pyx_t_3; + } else { + __pyx_t_5 = __pyx_t_4; + } + __pyx_v_ndim = __pyx_t_5; + + /* "View.MemoryView":1289 + * cdef int ndim = max(src_ndim, dst_ndim) + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: +*/ + __pyx_t_5 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_5; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) != (__pyx_v_dst.shape[__pyx_v_i])); + if (__pyx_t_2) { + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1292 + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: + * broadcasting = True # <<<<<<<<<<<<<< + * src.strides[i] = 0 + * else: +*/ + __pyx_v_broadcasting = 1; + + /* "View.MemoryView":1293 + * if src.shape[i] == 1: + * broadcasting = True + * src.strides[i] = 0 # <<<<<<<<<<<<<< + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) +*/ + (__pyx_v_src.strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":1295 + * src.strides[i] = 0 + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) # <<<<<<<<<<<<<< + * + * if src.suboffsets[i] >= 0: +*/ + /*else*/ { + __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1295, __pyx_L1_error) + } + __pyx_L7:; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + } + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + __pyx_t_2 = ((__pyx_v_src.suboffsets[__pyx_v_i]) >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":1298 + * + * if src.suboffsets[i] >= 0: + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) # <<<<<<<<<<<<<< + * + * if slices_overlap(&src, &dst, ndim, itemsize): +*/ + __pyx_t_6 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Dimension_d_is_not_direct, __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1298, __pyx_L1_error) + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + } + } + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + __pyx_t_2 = __pyx_slices_overlap((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + if (__pyx_t_2) { + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + __pyx_t_2 = (!__pyx_memviewslice_is_contig(__pyx_v_src, __pyx_v_order, __pyx_v_ndim)); + if (__pyx_t_2) { + + /* "View.MemoryView":1303 + * + * if not slice_is_contig(src, order, ndim): + * order = get_best_order(&dst, ndim) # <<<<<<<<<<<<<< + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim); + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + } + + /* "View.MemoryView":1305 + * order = get_best_order(&dst, ndim) + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) # <<<<<<<<<<<<<< + * src = tmp + * +*/ + __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1305, __pyx_L1_error) + __pyx_v_tmpdata = __pyx_t_7; + + /* "View.MemoryView":1306 + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) + * src = tmp # <<<<<<<<<<<<<< + * + * if not broadcasting: +*/ + __pyx_v_src = __pyx_v_tmp; + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (!__pyx_v_broadcasting); + if (__pyx_t_2) { + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'C', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1312 + * + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) # <<<<<<<<<<<<<< + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'C', __pyx_v_ndim); + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'F', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1314 + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) # <<<<<<<<<<<<<< + * + * if direct_copy: +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'F', __pyx_v_ndim); + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + } + __pyx_L12:; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + if (__pyx_v_direct_copy) { + + /* "View.MemoryView":1318 + * if direct_copy: + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1319 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) +*/ + (void)(memcpy(__pyx_v_dst.data, __pyx_v_src.data, __pyx_memoryview_slice_get_size((&__pyx_v_src), __pyx_v_ndim))); + + /* "View.MemoryView":1320 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * free(tmpdata) + * return 0 +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1321 + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1322 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * if order == 'F' == get_best_order(&dst, ndim): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (__pyx_v_order == 'F'); + if (__pyx_t_2) { + __pyx_t_2 = ('F' == __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim)); + } + if (__pyx_t_2) { + + /* "View.MemoryView":1327 + * + * + * transpose_memslice(&src) # <<<<<<<<<<<<<< + * transpose_memslice(&dst) + * +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1327, __pyx_L1_error) + + /* "View.MemoryView":1328 + * + * transpose_memslice(&src) + * transpose_memslice(&dst) # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1328, __pyx_L1_error) + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1330 + * transpose_memslice(&dst) + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1331 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * +*/ + copy_strided_to_strided((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1332 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * free(tmpdata) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1334 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1335 + * + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_broadcast_leading') +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_contents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim, int __pyx_v_ndim_other) { + int __pyx_v_i; + int __pyx_v_offset; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + + /* "View.MemoryView":1342 + * int ndim_other) noexcept nogil: + * cdef int i + * cdef int offset = ndim_other - ndim # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_offset = (__pyx_v_ndim_other - __pyx_v_ndim); + + /* "View.MemoryView":1344 + * cdef int offset = ndim_other - ndim + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1345 + * + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] # <<<<<<<<<<<<<< + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] +*/ + (__pyx_v_mslice->shape[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->shape[__pyx_v_i]); + + /* "View.MemoryView":1346 + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] # <<<<<<<<<<<<<< + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * +*/ + (__pyx_v_mslice->strides[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1347 + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] # <<<<<<<<<<<<<< + * + * for i in range(offset): +*/ + (__pyx_v_mslice->suboffsets[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->suboffsets[__pyx_v_i]); + } + + /* "View.MemoryView":1349 + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * + * for i in range(offset): # <<<<<<<<<<<<<< + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] +*/ + __pyx_t_1 = __pyx_v_offset; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1350 + * + * for i in range(offset): + * mslice.shape[i] = 1 # <<<<<<<<<<<<<< + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 +*/ + (__pyx_v_mslice->shape[__pyx_v_i]) = 1; + + /* "View.MemoryView":1351 + * for i in range(offset): + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] # <<<<<<<<<<<<<< + * mslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_mslice->strides[__pyx_v_i]) = (__pyx_v_mslice->strides[0]); + + /* "View.MemoryView":1352 + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_mslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_dtype_is_object, int __pyx_v_ndim, int __pyx_v_inc) { + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + if (__pyx_v_dtype_is_object) { + + /* "View.MemoryView":1363 + * + * if dtype_is_object: + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') +*/ + __pyx_memoryview_refcount_objects_in_slice_with_gil(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + } + + /* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1369 + * Py_ssize_t *strides, int ndim, + * bint inc) noexcept with gil: + * refcount_objects_in_slice(data, shape, strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, __pyx_v_shape, __pyx_v_strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ + __Pyx_PyGILState_Release(__pyx_gilstate_save); +} + +/* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + +static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1375 + * Py_ssize_t *strides, int ndim, bint inc) noexcept: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * + * for i in range(shape[0]): +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1377 + * cdef Py_ssize_t stride = strides[0] + * + * for i in range(shape[0]): # <<<<<<<<<<<<<< + * if ndim == 1: + * if inc: +*/ + __pyx_t_1 = (__pyx_v_shape[0]); + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + __pyx_t_4 = (__pyx_v_ndim == 1); + if (__pyx_t_4) { + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + if (__pyx_v_inc) { + + /* "View.MemoryView":1380 + * if ndim == 1: + * if inc: + * Py_INCREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * Py_DECREF(( data)[0]) +*/ + Py_INCREF((((PyObject **)__pyx_v_data)[0])); + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":1382 + * Py_INCREF(( data)[0]) + * else: + * Py_DECREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) +*/ + /*else*/ { + Py_DECREF((((PyObject **)__pyx_v_data)[0])); + } + __pyx_L6:; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":1384 + * Py_DECREF(( data)[0]) + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) # <<<<<<<<<<<<<< + * + * data += stride +*/ + /*else*/ { + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_inc); + } + __pyx_L5:; + + /* "View.MemoryView":1386 + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) + * + * data += stride # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item, int __pyx_v_dtype_is_object) { + + /* "View.MemoryView":1395 + * size_t itemsize, void *item, + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1396 + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) # <<<<<<<<<<<<<< + * refcount_copying(dst, dtype_is_object, ndim, inc=True) + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1397 + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_extent; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + + /* "View.MemoryView":1405 + * size_t itemsize, void *item) noexcept nogil: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t extent = shape[0] + * +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1406 + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] + * cdef Py_ssize_t extent = shape[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_extent = (__pyx_v_shape[0]); + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1409 + * + * if ndim == 1: + * for i in range(extent): # <<<<<<<<<<<<<< + * memcpy(data, item, itemsize) + * data += stride +*/ + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1410 + * if ndim == 1: + * for i in range(extent): + * memcpy(data, item, itemsize) # <<<<<<<<<<<<<< + * data += stride + * else: +*/ + (void)(memcpy(__pyx_v_data, __pyx_v_item, __pyx_v_itemsize)); + + /* "View.MemoryView":1411 + * for i in range(extent): + * memcpy(data, item, itemsize) + * data += stride # <<<<<<<<<<<<<< + * else: + * for i in range(extent): +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1413 + * data += stride + * else: + * for i in range(extent): # <<<<<<<<<<<<<< + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride +*/ + /*else*/ { + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1414 + * else: + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) # <<<<<<<<<<<<<< + * data += stride + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1415 + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride # <<<<<<<<<<<<<< + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ +} + +/* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum = {"__pyx_unpickle_Enum", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 4, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_Enum", 0) < (0)) __PYX_ERR(1, 4, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, i); __PYX_ERR(1, 4, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_v___pyx_state = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, __pyx_nargs); __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_r = __pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum", 0); + + /* "(tree fragment)":6 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') # <<<<<<<<<<<<<< + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: +*/ + __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, ((char const *)"name")); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(1, 6, __pyx_L1_error) + + /* "(tree fragment)":7 + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) +*/ + __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_MemviewEnum_type); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v___pyx_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None)); + if (__pyx_t_5) { + + /* "(tree fragment)":9 + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): +*/ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 9, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(((struct __pyx_MemviewEnum_obj *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + } + + /* "(tree fragment)":10 + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum__set_state", 0); + + /* "(tree fragment)":12 + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] # <<<<<<<<<<<<<< + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->name); + __Pyx_DECREF(__pyx_v___pyx_result->name); + __pyx_v___pyx_result->name = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":13 + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) # <<<<<<<<<<<<<< +*/ + __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 13, __pyx_L1_error) + + /* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":242 + * cdef int type_num + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_8itemsize_itemsize(PyArray_Descr *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":244 + * @property + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyDataType_ELSIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":242 + * cdef int type_num + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":246 + * return PyDataType_ELSIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_9alignment_alignment(PyArray_Descr *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":248 + * @property + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) # <<<<<<<<<<<<<< + * + * # Use fields/names with care as they may be NULL. You must check +*/ + __pyx_r = PyDataType_ALIGNMENT(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":246 + * return PyDataType_ELSIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":252 + * # Use fields/names with care as they may be NULL. You must check + * # for this using PyDataType_HASFIELDS. + * @property # <<<<<<<<<<<<<< + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_6fields_fields(PyArray_Descr *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1; + __Pyx_RefNannySetupContext("fields", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":254 + * @property + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyDataType_FIELDS(__pyx_v_self); + __Pyx_INCREF(((PyObject *)__pyx_t_1)); + __pyx_r = ((PyObject *)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":252 + * # Use fields/names with care as they may be NULL. You must check + * # for this using PyDataType_HASFIELDS. + * @property # <<<<<<<<<<<<<< + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":256 + * return PyDataType_FIELDS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_5names_names(PyArray_Descr *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1; + __Pyx_RefNannySetupContext("names", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":258 + * @property + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) # <<<<<<<<<<<<<< + * + * # Use PyDataType_HASSUBARRAY to test whether this field is +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyDataType_NAMES(__pyx_v_self); + __Pyx_INCREF(((PyObject*)__pyx_t_1)); + __pyx_r = ((PyObject*)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":256 + * return PyDataType_FIELDS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":263 + * # valid (the pointer can be NULL). Most users should access + * # this field via the inline helper method PyDataType_SHAPE. + * @property # <<<<<<<<<<<<<< + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) +*/ + +static CYTHON_INLINE PyArray_ArrayDescr *__pyx_f_5numpy_5dtype_8subarray_subarray(PyArray_Descr *__pyx_v_self) { + PyArray_ArrayDescr *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":265 + * @property + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyDataType_SUBARRAY(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":263 + * # valid (the pointer can be NULL). Most users should access + * # this field via the inline helper method PyDataType_SHAPE. + * @property # <<<<<<<<<<<<<< + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":267 + * return PyDataType_SUBARRAY(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" +*/ + +static CYTHON_INLINE npy_uint64 __pyx_f_5numpy_5dtype_5flags_flags(PyArray_Descr *__pyx_v_self) { + npy_uint64 __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":270 + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" + * return PyDataType_FLAGS(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyDataType_FLAGS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":267 + * return PyDataType_SUBARRAY(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":279 + * ctypedef class numpy.broadcast [object PyArrayMultiIterObject, check_size ignore]: + * + * @property # <<<<<<<<<<<<<< + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_7numiter_numiter(PyArrayMultiIterObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":282 + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" + * return PyArray_MultiIter_NUMITER(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_NUMITER(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":279 + * ctypedef class numpy.broadcast [object PyArrayMultiIterObject, check_size ignore]: + * + * @property # <<<<<<<<<<<<<< + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":284 + * return PyArray_MultiIter_NUMITER(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_4size_size(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":287 + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" + * return PyArray_MultiIter_SIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_SIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":284 + * return PyArray_MultiIter_NUMITER(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":289 + * return PyArray_MultiIter_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_5index_index(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":292 + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" + * return PyArray_MultiIter_INDEX(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_INDEX(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":289 + * return PyArray_MultiIter_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":294 + * return PyArray_MultiIter_INDEX(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_2nd_nd(PyArrayMultiIterObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":297 + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" + * return PyArray_MultiIter_NDIM(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_NDIM(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":294 + * return PyArray_MultiIter_INDEX(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":299 + * return PyArray_MultiIter_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_9broadcast_10dimensions_dimensions(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":302 + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" + * return PyArray_MultiIter_DIMS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_DIMS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":299 + * return PyArray_MultiIter_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":304 + * return PyArray_MultiIter_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline void** iters(self) noexcept nogil: + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. +*/ + +static CYTHON_INLINE void **__pyx_f_5numpy_9broadcast_5iters_iters(PyArrayMultiIterObject *__pyx_v_self) { + void **__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":308 + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. + * On return, the iterators are adjusted for broadcasting.""" + * return PyArray_MultiIter_ITERS(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyArray_MultiIter_ITERS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":304 + * return PyArray_MultiIter_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline void** iters(self) noexcept nogil: + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":322 + * # Instead, we use properties that map to the corresponding C-API functions. + * + * @property # <<<<<<<<<<<<<< + * cdef inline PyObject* base(self) noexcept nogil: + * """Returns a borrowed reference to the object owning the data/memory. +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_7ndarray_4base_base(PyArrayObject *__pyx_v_self) { + PyObject *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":326 + * """Returns a borrowed reference to the object owning the data/memory. + * """ + * return PyArray_BASE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_BASE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":322 + * # Instead, we use properties that map to the corresponding C-API functions. + * + * @property # <<<<<<<<<<<<<< + * cdef inline PyObject* base(self) noexcept nogil: + * """Returns a borrowed reference to the object owning the data/memory. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":328 + * return PyArray_BASE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline dtype descr(self): + * """Returns an owned reference to the dtype of the array. +*/ + +static CYTHON_INLINE PyArray_Descr *__pyx_f_5numpy_7ndarray_5descr_descr(PyArrayObject *__pyx_v_self) { + PyArray_Descr *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyArray_Descr *__pyx_t_1; + __Pyx_RefNannySetupContext("descr", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":332 + * """Returns an owned reference to the dtype of the array. + * """ + * return PyArray_DESCR(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __pyx_t_1 = PyArray_DESCR(__pyx_v_self); + __Pyx_INCREF((PyObject *)((PyArray_Descr *)__pyx_t_1)); + __pyx_r = ((PyArray_Descr *)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":328 + * return PyArray_BASE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline dtype descr(self): + * """Returns an owned reference to the dtype of the array. +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":334 + * return PyArray_DESCR(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int ndim(self) noexcept nogil: + * """Returns the number of dimensions in the array. +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_7ndarray_4ndim_ndim(PyArrayObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":338 + * """Returns the number of dimensions in the array. + * """ + * return PyArray_NDIM(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_NDIM(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":334 + * return PyArray_DESCR(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int ndim(self) noexcept nogil: + * """Returns the number of dimensions in the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":340 + * return PyArray_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *shape(self) noexcept nogil: + * """Returns a pointer to the dimensions/shape of the array. +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_5shape_shape(PyArrayObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":346 + * Can return NULL for 0-dimensional arrays. + * """ + * return PyArray_DIMS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_DIMS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":340 + * return PyArray_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *shape(self) noexcept nogil: + * """Returns a pointer to the dimensions/shape of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":348 + * return PyArray_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *strides(self) noexcept nogil: + * """Returns a pointer to the strides of the array. +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_7strides_strides(PyArrayObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":353 + * The number of elements matches the number of dimensions of the array (ndim). + * """ + * return PyArray_STRIDES(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_STRIDES(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":348 + * return PyArray_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *strides(self) noexcept nogil: + * """Returns a pointer to the strides of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":355 + * return PyArray_STRIDES(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """Returns the total size (in number of elements) of the array. +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_7ndarray_4size_size(PyArrayObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":359 + * """Returns the total size (in number of elements) of the array. + * """ + * return PyArray_SIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_SIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":355 + * return PyArray_STRIDES(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """Returns the total size (in number of elements) of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":361 + * return PyArray_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline char* data(self) noexcept nogil: + * """The pointer to the data buffer as a char*. +*/ + +static CYTHON_INLINE char *__pyx_f_5numpy_7ndarray_4data_data(PyArrayObject *__pyx_v_self) { + char *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":368 + * of `PyArray_DATA()` instead, which returns a 'void*'. + * """ + * return PyArray_BYTES(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyArray_BYTES(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":361 + * return PyArray_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline char* data(self) noexcept nogil: + * """The pointer to the data buffer as a char*. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":776 + * ctypedef long double complex clongdouble_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__pyx_v_a) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":777 + * + * cdef inline object PyArray_MultiIterNew1(a): + * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew2(a, b): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(1, ((void *)__pyx_v_a)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 777, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":776 + * ctypedef long double complex clongdouble_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew1", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":779 + * return PyArray_MultiIterNew(1, a) + * + * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(2, a, b) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__pyx_v_a, PyObject *__pyx_v_b) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":780 + * + * cdef inline object PyArray_MultiIterNew2(a, b): + * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(2, ((void *)__pyx_v_a), ((void *)__pyx_v_b)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":779 + * return PyArray_MultiIterNew(1, a) + * + * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(2, a, b) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":782 + * return PyArray_MultiIterNew(2, a, b) + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(3, a, b, c) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":783 + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): + * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(3, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 783, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":782 + * return PyArray_MultiIterNew(2, a, b) + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(3, a, b, c) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":785 + * return PyArray_MultiIterNew(3, a, b, c) + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(4, a, b, c, d) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":786 + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): + * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(4, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 786, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":785 + * return PyArray_MultiIterNew(3, a, b, c) + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(4, a, b, c, d) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew4", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":788 + * return PyArray_MultiIterNew(4, a, b, c, d) + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d, PyObject *__pyx_v_e) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":789 + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): + * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< + * + * cdef inline tuple PyDataType_SHAPE(dtype d): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(5, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d), ((void *)__pyx_v_e)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":788 + * return PyArray_MultiIterNew(4, a, b, c, d) + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew5", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":791 + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * + * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__pyx_v_d) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2; + __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":792 + * + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< + * return d.subarray.shape + * else: +*/ + __pyx_t_1 = PyDataType_HASSUBARRAY(__pyx_v_d); + if (__pyx_t_1) { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":793 + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape # <<<<<<<<<<<<<< + * else: + * return () +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_f_5numpy_5dtype_8subarray_subarray(__pyx_v_d)->shape; + __Pyx_INCREF(((PyObject*)__pyx_t_2)); + __pyx_r = ((PyObject*)__pyx_t_2); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":792 + * + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< + * return d.subarray.shape + * else: +*/ + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":795 + * return d.subarray.shape + * else: + * return () # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_empty_tuple); + __pyx_r = __pyx_mstate_global->__pyx_empty_tuple; + goto __pyx_L0; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":791 + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * + * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":994 + * int _import_umath() except -1 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: # <<<<<<<<<<<<<< + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) +*/ + +static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_arr, PyObject *__pyx_v_base) { + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":995 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: + * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< + * PyArray_SetBaseObject(arr, base) + * +*/ + Py_INCREF(__pyx_v_base); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":996 + * cdef inline void set_array_base(ndarray arr, object base) except *: + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< + * + * cdef inline object get_array_base(ndarray arr): +*/ + __pyx_t_1 = PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 996, __pyx_L1_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":994 + * int _import_umath() except -1 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: # <<<<<<<<<<<<<< + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("numpy.set_array_base", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":998 + * PyArray_SetBaseObject(arr, base) + * + * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< + * base = PyArray_BASE(arr) + * if base is NULL: +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__pyx_v_arr) { + PyObject *__pyx_v_base; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("get_array_base", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":999 + * + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< + * if base is NULL: + * return None +*/ + __pyx_v_base = PyArray_BASE(__pyx_v_arr); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1000 + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) + * if base is NULL: # <<<<<<<<<<<<<< + * return None + * return base +*/ + __pyx_t_1 = (__pyx_v_base == NULL); + if (__pyx_t_1) { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1001 + * base = PyArray_BASE(arr) + * if base is NULL: + * return None # <<<<<<<<<<<<<< + * return base + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1000 + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) + * if base is NULL: # <<<<<<<<<<<<<< + * return None + * return base +*/ + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1002 + * if base is NULL: + * return None + * return base # <<<<<<<<<<<<<< + * + * # Versions of the import_* functions which are more suitable for +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_base)); + __pyx_r = ((PyObject *)__pyx_v_base); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":998 + * PyArray_SetBaseObject(arr, base) + * + * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< + * base = PyArray_BASE(arr) + * if base is NULL: +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1006 + * # Versions of the import_* functions which are more suitable for + * # Cython code. + * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< + * try: + * __pyx_import_array() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_array", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1008 + * cdef inline int import_array() except -1: + * try: + * __pyx_import_array() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.multiarray failed to import") +*/ + __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1008, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1009 + * try: + * __pyx_import_array() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.multiarray failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_array", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1009, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1010 + * __pyx_import_array() + * except Exception: + * raise ImportError("numpy._core.multiarray failed to import") # <<<<<<<<<<<<<< + * + * cdef inline int import_umath() except -1: +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_multiarray_failed_to}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1010, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1010, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1006 + * # Versions of the import_* functions which are more suitable for + * # Cython code. + * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< + * try: + * __pyx_import_array() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_array", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1012 + * raise ImportError("numpy._core.multiarray failed to import") + * + * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_umath", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1014 + * cdef inline int import_umath() except -1: + * try: + * _import_umath() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.umath failed to import") +*/ + __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1014, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1015 + * try: + * _import_umath() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.umath failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_umath", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1015, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1016 + * _import_umath() + * except Exception: + * raise ImportError("numpy._core.umath failed to import") # <<<<<<<<<<<<<< + * + * cdef inline int import_ufunc() except -1: +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_umath_failed_to_impo}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1016, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1016, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1012 + * raise ImportError("numpy._core.multiarray failed to import") + * + * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_umath", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1018 + * raise ImportError("numpy._core.umath failed to import") + * + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_ufunc", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1020 + * cdef inline int import_ufunc() except -1: + * try: + * _import_umath() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.umath failed to import") +*/ + __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1020, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1021 + * try: + * _import_umath() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.umath failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_ufunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1021, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1022 + * _import_umath() + * except Exception: + * raise ImportError("numpy._core.umath failed to import") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_umath_failed_to_impo}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1022, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1022, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1018 + * raise ImportError("numpy._core.umath failed to import") + * + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_ufunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1025 + * + * + * cdef inline bint is_timedelta64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.timedelta64)` +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_is_timedelta64_object(PyObject *__pyx_v_obj) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1037 + * bool + * """ + * return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyObject_TypeCheck(__pyx_v_obj, (&PyTimedeltaArrType_Type)); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1025 + * + * + * cdef inline bint is_timedelta64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.timedelta64)` +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1040 + * + * + * cdef inline bint is_datetime64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.datetime64)` +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_is_datetime64_object(PyObject *__pyx_v_obj) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1052 + * bool + * """ + * return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyObject_TypeCheck(__pyx_v_obj, (&PyDatetimeArrType_Type)); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1040 + * + * + * cdef inline bint is_datetime64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.datetime64)` +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1055 + * + * + * cdef inline npy_datetime get_datetime64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy datetime64 object +*/ + +static CYTHON_INLINE npy_datetime __pyx_f_5numpy_get_datetime64_value(PyObject *__pyx_v_obj) { + npy_datetime __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1062 + * also needed. That can be found using `get_datetime64_unit`. + * """ + * return (obj).obval # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((PyDatetimeScalarObject *)__pyx_v_obj)->obval; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1055 + * + * + * cdef inline npy_datetime get_datetime64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy datetime64 object +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1065 + * + * + * cdef inline npy_timedelta get_timedelta64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy timedelta64 object +*/ + +static CYTHON_INLINE npy_timedelta __pyx_f_5numpy_get_timedelta64_value(PyObject *__pyx_v_obj) { + npy_timedelta __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1069 + * returns the int64 value underlying scalar numpy timedelta64 object + * """ + * return (obj).obval # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((PyTimedeltaScalarObject *)__pyx_v_obj)->obval; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1065 + * + * + * cdef inline npy_timedelta get_timedelta64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy timedelta64 object +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1072 + * + * + * cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the unit part of the dtype for a numpy datetime64 object. +*/ + +static CYTHON_INLINE NPY_DATETIMEUNIT __pyx_f_5numpy_get_datetime64_unit(PyObject *__pyx_v_obj) { + NPY_DATETIMEUNIT __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1076 + * returns the unit part of the dtype for a numpy datetime64 object. + * """ + * return (obj).obmeta.base # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((NPY_DATETIMEUNIT)((PyDatetimeScalarObject *)__pyx_v_obj)->obmeta.base); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1072 + * + * + * cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the unit part of the dtype for a numpy datetime64 object. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":31 + * + * cdef class Vec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_3Vec_arg_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mode; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":33 + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: # <<<<<<<<<<<<<< + * return 0 if scores[0] > scores[1] else 1 + * cdef int i +*/ + __pyx_t_1 = (__pyx_v_n_classes == 2); + if (__pyx_t_1) { + + /* "linalg.pxd":34 + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: + * return 0 if scores[0] > scores[1] else 1 # <<<<<<<<<<<<<< + * cdef int i + * cdef int best = 0 +*/ + __pyx_t_1 = ((__pyx_v_scores[0]) > (__pyx_v_scores[1])); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = 1; + } + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "linalg.pxd":33 + * @staticmethod + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: # <<<<<<<<<<<<<< + * return 0 if scores[0] > scores[1] else 1 + * cdef int i +*/ + } + + /* "linalg.pxd":36 + * return 0 if scores[0] > scores[1] else 1 + * cdef int i + * cdef int best = 0 # <<<<<<<<<<<<<< + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): +*/ + __pyx_v_best = 0; + + /* "linalg.pxd":37 + * cdef int i + * cdef int best = 0 + * cdef weight_t mode = scores[0] # <<<<<<<<<<<<<< + * for i in range(1, n_classes): + * if scores[i] > mode: +*/ + __pyx_v_mode = (__pyx_v_scores[0]); + + /* "linalg.pxd":38 + * cdef int best = 0 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): # <<<<<<<<<<<<<< + * if scores[i] > mode: + * mode = scores[i] +*/ + __pyx_t_2 = __pyx_v_n_classes; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 1; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "linalg.pxd":39 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): + * if scores[i] > mode: # <<<<<<<<<<<<<< + * mode = scores[i] + * best = i +*/ + __pyx_t_1 = ((__pyx_v_scores[__pyx_v_i]) > __pyx_v_mode); + if (__pyx_t_1) { + + /* "linalg.pxd":40 + * for i in range(1, n_classes): + * if scores[i] > mode: + * mode = scores[i] # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_v_mode = (__pyx_v_scores[__pyx_v_i]); + + /* "linalg.pxd":41 + * if scores[i] > mode: + * mode = scores[i] + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "linalg.pxd":39 + * cdef weight_t mode = scores[0] + * for i in range(1, n_classes): + * if scores[i] > mode: # <<<<<<<<<<<<<< + * mode = scores[i] + * best = i +*/ + } + } + + /* "linalg.pxd":42 + * mode = scores[i] + * best = i + * return best # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "linalg.pxd":31 + * + * cdef class Vec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max(const weight_t* scores, const int n_classes) noexcept nogil: + * if n_classes == 2: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":44 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_max(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mode; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":46 + * @staticmethod + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: # <<<<<<<<<<<<<< + * return 0 + * cdef int i +*/ + __pyx_t_1 = (__pyx_v_nr == 0); + if (__pyx_t_1) { + + /* "linalg.pxd":47 + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: + * return 0 # <<<<<<<<<<<<<< + * cdef int i + * cdef weight_t mode = x[0] +*/ + __pyx_r = 0.0; + goto __pyx_L0; + + /* "linalg.pxd":46 + * @staticmethod + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: # <<<<<<<<<<<<<< + * return 0 + * cdef int i +*/ + } + + /* "linalg.pxd":49 + * return 0 + * cdef int i + * cdef weight_t mode = x[0] # <<<<<<<<<<<<<< + * for i in range(1, nr): + * if x[i] > mode: +*/ + __pyx_v_mode = (__pyx_v_x[0]); + + /* "linalg.pxd":50 + * cdef int i + * cdef weight_t mode = x[0] + * for i in range(1, nr): # <<<<<<<<<<<<<< + * if x[i] > mode: + * mode = x[i] +*/ + __pyx_t_2 = __pyx_v_nr; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 1; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "linalg.pxd":51 + * cdef weight_t mode = x[0] + * for i in range(1, nr): + * if x[i] > mode: # <<<<<<<<<<<<<< + * mode = x[i] + * return mode +*/ + __pyx_t_1 = ((__pyx_v_x[__pyx_v_i]) > __pyx_v_mode); + if (__pyx_t_1) { + + /* "linalg.pxd":52 + * for i in range(1, nr): + * if x[i] > mode: + * mode = x[i] # <<<<<<<<<<<<<< + * return mode + * +*/ + __pyx_v_mode = (__pyx_v_x[__pyx_v_i]); + + /* "linalg.pxd":51 + * cdef weight_t mode = x[0] + * for i in range(1, nr): + * if x[i] > mode: # <<<<<<<<<<<<<< + * mode = x[i] + * return mode +*/ + } + } + + /* "linalg.pxd":53 + * if x[i] > mode: + * mode = x[i] + * return mode # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_mode; + goto __pyx_L0; + + /* "linalg.pxd":44 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t max(const weight_t* x, int32_t nr) noexcept nogil: + * if nr == 0: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":55 + * return mode + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_sum(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "linalg.pxd":58 + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += vec[i] +*/ + __pyx_v_total = 0.0; + + /* "linalg.pxd":59 + * cdef int i + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += vec[i] + * return total +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":60 + * cdef weight_t total = 0 + * for i in range(nr): + * total += vec[i] # <<<<<<<<<<<<<< + * return total + * +*/ + __pyx_v_total = (__pyx_v_total + (__pyx_v_vec[__pyx_v_i])); + } + + /* "linalg.pxd":61 + * for i in range(nr): + * total += vec[i] + * return total # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_total; + goto __pyx_L0; + + /* "linalg.pxd":55 + * return mode + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t sum(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":63 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_norm(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + int32_t __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + + /* "linalg.pxd":65 + * @staticmethod + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += vec[i] ** 2 +*/ + __pyx_v_total = 0.0; + + /* "linalg.pxd":66 + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += vec[i] ** 2 + * return sqrt(total) +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":67 + * cdef weight_t total = 0 + * for i in range(nr): + * total += vec[i] ** 2 # <<<<<<<<<<<<<< + * return sqrt(total) + * +*/ + __pyx_v_total = (__pyx_v_total + powf(((__pyx_t_5thinc_8backends_6linalg_weight_t)(__pyx_v_vec[__pyx_v_i])), 2.0)); + } + + /* "linalg.pxd":68 + * for i in range(nr): + * total += vec[i] ** 2 + * return sqrt(total) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = sqrt(__pyx_v_total); + goto __pyx_L0; + + /* "linalg.pxd":63 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t norm(const weight_t* vec, int32_t nr) noexcept nogil: + * cdef weight_t total = 0 +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":70 + * return sqrt(total) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr) { + + /* "linalg.pxd":73 + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.add_i(output, inc, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":74 + * weight_t inc, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * Vec.add_i(output, inc, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_v_output, __pyx_v_inc, __pyx_v_nr); + + /* "linalg.pxd":70 + * return sqrt(total) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, const weight_t* x, + * weight_t inc, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":76 + * Vec.add_i(output, inc, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_inc, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":79 + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] += inc + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":80 + * cdef int i + * for i in range(nr): + * vec[i] += inc # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) + __pyx_v_inc); + } + + /* "linalg.pxd":76 + * Vec.add_i(output, inc, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* vec, weight_t inc, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "linalg.pxd":82 + * vec[i] += inc + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "linalg.pxd":85 + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.mul_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":86 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.mul_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "linalg.pxd":82 + * vec[i] += inc + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":88 + * Vec.mul_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* vec, weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":94 + * blis.cy.scalv(BLIS_NO_CONJUGATE, nr, scal, vec, 1) + * ELSE: + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] *= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":95 + * ELSE: + * for i in range(nr): + * vec[i] *= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) * __pyx_v_scal); + } + + /* "linalg.pxd":88 + * Vec.mul_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* vec, weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "linalg.pxd":97 + * vec[i] *= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "linalg.pxd":100 + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.pow_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":101 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.pow_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "linalg.pxd":97 + * vec[i] *= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":103 + * Vec.pow_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":106 + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] **= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":107 + * cdef int i + * for i in range(nr): + * vec[i] **= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = powf((__pyx_v_vec[__pyx_t_4]), ((__pyx_t_5thinc_8backends_6linalg_weight_t)__pyx_v_scal)); + } + + /* "linalg.pxd":103 + * Vec.pow_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void pow_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "linalg.pxd":109 + * vec[i] **= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scal, int32_t __pyx_v_nr) { + + /* "linalg.pxd":113 + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.div_i(output, scal, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":114 + * int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.div_i(output, scal, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_v_output, __pyx_v_scal, __pyx_v_nr); + + /* "linalg.pxd":109 + * vec[i] **= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div(weight_t* output, const weight_t* vec, weight_t scal, +*/ + + /* function exit code */ +} + +/* "linalg.pxd":116 + * Vec.div_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_div_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, __pyx_t_5thinc_8backends_6linalg_weight_t const __pyx_v_scal, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":120 + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] /= scal + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":121 + * cdef int i + * for i in range(nr): + * vec[i] /= scal # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_vec[__pyx_t_4]) = ((__pyx_v_vec[__pyx_t_4]) / __pyx_v_scal); + } + + /* "linalg.pxd":116 + * Vec.div_i(output, scal, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * @cython.cdivision(True) + * cdef inline void div_i(weight_t* vec, const weight_t scal, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":123 + * vec[i] /= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_vec, int32_t __pyx_v_nr) { + + /* "linalg.pxd":125 + * @staticmethod + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * Vec.exp_i(output, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_vec, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":126 + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) + * Vec.exp_i(output, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_v_output, __pyx_v_nr); + + /* "linalg.pxd":123 + * vec[i] /= scal + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp(weight_t* output, const weight_t* vec, int32_t nr) noexcept nogil: + * memcpy(output, vec, sizeof(output[0]) * nr) +*/ + + /* function exit code */ +} + +/* "linalg.pxd":128 + * Vec.exp_i(output, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_exp_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "linalg.pxd":131 + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] = exp(vec[i]) + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":132 + * cdef int i + * for i in range(nr): + * vec[i] = exp(vec[i]) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + (__pyx_v_vec[__pyx_v_i]) = exp((__pyx_v_vec[__pyx_v_i])); + } + + /* "linalg.pxd":128 + * Vec.exp_i(output, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void exp_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "linalg.pxd":134 + * vec[i] = exp(vec[i]) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Vec_reciprocal_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_vec, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "linalg.pxd":137 + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * vec[i] = 1.0 / vec[i] + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":138 + * cdef int i + * for i in range(nr): + * vec[i] = 1.0 / vec[i] # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + (__pyx_v_vec[__pyx_v_i]) = (1.0 / (__pyx_v_vec[__pyx_v_i])); + } + + /* "linalg.pxd":134 + * vec[i] = exp(vec[i]) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void reciprocal_i(weight_t* vec, int32_t nr) noexcept nogil: + * cdef int i +*/ + + /* function exit code */ +} + +/* "linalg.pxd":140 + * vec[i] = 1.0 / vec[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_mean(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_mean; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_x; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_1; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_2; + __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_t_3; + + /* "linalg.pxd":142 + * @staticmethod + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. # <<<<<<<<<<<<<< + * for x in X[:nr_dim]: + * mean += x +*/ + __pyx_v_mean = 0.; + + /* "linalg.pxd":143 + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. + * for x in X[:nr_dim]: # <<<<<<<<<<<<<< + * mean += x + * return mean / nr_dim +*/ + __pyx_t_2 = (__pyx_v_X + __pyx_v_nr_dim); + for (__pyx_t_3 = __pyx_v_X; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_x = (__pyx_t_1[0]); + + /* "linalg.pxd":144 + * cdef weight_t mean = 0. + * for x in X[:nr_dim]: + * mean += x # <<<<<<<<<<<<<< + * return mean / nr_dim + * +*/ + __pyx_v_mean = (__pyx_v_mean + __pyx_v_x); + } + + /* "linalg.pxd":145 + * for x in X[:nr_dim]: + * mean += x + * return mean / nr_dim # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = (__pyx_v_mean / __pyx_v_nr_dim); + goto __pyx_L0; + + /* "linalg.pxd":140 + * vec[i] = 1.0 / vec[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t mean(const weight_t* X, int32_t nr_dim) noexcept nogil: + * cdef weight_t mean = 0. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":147 + * return mean / nr_dim + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_3Vec_variance(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_X, int32_t __pyx_v_nr_dim) { + double __pyx_v_m; + double __pyx_v_v; + long __pyx_v_i; + double __pyx_v_diff; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + long __pyx_t_3; + + /* "linalg.pxd":150 + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ + * cdef double m = X[0] # <<<<<<<<<<<<<< + * cdef double v = 0. + * for i in range(1, nr_dim): +*/ + __pyx_v_m = (__pyx_v_X[0]); + + /* "linalg.pxd":151 + * # See https://www.johndcook.com/blog/standard_deviation/ + * cdef double m = X[0] + * cdef double v = 0. # <<<<<<<<<<<<<< + * for i in range(1, nr_dim): + * diff = X[i]-m +*/ + __pyx_v_v = 0.; + + /* "linalg.pxd":152 + * cdef double m = X[0] + * cdef double v = 0. + * for i in range(1, nr_dim): # <<<<<<<<<<<<<< + * diff = X[i]-m + * m += diff / (i+1) +*/ + __pyx_t_1 = __pyx_v_nr_dim; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 1; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":153 + * cdef double v = 0. + * for i in range(1, nr_dim): + * diff = X[i]-m # <<<<<<<<<<<<<< + * m += diff / (i+1) + * v += diff * (X[i] - m) +*/ + __pyx_v_diff = ((__pyx_v_X[__pyx_v_i]) - __pyx_v_m); + + /* "linalg.pxd":154 + * for i in range(1, nr_dim): + * diff = X[i]-m + * m += diff / (i+1) # <<<<<<<<<<<<<< + * v += diff * (X[i] - m) + * return v / nr_dim +*/ + __pyx_v_m = (__pyx_v_m + (__pyx_v_diff / (__pyx_v_i + 1))); + + /* "linalg.pxd":155 + * diff = X[i]-m + * m += diff / (i+1) + * v += diff * (X[i] - m) # <<<<<<<<<<<<<< + * return v / nr_dim + * +*/ + __pyx_v_v = (__pyx_v_v + (__pyx_v_diff * ((__pyx_v_X[__pyx_v_i]) - __pyx_v_m))); + } + + /* "linalg.pxd":156 + * m += diff / (i+1) + * v += diff * (X[i] - m) + * return v / nr_dim # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = (__pyx_v_v / __pyx_v_nr_dim); + goto __pyx_L0; + + /* "linalg.pxd":147 + * return mean / nr_dim + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t variance(const weight_t* X, int32_t nr_dim) noexcept nogil: + * # See https://www.johndcook.com/blog/standard_deviation/ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":160 + * + * cdef class VecVec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, + * const weight_t* x, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr) { + + /* "linalg.pxd":166 + * weight_t scale, + * int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.add_i(output, y, scale, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":167 + * int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.add_i(output, y, scale, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_output, __pyx_v_y, __pyx_v_scale, __pyx_v_nr); + + /* "linalg.pxd":160 + * + * cdef class VecVec: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add(weight_t* output, + * const weight_t* x, +*/ + + /* function exit code */ +} + +/* "linalg.pxd":169 + * VecVec.add_i(output, y, scale, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* x, + * const weight_t* y, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":178 + * blis.cy.axpyv(BLIS_NO_CONJUGATE, nr, scale, y, 1, x, 1) + * ELSE: + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] += y[i] * scale + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":179 + * ELSE: + * for i in range(nr): + * x[i] += y[i] * scale # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_x[__pyx_t_4]) = ((__pyx_v_x[__pyx_t_4]) + ((__pyx_v_y[__pyx_v_i]) * __pyx_v_scale)); + } + + /* "linalg.pxd":169 + * VecVec.add_i(output, y, scale, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_i(weight_t* x, + * const weight_t* y, +*/ + + /* function exit code */ +} + +/* "linalg.pxd":181 + * x[i] += y[i] * scale + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void batch_add_i(weight_t* x, + * const weight_t* y, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_batch_add_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_scale, int32_t __pyx_v_nr, int32_t __pyx_v_nr_batch) { + CYTHON_UNUSED int __pyx_v__; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "linalg.pxd":188 + * # For fixed x, matrix of y + * cdef int i, _ + * for _ in range(nr_batch): # <<<<<<<<<<<<<< + * VecVec.add_i(x, + * y, scale, nr) +*/ + __pyx_t_1 = __pyx_v_nr_batch; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v__ = __pyx_t_3; + + /* "linalg.pxd":189 + * cdef int i, _ + * for _ in range(nr_batch): + * VecVec.add_i(x, # <<<<<<<<<<<<<< + * y, scale, nr) + * y += nr +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_x, __pyx_v_y, __pyx_v_scale, __pyx_v_nr); + + /* "linalg.pxd":191 + * VecVec.add_i(x, + * y, scale, nr) + * y += nr # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_v_y = (__pyx_v_y + __pyx_v_nr); + } + + /* "linalg.pxd":181 + * x[i] += y[i] * scale + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void batch_add_i(weight_t* x, + * const weight_t* y, +*/ + + /* function exit code */ +} + +/* "linalg.pxd":193 + * y += nr + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr) { + + /* "linalg.pxd":196 + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.add_pow_i(output, y, power, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":197 + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.add_pow_i(output, y, power, nr) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_v_output, __pyx_v_y, __pyx_v_power, __pyx_v_nr); + + /* "linalg.pxd":193 + * y += nr + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow(weight_t* output, + * const weight_t* x, const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":200 + * + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow_i(weight_t* x, + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_add_pow_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_power, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + double __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "linalg.pxd":204 + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] += y[i] ** power + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":205 + * cdef int i + * for i in range(nr): + * x[i] += y[i] ** power # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + __pyx_t_5 = __Pyx_SoftComplexToDouble(__Pyx_c_sum_double(__pyx_t_double_complex_from_parts((__pyx_v_x[__pyx_t_4]), 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts((__pyx_v_y[__pyx_v_i]), 0), __pyx_t_double_complex_from_parts(__pyx_v_power, 0))), 2); if (unlikely(__pyx_t_5 == ((double)-1) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(3, 205, __pyx_L1_error) + (__pyx_v_x[__pyx_t_4]) = __pyx_t_5; + } + + /* "linalg.pxd":200 + * + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void add_pow_i(weight_t* x, + * const weight_t* y, weight_t power, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_WriteUnraisable("thinc.backends.linalg.VecVec.add_pow_i", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; +} + +/* "linalg.pxd":207 + * x[i] += y[i] ** power + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_output, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + + /* "linalg.pxd":210 + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) # <<<<<<<<<<<<<< + * VecVec.mul_i(output, y, nr) + * +*/ + (void)(memcpy(__pyx_v_output, __pyx_v_x, ((sizeof((__pyx_v_output[0]))) * __pyx_v_nr))); + + /* "linalg.pxd":211 + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * memcpy(output, x, sizeof(output[0]) * nr) + * VecVec.mul_i(output, y, nr) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_v_output, __pyx_v_y, __pyx_v_nr); + + /* "linalg.pxd":207 + * x[i] += y[i] ** power + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul(weight_t* output, + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":213 + * VecVec.mul_i(output, y, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* x, + * const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_6VecVec_mul_i(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + int __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "linalg.pxd":217 + * const weight_t* y, int32_t nr) noexcept nogil: + * cdef int i + * for i in range(nr): # <<<<<<<<<<<<<< + * x[i] *= y[i] + * +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":218 + * cdef int i + * for i in range(nr): + * x[i] *= y[i] # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_t_4 = __pyx_v_i; + (__pyx_v_x[__pyx_t_4]) = ((__pyx_v_x[__pyx_t_4]) * (__pyx_v_y[__pyx_v_i])); + } + + /* "linalg.pxd":213 + * VecVec.mul_i(output, y, nr) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mul_i(weight_t* x, + * const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":220 + * x[i] *= y[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t dot( + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + +static CYTHON_INLINE __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_f_5thinc_8backends_6linalg_6VecVec_dot(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_x, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_y, int32_t __pyx_v_nr) { + int __pyx_v_i; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_total; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_r; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int __pyx_t_3; + + /* "linalg.pxd":224 + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: + * cdef int i + * cdef weight_t total = 0 # <<<<<<<<<<<<<< + * for i in range(nr): + * total += x[i] * y[i] +*/ + __pyx_v_total = 0.0; + + /* "linalg.pxd":225 + * cdef int i + * cdef weight_t total = 0 + * for i in range(nr): # <<<<<<<<<<<<<< + * total += x[i] * y[i] + * return total +*/ + __pyx_t_1 = __pyx_v_nr; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":226 + * cdef weight_t total = 0 + * for i in range(nr): + * total += x[i] * y[i] # <<<<<<<<<<<<<< + * return total + * +*/ + __pyx_v_total = (__pyx_v_total + ((__pyx_v_x[__pyx_v_i]) * (__pyx_v_y[__pyx_v_i]))); + } + + /* "linalg.pxd":227 + * for i in range(nr): + * total += x[i] * y[i] + * return total # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_total; + goto __pyx_L0; + + /* "linalg.pxd":220 + * x[i] *= y[i] + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline weight_t dot( + * const weight_t* x, const weight_t* y, int32_t nr) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":229 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_true( + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_true(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + + /* "linalg.pxd":233 + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: + * cdef int i + * cdef int best = -1 # <<<<<<<<<<<<<< + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): +*/ + __pyx_v_best = -1; + + /* "linalg.pxd":234 + * cdef int i + * cdef int best = -1 + * for i in range(n_classes): # <<<<<<<<<<<<<< + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i +*/ + __pyx_t_1 = __pyx_v_n_classes; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":235 + * cdef int best = -1 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_t_5 = ((__pyx_v_is_valid[__pyx_v_i]) != 0); + if (__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = (__pyx_v_best == -1L); + if (!__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = ((__pyx_v_scores[__pyx_v_i]) > (__pyx_v_scores[__pyx_v_best])); + __pyx_t_4 = __pyx_t_5; + __pyx_L6_bool_binop_done:; + if (__pyx_t_4) { + + /* "linalg.pxd":236 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "linalg.pxd":235 + * cdef int best = -1 + * for i in range(n_classes): + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + } + } + + /* "linalg.pxd":237 + * if is_valid[i] and (best == -1 or scores[i] > scores[best]): + * best = i + * return best # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "linalg.pxd":229 + * return total + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_true( + * const weight_t* scores, const int* is_valid, const int n_classes) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":239 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_zero( + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: +*/ + +static CYTHON_INLINE int __pyx_f_5thinc_8backends_6linalg_6VecVec_arg_max_if_zero(__pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_scores, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_costs, int const __pyx_v_n_classes) { + int __pyx_v_i; + int __pyx_v_best; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + + /* "linalg.pxd":243 + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: + * cdef int i + * cdef int best = -1 # <<<<<<<<<<<<<< + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): +*/ + __pyx_v_best = -1; + + /* "linalg.pxd":244 + * cdef int i + * cdef int best = -1 + * for i in range(n_classes): # <<<<<<<<<<<<<< + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i +*/ + __pyx_t_1 = __pyx_v_n_classes; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":245 + * cdef int best = -1 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + __pyx_t_5 = ((__pyx_v_costs[__pyx_v_i]) == 0.0); + if (__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = (__pyx_v_best == -1L); + if (!__pyx_t_5) { + } else { + __pyx_t_4 = __pyx_t_5; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_5 = ((__pyx_v_scores[__pyx_v_i]) > (__pyx_v_scores[__pyx_v_best])); + __pyx_t_4 = __pyx_t_5; + __pyx_L6_bool_binop_done:; + if (__pyx_t_4) { + + /* "linalg.pxd":246 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i # <<<<<<<<<<<<<< + * return best + * +*/ + __pyx_v_best = __pyx_v_i; + + /* "linalg.pxd":245 + * cdef int best = -1 + * for i in range(n_classes): + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): # <<<<<<<<<<<<<< + * best = i + * return best +*/ + } + } + + /* "linalg.pxd":247 + * if costs[i] == 0 and (best == -1 or scores[i] > scores[best]): + * best = i + * return best # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_best; + goto __pyx_L0; + + /* "linalg.pxd":239 + * return best + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline int arg_max_if_zero( + * const weight_t* scores, const weight_t* costs, const int n_classes) noexcept nogil: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "linalg.pxd":251 + * + * cdef class Mat: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_mean_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Ex, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col) { + int32_t __pyx_v_i; + int32_t __pyx_t_1; + int32_t __pyx_t_2; + int32_t __pyx_t_3; + + /* "linalg.pxd":254 + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) # <<<<<<<<<<<<<< + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) +*/ + (void)(memset(__pyx_v_Ex, 0, ((sizeof((__pyx_v_Ex[0]))) * __pyx_v_nr_col))); + + /* "linalg.pxd":255 + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) + * for i in range(nr_row): # <<<<<<<<<<<<<< + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) +*/ + __pyx_t_1 = __pyx_v_nr_row; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "linalg.pxd":256 + * memset(Ex, 0, sizeof(Ex[0]) * nr_col) + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) # <<<<<<<<<<<<<< + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_Ex, (&(__pyx_v_mat[(__pyx_v_i * __pyx_v_nr_col)])), 1.0, __pyx_v_nr_col); + } + + /* "linalg.pxd":257 + * for i in range(nr_row): + * VecVec.add_i(Ex, &mat[i * nr_col], 1.0, nr_col) + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) # <<<<<<<<<<<<<< + * + * @staticmethod +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_Ex, (1.0 / __pyx_v_nr_row), __pyx_v_nr_col); + + /* "linalg.pxd":251 + * + * cdef class Mat: + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void mean_row(weight_t* Ex, + * const weight_t* mat, int32_t nr_row, int32_t nr_col) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "linalg.pxd":259 + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void var_row(weight_t* Vx, + * const weight_t* mat, const weight_t* Ex, +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_8backends_6linalg_3Mat_var_row(__pyx_t_5thinc_8backends_6linalg_weight_t *__pyx_v_Vx, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_mat, __pyx_t_5thinc_8backends_6linalg_weight_t const *__pyx_v_Ex, int32_t __pyx_v_nr_row, int32_t __pyx_v_nr_col, __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_eps) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_sum_; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_sum2; + int32_t __pyx_v_i; + int32_t __pyx_v_j; + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_x; + int __pyx_t_1; + int __pyx_t_2; + int32_t __pyx_t_3; + int32_t __pyx_t_4; + int32_t __pyx_t_5; + int32_t __pyx_t_6; + int32_t __pyx_t_7; + int32_t __pyx_t_8; + + /* "linalg.pxd":264 + * int32_t nr_row, int32_t nr_col, weight_t eps) noexcept nogil: + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: # <<<<<<<<<<<<<< + * return + * cdef weight_t sum_, sum2 +*/ + __pyx_t_2 = (__pyx_v_nr_row == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_nr_col == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "linalg.pxd":265 + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: + * return # <<<<<<<<<<<<<< + * cdef weight_t sum_, sum2 + * for i in range(nr_col): +*/ + goto __pyx_L0; + + /* "linalg.pxd":264 + * int32_t nr_row, int32_t nr_col, weight_t eps) noexcept nogil: + * # From https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + * if nr_row == 0 or nr_col == 0: # <<<<<<<<<<<<<< + * return + * cdef weight_t sum_, sum2 +*/ + } + + /* "linalg.pxd":267 + * return + * cdef weight_t sum_, sum2 + * for i in range(nr_col): # <<<<<<<<<<<<<< + * sum_ = 0.0 + * sum2 = 0.0 +*/ + __pyx_t_3 = __pyx_v_nr_col; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "linalg.pxd":268 + * cdef weight_t sum_, sum2 + * for i in range(nr_col): + * sum_ = 0.0 # <<<<<<<<<<<<<< + * sum2 = 0.0 + * for j in range(nr_row): +*/ + __pyx_v_sum_ = 0.0; + + /* "linalg.pxd":269 + * for i in range(nr_col): + * sum_ = 0.0 + * sum2 = 0.0 # <<<<<<<<<<<<<< + * for j in range(nr_row): + * x = mat[j * nr_col + i] +*/ + __pyx_v_sum2 = 0.0; + + /* "linalg.pxd":270 + * sum_ = 0.0 + * sum2 = 0.0 + * for j in range(nr_row): # <<<<<<<<<<<<<< + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 +*/ + __pyx_t_6 = __pyx_v_nr_row; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_j = __pyx_t_8; + + /* "linalg.pxd":271 + * sum2 = 0.0 + * for j in range(nr_row): + * x = mat[j * nr_col + i] # <<<<<<<<<<<<<< + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] +*/ + __pyx_v_x = (__pyx_v_mat[((__pyx_v_j * __pyx_v_nr_col) + __pyx_v_i)]); + + /* "linalg.pxd":272 + * for j in range(nr_row): + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 # <<<<<<<<<<<<<< + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row +*/ + __pyx_v_sum2 = (__pyx_v_sum2 + powf((__pyx_v_x - (__pyx_v_Ex[__pyx_v_i])), 2.0)); + + /* "linalg.pxd":273 + * x = mat[j * nr_col + i] + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] # <<<<<<<<<<<<<< + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row + * Vx[i] += eps +*/ + __pyx_v_sum_ = (__pyx_v_sum_ + (__pyx_v_x - (__pyx_v_Ex[__pyx_v_i]))); + } + + /* "linalg.pxd":274 + * sum2 += (x - Ex[i]) ** 2 + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row # <<<<<<<<<<<<<< + * Vx[i] += eps +*/ + (__pyx_v_Vx[__pyx_v_i]) = ((__pyx_v_sum2 - (powf(__pyx_v_sum_, 2.0) / __pyx_v_nr_row)) / __pyx_v_nr_row); + + /* "linalg.pxd":275 + * sum_ += x - Ex[i] + * Vx[i] = (sum2 - sum_**2 / nr_row) / nr_row + * Vx[i] += eps # <<<<<<<<<<<<<< +*/ + __pyx_t_6 = __pyx_v_i; + (__pyx_v_Vx[__pyx_t_6]) = ((__pyx_v_Vx[__pyx_t_6]) + __pyx_v_eps); + } + + /* "linalg.pxd":259 + * Vec.mul_i(Ex, 1.0 / nr_row, nr_col) + * + * @staticmethod # <<<<<<<<<<<<<< + * cdef inline void var_row(weight_t* Vx, + * const weight_t* mat, const weight_t* Ex, +*/ + + /* function exit code */ + __pyx_L0:; +} + +/* "thinc/backends/numpy_ops.pyx":52 + * xp = numpy + * + * def __init__( # <<<<<<<<<<<<<< + * self, + * device_type: DeviceTypes = "cpu", +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_1__init__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps___init__, "NumpyOps.__init__(self, device_type: DeviceTypes = 'cpu', device_id: int = -1, *, use_blis: bool = True) -> None"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_1__init__ = {"__init__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_1__init__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps___init__}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_1__init__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyObject *__pyx_v_device_type = 0; + PyObject *__pyx_v_device_id = 0; + PyObject *__pyx_v_use_blis = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_device_type,&__pyx_mstate_global->__pyx_n_u_device_id,&__pyx_mstate_global->__pyx_n_u_use_blis,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 52, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 52, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 52, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 52, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 52, __pyx_L3_error) + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_n_u_cpu))); + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_neg_1))); + + /* "thinc/backends/numpy_ops.pyx":57 + * device_id: int = -1, + * *, + * use_blis: bool = True # <<<<<<<<<<<<<< + * ) -> None: + * self.device_type = device_type +*/ + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_True))); + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 3, i); __PYX_ERR(0, 52, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 52, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 52, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 52, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_n_u_cpu))); + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_neg_1))); + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_True))); + } + __pyx_v_self = values[0]; + __pyx_v_device_type = values[1]; + __pyx_v_device_id = values[2]; + __pyx_v_use_blis = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 3, __pyx_nargs); __PYX_ERR(0, 52, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps___init__(__pyx_self, __pyx_v_self, __pyx_v_device_type, __pyx_v_device_id, __pyx_v_use_blis); + + /* "thinc/backends/numpy_ops.pyx":52 + * xp = numpy + * + * def __init__( # <<<<<<<<<<<<<< + * self, + * device_type: DeviceTypes = "cpu", +*/ + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_device_type, PyObject *__pyx_v_device_id, PyObject *__pyx_v_use_blis) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "thinc/backends/numpy_ops.pyx":59 + * use_blis: bool = True + * ) -> None: + * self.device_type = device_type # <<<<<<<<<<<<<< + * self.device_id = device_id + * self.use_blis = use_blis +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_device_type, __pyx_v_device_type) < (0)) __PYX_ERR(0, 59, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":60 + * ) -> None: + * self.device_type = device_type + * self.device_id = device_id # <<<<<<<<<<<<<< + * self.use_blis = use_blis + * if self.use_blis and not has_blis: +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_device_id, __pyx_v_device_id) < (0)) __PYX_ERR(0, 60, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":61 + * self.device_type = device_type + * self.device_id = device_id + * self.use_blis = use_blis # <<<<<<<<<<<<<< + * if self.use_blis and not has_blis: + * raise ValueError("BLIS support requires blis: pip install blis") +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_use_blis, __pyx_v_use_blis) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":62 + * self.device_id = device_id + * self.use_blis = use_blis + * if self.use_blis and not has_blis: # <<<<<<<<<<<<<< + * raise ValueError("BLIS support requires blis: pip install blis") + * +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_use_blis); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_has_blis); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = (!__pyx_t_3); + __pyx_t_1 = __pyx_t_4; + __pyx_L4_bool_binop_done:; + if (unlikely(__pyx_t_1)) { + + /* "thinc/backends/numpy_ops.pyx":63 + * self.use_blis = use_blis + * if self.use_blis and not has_blis: + * raise ValueError("BLIS support requires blis: pip install blis") # <<<<<<<<<<<<<< + * + * def asarray(self, data, dtype=None): +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_BLIS_support_requires_blis_pip_i}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 63, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 63, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":62 + * self.device_id = device_id + * self.use_blis = use_blis + * if self.use_blis and not has_blis: # <<<<<<<<<<<<<< + * raise ValueError("BLIS support requires blis: pip install blis") + * +*/ + } + + /* "thinc/backends/numpy_ops.pyx":52 + * xp = numpy + * + * def __init__( # <<<<<<<<<<<<<< + * self, + * device_type: DeviceTypes = "cpu", +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":65 + * raise ValueError("BLIS support requires blis: pip install blis") + * + * def asarray(self, data, dtype=None): # <<<<<<<<<<<<<< + * if isinstance(data, self.xp.ndarray): + * array = data +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_3asarray(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_2asarray, "NumpyOps.asarray(self, data, dtype=None)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_3asarray = {"asarray", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_3asarray, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_2asarray}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_3asarray(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyObject *__pyx_v_data = 0; + PyObject *__pyx_v_dtype = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("asarray (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_data,&__pyx_mstate_global->__pyx_n_u_dtype,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 65, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 65, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 65, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 65, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "asarray", 0) < (0)) __PYX_ERR(0, 65, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("asarray", 0, 2, 3, i); __PYX_ERR(0, 65, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 65, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 65, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 65, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + } + __pyx_v_self = values[0]; + __pyx_v_data = values[1]; + __pyx_v_dtype = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("asarray", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 65, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.asarray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_2asarray(__pyx_self, __pyx_v_self, __pyx_v_data, __pyx_v_dtype); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_2asarray(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_data, PyObject *__pyx_v_dtype) { + PyObject *__pyx_v_array = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("asarray", 0); + + /* "thinc/backends/numpy_ops.pyx":66 + * + * def asarray(self, data, dtype=None): + * if isinstance(data, self.xp.ndarray): # <<<<<<<<<<<<<< + * array = data + * elif hasattr(data, 'numpy'): +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_ndarray); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = PyObject_IsInstance(__pyx_v_data, __pyx_t_2); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + + /* "thinc/backends/numpy_ops.pyx":67 + * def asarray(self, data, dtype=None): + * if isinstance(data, self.xp.ndarray): + * array = data # <<<<<<<<<<<<<< + * elif hasattr(data, 'numpy'): + * # Handles PyTorch Tensor +*/ + __Pyx_INCREF(__pyx_v_data); + __pyx_v_array = __pyx_v_data; + + /* "thinc/backends/numpy_ops.pyx":66 + * + * def asarray(self, data, dtype=None): + * if isinstance(data, self.xp.ndarray): # <<<<<<<<<<<<<< + * array = data + * elif hasattr(data, 'numpy'): +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":68 + * if isinstance(data, self.xp.ndarray): + * array = data + * elif hasattr(data, 'numpy'): # <<<<<<<<<<<<<< + * # Handles PyTorch Tensor + * array = data.numpy() +*/ + __pyx_t_3 = __Pyx_HasAttr(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(0, 68, __pyx_L1_error) + if (__pyx_t_3) { + + /* "thinc/backends/numpy_ops.pyx":70 + * elif hasattr(data, 'numpy'): + * # Handles PyTorch Tensor + * array = data.numpy() # <<<<<<<<<<<<<< + * elif hasattr(data, "get"): + * array = data.get() +*/ + __pyx_t_1 = __pyx_v_data; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_numpy, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 70, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_array = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":68 + * if isinstance(data, self.xp.ndarray): + * array = data + * elif hasattr(data, 'numpy'): # <<<<<<<<<<<<<< + * # Handles PyTorch Tensor + * array = data.numpy() +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":71 + * # Handles PyTorch Tensor + * array = data.numpy() + * elif hasattr(data, "get"): # <<<<<<<<<<<<<< + * array = data.get() + * else: +*/ + __pyx_t_3 = __Pyx_HasAttr(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_get); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(0, 71, __pyx_L1_error) + if (__pyx_t_3) { + + /* "thinc/backends/numpy_ops.pyx":72 + * array = data.numpy() + * elif hasattr(data, "get"): + * array = data.get() # <<<<<<<<<<<<<< + * else: + * array = self.xp.array(data, dtype=dtype) +*/ + __pyx_t_1 = __pyx_v_data; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 72, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_array = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":71 + * # Handles PyTorch Tensor + * array = data.numpy() + * elif hasattr(data, "get"): # <<<<<<<<<<<<<< + * array = data.get() + * else: +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":74 + * array = data.get() + * else: + * array = self.xp.array(data, dtype=dtype) # <<<<<<<<<<<<<< + * + * if dtype is not None: +*/ + /*else*/ { + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_1, __pyx_v_data}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_v_dtype, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_array, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_array = __pyx_t_2; + __pyx_t_2 = 0; + } + __pyx_L3:; + + /* "thinc/backends/numpy_ops.pyx":76 + * array = self.xp.array(data, dtype=dtype) + * + * if dtype is not None: # <<<<<<<<<<<<<< + * array = array.astype(dtype=dtype, copy=False) + * +*/ + __pyx_t_3 = (__pyx_v_dtype != Py_None); + if (__pyx_t_3) { + + /* "thinc/backends/numpy_ops.pyx":77 + * + * if dtype is not None: + * array = array.astype(dtype=dtype, copy=False) # <<<<<<<<<<<<<< + * + * return array +*/ + __pyx_t_5 = __pyx_v_array; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_5, NULL}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 77, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_v_dtype, __pyx_t_6, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 77, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_copy, Py_False, __pyx_t_6, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 77, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_astype, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 77, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_array, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":76 + * array = self.xp.array(data, dtype=dtype) + * + * if dtype is not None: # <<<<<<<<<<<<<< + * array = array.astype(dtype=dtype, copy=False) + * +*/ + } + + /* "thinc/backends/numpy_ops.pyx":79 + * array = array.astype(dtype=dtype, copy=False) + * + * return array # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_array); + __pyx_r = __pyx_v_array; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":65 + * raise ValueError("BLIS support requires blis: pip install blis") + * + * def asarray(self, data, dtype=None): # <<<<<<<<<<<<<< + * if isinstance(data, self.xp.ndarray): + * array = data +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.asarray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_array); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":82 + * + * + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: # <<<<<<<<<<<<<< + * if zeros: + * return self.xp.zeros(shape, dtype=dtype) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_5alloc(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_4alloc, "NumpyOps.alloc(self, shape: Shape, *, dtype: Optional[DTypes] = 'float32', zeros: bool = True) -> ArrayXd"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_5alloc = {"alloc", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_5alloc, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_4alloc}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_5alloc(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyObject *__pyx_v_shape = 0; + PyObject *__pyx_v_dtype = 0; + PyObject *__pyx_v_zeros = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("alloc (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_shape,&__pyx_mstate_global->__pyx_n_u_dtype,&__pyx_mstate_global->__pyx_n_u_zeros,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 82, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "alloc", 0) < (0)) __PYX_ERR(0, 82, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_n_u_float32))); + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_True))); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("alloc", 1, 2, 2, i); __PYX_ERR(0, 82, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 82, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 82, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_n_u_float32))); + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_True))); + } + __pyx_v_self = values[0]; + __pyx_v_shape = values[1]; + __pyx_v_dtype = values[2]; + __pyx_v_zeros = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("alloc", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 82, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.alloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_4alloc(__pyx_self, __pyx_v_self, __pyx_v_shape, __pyx_v_dtype, __pyx_v_zeros); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_4alloc(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_shape, PyObject *__pyx_v_dtype, PyObject *__pyx_v_zeros) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("alloc", 0); + + /* "thinc/backends/numpy_ops.pyx":83 + * + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: + * if zeros: # <<<<<<<<<<<<<< + * return self.xp.zeros(shape, dtype=dtype) + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_zeros); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 83, __pyx_L1_error) + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":84 + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: + * if zeros: + * return self.xp.zeros(shape, dtype=dtype) # <<<<<<<<<<<<<< + * else: + * return self.xp.empty(shape, dtype=dtype) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_v_shape}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_v_dtype, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 84, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":83 + * + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: + * if zeros: # <<<<<<<<<<<<<< + * return self.xp.zeros(shape, dtype=dtype) + * else: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":86 + * return self.xp.zeros(shape, dtype=dtype) + * else: + * return self.xp.empty(shape, dtype=dtype) # <<<<<<<<<<<<<< + * + * def cblas(self) -> CBlas: +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_4, __pyx_v_shape}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_v_dtype, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 86, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_empty, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":82 + * + * + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: # <<<<<<<<<<<<<< + * if zeros: + * return self.xp.zeros(shape, dtype=dtype) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.alloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":88 + * return self.xp.empty(shape, dtype=dtype) + * + * def cblas(self) -> CBlas: # <<<<<<<<<<<<<< + * return CBlas() + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_7cblas(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_6cblas, "NumpyOps.cblas(self) -> CBlas"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_7cblas = {"cblas", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_7cblas, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_6cblas}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_7cblas(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("cblas (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 88, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 88, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "cblas", 0) < (0)) __PYX_ERR(0, 88, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("cblas", 1, 1, 1, i); __PYX_ERR(0, 88, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 88, __pyx_L3_error) + } + __pyx_v_self = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("cblas", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 88, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.cblas", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_6cblas(__pyx_self, __pyx_v_self); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_6cblas(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("cblas", 0); + + /* "thinc/backends/numpy_ops.pyx":89 + * + * def cblas(self) -> CBlas: + * return CBlas() # <<<<<<<<<<<<<< + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas, __pyx_callargs+__pyx_t_3, (1-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 89, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_r = ((PyObject *)__pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":88 + * return self.xp.empty(shape, dtype=dtype) + * + * def cblas(self) -> CBlas: # <<<<<<<<<<<<<< + * return CBlas() + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.cblas", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":91 + * return CBlas() + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): # <<<<<<<<<<<<<< + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_9gemm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_8gemm, "NumpyOps.gemm(self, ndarray x, ndarray y, *, ndarray out=None, trans1=False, trans2=False)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_9gemm = {"gemm", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_9gemm, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_8gemm}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_9gemm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_x = 0; + PyArrayObject *__pyx_v_y = 0; + PyArrayObject *__pyx_v_out = 0; + PyObject *__pyx_v_trans1 = 0; + PyObject *__pyx_v_trans2 = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[6] = {0,0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("gemm (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_x,&__pyx_mstate_global->__pyx_n_u_y,&__pyx_mstate_global->__pyx_n_u_out,&__pyx_mstate_global->__pyx_n_u_trans1,&__pyx_mstate_global->__pyx_n_u_trans2,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 91, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 91, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 91, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 91, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "gemm", 0) < (0)) __PYX_ERR(0, 91, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((PyArrayObject *)Py_None)); + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + if (!values[5]) values[5] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("gemm", 1, 3, 3, i); __PYX_ERR(0, 91, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 91, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 91, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 91, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef((PyObject *)((PyArrayObject *)Py_None)); + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + if (!values[5]) values[5] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + } + __pyx_v_self = values[0]; + __pyx_v_x = ((PyArrayObject *)values[1]); + __pyx_v_y = ((PyArrayObject *)values[2]); + __pyx_v_out = ((PyArrayObject *)values[3]); + __pyx_v_trans1 = values[4]; + __pyx_v_trans2 = values[5]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("gemm", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 91, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gemm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "x", 0))) __PYX_ERR(0, 91, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "y", 0))) __PYX_ERR(0, 91, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_out), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "out", 0))) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_8gemm(__pyx_self, __pyx_v_self, __pyx_v_x, __pyx_v_y, __pyx_v_out, __pyx_v_trans1, __pyx_v_trans2); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_8gemm(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_x, PyArrayObject *__pyx_v_y, PyArrayObject *__pyx_v_out, PyObject *__pyx_v_trans1, PyObject *__pyx_v_trans2) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5[3]; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + int __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("gemm", 0); + __Pyx_INCREF((PyObject *)__pyx_v_x); + __Pyx_INCREF((PyObject *)__pyx_v_y); + __Pyx_INCREF((PyObject *)__pyx_v_out); + + /* "thinc/backends/numpy_ops.pyx":92 + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): + * if x.ndim != 2: # <<<<<<<<<<<<<< + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + * if y.ndim != 2: +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_x) != 2); + if (unlikely(__pyx_t_1)) { + + /* "thinc/backends/numpy_ops.pyx":93 + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") # <<<<<<<<<<<<<< + * if y.ndim != 2: + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") +*/ + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyUnicode_From_int(__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_x), 0, ' ', 'd'); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5[0] = __pyx_mstate_global->__pyx_kp_u_Provided_x_array_should_be_2_dim; + __pyx_t_5[1] = __pyx_t_4; + __pyx_t_5[2] = __pyx_mstate_global->__pyx_kp_u_dimension_s; + __pyx_t_6 = __Pyx_PyUnicode_Join(__pyx_t_5, 3, 54 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4) + 14, 127); + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_7 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_6}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 93, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":92 + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): + * if x.ndim != 2: # <<<<<<<<<<<<<< + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + * if y.ndim != 2: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":94 + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + * if y.ndim != 2: # <<<<<<<<<<<<<< + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + * if not self.use_blis: # delegate to base Ops +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_y) != 2); + if (unlikely(__pyx_t_1)) { + + /* "thinc/backends/numpy_ops.pyx":95 + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + * if y.ndim != 2: + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") # <<<<<<<<<<<<<< + * if not self.use_blis: # delegate to base Ops + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) +*/ + __pyx_t_6 = NULL; + __pyx_t_3 = __Pyx_PyUnicode_From_int(__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_y), 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 95, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5[0] = __pyx_mstate_global->__pyx_kp_u_Provided_y_array_should_be_2_dim; + __pyx_t_5[1] = __pyx_t_3; + __pyx_t_5[2] = __pyx_mstate_global->__pyx_kp_u_dimension_s; + __pyx_t_4 = __Pyx_PyUnicode_Join(__pyx_t_5, 3, 54 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 14, 127); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 95, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_7 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 95, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 95, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":94 + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + * if y.ndim != 2: # <<<<<<<<<<<<<< + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + * if not self.use_blis: # delegate to base Ops +*/ + } + + /* "thinc/backends/numpy_ops.pyx":96 + * if y.ndim != 2: + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + * if not self.use_blis: # delegate to base Ops # <<<<<<<<<<<<<< + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) + * x = self.as_contig(x) +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_use_blis); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 96, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 96, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_8 = (!__pyx_t_1); + if (__pyx_t_8) { + + /* "thinc/backends/numpy_ops.pyx":97 + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + * if not self.use_blis: # delegate to base Ops + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) # <<<<<<<<<<<<<< + * x = self.as_contig(x) + * y = self.as_contig(y) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = NULL; + __pyx_t_9 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_9) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 97, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_9); + __pyx_t_7 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_t_9, __pyx_v_self}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_4 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_4, ((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y)}; + __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_out, ((PyObject *)__pyx_v_out), __pyx_t_9, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 97, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_trans1, __pyx_v_trans1, __pyx_t_9, __pyx_callargs+3, 1) < (0)) __PYX_ERR(0, 97, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_trans2, __pyx_v_trans2, __pyx_t_9, __pyx_callargs+3, 2) < (0)) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_gemm, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":96 + * if y.ndim != 2: + * raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + * if not self.use_blis: # delegate to base Ops # <<<<<<<<<<<<<< + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) + * x = self.as_contig(x) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":98 + * if not self.use_blis: # delegate to base Ops + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) + * x = self.as_contig(x) # <<<<<<<<<<<<<< + * y = self.as_contig(y) + * if out is not None: +*/ + __pyx_t_6 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, ((PyObject *)__pyx_v_x)}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_as_contig, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 98, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 98, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_x, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":99 + * return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) + * x = self.as_contig(x) + * y = self.as_contig(y) # <<<<<<<<<<<<<< + * if out is not None: + * out = self.as_contig(out) +*/ + __pyx_t_6 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, ((PyObject *)__pyx_v_y)}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_as_contig, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_y, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":100 + * x = self.as_contig(x) + * y = self.as_contig(y) + * if out is not None: # <<<<<<<<<<<<<< + * out = self.as_contig(out) + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) +*/ + __pyx_t_8 = (((PyObject *)__pyx_v_out) != Py_None); + if (__pyx_t_8) { + + /* "thinc/backends/numpy_ops.pyx":101 + * y = self.as_contig(y) + * if out is not None: + * out = self.as_contig(out) # <<<<<<<<<<<<<< + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + * +*/ + __pyx_t_6 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, ((PyObject *)__pyx_v_out)}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_as_contig, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 101, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 101, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_out, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":100 + * x = self.as_contig(x) + * y = self.as_contig(y) + * if out is not None: # <<<<<<<<<<<<<< + * out = self.as_contig(out) + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":102 + * if out is not None: + * out = self.as_contig(out) + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) # <<<<<<<<<<<<<< + * + * def relu(self, np.ndarray X, inplace=False): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_blis); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_py); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_6 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 4 : 0)] = {__pyx_t_6, ((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y)}; + __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(4); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_out, ((PyObject *)__pyx_v_out), __pyx_t_9, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 102, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_trans1, __pyx_v_trans1, __pyx_t_9, __pyx_callargs+3, 1) < (0)) __PYX_ERR(0, 102, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_trans2, __pyx_v_trans2, __pyx_t_9, __pyx_callargs+3, 2) < (0)) __PYX_ERR(0, 102, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_beta, __pyx_mstate_global->__pyx_float_0_, __pyx_t_9, __pyx_callargs+3, 3) < (0)) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_gemm, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":91 + * return CBlas() + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): # <<<<<<<<<<<<<< + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gemm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_x); + __Pyx_XDECREF((PyObject *)__pyx_v_y); + __Pyx_XDECREF((PyObject *)__pyx_v_out); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":104 + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + * + * def relu(self, np.ndarray X, inplace=False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_11relu(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_10relu, "NumpyOps.relu(self, ndarray X, inplace=False)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_11relu = {"relu", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_11relu, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_10relu}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_11relu(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_X = 0; + PyObject *__pyx_v_inplace = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("relu (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_inplace,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 104, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 104, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 104, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 104, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "relu", 0) < (0)) __PYX_ERR(0, 104, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("relu", 0, 2, 3, i); __PYX_ERR(0, 104, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 104, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 104, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 104, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + } + __pyx_v_self = values[0]; + __pyx_v_X = ((PyArrayObject *)values[1]); + __pyx_v_inplace = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("relu", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 104, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.relu", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 104, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_10relu(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_inplace); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_10relu(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_inplace) { + PyArrayObject *__pyx_v_Y = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("relu", 0); + + /* "thinc/backends/numpy_ops.pyx":107 + * cdef np.ndarray Y + * + * if X.dtype == "float32": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 107, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":108 + * + * if X.dtype == "float32": + * Y = _inplace_or_copy(X, inplace) # <<<<<<<<<<<<<< + * cpu_relu(Y.data, Y.size) + * return Y +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_X, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":109 + * if X.dtype == "float32": + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) # <<<<<<<<<<<<<< + * return Y + * elif X.dtype == "float64": +*/ + cpu_relu(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Y)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_Y))); + + /* "thinc/backends/numpy_ops.pyx":110 + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) + * return Y # <<<<<<<<<<<<<< + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_Y); + __pyx_r = ((PyObject *)__pyx_v_Y); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":107 + * cdef np.ndarray Y + * + * if X.dtype == "float32": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":111 + * cpu_relu(Y.data, Y.size) + * return Y + * elif X.dtype == "float64": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 111, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 111, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":112 + * return Y + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) # <<<<<<<<<<<<<< + * cpu_relu(Y.data, Y.size) + * return Y +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_X, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":113 + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) # <<<<<<<<<<<<<< + * return Y + * else: +*/ + cpu_relu(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Y)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_Y))); + + /* "thinc/backends/numpy_ops.pyx":114 + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) + * return Y # <<<<<<<<<<<<<< + * else: + * return super().relu(X, inplace=inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_Y); + __pyx_r = ((PyObject *)__pyx_v_Y); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":111 + * cpu_relu(Y.data, Y.size) + * return Y + * elif X.dtype == "float64": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_relu(Y.data, Y.size) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":116 + * return Y + * else: + * return super().relu(X, inplace=inplace) # <<<<<<<<<<<<<< + * + * def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_5 = NULL; + __pyx_t_6 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_6) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 116, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_t_6, __pyx_v_self}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_3 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, ((PyObject *)__pyx_v_X)}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_inplace, __pyx_v_inplace, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 116, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_relu, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 116, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":104 + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + * + * def relu(self, np.ndarray X, inplace=False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.relu", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_Y); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":118 + * return super().relu(X, inplace=inplace) + * + * def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, Y) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_13backprop_relu(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_12backprop_relu, "NumpyOps.backprop_relu(self, ndarray dY, ndarray Y, inplace=False)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_13backprop_relu = {"backprop_relu", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_13backprop_relu, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_12backprop_relu}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_13backprop_relu(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_dY = 0; + PyArrayObject *__pyx_v_Y = 0; + PyObject *__pyx_v_inplace = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_relu (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_Y,&__pyx_mstate_global->__pyx_n_u_inplace,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 118, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 118, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 118, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 118, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 118, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_relu", 0) < (0)) __PYX_ERR(0, 118, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_relu", 0, 3, 4, i); __PYX_ERR(0, 118, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 118, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 118, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 118, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 118, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + } + __pyx_v_self = values[0]; + __pyx_v_dY = ((PyArrayObject *)values[1]); + __pyx_v_Y = ((PyArrayObject *)values[2]); + __pyx_v_inplace = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_relu", 0, 3, 4, __pyx_nargs); __PYX_ERR(0, 118, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_relu", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "dY", 0))) __PYX_ERR(0, 118, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_Y), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "Y", 0))) __PYX_ERR(0, 118, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_12backprop_relu(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_Y, __pyx_v_inplace); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_12backprop_relu(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_Y, PyObject *__pyx_v_inplace) { + size_t __pyx_v_size; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t *__pyx_v_dX_ptr; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t const *__pyx_v_Y_ptr; + PyArrayObject *__pyx_v_dX = 0; + size_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + size_t __pyx_t_7; + size_t __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_relu", 0); + + /* "thinc/backends/numpy_ops.pyx":119 + * + * def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): + * _check_compatible_shape(dY, Y) # <<<<<<<<<<<<<< + * + * cdef size_t size = Y.size +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_check_compatible_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, ((PyObject *)__pyx_v_dY), ((PyObject *)__pyx_v_Y)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":121 + * _check_compatible_shape(dY, Y) + * + * cdef size_t size = Y.size # <<<<<<<<<<<<<< + * cdef weight_t* dX_ptr + * cdef const weight_t* Y_ptr = Y.data +*/ + __pyx_v_size = __pyx_f_5numpy_7ndarray_4size_size(__pyx_v_Y); + + /* "thinc/backends/numpy_ops.pyx":123 + * cdef size_t size = Y.size + * cdef weight_t* dX_ptr + * cdef const weight_t* Y_ptr = Y.data # <<<<<<<<<<<<<< + * cdef np.ndarray dX + * if dY.dtype == "float32" and Y.dtype == "float32": +*/ + __pyx_v_Y_ptr = ((__pyx_t_5thinc_8backends_9numpy_ops_weight_t const *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Y)); + + /* "thinc/backends/numpy_ops.pyx":125 + * cdef const weight_t* Y_ptr = Y.data + * cdef np.ndarray dX + * if dY.dtype == "float32" and Y.dtype == "float32": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * dX_ptr = dX.data +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_6) { + } else { + __pyx_t_5 = __pyx_t_6; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_Y), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 125, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __pyx_t_6; + __pyx_L4_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":126 + * cdef np.ndarray dX + * if dY.dtype == "float32" and Y.dtype == "float32": + * dX = _inplace_or_copy(dY, inplace) # <<<<<<<<<<<<<< + * dX_ptr = dX.data + * for i in range(size): +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_dY, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 126, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_dX = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":127 + * if dY.dtype == "float32" and Y.dtype == "float32": + * dX = _inplace_or_copy(dY, inplace) + * dX_ptr = dX.data # <<<<<<<<<<<<<< + * for i in range(size): + * if Y_ptr[i] <= 0: +*/ + __pyx_v_dX_ptr = ((__pyx_t_5thinc_8backends_9numpy_ops_weight_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)); + + /* "thinc/backends/numpy_ops.pyx":128 + * dX = _inplace_or_copy(dY, inplace) + * dX_ptr = dX.data + * for i in range(size): # <<<<<<<<<<<<<< + * if Y_ptr[i] <= 0: + * dX_ptr[i] = 0. +*/ + __pyx_t_4 = __pyx_v_size; + __pyx_t_7 = __pyx_t_4; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + + /* "thinc/backends/numpy_ops.pyx":129 + * dX_ptr = dX.data + * for i in range(size): + * if Y_ptr[i] <= 0: # <<<<<<<<<<<<<< + * dX_ptr[i] = 0. + * return dX +*/ + __pyx_t_5 = ((__pyx_v_Y_ptr[__pyx_v_i]) <= 0.0); + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":130 + * for i in range(size): + * if Y_ptr[i] <= 0: + * dX_ptr[i] = 0. # <<<<<<<<<<<<<< + * return dX + * else: +*/ + (__pyx_v_dX_ptr[__pyx_v_i]) = 0.; + + /* "thinc/backends/numpy_ops.pyx":129 + * dX_ptr = dX.data + * for i in range(size): + * if Y_ptr[i] <= 0: # <<<<<<<<<<<<<< + * dX_ptr[i] = 0. + * return dX +*/ + } + } + + /* "thinc/backends/numpy_ops.pyx":131 + * if Y_ptr[i] <= 0: + * dX_ptr[i] = 0. + * return dX # <<<<<<<<<<<<<< + * else: + * return super().backprop_relu(dY, Y, inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":125 + * cdef const weight_t* Y_ptr = Y.data + * cdef np.ndarray dX + * if dY.dtype == "float32" and Y.dtype == "float32": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * dX_ptr = dX.data +*/ + } + + /* "thinc/backends/numpy_ops.pyx":133 + * return dX + * else: + * return super().backprop_relu(dY, Y, inplace) # <<<<<<<<<<<<<< + * + * def lstm_forward_training( +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_10) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 133, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_10); + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_9, __pyx_t_10, __pyx_v_self}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_3 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[4] = {__pyx_t_3, ((PyObject *)__pyx_v_dY), ((PyObject *)__pyx_v_Y), __pyx_v_inplace}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_backprop_relu, __pyx_callargs+__pyx_t_4, (4-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":118 + * return super().relu(X, inplace=inplace) + * + * def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, Y) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_relu", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":135 + * return super().backprop_relu(dY, Y, inplace) + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_15lstm_forward_training(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_14lstm_forward_training, "NumpyOps.lstm_forward_training(self, ndarray params, ndarray H0, ndarray C0, ndarray X, ndarray size_at_t)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_15lstm_forward_training = {"lstm_forward_training", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_15lstm_forward_training, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_14lstm_forward_training}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_15lstm_forward_training(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_params = 0; + PyArrayObject *__pyx_v_H0 = 0; + PyArrayObject *__pyx_v_C0 = 0; + PyArrayObject *__pyx_v_X = 0; + PyArrayObject *__pyx_v_size_at_t = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[6] = {0,0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lstm_forward_training (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_H0,&__pyx_mstate_global->__pyx_n_u_C0,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_size_at_t,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 135, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 6: + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 135, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "lstm_forward_training", 0) < (0)) __PYX_ERR(0, 135, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 6; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("lstm_forward_training", 1, 6, 6, i); __PYX_ERR(0, 135, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 6)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 135, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 135, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 135, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 135, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 135, __pyx_L3_error) + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 135, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_params = ((PyArrayObject *)values[1]); + __pyx_v_H0 = ((PyArrayObject *)values[2]); + __pyx_v_C0 = ((PyArrayObject *)values[3]); + __pyx_v_X = ((PyArrayObject *)values[4]); + __pyx_v_size_at_t = ((PyArrayObject *)values[5]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("lstm_forward_training", 1, 6, 6, __pyx_nargs); __PYX_ERR(0, 135, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.lstm_forward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 137, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_H0), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "H0", 0))) __PYX_ERR(0, 138, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_C0), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "C0", 0))) __PYX_ERR(0, 139, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 140, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_size_at_t), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "size_at_t", 0))) __PYX_ERR(0, 141, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_14lstm_forward_training(__pyx_self, __pyx_v_self, __pyx_v_params, __pyx_v_H0, __pyx_v_C0, __pyx_v_X, __pyx_v_size_at_t); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_14lstm_forward_training(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_H0, PyArrayObject *__pyx_v_C0, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_size_at_t) { + PyObject *__pyx_v_Y = NULL; + PyObject *__pyx_v_fwd_state = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("lstm_forward_training", 0); + + /* "thinc/backends/numpy_ops.pyx":143 + * np.ndarray size_at_t + * ): + * assert H0.shape[0] == C0.shape[0] # <<<<<<<<<<<<<< + * assert H0.shape[1] == C0.shape[1] + * Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_H0)[0]) == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C0)[0])); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 143, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 143, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":144 + * ): + * assert H0.shape[0] == C0.shape[0] + * assert H0.shape[1] == C0.shape[1] # <<<<<<<<<<<<<< + * Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) + * return Y, fwd_state +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_H0)[1]) == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C0)[1])); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 144, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 144, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":145 + * assert H0.shape[0] == C0.shape[0] + * assert H0.shape[1] == C0.shape[1] + * Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) # <<<<<<<<<<<<<< + * return Y, fwd_state + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_lstm_forward_training); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_3, ((PyObject *)__pyx_v_params), ((PyObject *)__pyx_v_H0), ((PyObject *)__pyx_v_C0), ((PyObject *)__pyx_v_X), ((PyObject *)__pyx_v_size_at_t)}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (6-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 145, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_6 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); + index = 0; __pyx_t_4 = __pyx_t_7(__pyx_t_6); if (unlikely(!__pyx_t_4)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_3 = __pyx_t_7(__pyx_t_6); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_7(__pyx_t_6), 2) < (0)) __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_t_7 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_7 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 145, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_Y = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_v_fwd_state = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":146 + * assert H0.shape[1] == C0.shape[1] + * Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) + * return Y, fwd_state # <<<<<<<<<<<<<< + * + * def lstm_forward_inference( +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_v_Y); + __Pyx_GIVEREF(__pyx_v_Y); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_Y) != (0)) __PYX_ERR(0, 146, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_fwd_state); + __Pyx_GIVEREF(__pyx_v_fwd_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_fwd_state) != (0)) __PYX_ERR(0, 146, __pyx_L1_error); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":135 + * return super().backprop_relu(dY, Y, inplace) + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.lstm_forward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_Y); + __Pyx_XDECREF(__pyx_v_fwd_state); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":148 + * return Y, fwd_state + * + * def lstm_forward_inference( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_17lstm_forward_inference(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_16lstm_forward_inference, "NumpyOps.lstm_forward_inference(self, ndarray params, ndarray H0, ndarray C0, ndarray X, ndarray size_at_t)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_17lstm_forward_inference = {"lstm_forward_inference", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_17lstm_forward_inference, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_16lstm_forward_inference}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_17lstm_forward_inference(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_params = 0; + PyArrayObject *__pyx_v_H0 = 0; + PyArrayObject *__pyx_v_C0 = 0; + PyArrayObject *__pyx_v_X = 0; + PyArrayObject *__pyx_v_size_at_t = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[6] = {0,0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lstm_forward_inference (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_H0,&__pyx_mstate_global->__pyx_n_u_C0,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_size_at_t,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 148, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 6: + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 148, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "lstm_forward_inference", 0) < (0)) __PYX_ERR(0, 148, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 6; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("lstm_forward_inference", 1, 6, 6, i); __PYX_ERR(0, 148, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 6)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 148, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 148, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 148, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 148, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 148, __pyx_L3_error) + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 148, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_params = ((PyArrayObject *)values[1]); + __pyx_v_H0 = ((PyArrayObject *)values[2]); + __pyx_v_C0 = ((PyArrayObject *)values[3]); + __pyx_v_X = ((PyArrayObject *)values[4]); + __pyx_v_size_at_t = ((PyArrayObject *)values[5]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("lstm_forward_inference", 1, 6, 6, __pyx_nargs); __PYX_ERR(0, 148, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.lstm_forward_inference", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 150, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_H0), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "H0", 0))) __PYX_ERR(0, 151, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_C0), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "C0", 0))) __PYX_ERR(0, 152, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 153, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_size_at_t), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "size_at_t", 0))) __PYX_ERR(0, 154, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_16lstm_forward_inference(__pyx_self, __pyx_v_self, __pyx_v_params, __pyx_v_H0, __pyx_v_C0, __pyx_v_X, __pyx_v_size_at_t); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_16lstm_forward_inference(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_H0, PyArrayObject *__pyx_v_C0, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_size_at_t) { + PyObject *__pyx_v_Y = NULL; + CYTHON_UNUSED PyObject *__pyx_v__ = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("lstm_forward_inference", 0); + + /* "thinc/backends/numpy_ops.pyx":156 + * np.ndarray size_at_t + * ): + * Y, _ = lstm_forward_training(params, H0, C0, X, size_at_t) # <<<<<<<<<<<<<< + * return Y + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_lstm_forward_training); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_2, ((PyObject *)__pyx_v_params), ((PyObject *)__pyx_v_H0), ((PyObject *)__pyx_v_C0), ((PyObject *)__pyx_v_X), ((PyObject *)__pyx_v_size_at_t)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (6-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 156, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_2 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < (0)) __PYX_ERR(0, 156, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 156, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_Y = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v__ = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":157 + * ): + * Y, _ = lstm_forward_training(params, H0, C0, X, size_at_t) + * return Y # <<<<<<<<<<<<<< + * + * def backprop_lstm( +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_Y); + __pyx_r = __pyx_v_Y; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":148 + * return Y, fwd_state + * + * def lstm_forward_inference( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.lstm_forward_inference", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_Y); + __Pyx_XDECREF(__pyx_v__); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":159 + * return Y + * + * def backprop_lstm( # <<<<<<<<<<<<<< + * self, np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state + * ): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_19backprop_lstm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_18backprop_lstm, "NumpyOps.backprop_lstm(self, ndarray dY, ndarray lengths, ndarray params, fwd_state)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_19backprop_lstm = {"backprop_lstm", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_19backprop_lstm, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_18backprop_lstm}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_19backprop_lstm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_dY = 0; + PyArrayObject *__pyx_v_lengths = 0; + PyArrayObject *__pyx_v_params = 0; + PyObject *__pyx_v_fwd_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_lstm (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_lengths,&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_fwd_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 159, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 159, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 159, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 159, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 159, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 159, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_lstm", 0) < (0)) __PYX_ERR(0, 159, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 5; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_lstm", 1, 5, 5, i); __PYX_ERR(0, 159, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 5)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 159, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 159, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 159, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 159, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 159, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_dY = ((PyArrayObject *)values[1]); + __pyx_v_lengths = ((PyArrayObject *)values[2]); + __pyx_v_params = ((PyArrayObject *)values[3]); + __pyx_v_fwd_state = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_lstm", 1, 5, 5, __pyx_nargs); __PYX_ERR(0, 159, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_lstm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "dY", 0))) __PYX_ERR(0, 160, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lengths), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "lengths", 0))) __PYX_ERR(0, 160, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 160, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_18backprop_lstm(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_lengths, __pyx_v_params, __pyx_v_fwd_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_18backprop_lstm(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_lengths, PyArrayObject *__pyx_v_params, PyObject *__pyx_v_fwd_state) { + PyObject *__pyx_v_dX = NULL; + PyObject *__pyx_v_d_params = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_lstm", 0); + + /* "thinc/backends/numpy_ops.pyx":162 + * self, np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state + * ): + * dX, d_params = backprop_lstm(dY, lengths, params, fwd_state) # <<<<<<<<<<<<<< + * return dX, d_params + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_backprop_lstm); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_2, ((PyObject *)__pyx_v_dY), ((PyObject *)__pyx_v_lengths), ((PyObject *)__pyx_v_params), __pyx_v_fwd_state}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (5-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 162, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 162, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_2 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < (0)) __PYX_ERR(0, 162, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 162, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_dX = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_d_params = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":163 + * ): + * dX, d_params = backprop_lstm(dY, lengths, params, fwd_state) + * return dX, d_params # <<<<<<<<<<<<<< + * + * def maxout(self, reals3d_ft X): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_dX); + __Pyx_GIVEREF(__pyx_v_dX); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_dX) != (0)) __PYX_ERR(0, 163, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_d_params); + __Pyx_GIVEREF(__pyx_v_d_params); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_d_params) != (0)) __PYX_ERR(0, 163, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":159 + * return Y + * + * def backprop_lstm( # <<<<<<<<<<<<<< + * self, np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state + * ): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_lstm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_dX); + __Pyx_XDECREF(__pyx_v_d_params); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":165 + * return dX, d_params + * + * def maxout(self, reals3d_ft X): # <<<<<<<<<<<<<< + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_21maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout, "NumpyOps.maxout(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_21maxout = {"maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_21maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_21maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 165, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 165, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 165, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 165, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 165, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 165, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("maxout", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_X, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_X); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(2, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 165, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_f17e6b_2_2_48de32__aa3eff__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_57maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_57maxout = {"__pyx_fuse_0maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_57maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_57maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float3d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("maxout (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 165, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "maxout", 0) < (0)) __PYX_ERR(0, 165, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("maxout", 1, 2, 2, i); __PYX_ERR(0, 165, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 165, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("maxout", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 165, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_56maxout(__pyx_self, __pyx_v_self, __pyx_v_X); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_56maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float3d_t __pyx_v_X) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_P; + PyArrayObject *__pyx_v_best = 0; + PyArrayObject *__pyx_v_which = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0maxout", 0); + + /* "thinc/backends/numpy_ops.pyx":166 + * + * def maxout(self, reals3d_ft X): + * cdef int B = X.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int P = X.shape[2] +*/ + __pyx_v_B = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":167 + * def maxout(self, reals3d_ft X): + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int P = X.shape[2] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":168 + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] + * cdef int P = X.shape[2] # <<<<<<<<<<<<<< + * + * cdef np.ndarray best +*/ + __pyx_v_P = (__pyx_v_X.shape[2]); + + /* "thinc/backends/numpy_ops.pyx":171 + * + * cdef np.ndarray best + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype='int32', zeros=False) # <<<<<<<<<<<<<< + * if reals3d_ft is float3d_t: + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 171, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 171, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_2, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_int32, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_4, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 171, __pyx_L1_error) + __pyx_v_which = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":173 + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype='int32', zeros=False) + * if reals3d_ft is float3d_t: + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) # <<<<<<<<<<<<<< + * if len(X) > 0: + * cpu_maxout(best.data, which.data, +*/ + __pyx_t_4 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 173, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 173, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_2 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_2 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_3, __pyx_t_2, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 173, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_2, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 173, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_2, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 173, __pyx_L1_error) + __pyx_v_best = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":174 + * if reals3d_ft is float3d_t: + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) + * if len(X) > 0: # <<<<<<<<<<<<<< + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) +*/ + __pyx_t_7 = __Pyx_MemoryView_Len(__pyx_v_X); + __pyx_t_8 = (__pyx_t_7 > 0); + if (__pyx_t_8) { + + /* "thinc/backends/numpy_ops.pyx":176 + * if len(X) > 0: + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) # <<<<<<<<<<<<<< + * else: + * best = self.alloc(shape=(B, O), dtype="float64", zeros=False) +*/ + __pyx_t_9 = 0; + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_9 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[2]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 2; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[2])) __pyx_t_12 = 2; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 176, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":175 + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) + * if len(X) > 0: + * cpu_maxout(best.data, which.data, # <<<<<<<<<<<<<< + * &X[0, 0, 0], B, O, P) + * else: +*/ + cpu_maxout(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_best)), ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_which)), (&(*((float *) ( /* dim=2 */ ((char *) (((float *) ( /* dim=1 */ (( /* dim=0 */ (__pyx_v_X.data + __pyx_t_9 * __pyx_v_X.strides[0]) ) + __pyx_t_10 * __pyx_v_X.strides[1]) )) + __pyx_t_11)) )))), __pyx_v_B, __pyx_v_O, __pyx_v_P); + + /* "thinc/backends/numpy_ops.pyx":174 + * if reals3d_ft is float3d_t: + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) + * if len(X) > 0: # <<<<<<<<<<<<<< + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":182 + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) + * return best, which # <<<<<<<<<<<<<< + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 182, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF((PyObject *)__pyx_v_best); + __Pyx_GIVEREF((PyObject *)__pyx_v_best); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_best)) != (0)) __PYX_ERR(0, 182, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 182, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":165 + * return dX, d_params + * + * def maxout(self, reals3d_ft X): # <<<<<<<<<<<<<< + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_best); + __Pyx_XDECREF((PyObject *)__pyx_v_which); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_59maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_59maxout = {"__pyx_fuse_1maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_59maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_20maxout}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_59maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double3d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("maxout (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 165, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "maxout", 0) < (0)) __PYX_ERR(0, 165, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("maxout", 1, 2, 2, i); __PYX_ERR(0, 165, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 165, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 165, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 165, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("maxout", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 165, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_58maxout(__pyx_self, __pyx_v_self, __pyx_v_X); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_58maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double3d_t __pyx_v_X) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_P; + PyArrayObject *__pyx_v_best = 0; + PyArrayObject *__pyx_v_which = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1maxout", 0); + + /* "thinc/backends/numpy_ops.pyx":166 + * + * def maxout(self, reals3d_ft X): + * cdef int B = X.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int P = X.shape[2] +*/ + __pyx_v_B = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":167 + * def maxout(self, reals3d_ft X): + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int P = X.shape[2] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":168 + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] + * cdef int P = X.shape[2] # <<<<<<<<<<<<<< + * + * cdef np.ndarray best +*/ + __pyx_v_P = (__pyx_v_X.shape[2]); + + /* "thinc/backends/numpy_ops.pyx":171 + * + * cdef np.ndarray best + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype='int32', zeros=False) # <<<<<<<<<<<<<< + * if reals3d_ft is float3d_t: + * best = self.alloc(shape=(B, O), dtype="float32", zeros=False) +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 171, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 171, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_2, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_int32, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_4, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 171, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 171, __pyx_L1_error) + __pyx_v_which = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":178 + * &X[0, 0, 0], B, O, P) + * else: + * best = self.alloc(shape=(B, O), dtype="float64", zeros=False) # <<<<<<<<<<<<<< + * if len(X) > 0: + * cpu_maxout(best.data, which.data, +*/ + __pyx_t_4 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 178, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 178, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 178, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 178, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 178, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_2 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_2 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 178, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_3, __pyx_t_2, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 178, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_2, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 178, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_2, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 178, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 178, __pyx_L1_error) + __pyx_v_best = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":179 + * else: + * best = self.alloc(shape=(B, O), dtype="float64", zeros=False) + * if len(X) > 0: # <<<<<<<<<<<<<< + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) +*/ + __pyx_t_7 = __Pyx_MemoryView_Len(__pyx_v_X); + __pyx_t_8 = (__pyx_t_7 > 0); + if (__pyx_t_8) { + + /* "thinc/backends/numpy_ops.pyx":181 + * if len(X) > 0: + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) # <<<<<<<<<<<<<< + * return best, which + * +*/ + __pyx_t_9 = 0; + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_9 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[2]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 2; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[2])) __pyx_t_12 = 2; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 181, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":180 + * best = self.alloc(shape=(B, O), dtype="float64", zeros=False) + * if len(X) > 0: + * cpu_maxout(best.data, which.data, # <<<<<<<<<<<<<< + * &X[0, 0, 0], B, O, P) + * return best, which +*/ + cpu_maxout(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_best)), ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_which)), (&(*((double *) ( /* dim=2 */ ((char *) (((double *) ( /* dim=1 */ (( /* dim=0 */ (__pyx_v_X.data + __pyx_t_9 * __pyx_v_X.strides[0]) ) + __pyx_t_10 * __pyx_v_X.strides[1]) )) + __pyx_t_11)) )))), __pyx_v_B, __pyx_v_O, __pyx_v_P); + + /* "thinc/backends/numpy_ops.pyx":179 + * else: + * best = self.alloc(shape=(B, O), dtype="float64", zeros=False) + * if len(X) > 0: # <<<<<<<<<<<<<< + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":182 + * cpu_maxout(best.data, which.data, + * &X[0, 0, 0], B, O, P) + * return best, which # <<<<<<<<<<<<<< + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 182, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF((PyObject *)__pyx_v_best); + __Pyx_GIVEREF((PyObject *)__pyx_v_best); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_best)) != (0)) __PYX_ERR(0, 182, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 182, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":165 + * return dX, d_params + * + * def maxout(self, reals3d_ft X): # <<<<<<<<<<<<<< + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_best); + __Pyx_XDECREF((PyObject *)__pyx_v_which); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":184 + * return best, which + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): # <<<<<<<<<<<<<< + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_23backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout, "NumpyOps.backprop_maxout(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_23backprop_maxout = {"backprop_maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_23backprop_maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_23backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 184, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 184, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 184, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_maxout", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_dY, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_dY); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(4, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 184, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_63backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_63backprop_maxout = {"__pyx_fuse_0backprop_maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_63backprop_maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_63backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_dY = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_which = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_v_P; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_maxout (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_which,&__pyx_mstate_global->__pyx_n_u_P,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 184, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_maxout", 0) < (0)) __PYX_ERR(0, 184, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_maxout", 1, 4, 4, i); __PYX_ERR(0, 184, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_dY = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_dY.memview)) __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_v_which = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_which.memview)) __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_v_P = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_P == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 184, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_maxout", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_dY, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_62backprop_maxout(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_which, __pyx_v_P); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_dY, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_62backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_dY, __Pyx_memviewslice __pyx_v_which, int __pyx_v_P) { + int __pyx_v_B; + int __pyx_v_O; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + size_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + int __pyx_t_11; + Py_ssize_t __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0backprop_maxout", 0); + + /* "thinc/backends/numpy_ops.pyx":185 + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): + * cdef int B = dY.shape[0] # <<<<<<<<<<<<<< + * cdef int O = dY.shape[1] + * +*/ + __pyx_v_B = (__pyx_v_dY.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":186 + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __pyx_v_O = (__pyx_v_dY.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":190 + * cdef np.ndarray dX + * if reals2d_ft == float2d_t: + * dX = numpy.zeros((B, O, P), dtype='float32') # <<<<<<<<<<<<<< + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + * else: +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_v_P); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 190, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 190, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 190, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 190, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 190, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":191 + * if reals2d_ft == float2d_t: + * dX = numpy.zeros((B, O, P), dtype='float32') + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) # <<<<<<<<<<<<<< + * else: + * dX = numpy.zeros((B, O, P), dtype='float64') +*/ + __pyx_t_9 = 0; + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_dY.shape[0]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_9 >= __pyx_v_dY.shape[0])) __pyx_t_11 = 0; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_dY.shape[1]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 1; + } else if (unlikely(__pyx_t_10 >= __pyx_v_dY.shape[1])) __pyx_t_11 = 1; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 191, __pyx_L1_error) + } + __pyx_t_12 = 0; + __pyx_t_13 = 0; + __pyx_t_11 = -1; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_which.shape[0]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_12 >= __pyx_v_which.shape[0])) __pyx_t_11 = 0; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_which.shape[1]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_11 = 1; + } else if (unlikely(__pyx_t_13 >= __pyx_v_which.shape[1])) __pyx_t_11 = 1; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 191, __pyx_L1_error) + } + try { + cpu_backprop_maxout(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((float *)(&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_dY.data + __pyx_t_9 * __pyx_v_dY.strides[0]) )) + __pyx_t_10)) ))))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_which.data + __pyx_t_12 * __pyx_v_which.strides[0]) )) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_O, __pyx_v_P); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 191, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":196 + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + * + * return dX # <<<<<<<<<<<<<< + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":184 + * return best, which + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): # <<<<<<<<<<<<<< + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_65backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_65backprop_maxout = {"__pyx_fuse_1backprop_maxout", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_65backprop_maxout, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_22backprop_maxout}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_65backprop_maxout(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_dY = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_which = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_v_P; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_maxout (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_which,&__pyx_mstate_global->__pyx_n_u_P,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 184, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_maxout", 0) < (0)) __PYX_ERR(0, 184, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_maxout", 1, 4, 4, i); __PYX_ERR(0, 184, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 184, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 184, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 184, __pyx_L3_error) + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 184, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_dY = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_dY.memview)) __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_v_which = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_which.memview)) __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_v_P = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_P == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 184, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_maxout", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 184, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_dY, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_64backprop_maxout(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_which, __pyx_v_P); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_dY, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_64backprop_maxout(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_dY, __Pyx_memviewslice __pyx_v_which, int __pyx_v_P) { + int __pyx_v_B; + int __pyx_v_O; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + size_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + int __pyx_t_11; + Py_ssize_t __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1backprop_maxout", 0); + + /* "thinc/backends/numpy_ops.pyx":185 + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): + * cdef int B = dY.shape[0] # <<<<<<<<<<<<<< + * cdef int O = dY.shape[1] + * +*/ + __pyx_v_B = (__pyx_v_dY.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":186 + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __pyx_v_O = (__pyx_v_dY.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":193 + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + * else: + * dX = numpy.zeros((B, O, P), dtype='float64') # <<<<<<<<<<<<<< + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_v_P); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 193, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 193, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 193, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 193, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 193, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":194 + * else: + * dX = numpy.zeros((B, O, P), dtype='float64') + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) # <<<<<<<<<<<<<< + * + * return dX +*/ + __pyx_t_9 = 0; + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_dY.shape[0]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_9 >= __pyx_v_dY.shape[0])) __pyx_t_11 = 0; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_dY.shape[1]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 1; + } else if (unlikely(__pyx_t_10 >= __pyx_v_dY.shape[1])) __pyx_t_11 = 1; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 194, __pyx_L1_error) + } + __pyx_t_12 = 0; + __pyx_t_13 = 0; + __pyx_t_11 = -1; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_which.shape[0]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_12 >= __pyx_v_which.shape[0])) __pyx_t_11 = 0; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_which.shape[1]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_11 = 1; + } else if (unlikely(__pyx_t_13 >= __pyx_v_which.shape[1])) __pyx_t_11 = 1; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 194, __pyx_L1_error) + } + try { + cpu_backprop_maxout(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((double *)(&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_dY.data + __pyx_t_9 * __pyx_v_dY.strides[0]) )) + __pyx_t_10)) ))))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_which.data + __pyx_t_12 * __pyx_v_which.strides[0]) )) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_O, __pyx_v_P); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 194, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":196 + * cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + * + * return dX # <<<<<<<<<<<<<< + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":184 + * return best, which + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): # <<<<<<<<<<<<<< + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_maxout", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":198 + * return dX + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_25mish(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_24mish, "NumpyOps.mish(self, ndarray X, threshold=20.0, inplace: bool = False)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_25mish = {"mish", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_25mish, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_24mish}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_25mish(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_X = 0; + PyObject *__pyx_v_threshold = 0; + PyObject *__pyx_v_inplace = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("mish (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_threshold,&__pyx_mstate_global->__pyx_n_u_inplace,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 198, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "mish", 0) < (0)) __PYX_ERR(0, 198, __pyx_L3_error) + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_20_0))); + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("mish", 0, 2, 4, i); __PYX_ERR(0, 198, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 198, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 198, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 198, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_20_0))); + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + } + __pyx_v_self = values[0]; + __pyx_v_X = ((PyArrayObject *)values[1]); + __pyx_v_threshold = values[2]; + __pyx_v_inplace = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("mish", 0, 2, 4, __pyx_nargs); __PYX_ERR(0, 198, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.mish", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_24mish(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_threshold, __pyx_v_inplace); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_24mish(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_threshold, PyObject *__pyx_v_inplace) { + PyArrayObject *__pyx_v_Y = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + float __pyx_t_3; + double __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("mish", 0); + + /* "thinc/backends/numpy_ops.pyx":201 + * cdef np.ndarray Y + * + * if X.dtype == "float32": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 201, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 201, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":202 + * + * if X.dtype == "float32": + * Y = _inplace_or_copy(X, inplace) # <<<<<<<<<<<<<< + * cpu_mish(Y.data, Y.size, threshold) + * return Y +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_X, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":203 + * if X.dtype == "float32": + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) # <<<<<<<<<<<<<< + * return Y + * elif X.dtype == "float64": +*/ + __pyx_t_3 = __Pyx_PyFloat_AsFloat(__pyx_v_threshold); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 203, __pyx_L1_error) + cpu_mish(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Y)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_Y)), ((float)__pyx_t_3)); + + /* "thinc/backends/numpy_ops.pyx":204 + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) + * return Y # <<<<<<<<<<<<<< + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_Y); + __pyx_r = ((PyObject *)__pyx_v_Y); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":201 + * cdef np.ndarray Y + * + * if X.dtype == "float32": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":205 + * cpu_mish(Y.data, Y.size, threshold) + * return Y + * elif X.dtype == "float64": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 205, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 205, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":206 + * return Y + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) # <<<<<<<<<<<<<< + * cpu_mish(Y.data, Y.size, threshold) + * return Y +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_X, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":207 + * elif X.dtype == "float64": + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) # <<<<<<<<<<<<<< + * return Y + * else: +*/ + __pyx_t_4 = __Pyx_PyFloat_AsDouble(__pyx_v_threshold); if (unlikely((__pyx_t_4 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 207, __pyx_L1_error) + cpu_mish(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Y)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_Y)), ((double)__pyx_t_4)); + + /* "thinc/backends/numpy_ops.pyx":208 + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) + * return Y # <<<<<<<<<<<<<< + * else: + * return super().mish(X, inplace=inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_Y); + __pyx_r = ((PyObject *)__pyx_v_Y); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":205 + * cpu_mish(Y.data, Y.size, threshold) + * return Y + * elif X.dtype == "float64": # <<<<<<<<<<<<<< + * Y = _inplace_or_copy(X, inplace) + * cpu_mish(Y.data, Y.size, threshold) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":210 + * return Y + * else: + * return super().mish(X, inplace=inplace) # <<<<<<<<<<<<<< + * + * def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_7 = NULL; + __pyx_t_8 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_8) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 210, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_8); + __pyx_t_9 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_7, __pyx_t_8, __pyx_v_self}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 210, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_5 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, ((PyObject *)__pyx_v_X)}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 210, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_inplace, __pyx_v_inplace, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 210, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_mish, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 210, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":198 + * return dX + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.mish", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_Y); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":212 + * return super().mish(X, inplace=inplace) + * + * def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, X) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_27backprop_mish(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_26backprop_mish, "NumpyOps.backprop_mish(self, ndarray dY, ndarray X, threshold=20.0, inplace=False)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_27backprop_mish = {"backprop_mish", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_27backprop_mish, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_26backprop_mish}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_27backprop_mish(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_dY = 0; + PyArrayObject *__pyx_v_X = 0; + PyObject *__pyx_v_threshold = 0; + PyObject *__pyx_v_inplace = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_mish (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_threshold,&__pyx_mstate_global->__pyx_n_u_inplace,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 212, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_mish", 0) < (0)) __PYX_ERR(0, 212, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_20_0))); + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_mish", 0, 3, 5, i); __PYX_ERR(0, 212, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 212, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 212, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 212, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 212, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_float_20_0))); + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)((PyObject*)Py_False))); + } + __pyx_v_self = values[0]; + __pyx_v_dY = ((PyArrayObject *)values[1]); + __pyx_v_X = ((PyArrayObject *)values[2]); + __pyx_v_threshold = values[3]; + __pyx_v_inplace = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_mish", 0, 3, 5, __pyx_nargs); __PYX_ERR(0, 212, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_mish", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "dY", 0))) __PYX_ERR(0, 212, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 212, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_26backprop_mish(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_X, __pyx_v_threshold, __pyx_v_inplace); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_26backprop_mish(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_X, PyObject *__pyx_v_threshold, PyObject *__pyx_v_inplace) { + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + float __pyx_t_7; + double __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_mish", 0); + + /* "thinc/backends/numpy_ops.pyx":213 + * + * def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): + * _check_compatible_shape(dY, X) # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_check_compatible_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 213, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, ((PyObject *)__pyx_v_dY), ((PyObject *)__pyx_v_X)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 213, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":217 + * cdef np.ndarray dX + * + * if dY.dtype == "float32" and X.dtype == "float32": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_6) { + } else { + __pyx_t_5 = __pyx_t_6; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __pyx_t_6; + __pyx_L4_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":218 + * + * if dY.dtype == "float32" and X.dtype == "float32": + * dX = _inplace_or_copy(dY, inplace) # <<<<<<<<<<<<<< + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_dY, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 218, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_dX = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":219 + * if dY.dtype == "float32" and X.dtype == "float32": + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) # <<<<<<<<<<<<<< + * return dX + * elif dY.dtype == "float64" and X.dtype == "float64": +*/ + __pyx_t_7 = __Pyx_PyFloat_AsFloat(__pyx_v_threshold); if (unlikely((__pyx_t_7 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 219, __pyx_L1_error) + cpu_backprop_mish(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_X)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_X)), ((float)__pyx_t_7)); + + /* "thinc/backends/numpy_ops.pyx":220 + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX # <<<<<<<<<<<<<< + * elif dY.dtype == "float64" and X.dtype == "float64": + * dX = _inplace_or_copy(dY, inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":217 + * cdef np.ndarray dX + * + * if dY.dtype == "float32" and X.dtype == "float32": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":221 + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX + * elif dY.dtype == "float64" and X.dtype == "float64": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 221, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 221, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_6) { + } else { + __pyx_t_5 = __pyx_t_6; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 221, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 221, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __pyx_t_6; + __pyx_L6_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":222 + * return dX + * elif dY.dtype == "float64" and X.dtype == "float64": + * dX = _inplace_or_copy(dY, inplace) # <<<<<<<<<<<<<< + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX +*/ + __pyx_t_1 = ((PyObject *)__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(__pyx_v_dY, __pyx_v_inplace)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 222, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_dX = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":223 + * elif dY.dtype == "float64" and X.dtype == "float64": + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) # <<<<<<<<<<<<<< + * return dX + * else: +*/ + __pyx_t_8 = __Pyx_PyFloat_AsDouble(__pyx_v_threshold); if (unlikely((__pyx_t_8 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 223, __pyx_L1_error) + cpu_backprop_mish(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_X)), ((int)__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_X)), ((double)__pyx_t_8)); + + /* "thinc/backends/numpy_ops.pyx":224 + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX # <<<<<<<<<<<<<< + * else: + * return super().backprop_mish(dY, X, threshold, inplace) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":221 + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) + * return dX + * elif dY.dtype == "float64" and X.dtype == "float64": # <<<<<<<<<<<<<< + * dX = _inplace_or_copy(dY, inplace) + * cpu_backprop_mish(dX.data, X.data, X.size, threshold) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":226 + * return dX + * else: + * return super().backprop_mish(dY, X, threshold, inplace) # <<<<<<<<<<<<<< + * + * def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_10) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 226, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_10); + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_9, __pyx_t_10, __pyx_v_self}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 226, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_3 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[5] = {__pyx_t_3, ((PyObject *)__pyx_v_dY), ((PyObject *)__pyx_v_X), __pyx_v_threshold, __pyx_v_inplace}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_backprop_mish, __pyx_callargs+__pyx_t_4, (5-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 226, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":212 + * return super().mish(X, inplace=inplace) + * + * def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, X) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_mish", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":228 + * return super().backprop_mish(dY, X, threshold, inplace) + * + * def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + * sequence. The new sequence is constructed by concatenating nW preceding +*/ + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_20__defaults__(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__defaults__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_memoryview_fromslice(__Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_self)->arg0, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_lengths, __pyx_t_2) < (0)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, Py_None) != (0)) __PYX_ERR(0, 228, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 228, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("thinc.backends.numpy_ops.__defaults__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_29seq2col(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_28seq2col, "NumpyOps.seq2col(self, ndarray seq, int nW, *, int[::1] lengths=None)\n\nGiven an (M, N) sequence of vectors, return an (M, N*(nW*2+1))\nsequence. The new sequence is constructed by concatenating nW preceding\nand succeeding vectors onto each column in the sequence, to extract a\n window of features."); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_29seq2col = {"seq2col", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_29seq2col, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_28seq2col}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_29seq2col(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_seq = 0; + int __pyx_v_nW; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("seq2col (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_seq,&__pyx_mstate_global->__pyx_n_u_nW,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + struct __pyx_defaults1 *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 228, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 228, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 228, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 228, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "seq2col", 0) < (0)) __PYX_ERR(0, 228, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("seq2col", 1, 3, 3, i); __PYX_ERR(0, 228, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 228, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 228, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 228, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_seq = ((PyArrayObject *)values[1]); + __pyx_v_nW = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_nW == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 228, __pyx_L3_error) + if (values[3]) { + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[3], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 228, __pyx_L3_error) + } else { + __pyx_v_lengths = __pyx_dynamic_args->arg0; + __PYX_INC_MEMVIEW(&__pyx_v_lengths, 1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("seq2col", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 228, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.seq2col", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_seq), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "seq", 0))) __PYX_ERR(0, 228, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_28seq2col(__pyx_self, __pyx_v_self, __pyx_v_seq, __pyx_v_nW, __pyx_v_lengths); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_28seq2col(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_seq, int __pyx_v_nW, __Pyx_memviewslice __pyx_v_lengths) { + PyObject *__pyx_v_msg = NULL; + int __pyx_v_B; + int __pyx_v_I; + int __pyx_v_nL; + PyArrayObject *__pyx_v_cols = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + __Pyx_memviewslice __pyx_t_9 = { 0, 0, { 0 }, { 0 }, { 0 } }; + Py_ssize_t __pyx_t_10; + int __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("seq2col", 0); + __PYX_INC_MEMVIEW(&__pyx_v_lengths, 1); + + /* "thinc/backends/numpy_ops.pyx":240 + * # type. + * + * if seq.ndim != 2: # <<<<<<<<<<<<<< + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + * raise ValueError(msg) +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_seq) != 2); + if (unlikely(__pyx_t_1)) { + + /* "thinc/backends/numpy_ops.pyx":241 + * + * if seq.ndim != 2: + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" # <<<<<<<<<<<<<< + * raise ValueError(msg) + * if not seq.flags.c_contiguous: +*/ + __pyx_t_2 = __Pyx_PyUnicode_From_int(__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_seq), 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 241, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_seq2col_requires_sequence_array, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 241, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_msg = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":242 + * if seq.ndim != 2: + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + * raise ValueError(msg) # <<<<<<<<<<<<<< + * if not seq.flags.c_contiguous: + * msg = f"seq2col requires sequence array that is in C order and contiguous" +*/ + __pyx_t_2 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_msg}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 242, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 242, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":240 + * # type. + * + * if seq.ndim != 2: # <<<<<<<<<<<<<< + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + * raise ValueError(msg) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":243 + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + * raise ValueError(msg) + * if not seq.flags.c_contiguous: # <<<<<<<<<<<<<< + * msg = f"seq2col requires sequence array that is in C order and contiguous" + * raise ValueError(msg) +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_seq), __pyx_mstate_global->__pyx_n_u_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_c_contiguous); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (!__pyx_t_1); + if (unlikely(__pyx_t_5)) { + + /* "thinc/backends/numpy_ops.pyx":244 + * raise ValueError(msg) + * if not seq.flags.c_contiguous: + * msg = f"seq2col requires sequence array that is in C order and contiguous" # <<<<<<<<<<<<<< + * raise ValueError(msg) + * +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_seq2col_requires_sequence_array_2); + __pyx_v_msg = __pyx_mstate_global->__pyx_kp_u_seq2col_requires_sequence_array_2; + + /* "thinc/backends/numpy_ops.pyx":245 + * if not seq.flags.c_contiguous: + * msg = f"seq2col requires sequence array that is in C order and contiguous" + * raise ValueError(msg) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_msg}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 245, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 245, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":243 + * msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + * raise ValueError(msg) + * if not seq.flags.c_contiguous: # <<<<<<<<<<<<<< + * msg = f"seq2col requires sequence array that is in C order and contiguous" + * raise ValueError(msg) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":248 + * + * + * cdef int B = seq.shape[0] # <<<<<<<<<<<<<< + * cdef int I = seq.shape[1] + * +*/ + __pyx_v_B = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_seq)[0]); + + /* "thinc/backends/numpy_ops.pyx":249 + * + * cdef int B = seq.shape[0] + * cdef int I = seq.shape[1] # <<<<<<<<<<<<<< + * + * lengths = check_seq2col_lengths(self, lengths, B) +*/ + __pyx_v_I = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_seq)[1]); + + /* "thinc/backends/numpy_ops.pyx":251 + * cdef int I = seq.shape[1] + * + * lengths = check_seq2col_lengths(self, lengths, B) # <<<<<<<<<<<<<< + * cdef int nL = lengths.shape[0] + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_check_seq2col_lengths); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[4] = {__pyx_t_3, __pyx_v_self, __pyx_t_7, __pyx_t_8}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_4, (4-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_9 = __Pyx_PyObject_to_MemoryviewSlice_dc_int(__pyx_t_2, PyBUF_WRITABLE); if (unlikely(!__pyx_t_9.memview)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __pyx_v_lengths = __pyx_t_9; + __pyx_t_9.memview = NULL; + __pyx_t_9.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":252 + * + * lengths = check_seq2col_lengths(self, lengths, B) + * cdef int nL = lengths.shape[0] # <<<<<<<<<<<<<< + * + * cdef np.ndarray cols +*/ + __pyx_v_nL = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":255 + * + * cdef np.ndarray cols + * if seq.dtype == "float32": # <<<<<<<<<<<<<< + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + * if seq.size != 0 and lengths.size != 0: +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_seq), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = (__Pyx_PyUnicode_Equals(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":256 + * cdef np.ndarray cols + * if seq.dtype == "float32": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") # <<<<<<<<<<<<<< + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) +*/ + __pyx_t_6 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = __Pyx_PyLong_From_long((((2 * __pyx_v_nW) + 1) * __pyx_v_I)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 256, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 256, __pyx_L1_error); + __pyx_t_8 = 0; + __pyx_t_7 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_7, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 256, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 256, __pyx_L1_error) + __pyx_v_cols = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":257 + * if seq.dtype == "float32": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + * if seq.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_seq) != 0); + if (__pyx_t_1) { + } else { + __pyx_t_5 = __pyx_t_1; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = (__Pyx_PyLong_BoolNeObjC(__pyx_t_7, __pyx_mstate_global->__pyx_int_0, 0, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_5 = __pyx_t_1; + __pyx_L7_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":258 + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) # <<<<<<<<<<<<<< + * return cols + * elif seq.dtype == "float64": +*/ + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_lengths.shape[0])) __pyx_t_11 = 0; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 258, __pyx_L1_error) + } + seq2col(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_cols)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_seq)), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_10)) )))), __pyx_v_nW, __pyx_v_B, __pyx_v_I, __pyx_v_nL); + + /* "thinc/backends/numpy_ops.pyx":257 + * if seq.dtype == "float32": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + * if seq.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols +*/ + } + + /* "thinc/backends/numpy_ops.pyx":259 + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols # <<<<<<<<<<<<<< + * elif seq.dtype == "float64": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_cols); + __pyx_r = ((PyObject *)__pyx_v_cols); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":255 + * + * cdef np.ndarray cols + * if seq.dtype == "float32": # <<<<<<<<<<<<<< + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + * if seq.size != 0 and lengths.size != 0: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":260 + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols + * elif seq.dtype == "float64": # <<<<<<<<<<<<<< + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + * if seq.size != 0 and lengths.size != 0: +*/ + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_seq), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 260, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_5 = (__Pyx_PyUnicode_Equals(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 260, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":261 + * return cols + * elif seq.dtype == "float64": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") # <<<<<<<<<<<<<< + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyLong_From_long((((2 * __pyx_v_nW) + 1) * __pyx_v_I)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 261, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 261, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_8}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 261, __pyx_L1_error) + __pyx_t_7 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + } + if (!(likely(((__pyx_t_7) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_7, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 261, __pyx_L1_error) + __pyx_v_cols = ((PyArrayObject *)__pyx_t_7); + __pyx_t_7 = 0; + + /* "thinc/backends/numpy_ops.pyx":262 + * elif seq.dtype == "float64": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + * if seq.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_seq) != 0); + if (__pyx_t_1) { + } else { + __pyx_t_5 = __pyx_t_1; + goto __pyx_L10_bool_binop_done; + } + __pyx_t_7 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 262, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 262, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_1 = (__Pyx_PyLong_BoolNeObjC(__pyx_t_6, __pyx_mstate_global->__pyx_int_0, 0, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 262, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_5 = __pyx_t_1; + __pyx_L10_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":263 + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) # <<<<<<<<<<<<<< + * return cols + * else: +*/ + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_lengths.shape[0])) __pyx_t_11 = 0; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 263, __pyx_L1_error) + } + seq2col(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_cols)), ((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_seq)), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_10)) )))), __pyx_v_nW, __pyx_v_B, __pyx_v_I, __pyx_v_nL); + + /* "thinc/backends/numpy_ops.pyx":262 + * elif seq.dtype == "float64": + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + * if seq.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols +*/ + } + + /* "thinc/backends/numpy_ops.pyx":264 + * if seq.size != 0 and lengths.size != 0: + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols # <<<<<<<<<<<<<< + * else: + * return super().seq2col(seq, nW, lengths=lengths) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_cols); + __pyx_r = ((PyObject *)__pyx_v_cols); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":260 + * seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + * return cols + * elif seq.dtype == "float64": # <<<<<<<<<<<<<< + * cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + * if seq.size != 0 and lengths.size != 0: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":266 + * return cols + * else: + * return super().seq2col(seq, nW, lengths=lengths) # <<<<<<<<<<<<<< + * + * def backprop_seq2col(self, np.ndarray dY, int nW, *, int[::1] lengths=None): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_3) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 266, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_3, __pyx_v_self}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 266, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __pyx_t_7 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nW); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 266, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 266, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_7, ((PyObject *)__pyx_v_seq), __pyx_t_3}; + __pyx_t_12 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 266, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_lengths, __pyx_t_2, __pyx_t_12, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 266, __pyx_L1_error) + __pyx_t_6 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_seq2col, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_12); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 266, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":228 + * return super().backprop_mish(dY, X, threshold, inplace) + * + * def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + * sequence. The new sequence is constructed by concatenating nW preceding +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_9, 1); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.seq2col", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_XDECREF((PyObject *)__pyx_v_cols); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":268 + * return super().seq2col(seq, nW, lengths=lengths) + * + * def backprop_seq2col(self, np.ndarray dY, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * # Note: the type of dY should be changed to reals2d_ft once + * # cython/cython#4697 is fixed. The following checks can then be +*/ + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_22__defaults__(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__defaults__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_memoryview_fromslice(__Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_self)->arg0, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_lengths, __pyx_t_2) < (0)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, Py_None) != (0)) __PYX_ERR(0, 268, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 268, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("thinc.backends.numpy_ops.__defaults__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_31backprop_seq2col(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_30backprop_seq2col, "NumpyOps.backprop_seq2col(self, ndarray dY, int nW, *, int[::1] lengths=None)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_31backprop_seq2col = {"backprop_seq2col", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_31backprop_seq2col, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_30backprop_seq2col}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_31backprop_seq2col(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_dY = 0; + int __pyx_v_nW; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_seq2col (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_nW,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + struct __pyx_defaults1 *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 268, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 268, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 268, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 268, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_seq2col", 0) < (0)) __PYX_ERR(0, 268, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_seq2col", 1, 3, 3, i); __PYX_ERR(0, 268, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 268, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 268, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 268, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_dY = ((PyArrayObject *)values[1]); + __pyx_v_nW = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_nW == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 268, __pyx_L3_error) + if (values[3]) { + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[3], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 268, __pyx_L3_error) + } else { + __pyx_v_lengths = __pyx_dynamic_args->arg0; + __PYX_INC_MEMVIEW(&__pyx_v_lengths, 1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_seq2col", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 268, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_seq2col", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "dY", 0))) __PYX_ERR(0, 268, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_30backprop_seq2col(__pyx_self, __pyx_v_self, __pyx_v_dY, __pyx_v_nW, __pyx_v_lengths); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_30backprop_seq2col(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_dY, int __pyx_v_nW, __Pyx_memviewslice __pyx_v_lengths) { + PyObject *__pyx_v_msg = NULL; + int __pyx_v_B; + int __pyx_v_nF; + int __pyx_v_I; + int __pyx_v_nL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + __Pyx_memviewslice __pyx_t_9 = { 0, 0, { 0 }, { 0 }, { 0 } }; + Py_ssize_t __pyx_t_10; + int __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_seq2col", 0); + __PYX_INC_MEMVIEW(&__pyx_v_lengths, 1); + + /* "thinc/backends/numpy_ops.pyx":274 + * # type. + * + * if dY.ndim != 2: # <<<<<<<<<<<<<< + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + * raise ValueError(msg) +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_dY) != 2); + if (unlikely(__pyx_t_1)) { + + /* "thinc/backends/numpy_ops.pyx":275 + * + * if dY.ndim != 2: + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" # <<<<<<<<<<<<<< + * raise ValueError(msg) + * if not dY.flags.c_contiguous: +*/ + __pyx_t_2 = __Pyx_PyUnicode_From_int(__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_dY), 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_backprop_seq2col_requires_gradie, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_msg = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":276 + * if dY.ndim != 2: + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + * raise ValueError(msg) # <<<<<<<<<<<<<< + * if not dY.flags.c_contiguous: + * msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" +*/ + __pyx_t_2 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_msg}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 276, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":274 + * # type. + * + * if dY.ndim != 2: # <<<<<<<<<<<<<< + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + * raise ValueError(msg) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":277 + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + * raise ValueError(msg) + * if not dY.flags.c_contiguous: # <<<<<<<<<<<<<< + * msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" + * raise ValueError(msg) +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_c_contiguous); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = (!__pyx_t_1); + if (unlikely(__pyx_t_5)) { + + /* "thinc/backends/numpy_ops.pyx":278 + * raise ValueError(msg) + * if not dY.flags.c_contiguous: + * msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" # <<<<<<<<<<<<<< + * raise ValueError(msg) + * +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_backprop_seq2col_requires_gradie_2); + __pyx_v_msg = __pyx_mstate_global->__pyx_kp_u_backprop_seq2col_requires_gradie_2; + + /* "thinc/backends/numpy_ops.pyx":279 + * if not dY.flags.c_contiguous: + * msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" + * raise ValueError(msg) # <<<<<<<<<<<<<< + * + * cdef int B = dY.shape[0] +*/ + __pyx_t_3 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_msg}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 279, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 279, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":277 + * msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + * raise ValueError(msg) + * if not dY.flags.c_contiguous: # <<<<<<<<<<<<<< + * msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" + * raise ValueError(msg) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":281 + * raise ValueError(msg) + * + * cdef int B = dY.shape[0] # <<<<<<<<<<<<<< + * cdef int nF = nW*2+1 + * cdef int I = dY.shape[1] / nF +*/ + __pyx_v_B = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dY)[0]); + + /* "thinc/backends/numpy_ops.pyx":282 + * + * cdef int B = dY.shape[0] + * cdef int nF = nW*2+1 # <<<<<<<<<<<<<< + * cdef int I = dY.shape[1] / nF + * +*/ + __pyx_v_nF = ((__pyx_v_nW * 2) + 1); + + /* "thinc/backends/numpy_ops.pyx":283 + * cdef int B = dY.shape[0] + * cdef int nF = nW*2+1 + * cdef int I = dY.shape[1] / nF # <<<<<<<<<<<<<< + * + * lengths = check_seq2col_lengths(self, lengths, B) +*/ + __pyx_v_I = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dY)[1]) / __pyx_v_nF); + + /* "thinc/backends/numpy_ops.pyx":285 + * cdef int I = dY.shape[1] / nF + * + * lengths = check_seq2col_lengths(self, lengths, B) # <<<<<<<<<<<<<< + * cdef int nL = lengths.shape[0] + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_check_seq2col_lengths); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[4] = {__pyx_t_3, __pyx_v_self, __pyx_t_7, __pyx_t_8}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_4, (4-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_9 = __Pyx_PyObject_to_MemoryviewSlice_dc_int(__pyx_t_2, PyBUF_WRITABLE); if (unlikely(!__pyx_t_9.memview)) __PYX_ERR(0, 285, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __pyx_v_lengths = __pyx_t_9; + __pyx_t_9.memview = NULL; + __pyx_t_9.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":286 + * + * lengths = check_seq2col_lengths(self, lengths, B) + * cdef int nL = lengths.shape[0] # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __pyx_v_nL = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":289 + * + * cdef np.ndarray dX + * if dY.dtype == "float32": # <<<<<<<<<<<<<< + * dX = self.alloc((B, I), dtype='float32') + * if dY.size != 0 and lengths.size != 0: +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 289, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = (__Pyx_PyUnicode_Equals(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 289, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":290 + * cdef np.ndarray dX + * if dY.dtype == "float32": + * dX = self.alloc((B, I), dtype='float32') # <<<<<<<<<<<<<< + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) +*/ + __pyx_t_6 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_I); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 290, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 290, __pyx_L1_error); + __pyx_t_8 = 0; + __pyx_t_7 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_7, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 290, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 290, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 290, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":291 + * if dY.dtype == "float32": + * dX = self.alloc((B, I), dtype='float32') + * if dY.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_dY) != 0); + if (__pyx_t_1) { + } else { + __pyx_t_5 = __pyx_t_1; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = (__Pyx_PyLong_BoolNeObjC(__pyx_t_7, __pyx_mstate_global->__pyx_int_0, 0, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 291, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_5 = __pyx_t_1; + __pyx_L7_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":292 + * dX = self.alloc((B, I), dtype='float32') + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) # <<<<<<<<<<<<<< + * return dX + * elif dY.dtype == "float64": +*/ + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_lengths.shape[0])) __pyx_t_11 = 0; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 292, __pyx_L1_error) + } + backprop_seq2col(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dY)), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_10)) )))), __pyx_v_B, __pyx_v_I, __pyx_v_nW, __pyx_v_nL); + + /* "thinc/backends/numpy_ops.pyx":291 + * if dY.dtype == "float32": + * dX = self.alloc((B, I), dtype='float32') + * if dY.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX +*/ + } + + /* "thinc/backends/numpy_ops.pyx":293 + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX # <<<<<<<<<<<<<< + * elif dY.dtype == "float64": + * dX = self.alloc((B, I), dtype='float64') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":289 + * + * cdef np.ndarray dX + * if dY.dtype == "float32": # <<<<<<<<<<<<<< + * dX = self.alloc((B, I), dtype='float32') + * if dY.size != 0 and lengths.size != 0: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":294 + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX + * elif dY.dtype == "float64": # <<<<<<<<<<<<<< + * dX = self.alloc((B, I), dtype='float64') + * if dY.size != 0 and lengths.size != 0: +*/ + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_5 = (__Pyx_PyUnicode_Equals(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_float64, Py_EQ)); if (unlikely((__pyx_t_5 < 0))) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":295 + * return dX + * elif dY.dtype == "float64": + * dX = self.alloc((B, I), dtype='float64') # <<<<<<<<<<<<<< + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_v_I); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 295, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 295, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_8}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 295, __pyx_L1_error) + __pyx_t_7 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + } + if (!(likely(((__pyx_t_7) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_7, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 295, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_7); + __pyx_t_7 = 0; + + /* "thinc/backends/numpy_ops.pyx":296 + * elif dY.dtype == "float64": + * dX = self.alloc((B, I), dtype='float64') + * if dY.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX +*/ + __pyx_t_1 = (__pyx_f_5numpy_7ndarray_4size_size(__pyx_v_dY) != 0); + if (__pyx_t_1) { + } else { + __pyx_t_5 = __pyx_t_1; + goto __pyx_L10_bool_binop_done; + } + __pyx_t_7 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 296, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 296, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_1 = (__Pyx_PyLong_BoolNeObjC(__pyx_t_6, __pyx_mstate_global->__pyx_int_0, 0, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 296, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_5 = __pyx_t_1; + __pyx_L10_bool_binop_done:; + if (__pyx_t_5) { + + /* "thinc/backends/numpy_ops.pyx":297 + * dX = self.alloc((B, I), dtype='float64') + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) # <<<<<<<<<<<<<< + * return dX + * else: +*/ + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_lengths.shape[0])) __pyx_t_11 = 0; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 297, __pyx_L1_error) + } + backprop_seq2col(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dY)), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_10)) )))), __pyx_v_B, __pyx_v_I, __pyx_v_nW, __pyx_v_nL); + + /* "thinc/backends/numpy_ops.pyx":296 + * elif dY.dtype == "float64": + * dX = self.alloc((B, I), dtype='float64') + * if dY.size != 0 and lengths.size != 0: # <<<<<<<<<<<<<< + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX +*/ + } + + /* "thinc/backends/numpy_ops.pyx":298 + * if dY.size != 0 and lengths.size != 0: + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX # <<<<<<<<<<<<<< + * else: + * return super().backprop_seq2col(dY, nW, lengths=lengths) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":294 + * backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + * return dX + * elif dY.dtype == "float64": # <<<<<<<<<<<<<< + * dX = self.alloc((B, I), dtype='float64') + * if dY.size != 0 and lengths.size != 0: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":300 + * return dX + * else: + * return super().backprop_seq2col(dY, nW, lengths=lengths) # <<<<<<<<<<<<<< + * + * @cython.boundscheck(False) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_CyFunction_GetClassObj(__pyx_self); + if (!__pyx_t_3) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); __PYX_ERR(0, 300, __pyx_L1_error) } + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_t_3, __pyx_v_self}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_super, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __pyx_t_7 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nW); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_lengths, 1, (PyObject *(*)(char *)) __pyx_memview_get_int, (int (*)(char *, PyObject *)) __pyx_memview_set_int, 0);; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_7, ((PyObject *)__pyx_v_dY), __pyx_t_3}; + __pyx_t_12 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_lengths, __pyx_t_2, __pyx_t_12, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 300, __pyx_L1_error) + __pyx_t_6 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_backprop_seq2col, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_12); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":268 + * return super().seq2col(seq, nW, lengths=lengths) + * + * def backprop_seq2col(self, np.ndarray dY, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * # Note: the type of dY should be changed to reals2d_ft once + * # cython/cython#4697 is fixed. The following checks can then be +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_9, 1); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_seq2col", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":302 + * return super().backprop_seq2col(dY, nW, lengths=lengths) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def hash(self, const uint64_t[::1] ids, uint32_t seed): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_33hash(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_32hash, "NumpyOps.hash(self, const uint64_t[::1] ids, uint32_t seed)\n\nHash a sequence of 64-bit keys into a table with 4 32-bit keys."); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_33hash = {"hash", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_33hash, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_32hash}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_33hash(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + __Pyx_memviewslice __pyx_v_ids = { 0, 0, { 0 }, { 0 }, { 0 } }; + uint32_t __pyx_v_seed; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("hash (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_ids,&__pyx_mstate_global->__pyx_n_u_seed,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 302, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "hash", 0) < (0)) __PYX_ERR(0, 302, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("hash", 1, 3, 3, i); __PYX_ERR(0, 302, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 302, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 302, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 302, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_ids = __Pyx_PyObject_to_MemoryviewSlice_dc_nn_uint64_t__const__(values[1], 0); if (unlikely(!__pyx_v_ids.memview)) __PYX_ERR(0, 304, __pyx_L3_error) + __pyx_v_seed = __Pyx_PyLong_As_uint32_t(values[2]); if (unlikely((__pyx_v_seed == ((uint32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 304, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("hash", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 302, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_ids, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.hash", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_32hash(__pyx_self, __pyx_v_self, __pyx_v_ids, __pyx_v_seed); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_ids, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_32hash(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __Pyx_memviewslice __pyx_v_ids, uint32_t __pyx_v_seed) { + PyArrayObject *__pyx_v_keys = 0; + int __pyx_v_i; + uint32_t *__pyx_v_dest; + __Pyx_LocalBuf_ND __pyx_pybuffernd_keys; + __Pyx_Buffer __pyx_pybuffer_keys; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("hash", 0); + __pyx_pybuffer_keys.pybuffer.buf = NULL; + __pyx_pybuffer_keys.refcount = 0; + __pyx_pybuffernd_keys.data = NULL; + __pyx_pybuffernd_keys.rcbuffer = &__pyx_pybuffer_keys; + + /* "thinc/backends/numpy_ops.pyx":307 + * """Hash a sequence of 64-bit keys into a table with 4 32-bit keys.""" + * # Written to mirror the GPU implementation + * cdef np.ndarray[uint32_t, ndim=2] keys = self.alloc((ids.shape[0], 4), dtype='uint32') # <<<<<<<<<<<<<< + * cdef int i + * cdef uint32_t* dest = keys.data +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyLong_FromSsize_t((__pyx_v_ids.shape[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 307, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 307, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 307, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_4); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_mstate_global->__pyx_int_4) != (0)) __PYX_ERR(0, 307, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_4}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 307, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_uint32, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 307, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 307, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 307, __pyx_L1_error) + { + __Pyx_BufFmt_StackElem __pyx_stack[1]; + if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_keys.rcbuffer->pybuffer, (PyObject*)((PyArrayObject *)__pyx_t_1), &__Pyx_TypeInfo_nn_uint32_t, PyBUF_FORMAT| PyBUF_STRIDES, 2, 0, __pyx_stack) == -1)) { + __pyx_v_keys = ((PyArrayObject *)Py_None); __Pyx_INCREF(Py_None); __pyx_pybuffernd_keys.rcbuffer->pybuffer.buf = NULL; + __PYX_ERR(0, 307, __pyx_L1_error) + } else {__pyx_pybuffernd_keys.diminfo[0].strides = __pyx_pybuffernd_keys.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_keys.diminfo[0].shape = __pyx_pybuffernd_keys.rcbuffer->pybuffer.shape[0]; __pyx_pybuffernd_keys.diminfo[1].strides = __pyx_pybuffernd_keys.rcbuffer->pybuffer.strides[1]; __pyx_pybuffernd_keys.diminfo[1].shape = __pyx_pybuffernd_keys.rcbuffer->pybuffer.shape[1]; + } + } + __pyx_v_keys = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":309 + * cdef np.ndarray[uint32_t, ndim=2] keys = self.alloc((ids.shape[0], 4), dtype='uint32') + * cdef int i + * cdef uint32_t* dest = keys.data # <<<<<<<<<<<<<< + * for i in range(len(ids)): + * MurmurHash3_x86_128_uint64(ids[i], seed, &dest[i*4]) +*/ + __pyx_v_dest = ((uint32_t *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_keys))); + + /* "thinc/backends/numpy_ops.pyx":310 + * cdef int i + * cdef uint32_t* dest = keys.data + * for i in range(len(ids)): # <<<<<<<<<<<<<< + * MurmurHash3_x86_128_uint64(ids[i], seed, &dest[i*4]) + * return keys +*/ + __pyx_t_6 = __Pyx_MemoryView_Len(__pyx_v_ids); + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + + /* "thinc/backends/numpy_ops.pyx":311 + * cdef uint32_t* dest = keys.data + * for i in range(len(ids)): + * MurmurHash3_x86_128_uint64(ids[i], seed, &dest[i*4]) # <<<<<<<<<<<<<< + * return keys + * +*/ + __pyx_t_9 = __pyx_v_i; + __pyx_f_5thinc_8backends_9numpy_ops_MurmurHash3_x86_128_uint64((*((uint64_t const *) ( /* dim=0 */ ((char *) (((uint64_t const *) __pyx_v_ids.data) + __pyx_t_9)) ))), __pyx_v_seed, (&(__pyx_v_dest[(__pyx_v_i * 4)]))); + } + + /* "thinc/backends/numpy_ops.pyx":312 + * for i in range(len(ids)): + * MurmurHash3_x86_128_uint64(ids[i], seed, &dest[i*4]) + * return keys # <<<<<<<<<<<<<< + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_keys); + __pyx_r = ((PyObject *)__pyx_v_keys); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":302 + * return super().backprop_seq2col(dY, nW, lengths=lengths) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def hash(self, const uint64_t[::1] ids, uint32_t seed): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + { PyObject *__pyx_type, *__pyx_value, *__pyx_tb; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_keys.rcbuffer->pybuffer); + __Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);} + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.hash", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + goto __pyx_L2; + __pyx_L0:; + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_keys.rcbuffer->pybuffer); + __pyx_L2:; + __Pyx_XDECREF((PyObject *)__pyx_v_keys); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":314 + * return keys + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_35reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean, "NumpyOps.reduce_mean(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_35reduce_mean = {"reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_35reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_35reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 314, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 314, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 314, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 314, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 314, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("reduce_mean", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_X, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_X); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 314, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_69reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_69reduce_mean = {"__pyx_fuse_0reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_69reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_69reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_mean (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 314, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_mean", 0) < (0)) __PYX_ERR(0, 314, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_mean", 1, 3, 3, i); __PYX_ERR(0, 314, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 314, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 314, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_mean", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 314, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_68reduce_mean(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_68reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_means = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0reduce_mean", 0); + + /* "thinc/backends/numpy_ops.pyx":315 + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":316 + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":317 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":319 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + __pyx_t_2 = (__pyx_v_B == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_O == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":321 + * if B == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(B, O), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 321, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 321, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_7 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_8, __pyx_t_7, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 321, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_7, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 321, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":319 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":327 + * cdef np.ndarray means + * if reals2d_ft is float2d_t: + * means = numpy.zeros(shape=(B, O), dtype="float32") # <<<<<<<<<<<<<< + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + * else: +*/ + __pyx_t_6 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 327, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 327, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_4 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_6); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_6, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 327, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 327, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 327, __pyx_L1_error) + __pyx_v_means = ((PyArrayObject *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":328 + * if reals2d_ft is float2d_t: + * means = numpy.zeros(shape=(B, O), dtype="float32") + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * means = numpy.zeros(shape=(B, O), dtype="float64") +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 328, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 328, __pyx_L1_error) + } + try { + cpu_reduce_mean(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_means)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 328, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":333 + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + * + * return means # <<<<<<<<<<<<<< + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_means); + __pyx_r = ((PyObject *)__pyx_v_means); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":314 + * return keys + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_means); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_71reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_71reduce_mean = {"__pyx_fuse_1reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_71reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_34reduce_mean}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_71reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_mean (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 314, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_mean", 0) < (0)) __PYX_ERR(0, 314, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_mean", 1, 3, 3, i); __PYX_ERR(0, 314, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 314, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 314, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 314, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 314, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 314, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_mean", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 314, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_70reduce_mean(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_70reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_means = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1reduce_mean", 0); + + /* "thinc/backends/numpy_ops.pyx":315 + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":316 + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":317 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":319 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + __pyx_t_2 = (__pyx_v_B == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_O == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":323 + * return numpy.zeros(shape=(B, O), dtype="float32") + * else: + * return numpy.zeros(shape=(B, O), dtype="float64") # <<<<<<<<<<<<<< + * + * cdef np.ndarray means +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 323, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 323, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_7 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_8, __pyx_t_7, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 323, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_7, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 323, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":319 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":330 + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + * else: + * means = numpy.zeros(shape=(B, O), dtype="float64") # <<<<<<<<<<<<<< + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + * +*/ + __pyx_t_6 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 330, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 330, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_4 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_6); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_6, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 330, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 330, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 330, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 330, __pyx_L1_error) + __pyx_v_means = ((PyArrayObject *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":331 + * else: + * means = numpy.zeros(shape=(B, O), dtype="float64") + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return means +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 331, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 331, __pyx_L1_error) + } + try { + cpu_reduce_mean(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_means)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 331, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":333 + * cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + * + * return means # <<<<<<<<<<<<<< + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_means); + __pyx_r = ((PyObject *)__pyx_v_means); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":314 + * return keys + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_means); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":335 + * return means + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_37backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean, "NumpyOps.backprop_reduce_mean(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_37backprop_reduce_mean = {"backprop_reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_37backprop_reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_37backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 335, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 335, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 335, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 335, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 335, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_reduce_mean", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_d_means, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_d_means); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 335, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_75backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_75backprop_reduce_mean = {"__pyx_fuse_0backprop_reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_75backprop_reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_75backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_means = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_mean (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_means,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 335, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_mean", 0) < (0)) __PYX_ERR(0, 335, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_mean", 1, 3, 3, i); __PYX_ERR(0, 335, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_means = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_means.memview)) __PYX_ERR(0, 335, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 335, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_mean", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 335, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_means, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_74backprop_reduce_mean(__pyx_self, __pyx_v_self, __pyx_v_d_means, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_means, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_74backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_means, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0backprop_reduce_mean", 0); + + /* "thinc/backends/numpy_ops.pyx":336 + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_means.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":337 + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_means.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":338 + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":340 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 340, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 340, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 340, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":341 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LT); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 341, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 341, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":342 + * for length in lengths[:B]: + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 342, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":341 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":343 + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":345 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":347 + * if T == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 347, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 347, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 347, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 347, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":345 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":353 + * cdef np.ndarray dX + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + * else: +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 353, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 353, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 353, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 353, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":354 + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * dX = numpy.zeros((T, O), dtype="float64") +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_means.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_means.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_means.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_means.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 354, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 354, __pyx_L1_error) + } + cpu_backprop_reduce_mean(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_d_means.data + __pyx_t_7 * __pyx_v_d_means.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_16)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + + /* "thinc/backends/numpy_ops.pyx":359 + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":335 + * return means + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_77backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_77backprop_reduce_mean = {"__pyx_fuse_1backprop_reduce_mean", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_77backprop_reduce_mean, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_36backprop_reduce_mean}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_77backprop_reduce_mean(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_means = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_mean (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_means,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 335, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_mean", 0) < (0)) __PYX_ERR(0, 335, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_mean", 1, 3, 3, i); __PYX_ERR(0, 335, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 335, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 335, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 335, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_means = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_means.memview)) __PYX_ERR(0, 335, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 335, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_mean", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 335, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_means, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_76backprop_reduce_mean(__pyx_self, __pyx_v_self, __pyx_v_d_means, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_means, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_76backprop_reduce_mean(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_means, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1backprop_reduce_mean", 0); + + /* "thinc/backends/numpy_ops.pyx":336 + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_means.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":337 + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_means.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":338 + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":340 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 340, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 340, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 340, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":341 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LT); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 341, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 341, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":342 + * for length in lengths[:B]: + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 342, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 342, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":341 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":343 + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 343, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":345 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":349 + * return numpy.zeros(shape=(T, O), dtype="float32") + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 349, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 349, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 349, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 349, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":345 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":356 + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + * else: + * dX = numpy.zeros((T, O), dtype="float64") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + * +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 356, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 356, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 356, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 356, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":357 + * else: + * dX = numpy.zeros((T, O), dtype="float64") + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return dX +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_means.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_means.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_means.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_means.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 357, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 357, __pyx_L1_error) + } + cpu_backprop_reduce_mean(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_d_means.data + __pyx_t_7 * __pyx_v_d_means.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_16)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + + /* "thinc/backends/numpy_ops.pyx":359 + * cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":335 + * return means + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_mean", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":361 + * return dX + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_39reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum, "NumpyOps.reduce_sum(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_39reduce_sum = {"reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_39reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_39reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 361, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 361, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 361, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 361, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 361, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("reduce_sum", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 361, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 361, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_X, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 361, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_X); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 361, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 361, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_81reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_81reduce_sum = {"__pyx_fuse_0reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_81reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_81reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_sum (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 361, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_sum", 0) < (0)) __PYX_ERR(0, 361, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_sum", 1, 3, 3, i); __PYX_ERR(0, 361, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 361, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 361, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_sum", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 361, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_80reduce_sum(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_80reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_sums = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0reduce_sum", 0); + + /* "thinc/backends/numpy_ops.pyx":362 + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":363 + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":364 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":366 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + __pyx_t_2 = (__pyx_v_B == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_O == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":368 + * if B == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(B, O), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 368, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 368, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_7 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_8, __pyx_t_7, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 368, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_7, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 368, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":366 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":374 + * cdef np.ndarray sums + * if reals2d_ft is float2d_t: + * sums = numpy.zeros(shape=(B, O), dtype="float32") # <<<<<<<<<<<<<< + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + * else: +*/ + __pyx_t_6 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 374, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 374, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_4 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_6); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_6, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 374, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 374, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 374, __pyx_L1_error) + __pyx_v_sums = ((PyArrayObject *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":375 + * if reals2d_ft is float2d_t: + * sums = numpy.zeros(shape=(B, O), dtype="float32") + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * sums = numpy.zeros(shape=(B, O), dtype="float64") +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 375, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 375, __pyx_L1_error) + } + try { + cpu_reduce_sum(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_sums)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 375, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":380 + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + * + * return sums # <<<<<<<<<<<<<< + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_sums); + __pyx_r = ((PyObject *)__pyx_v_sums); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":361 + * return dX + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_sums); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_83reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_83reduce_sum = {"__pyx_fuse_1reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_83reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_38reduce_sum}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_83reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_sum (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 361, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_sum", 0) < (0)) __PYX_ERR(0, 361, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_sum", 1, 3, 3, i); __PYX_ERR(0, 361, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 361, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 361, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 361, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 361, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 361, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_sum", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 361, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_82reduce_sum(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_82reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_sums = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1reduce_sum", 0); + + /* "thinc/backends/numpy_ops.pyx":362 + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":363 + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":364 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":366 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + __pyx_t_2 = (__pyx_v_B == 0); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_O == 0); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":370 + * return numpy.zeros(shape=(B, O), dtype="float32") + * else: + * return numpy.zeros(shape=(B, O), dtype="float64") # <<<<<<<<<<<<<< + * + * cdef np.ndarray sums +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 370, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 370, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_7 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_8, __pyx_t_7, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 370, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_7, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 370, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":366 + * cdef int T = X.shape[0] + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":377 + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + * else: + * sums = numpy.zeros(shape=(B, O), dtype="float64") # <<<<<<<<<<<<<< + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + * +*/ + __pyx_t_6 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 377, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 377, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_4 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_6); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_6, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 377, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 377, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 377, __pyx_L1_error) + __pyx_v_sums = ((PyArrayObject *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":378 + * else: + * sums = numpy.zeros(shape=(B, O), dtype="float64") + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return sums +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 378, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 378, __pyx_L1_error) + } + try { + cpu_reduce_sum(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_sums)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 378, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":380 + * cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + * + * return sums # <<<<<<<<<<<<<< + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_sums); + __pyx_r = ((PyObject *)__pyx_v_sums); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":361 + * return dX + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_sums); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":382 + * return sums + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_41backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum, "NumpyOps.backprop_reduce_sum(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_41backprop_reduce_sum = {"backprop_reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_41backprop_reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_41backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 382, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 382, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 382, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 382, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 382, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_reduce_sum", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 382, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 382, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_d_sums, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 382, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_d_sums); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 382, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 382, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_87backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_87backprop_reduce_sum = {"__pyx_fuse_0backprop_reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_87backprop_reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_87backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_sums = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_sum (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_sums,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 382, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_sum", 0) < (0)) __PYX_ERR(0, 382, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_sum", 1, 3, 3, i); __PYX_ERR(0, 382, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_sums = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_sums.memview)) __PYX_ERR(0, 382, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 382, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_sum", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 382, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_sums, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_86backprop_reduce_sum(__pyx_self, __pyx_v_self, __pyx_v_d_sums, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_sums, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_86backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_sums, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0backprop_reduce_sum", 0); + + /* "thinc/backends/numpy_ops.pyx":383 + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_sums.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":384 + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_sums.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":385 + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":387 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 387, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 387, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":388 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LT); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 388, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":389 + * for length in lengths[:B]: + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 389, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":388 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":390 + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":392 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":394 + * if T == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 394, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 394, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 394, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 394, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":392 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":400 + * cdef np.ndarray dX + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + * else: +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 400, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 400, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 400, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 400, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":401 + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * dX = numpy.zeros((T, O), dtype="float64") +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_sums.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_sums.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_sums.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_sums.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 401, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 401, __pyx_L1_error) + } + cpu_backprop_reduce_sum(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_d_sums.data + __pyx_t_7 * __pyx_v_d_sums.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_16)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + + /* "thinc/backends/numpy_ops.pyx":406 + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":382 + * return sums + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_89backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_89backprop_reduce_sum = {"__pyx_fuse_1backprop_reduce_sum", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_89backprop_reduce_sum, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_40backprop_reduce_sum}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_89backprop_reduce_sum(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_sums = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_sum (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_sums,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 382, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_sum", 0) < (0)) __PYX_ERR(0, 382, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_sum", 1, 3, 3, i); __PYX_ERR(0, 382, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 382, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 382, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 382, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_sums = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_sums.memview)) __PYX_ERR(0, 382, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 382, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_sum", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 382, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_sums, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_88backprop_reduce_sum(__pyx_self, __pyx_v_self, __pyx_v_d_sums, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_sums, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_88backprop_reduce_sum(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_sums, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1backprop_reduce_sum", 0); + + /* "thinc/backends/numpy_ops.pyx":383 + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_sums.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":384 + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_sums.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":385 + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":387 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 387, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 387, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":388 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LT); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 388, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":389 + * for length in lengths[:B]: + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 389, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":388 + * + * for length in lengths[:B]: + * if length < 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":390 + * if length < 0: + * raise ValueError(f"all sequence lengths must be >= 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":392 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":396 + * return numpy.zeros(shape=(T, O), dtype="float32") + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 396, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 396, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 396, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 396, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":392 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":403 + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + * else: + * dX = numpy.zeros((T, O), dtype="float64") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + * +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 403, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 403, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 403, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 403, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":404 + * else: + * dX = numpy.zeros((T, O), dtype="float64") + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return dX +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_sums.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_sums.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_sums.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_sums.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 404, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 404, __pyx_L1_error) + } + cpu_backprop_reduce_sum(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_d_sums.data + __pyx_t_7 * __pyx_v_d_sums.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_16)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + + /* "thinc/backends/numpy_ops.pyx":406 + * cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":382 + * return sums + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_sum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":408 + * return dX + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_43reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max, "NumpyOps.reduce_max(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_43reduce_max = {"reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_43reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_43reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 408, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 408, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 408, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 408, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 408, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("reduce_max", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_X, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_X); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 408, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_93reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_93reduce_max = {"__pyx_fuse_0reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_93reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_93reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_max (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 408, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_max", 0) < (0)) __PYX_ERR(0, 408, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_max", 1, 3, 3, i); __PYX_ERR(0, 408, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 408, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 408, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_max", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 408, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_92reduce_max(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_92reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_which = 0; + PyArrayObject *__pyx_v_maxes = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0reduce_max", 0); + + /* "thinc/backends/numpy_ops.pyx":409 + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":410 + * def reduce_max(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":411 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * # Needs to be zero-initialized as we start by assuming that the first element is the max value. +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":414 + * + * # Needs to be zero-initialized as we start by assuming that the first element is the max value. + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 414, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 414, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_2, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_i, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_True, __pyx_t_4, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 414, __pyx_L1_error) + __pyx_v_which = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":416 + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32"), which +*/ + __pyx_t_8 = (__pyx_v_B == 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_8 = (__pyx_v_O == 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L4_bool_binop_done:; + if (__pyx_t_7) { + + /* "thinc/backends/numpy_ops.pyx":418 + * if B == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32"), which # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(B, O), dtype="float64"), which +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 418, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 418, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_3 = 0; + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_9, __pyx_t_3, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 418, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_3, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 418, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 418, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 418, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 418, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":416 + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32"), which +*/ + } + + /* "thinc/backends/numpy_ops.pyx":424 + * cdef np.ndarray maxes + * if reals2d_ft is float2d_t: + * maxes = self.alloc(shape=(B, O), dtype="float32", zeros=False) # <<<<<<<<<<<<<< + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + * else: +*/ + __pyx_t_1 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 424, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 424, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 424, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 424, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_9) != (0)) __PYX_ERR(0, 424, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_9 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_1, NULL}; + __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 424, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_4, __pyx_t_9, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 424, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_9, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 424, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_9, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 424, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 424, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 424, __pyx_L1_error) + __pyx_v_maxes = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":425 + * if reals2d_ft is float2d_t: + * maxes = self.alloc(shape=(B, O), dtype="float32", zeros=False) + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * maxes = self.alloc(shape=(B, O), dtype="float64", zeros=False) +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 425, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 425, __pyx_L1_error) + } + try { + cpu_reduce_max(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_maxes)), ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_which)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 425, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":430 + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + * + * return maxes, which # <<<<<<<<<<<<<< + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF((PyObject *)__pyx_v_maxes); + __Pyx_GIVEREF((PyObject *)__pyx_v_maxes); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_v_maxes)) != (0)) __PYX_ERR(0, 430, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 430, __pyx_L1_error); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":408 + * return dX + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_which); + __Pyx_XDECREF((PyObject *)__pyx_v_maxes); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_95reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_95reduce_max = {"__pyx_fuse_1reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_95reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_42reduce_max}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_95reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("reduce_max (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 408, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "reduce_max", 0) < (0)) __PYX_ERR(0, 408, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("reduce_max", 1, 3, 3, i); __PYX_ERR(0, 408, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 408, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 408, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 408, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_X = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_X.memview)) __PYX_ERR(0, 408, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 408, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("reduce_max", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 408, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_94reduce_max(__pyx_self, __pyx_v_self, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_X, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_94reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_X, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyArrayObject *__pyx_v_which = 0; + PyArrayObject *__pyx_v_maxes = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + Py_ssize_t __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1reduce_max", 0); + + /* "thinc/backends/numpy_ops.pyx":409 + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":410 + * def reduce_max(self, reals2d_ft X, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] # <<<<<<<<<<<<<< + * cdef int T = X.shape[0] + * +*/ + __pyx_v_O = (__pyx_v_X.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":411 + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] + * cdef int T = X.shape[0] # <<<<<<<<<<<<<< + * + * # Needs to be zero-initialized as we start by assuming that the first element is the max value. +*/ + __pyx_v_T = (__pyx_v_X.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":414 + * + * # Needs to be zero-initialized as we start by assuming that the first element is the max value. + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) # <<<<<<<<<<<<<< + * + * if B == 0 or O == 0: +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 414, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 414, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_2, NULL}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_5, __pyx_t_4, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_i, __pyx_t_4, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_True, __pyx_t_4, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 414, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 414, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 414, __pyx_L1_error) + __pyx_v_which = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":416 + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32"), which +*/ + __pyx_t_8 = (__pyx_v_B == 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_8 = (__pyx_v_O == 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L4_bool_binop_done:; + if (__pyx_t_7) { + + /* "thinc/backends/numpy_ops.pyx":420 + * return numpy.zeros(shape=(B, O), dtype="float32"), which + * else: + * return numpy.zeros(shape=(B, O), dtype="float64"), which # <<<<<<<<<<<<<< + * + * cdef np.ndarray maxes +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 420, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 420, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_3 = 0; + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_4, NULL}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_9, __pyx_t_3, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 420, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_3, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 420, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 420, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 420, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 420, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":416 + * cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) + * + * if B == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(B, O), dtype="float32"), which +*/ + } + + /* "thinc/backends/numpy_ops.pyx":427 + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + * else: + * maxes = self.alloc(shape=(B, O), dtype="float64", zeros=False) # <<<<<<<<<<<<<< + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + * +*/ + __pyx_t_1 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_B); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 427, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_9) != (0)) __PYX_ERR(0, 427, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_9 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_1, NULL}; + __pyx_t_9 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_4, __pyx_t_9, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 427, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_9, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 427, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_9, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 427, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_9); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 427, __pyx_L1_error) + __pyx_v_maxes = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":428 + * else: + * maxes = self.alloc(shape=(B, O), dtype="float64", zeros=False) + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return maxes, which +*/ + __pyx_t_10 = 0; + __pyx_t_11 = 0; + __pyx_t_12 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_10 += __pyx_v_X.shape[0]; + if (unlikely(__pyx_t_10 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_X.shape[0])) __pyx_t_12 = 0; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_X.shape[1]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_12 = 1; + } else if (unlikely(__pyx_t_11 >= __pyx_v_X.shape[1])) __pyx_t_12 = 1; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 428, __pyx_L1_error) + } + __pyx_t_13 = 0; + __pyx_t_12 = -1; + if (__pyx_t_13 < 0) { + __pyx_t_13 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_13 < 0)) __pyx_t_12 = 0; + } else if (unlikely(__pyx_t_13 >= __pyx_v_lengths.shape[0])) __pyx_t_12 = 0; + if (unlikely(__pyx_t_12 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_12); + __PYX_ERR(0, 428, __pyx_L1_error) + } + try { + cpu_reduce_max(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_maxes)), ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_which)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_X.data + __pyx_t_10 * __pyx_v_X.strides[0]) )) + __pyx_t_11)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_13)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 428, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":430 + * cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + * + * return maxes, which # <<<<<<<<<<<<<< + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF((PyObject *)__pyx_v_maxes); + __Pyx_GIVEREF((PyObject *)__pyx_v_maxes); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_v_maxes)) != (0)) __PYX_ERR(0, 430, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_which); + __Pyx_GIVEREF((PyObject *)__pyx_v_which); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, ((PyObject *)__pyx_v_which)) != (0)) __PYX_ERR(0, 430, __pyx_L1_error); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":408 + * return dX + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_which); + __Pyx_XDECREF((PyObject *)__pyx_v_maxes); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":432 + * return maxes, which + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_45backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max, "NumpyOps.backprop_reduce_max(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_45backprop_reduce_max = {"backprop_reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_45backprop_reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_45backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 432, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 432, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 432, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, CYTHON_UNUSED PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_reduce_max", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_d_maxes, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_d_maxes); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(4, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 432, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __pyx_ff_match_signatures_single(((PyObject*)__pyx_v_signatures), __pyx_v_dest_sig0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_99backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_99backprop_reduce_max = {"__pyx_fuse_0backprop_reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_99backprop_reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max}; +static PyObject *__pyx_fuse_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_99backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_maxes = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_which = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_max (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_maxes,&__pyx_mstate_global->__pyx_n_u_which,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 432, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_max", 0) < (0)) __PYX_ERR(0, 432, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_max", 1, 4, 4, i); __PYX_ERR(0, 432, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_maxes = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_maxes.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_v_which = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_which.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[3], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_max", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_maxes, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_98backprop_reduce_max(__pyx_self, __pyx_v_self, __pyx_v_d_maxes, __pyx_v_which, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_maxes, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_98backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_d_maxes, __Pyx_memviewslice __pyx_v_which, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + Py_ssize_t __pyx_t_17; + Py_ssize_t __pyx_t_18; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0backprop_reduce_max", 0); + + /* "thinc/backends/numpy_ops.pyx":433 + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_maxes.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":434 + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_maxes.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":435 + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":437 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 437, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 437, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 437, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":438 + * + * for length in lengths[:B]: + * if length <= 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 438, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 438, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":439 + * for length in lengths[:B]: + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g_2, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 439, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":438 + * + * for length in lengths[:B]: + * if length <= 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":440 + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":442 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":444 + * if T == 0 or O == 0: + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") # <<<<<<<<<<<<<< + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 444, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 444, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 444, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 444, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 444, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":442 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":450 + * cdef np.ndarray dX + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + * &lengths[0], B, T, O) +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 450, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 450, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 450, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 450, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 450, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":451 + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], # <<<<<<<<<<<<<< + * &lengths[0], B, T, O) + * else: +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_maxes.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_maxes.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_maxes.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_maxes.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 451, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_17 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_which.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_which.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_17 < 0) { + __pyx_t_17 += __pyx_v_which.shape[1]; + if (unlikely(__pyx_t_17 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_17 >= __pyx_v_which.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 451, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":452 + * dX = numpy.zeros((T, O), dtype="float32") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + * &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * else: + * dX = numpy.zeros((T, O), dtype="float64") +*/ + __pyx_t_18 = 0; + __pyx_t_2 = -1; + if (__pyx_t_18 < 0) { + __pyx_t_18 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_18 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_18 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 452, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":451 + * if reals2d_ft is float2d_t: + * dX = numpy.zeros((T, O), dtype="float32") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], # <<<<<<<<<<<<<< + * &lengths[0], B, T, O) + * else: +*/ + try { + cpu_backprop_reduce_max(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_d_maxes.data + __pyx_t_7 * __pyx_v_d_maxes.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_which.data + __pyx_t_16 * __pyx_v_which.strides[0]) )) + __pyx_t_17)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_18)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 451, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":458 + * &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":432 + * return maxes, which + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_101backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_101backprop_reduce_max = {"__pyx_fuse_1backprop_reduce_max", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_101backprop_reduce_max, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_44backprop_reduce_max}; +static PyObject *__pyx_fuse_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_101backprop_reduce_max(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_maxes = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_which = { 0, 0, { 0 }, { 0 }, { 0 } }; + __Pyx_memviewslice __pyx_v_lengths = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_reduce_max (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_maxes,&__pyx_mstate_global->__pyx_n_u_which,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 432, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_reduce_max", 0) < (0)) __PYX_ERR(0, 432, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_reduce_max", 1, 4, 4, i); __PYX_ERR(0, 432, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 432, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 432, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 432, __pyx_L3_error) + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 432, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_maxes = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_maxes.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_v_which = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_which.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_v_lengths = __Pyx_PyObject_to_MemoryviewSlice_dc_int(values[3], PyBUF_WRITABLE); if (unlikely(!__pyx_v_lengths.memview)) __PYX_ERR(0, 432, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_reduce_max", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 432, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_maxes, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_100backprop_reduce_max(__pyx_self, __pyx_v_self, __pyx_v_d_maxes, __pyx_v_which, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_maxes, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_which, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_lengths, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_100backprop_reduce_max(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_d_maxes, __Pyx_memviewslice __pyx_v_which, __Pyx_memviewslice __pyx_v_lengths) { + int __pyx_v_B; + int __pyx_v_O; + int __pyx_v_T; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + Py_ssize_t __pyx_t_15; + Py_ssize_t __pyx_t_16; + Py_ssize_t __pyx_t_17; + Py_ssize_t __pyx_t_18; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1backprop_reduce_max", 0); + + /* "thinc/backends/numpy_ops.pyx":433 + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): + * cdef int B = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int O = d_maxes.shape[1] + * cdef int T = 0 +*/ + __pyx_v_B = (__pyx_v_lengths.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":434 + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] # <<<<<<<<<<<<<< + * cdef int T = 0 + * +*/ + __pyx_v_O = (__pyx_v_d_maxes.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":435 + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] + * cdef int T = 0 # <<<<<<<<<<<<<< + * + * for length in lengths[:B]: +*/ + __pyx_v_T = 0; + + /* "thinc/backends/numpy_ops.pyx":437 + * cdef int T = 0 + * + * for length in lengths[:B]: # <<<<<<<<<<<<<< + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") +*/ + __pyx_t_1.data = __pyx_v_lengths.data; + __pyx_t_1.memview = __pyx_v_lengths.memview; + __PYX_INC_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_2 = -1; + if (unlikely(__pyx_memoryview_slice_memviewslice( + &__pyx_t_1, + __pyx_v_lengths.shape[0], __pyx_v_lengths.strides[0], __pyx_v_lengths.suboffsets[0], + 0, + 0, + &__pyx_t_2, + 0, + __pyx_v_B, + 0, + 0, + 1, + 0, + 1) < 0)) +{ + __PYX_ERR(0, 437, __pyx_L1_error) +} + +if (unlikely(((PyObject *) __pyx_t_1.memview) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(0, 437, __pyx_L1_error) + } + __pyx_t_3 = __Pyx_MemoryView_Len(__pyx_t_1); + __pyx_t_4 = 1; + __pyx_t_5 = -1L; + while (1) { + if (!__pyx_t_4) break; + __pyx_t_5 = (__pyx_t_5 + 1); + __pyx_t_6 = (__pyx_t_5 < __pyx_t_3); + if (__pyx_t_6) { + __pyx_t_7 = __pyx_t_5; + __pyx_t_8 = __Pyx_PyLong_From_int((*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_t_1.data) + __pyx_t_7)) )))); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 437, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_8); + __pyx_t_8 = 0; + goto __pyx_L5; + } + /*else*/ { + __pyx_t_4 = 0; + goto __pyx_L3_continue; + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":438 + * + * for length in lengths[:B]: + * if length <= 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length +*/ + __pyx_t_8 = PyObject_RichCompare(__pyx_v_length, __pyx_mstate_global->__pyx_int_0, Py_LE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 438, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_6 < 0))) __PYX_ERR(0, 438, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(__pyx_t_6)) { + + /* "thinc/backends/numpy_ops.pyx":439 + * for length in lengths[:B]: + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") # <<<<<<<<<<<<<< + * T += length + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = __Pyx_PyObject_FormatSimple(__pyx_v_length, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_all_sequence_lengths_must_be_0_g_2, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_t_11}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(0, 439, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":438 + * + * for length in lengths[:B]: + * if length <= 0: # <<<<<<<<<<<<<< + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length +*/ + } + + /* "thinc/backends/numpy_ops.pyx":440 + * if length <= 0: + * raise ValueError(f"all sequence lengths must be > 0, got {length}") + * T += length # <<<<<<<<<<<<<< + * + * if T == 0 or O == 0: +*/ + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyNumber_InPlaceAdd(__pyx_t_8, __pyx_v_length); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_11); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 440, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_T = __pyx_t_2; + __pyx_L3_continue:; + } + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __pyx_t_1.memview = NULL; __pyx_t_1.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":442 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + __pyx_t_6 = (__pyx_v_T == 0); + if (!__pyx_t_6) { + } else { + __pyx_t_4 = __pyx_t_6; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_6 = (__pyx_v_O == 0); + __pyx_t_4 = __pyx_t_6; + __pyx_L8_bool_binop_done:; + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":446 + * return numpy.zeros(shape=(T, O), dtype="float32") + * else: + * return numpy.zeros(shape=(T, O), dtype="float64") # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = PyTuple_New(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_9); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_9) != (0)) __PYX_ERR(0, 446, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_14, 1, __pyx_t_13) != (0)) __PYX_ERR(0, 446, __pyx_L1_error); + __pyx_t_9 = 0; + __pyx_t_13 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_10))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_10); + assert(__pyx_t_8); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_10); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_10, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 2 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_13 = __Pyx_MakeVectorcallBuilderKwds(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_shape, __pyx_t_14, __pyx_t_13, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 446, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_13, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 446, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_12, (1-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_13); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 446, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + __pyx_r = __pyx_t_11; + __pyx_t_11 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":442 + * T += length + * + * if T == 0 or O == 0: # <<<<<<<<<<<<<< + * if reals2d_ft is float2d_t: + * return numpy.zeros(shape=(T, O), dtype="float32") +*/ + } + + /* "thinc/backends/numpy_ops.pyx":454 + * &lengths[0], B, T, O) + * else: + * dX = numpy.zeros((T, O), dtype="float64") # <<<<<<<<<<<<<< + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + * &lengths[0], B, T, O) +*/ + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_13 = __Pyx_PyLong_From_int(__pyx_v_T); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_O); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_13); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_13) != (0)) __PYX_ERR(0, 454, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 454, __pyx_L1_error); + __pyx_t_13 = 0; + __pyx_t_8 = 0; + __pyx_t_12 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_10 = PyMethod_GET_SELF(__pyx_t_14); + assert(__pyx_t_10); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_10); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_14, __pyx__function); + __pyx_t_12 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_10, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 454, __pyx_L1_error) + __pyx_t_11 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_14, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 454, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (!(likely(((__pyx_t_11) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_11, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 454, __pyx_L1_error) + __pyx_v_dX = ((PyArrayObject *)__pyx_t_11); + __pyx_t_11 = 0; + + /* "thinc/backends/numpy_ops.pyx":455 + * else: + * dX = numpy.zeros((T, O), dtype="float64") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], # <<<<<<<<<<<<<< + * &lengths[0], B, T, O) + * +*/ + __pyx_t_7 = 0; + __pyx_t_15 = 0; + __pyx_t_2 = -1; + if (__pyx_t_7 < 0) { + __pyx_t_7 += __pyx_v_d_maxes.shape[0]; + if (unlikely(__pyx_t_7 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_7 >= __pyx_v_d_maxes.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_15 < 0) { + __pyx_t_15 += __pyx_v_d_maxes.shape[1]; + if (unlikely(__pyx_t_15 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_15 >= __pyx_v_d_maxes.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 455, __pyx_L1_error) + } + __pyx_t_16 = 0; + __pyx_t_17 = 0; + __pyx_t_2 = -1; + if (__pyx_t_16 < 0) { + __pyx_t_16 += __pyx_v_which.shape[0]; + if (unlikely(__pyx_t_16 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_16 >= __pyx_v_which.shape[0])) __pyx_t_2 = 0; + if (__pyx_t_17 < 0) { + __pyx_t_17 += __pyx_v_which.shape[1]; + if (unlikely(__pyx_t_17 < 0)) __pyx_t_2 = 1; + } else if (unlikely(__pyx_t_17 >= __pyx_v_which.shape[1])) __pyx_t_2 = 1; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 455, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":456 + * dX = numpy.zeros((T, O), dtype="float64") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + * &lengths[0], B, T, O) # <<<<<<<<<<<<<< + * + * return dX +*/ + __pyx_t_18 = 0; + __pyx_t_2 = -1; + if (__pyx_t_18 < 0) { + __pyx_t_18 += __pyx_v_lengths.shape[0]; + if (unlikely(__pyx_t_18 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_18 >= __pyx_v_lengths.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 456, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":455 + * else: + * dX = numpy.zeros((T, O), dtype="float64") + * cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], # <<<<<<<<<<<<<< + * &lengths[0], B, T, O) + * +*/ + try { + cpu_backprop_reduce_max(((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_d_maxes.data + __pyx_t_7 * __pyx_v_d_maxes.strides[0]) )) + __pyx_t_15)) )))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_which.data + __pyx_t_16 * __pyx_v_which.strides[0]) )) + __pyx_t_17)) )))), (&(*((int *) ( /* dim=0 */ ((char *) (((int *) __pyx_v_lengths.data) + __pyx_t_18)) )))), __pyx_v_B, __pyx_v_T, __pyx_v_O); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 455, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":458 + * &lengths[0], B, T, O) + * + * return dX # <<<<<<<<<<<<<< + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __pyx_r = ((PyObject *)__pyx_v_dX); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":432 + * return maxes, which + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __PYX_XCLEAR_MEMVIEW(&__pyx_t_1, 1); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.backprop_reduce_max", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_47gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add, "NumpyOps.gather_add(signatures, args, kwargs, defaults, _fused_sigindex={})"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_47gather_add = {"gather_add", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_47gather_add, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_47gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_signatures = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + CYTHON_UNUSED PyObject *__pyx_v_defaults = 0; + PyObject *__pyx_v__fused_sigindex = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_fused_cpdef (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_signatures,&__pyx_mstate_global->__pyx_n_u_args,&__pyx_mstate_global->__pyx_n_u_kwargs,&__pyx_mstate_global->__pyx_n_u_defaults,&__pyx_mstate_global->__pyx_n_u_fused_sigindex,0}; + struct __pyx_defaults *__pyx_dynamic_args = __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_self); + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 460, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_fused_cpdef", 0) < (0)) __PYX_ERR(0, 460, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, i); __PYX_ERR(0, 460, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 460, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(__pyx_dynamic_args->arg0); + } + __pyx_v_signatures = values[0]; + __pyx_v_args = values[1]; + __pyx_v_kwargs = values[2]; + __pyx_v_defaults = values[3]; + __pyx_v__fused_sigindex = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_fused_cpdef", 0, 4, 5, __pyx_nargs); __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add(__pyx_self, __pyx_v_signatures, __pyx_v_args, __pyx_v_kwargs, __pyx_v_defaults, __pyx_v__fused_sigindex); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_signatures, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs, CYTHON_UNUSED PyObject *__pyx_v_defaults, PyObject *__pyx_v__fused_sigindex) { + PyTypeObject *__pyx_v_ndarray = 0; + PyObject *__pyx_v_arg = NULL; + PyObject *__pyx_v_dest_sig0 = NULL; + PyObject *__pyx_v_dest_sig1 = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9[4]; + PyObject *__pyx_t_10 = NULL; + size_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("gather_add", 0); + __Pyx_INCREF(__pyx_v_kwargs); + __pyx_t_2 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_kwargs); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_3 = (!__pyx_t_2); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + __Pyx_INCREF(Py_None); + __Pyx_DECREF_SET(__pyx_v_kwargs, Py_None); + } + __pyx_t_4 = ((PyObject *)__Pyx_ImportNumPyArrayTypeIfAvailable()); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_ndarray = ((PyTypeObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_1 = (1 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L7_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_table, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L7_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_table); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_arg = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L6; + } + /*else*/ { + __pyx_t_6 = NULL; + __pyx_t_7 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_7; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_10 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_7) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_10}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_L6:; + __pyx_t_4 = __pyx_ff_map_fused_8b136e_2_2_a0ee6f__93948b__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig0 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_1 = (2 < __pyx_t_5); + if (__pyx_t_1) { + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_Tuple(((PyObject*)__pyx_v_args), 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF_SET(__pyx_v_arg, __pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L9; + } + __pyx_t_3 = (__pyx_v_kwargs != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L10_bool_binop_done; + } + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_3 = (__Pyx_PyDict_ContainsTF(__pyx_mstate_global->__pyx_n_u_indices, ((PyObject*)__pyx_v_kwargs), Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_1 = __pyx_t_3; + __pyx_L10_bool_binop_done:; + if (likely(__pyx_t_1)) { + if (unlikely(__pyx_v_kwargs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_PyDict_GetItem(((PyObject*)__pyx_v_kwargs), __pyx_mstate_global->__pyx_n_u_indices); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF_SET(__pyx_v_arg, __pyx_t_4); + __pyx_t_4 = 0; + goto __pyx_L9; + } + /*else*/ { + __pyx_t_10 = NULL; + __pyx_t_6 = __Pyx_PyUnicode_From_long(3, 0, ' ', 'd'); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__pyx_v_args == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(((PyObject*)__pyx_v_args)); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_8 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_t_5, 0, ' ', 'd'); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9[0] = __pyx_mstate_global->__pyx_kp_u_Expected_at_least; + __pyx_t_9[1] = __pyx_t_6; + __pyx_t_9[2] = __pyx_mstate_global->__pyx_kp_u_arguments_got; + __pyx_t_9[3] = __pyx_t_8; + __pyx_t_7 = __Pyx_PyUnicode_Join(__pyx_t_9, 4, 18 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_6) + 16 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8), 127); + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_10, __pyx_t_7}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_TypeError)), __pyx_callargs+__pyx_t_11, (2-__pyx_t_11) | (__pyx_t_11*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(0, 460, __pyx_L1_error) + } + __pyx_L9:; + __pyx_t_4 = __pyx_ff_map_fused_d9e37b_2_2_ea222d__48c62c__5thinc_8backends_9numpy_ops__dunder_py__etc(__pyx_v_arg, __pyx_v_ndarray); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_dest_sig1 = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_dest_sig0); + __Pyx_GIVEREF(__pyx_v_dest_sig0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_dest_sig0) != (0)) __PYX_ERR(0, 460, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_dest_sig1); + __Pyx_GIVEREF(__pyx_v_dest_sig1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_dest_sig1) != (0)) __PYX_ERR(0, 460, __pyx_L1_error); + __pyx_t_7 = __pyx_ff_match_signatures(((PyObject*)__pyx_v_signatures), ((PyObject*)__pyx_t_4), ((PyObject*)__pyx_v__fused_sigindex)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.__pyx_fused_cpdef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_ndarray); + __Pyx_XDECREF(__pyx_v_arg); + __Pyx_XDECREF(__pyx_v_dest_sig0); + __Pyx_XDECREF(__pyx_v_dest_sig1); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_105gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_105gather_add = {"__pyx_fuse_0_0gather_add", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_105gather_add, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add}; +static PyObject *__pyx_fuse_0_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_105gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("gather_add (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_table,&__pyx_mstate_global->__pyx_n_u_indices,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 460, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "gather_add", 0) < (0)) __PYX_ERR(0, 460, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, i); __PYX_ERR(0, 460, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_table = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_table.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_v_indices = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_indices.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_104gather_add(__pyx_self, __pyx_v_self, __pyx_v_table, __pyx_v_indices); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_104gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas = 0; + Py_ssize_t __pyx_v_rows; + Py_ssize_t __pyx_v_dims; + PyArrayObject *__pyx_v_output = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + Py_ssize_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0_0gather_add", 0); + + /* "thinc/backends/numpy_ops.pyx":461 + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() # <<<<<<<<<<<<<< + * rows = indices.shape[0] + * dims = table.shape[1] +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cblas, __pyx_callargs+__pyx_t_3, (1-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 461, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas))))) __PYX_ERR(0, 461, __pyx_L1_error) + __pyx_v_cblas = ((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":462 + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] # <<<<<<<<<<<<<< + * dims = table.shape[1] + * +*/ + __pyx_v_rows = (__pyx_v_indices.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":463 + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] + * dims = table.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray output +*/ + __pyx_v_dims = (__pyx_v_table.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":467 + * cdef np.ndarray output + * if reals2d_ft is float2d_t: + * output = self.xp.zeros((rows, dims), dtype="float32") # <<<<<<<<<<<<<< + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_v_rows); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_v_dims); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 467, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 467, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 467, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 467, __pyx_L1_error) + __pyx_v_output = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":468 + * if reals2d_ft is float2d_t: + * output = self.xp.zeros((rows, dims), dtype="float32") + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], # <<<<<<<<<<<<<< + * table.shape[0], dims, rows, indices.shape[1]) + * else: +*/ + __pyx_t_8 = 0; + __pyx_t_9 = 0; + __pyx_t_10 = -1; + if (__pyx_t_8 < 0) { + __pyx_t_8 += __pyx_v_table.shape[0]; + if (unlikely(__pyx_t_8 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_8 >= __pyx_v_table.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_table.shape[1]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_9 >= __pyx_v_table.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 468, __pyx_L1_error) + } + __pyx_t_11 = 0; + __pyx_t_12 = 0; + __pyx_t_10 = -1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_indices.shape[0]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_11 >= __pyx_v_indices.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_indices.shape[1]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_12 >= __pyx_v_indices.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 468, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":469 + * output = self.xp.zeros((rows, dims), dtype="float32") + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) # <<<<<<<<<<<<<< + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") +*/ + try { + cpu_gather_add(__pyx_f_5thinc_8backends_5cblas_saxpy(__pyx_v_cblas), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_output)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_table.data + __pyx_t_8 * __pyx_v_table.strides[0]) )) + __pyx_t_9)) )))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_11 * __pyx_v_indices.strides[0]) )) + __pyx_t_12)) )))), (__pyx_v_table.shape[0]), __pyx_v_dims, __pyx_v_rows, (__pyx_v_indices.shape[1])); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 468, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":475 + * table.shape[0], dims, rows, indices.shape[1]) + * + * return output # <<<<<<<<<<<<<< + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_output); + __pyx_r = ((PyObject *)__pyx_v_output); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_cblas); + __Pyx_XDECREF((PyObject *)__pyx_v_output); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_0_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_107gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_0_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_107gather_add = {"__pyx_fuse_0_1gather_add", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_0_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_107gather_add, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add}; +static PyObject *__pyx_fuse_0_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_107gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("gather_add (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_table,&__pyx_mstate_global->__pyx_n_u_indices,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 460, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "gather_add", 0) < (0)) __PYX_ERR(0, 460, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, i); __PYX_ERR(0, 460, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_table = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_table.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_v_indices = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_indices.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_106gather_add(__pyx_self, __pyx_v_self, __pyx_v_table, __pyx_v_indices); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_106gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_float2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas = 0; + Py_ssize_t __pyx_v_rows; + Py_ssize_t __pyx_v_dims; + PyArrayObject *__pyx_v_output = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + Py_ssize_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0_1gather_add", 0); + + /* "thinc/backends/numpy_ops.pyx":461 + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() # <<<<<<<<<<<<<< + * rows = indices.shape[0] + * dims = table.shape[1] +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cblas, __pyx_callargs+__pyx_t_3, (1-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 461, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas))))) __PYX_ERR(0, 461, __pyx_L1_error) + __pyx_v_cblas = ((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":462 + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] # <<<<<<<<<<<<<< + * dims = table.shape[1] + * +*/ + __pyx_v_rows = (__pyx_v_indices.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":463 + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] + * dims = table.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray output +*/ + __pyx_v_dims = (__pyx_v_table.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":467 + * cdef np.ndarray output + * if reals2d_ft is float2d_t: + * output = self.xp.zeros((rows, dims), dtype="float32") # <<<<<<<<<<<<<< + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_v_rows); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_v_dims); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 467, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 467, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 467, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 467, __pyx_L1_error) + __pyx_v_output = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":468 + * if reals2d_ft is float2d_t: + * output = self.xp.zeros((rows, dims), dtype="float32") + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], # <<<<<<<<<<<<<< + * table.shape[0], dims, rows, indices.shape[1]) + * else: +*/ + __pyx_t_8 = 0; + __pyx_t_9 = 0; + __pyx_t_10 = -1; + if (__pyx_t_8 < 0) { + __pyx_t_8 += __pyx_v_table.shape[0]; + if (unlikely(__pyx_t_8 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_8 >= __pyx_v_table.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_table.shape[1]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_9 >= __pyx_v_table.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 468, __pyx_L1_error) + } + __pyx_t_11 = 0; + __pyx_t_12 = 0; + __pyx_t_10 = -1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_indices.shape[0]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_11 >= __pyx_v_indices.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_indices.shape[1]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_12 >= __pyx_v_indices.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 468, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":469 + * output = self.xp.zeros((rows, dims), dtype="float32") + * cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) # <<<<<<<<<<<<<< + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") +*/ + try { + cpu_gather_add(__pyx_f_5thinc_8backends_5cblas_saxpy(__pyx_v_cblas), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_output)), (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_table.data + __pyx_t_8 * __pyx_v_table.strides[0]) )) + __pyx_t_9)) )))), (&(*((unsigned int *) ( /* dim=1 */ ((char *) (((unsigned int *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_11 * __pyx_v_indices.strides[0]) )) + __pyx_t_12)) )))), (__pyx_v_table.shape[0]), __pyx_v_dims, __pyx_v_rows, (__pyx_v_indices.shape[1])); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 468, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":475 + * table.shape[0], dims, rows, indices.shape[1]) + * + * return output # <<<<<<<<<<<<<< + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_output); + __pyx_r = ((PyObject *)__pyx_v_output); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_cblas); + __Pyx_XDECREF((PyObject *)__pyx_v_output); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_109gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_109gather_add = {"__pyx_fuse_1_0gather_add", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_109gather_add, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add}; +static PyObject *__pyx_fuse_1_0__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_109gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("gather_add (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_table,&__pyx_mstate_global->__pyx_n_u_indices,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 460, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "gather_add", 0) < (0)) __PYX_ERR(0, 460, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, i); __PYX_ERR(0, 460, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_table = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_table.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_v_indices = __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_indices.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_108gather_add(__pyx_self, __pyx_v_self, __pyx_v_table, __pyx_v_indices); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_108gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_int2d_t __pyx_v_indices) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas = 0; + Py_ssize_t __pyx_v_rows; + Py_ssize_t __pyx_v_dims; + PyArrayObject *__pyx_v_output = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + Py_ssize_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1_0gather_add", 0); + + /* "thinc/backends/numpy_ops.pyx":461 + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() # <<<<<<<<<<<<<< + * rows = indices.shape[0] + * dims = table.shape[1] +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cblas, __pyx_callargs+__pyx_t_3, (1-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 461, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas))))) __PYX_ERR(0, 461, __pyx_L1_error) + __pyx_v_cblas = ((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":462 + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] # <<<<<<<<<<<<<< + * dims = table.shape[1] + * +*/ + __pyx_v_rows = (__pyx_v_indices.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":463 + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] + * dims = table.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray output +*/ + __pyx_v_dims = (__pyx_v_table.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":471 + * table.shape[0], dims, rows, indices.shape[1]) + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") # <<<<<<<<<<<<<< + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_v_rows); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_v_dims); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 471, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 471, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 471, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 471, __pyx_L1_error) + __pyx_v_output = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":472 + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], # <<<<<<<<<<<<<< + * table.shape[0], dims, rows, indices.shape[1]) + * +*/ + __pyx_t_8 = 0; + __pyx_t_9 = 0; + __pyx_t_10 = -1; + if (__pyx_t_8 < 0) { + __pyx_t_8 += __pyx_v_table.shape[0]; + if (unlikely(__pyx_t_8 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_8 >= __pyx_v_table.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_table.shape[1]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_9 >= __pyx_v_table.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 472, __pyx_L1_error) + } + __pyx_t_11 = 0; + __pyx_t_12 = 0; + __pyx_t_10 = -1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_indices.shape[0]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_11 >= __pyx_v_indices.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_indices.shape[1]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_12 >= __pyx_v_indices.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 472, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":473 + * output = self.xp.zeros((rows, dims), dtype="float64") + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) # <<<<<<<<<<<<<< + * + * return output +*/ + try { + cpu_gather_add(__pyx_f_5thinc_8backends_5cblas_daxpy(__pyx_v_cblas), ((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_output)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_table.data + __pyx_t_8 * __pyx_v_table.strides[0]) )) + __pyx_t_9)) )))), (&(*((int *) ( /* dim=1 */ ((char *) (((int *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_11 * __pyx_v_indices.strides[0]) )) + __pyx_t_12)) )))), (__pyx_v_table.shape[0]), __pyx_v_dims, __pyx_v_rows, (__pyx_v_indices.shape[1])); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 472, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":475 + * table.shape[0], dims, rows, indices.shape[1]) + * + * return output # <<<<<<<<<<<<<< + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_output); + __pyx_r = ((PyObject *)__pyx_v_output); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_cblas); + __Pyx_XDECREF((PyObject *)__pyx_v_output); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_fuse_1_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_111gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_fuse_1_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_111gather_add = {"__pyx_fuse_1_1gather_add", (PyCFunction)(void(*)(void))(PyCFunctionWithKeywords)__pyx_fuse_1_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_111gather_add, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_46gather_add}; +static PyObject *__pyx_fuse_1_1__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_111gather_add(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_self = 0; + __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices = { 0, 0, { 0 }, { 0 }, { 0 } }; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("gather_add (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_table,&__pyx_mstate_global->__pyx_n_u_indices,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 460, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "gather_add", 0) < (0)) __PYX_ERR(0, 460, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, i); __PYX_ERR(0, 460, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 460, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 460, __pyx_L3_error) + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 460, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_table = __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_table.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_v_indices = __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(values[2], PyBUF_WRITABLE); if (unlikely(!__pyx_v_indices.memview)) __PYX_ERR(0, 460, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("gather_add", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 460, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_110gather_add(__pyx_self, __pyx_v_self, __pyx_v_table, __pyx_v_indices); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_table, 1); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_indices, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_110gather_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __pyx_t_5thinc_8backends_9numpy_ops_double2d_t __pyx_v_table, __pyx_t_5thinc_8backends_9numpy_ops_uint2d_t __pyx_v_indices) { + struct __pyx_obj_5thinc_8backends_5cblas_CBlas *__pyx_v_cblas = 0; + Py_ssize_t __pyx_v_rows; + Py_ssize_t __pyx_v_dims; + PyArrayObject *__pyx_v_output = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + Py_ssize_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1_1gather_add", 0); + + /* "thinc/backends/numpy_ops.pyx":461 + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() # <<<<<<<<<<<<<< + * rows = indices.shape[0] + * dims = table.shape[1] +*/ + __pyx_t_2 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_cblas, __pyx_callargs+__pyx_t_3, (1-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 461, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_5cblas_CBlas))))) __PYX_ERR(0, 461, __pyx_L1_error) + __pyx_v_cblas = ((struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":462 + * def gather_add(self, reals2d_ft table, ints2d_ft indices): + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] # <<<<<<<<<<<<<< + * dims = table.shape[1] + * +*/ + __pyx_v_rows = (__pyx_v_indices.shape[0]); + + /* "thinc/backends/numpy_ops.pyx":463 + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] + * dims = table.shape[1] # <<<<<<<<<<<<<< + * + * cdef np.ndarray output +*/ + __pyx_v_dims = (__pyx_v_table.shape[1]); + + /* "thinc/backends/numpy_ops.pyx":471 + * table.shape[0], dims, rows, indices.shape[1]) + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") # <<<<<<<<<<<<<< + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_v_rows); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_v_dims); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 471, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 471, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float64, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 471, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 471, __pyx_L1_error) + __pyx_v_output = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":472 + * else: + * output = self.xp.zeros((rows, dims), dtype="float64") + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], # <<<<<<<<<<<<<< + * table.shape[0], dims, rows, indices.shape[1]) + * +*/ + __pyx_t_8 = 0; + __pyx_t_9 = 0; + __pyx_t_10 = -1; + if (__pyx_t_8 < 0) { + __pyx_t_8 += __pyx_v_table.shape[0]; + if (unlikely(__pyx_t_8 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_8 >= __pyx_v_table.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_9 < 0) { + __pyx_t_9 += __pyx_v_table.shape[1]; + if (unlikely(__pyx_t_9 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_9 >= __pyx_v_table.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 472, __pyx_L1_error) + } + __pyx_t_11 = 0; + __pyx_t_12 = 0; + __pyx_t_10 = -1; + if (__pyx_t_11 < 0) { + __pyx_t_11 += __pyx_v_indices.shape[0]; + if (unlikely(__pyx_t_11 < 0)) __pyx_t_10 = 0; + } else if (unlikely(__pyx_t_11 >= __pyx_v_indices.shape[0])) __pyx_t_10 = 0; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_indices.shape[1]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_10 = 1; + } else if (unlikely(__pyx_t_12 >= __pyx_v_indices.shape[1])) __pyx_t_10 = 1; + if (unlikely(__pyx_t_10 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_10); + __PYX_ERR(0, 472, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":473 + * output = self.xp.zeros((rows, dims), dtype="float64") + * cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + * table.shape[0], dims, rows, indices.shape[1]) # <<<<<<<<<<<<<< + * + * return output +*/ + try { + cpu_gather_add(__pyx_f_5thinc_8backends_5cblas_daxpy(__pyx_v_cblas), ((double *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_output)), (&(*((double *) ( /* dim=1 */ ((char *) (((double *) ( /* dim=0 */ (__pyx_v_table.data + __pyx_t_8 * __pyx_v_table.strides[0]) )) + __pyx_t_9)) )))), (&(*((unsigned int *) ( /* dim=1 */ ((char *) (((unsigned int *) ( /* dim=0 */ (__pyx_v_indices.data + __pyx_t_11 * __pyx_v_indices.strides[0]) )) + __pyx_t_12)) )))), (__pyx_v_table.shape[0]), __pyx_v_dims, __pyx_v_rows, (__pyx_v_indices.shape[1])); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 472, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":475 + * table.shape[0], dims, rows, indices.shape[1]) + * + * return output # <<<<<<<<<<<<<< + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_output); + __pyx_r = ((PyObject *)__pyx_v_output); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.gather_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_cblas); + __Pyx_XDECREF((PyObject *)__pyx_v_output); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":477 + * return output + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): # <<<<<<<<<<<<<< + * if table.dtype == 'float32' \ + * and indices.dtype == 'int32' \ +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_49scatter_add(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_48scatter_add, "NumpyOps.scatter_add(self, ndarray table, ndarray indices, ndarray values)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_49scatter_add = {"scatter_add", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_49scatter_add, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_48scatter_add}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_49scatter_add(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_table = 0; + PyArrayObject *__pyx_v_indices = 0; + PyArrayObject *__pyx_v_values = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("scatter_add (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_table,&__pyx_mstate_global->__pyx_n_u_indices,&__pyx_mstate_global->__pyx_n_u_values,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 477, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 477, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 477, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 477, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 477, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "scatter_add", 0) < (0)) __PYX_ERR(0, 477, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("scatter_add", 1, 4, 4, i); __PYX_ERR(0, 477, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 477, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 477, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 477, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 477, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_table = ((PyArrayObject *)values[1]); + __pyx_v_indices = ((PyArrayObject *)values[2]); + __pyx_v_values = ((PyArrayObject *)values[3]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("scatter_add", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 477, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.scatter_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_table), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "table", 0))) __PYX_ERR(0, 477, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_indices), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "indices", 0))) __PYX_ERR(0, 477, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_values), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "values", 0))) __PYX_ERR(0, 477, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_48scatter_add(__pyx_self, __pyx_v_self, __pyx_v_table, __pyx_v_indices, __pyx_v_values); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_48scatter_add(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyArrayObject *__pyx_v_table, PyArrayObject *__pyx_v_indices, PyArrayObject *__pyx_v_values) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("scatter_add", 0); + + /* "thinc/backends/numpy_ops.pyx":478 + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): + * if table.dtype == 'float32' \ # <<<<<<<<<<<<<< + * and indices.dtype == 'int32' \ + * and values.dtype == 'float32' \ +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_table), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 478, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 478, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":479 + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): + * if table.dtype == 'float32' \ + * and indices.dtype == 'int32' \ # <<<<<<<<<<<<<< + * and values.dtype == 'float32' \ + * and table.flags.c_contiguous \ +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_indices), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_int32, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 479, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":480 + * if table.dtype == 'float32' \ + * and indices.dtype == 'int32' \ + * and values.dtype == 'float32' \ # <<<<<<<<<<<<<< + * and table.flags.c_contiguous \ + * and indices.flags.c_contiguous \ +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_values), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 480, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_float32, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 480, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":481 + * and indices.dtype == 'int32' \ + * and values.dtype == 'float32' \ + * and table.flags.c_contiguous \ # <<<<<<<<<<<<<< + * and indices.flags.c_contiguous \ + * and values.flags.c_contiguous \ +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_table), __pyx_mstate_global->__pyx_n_u_flags); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 481, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_c_contiguous); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 481, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 481, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":482 + * and values.dtype == 'float32' \ + * and table.flags.c_contiguous \ + * and indices.flags.c_contiguous \ # <<<<<<<<<<<<<< + * and values.flags.c_contiguous \ + * and indices.ndim == 1 \ +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_indices), __pyx_mstate_global->__pyx_n_u_flags); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 482, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_c_contiguous); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 482, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 482, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":483 + * and table.flags.c_contiguous \ + * and indices.flags.c_contiguous \ + * and values.flags.c_contiguous \ # <<<<<<<<<<<<<< + * and indices.ndim == 1 \ + * and table.ndim == 2 \ +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_values), __pyx_mstate_global->__pyx_n_u_flags); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 483, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_c_contiguous); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 483, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(0, 483, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":484 + * and indices.flags.c_contiguous \ + * and values.flags.c_contiguous \ + * and indices.ndim == 1 \ # <<<<<<<<<<<<<< + * and table.ndim == 2 \ + * and values.ndim == 2 \ +*/ + __pyx_t_3 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_indices) == 1); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":485 + * and values.flags.c_contiguous \ + * and indices.ndim == 1 \ + * and table.ndim == 2 \ # <<<<<<<<<<<<<< + * and values.ndim == 2 \ + * and values.shape[0] == indices.shape[0] \ +*/ + __pyx_t_3 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_table) == 2); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":486 + * and indices.ndim == 1 \ + * and table.ndim == 2 \ + * and values.ndim == 2 \ # <<<<<<<<<<<<<< + * and values.shape[0] == indices.shape[0] \ + * and values.shape[1] == table.shape[1]: +*/ + __pyx_t_3 = (__pyx_f_5numpy_7ndarray_4ndim_ndim(__pyx_v_values) == 2); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":487 + * and table.ndim == 2 \ + * and values.ndim == 2 \ + * and values.shape[0] == indices.shape[0] \ # <<<<<<<<<<<<<< + * and values.shape[1] == table.shape[1]: + * cpu_scatter_add(table.data, +*/ + __pyx_t_3 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_values)[0]) == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_indices)[0])); + if (__pyx_t_3) { + } else { + __pyx_t_1 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "thinc/backends/numpy_ops.pyx":488 + * and values.ndim == 2 \ + * and values.shape[0] == indices.shape[0] \ + * and values.shape[1] == table.shape[1]: # <<<<<<<<<<<<<< + * cpu_scatter_add(table.data, + * indices.data, values.data, +*/ + __pyx_t_3 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_values)[1]) == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_table)[1])); + __pyx_t_1 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + + /* "thinc/backends/numpy_ops.pyx":478 + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): + * if table.dtype == 'float32' \ # <<<<<<<<<<<<<< + * and indices.dtype == 'int32' \ + * and values.dtype == 'float32' \ +*/ + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":489 + * and values.shape[0] == indices.shape[0] \ + * and values.shape[1] == table.shape[1]: + * cpu_scatter_add(table.data, # <<<<<<<<<<<<<< + * indices.data, values.data, + * indices.shape[0], table.shape[1]) +*/ + __pyx_f_5thinc_8backends_9numpy_ops_cpu_scatter_add(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_table)), ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_indices)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_values)), (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_indices)[0]), (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_table)[1])); + + /* "thinc/backends/numpy_ops.pyx":478 + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): + * if table.dtype == 'float32' \ # <<<<<<<<<<<<<< + * and indices.dtype == 'int32' \ + * and values.dtype == 'float32' \ +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":493 + * indices.shape[0], table.shape[1]) + * else: + * self.xp.add.at(table, indices, values) # <<<<<<<<<<<<<< + * + * @cython.boundscheck(False) +*/ + /*else*/ { + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 493, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_add); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 493, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_2 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, ((PyObject *)__pyx_v_table), ((PyObject *)__pyx_v_indices), ((PyObject *)__pyx_v_values)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_at, __pyx_callargs+__pyx_t_7, (4-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 493, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __pyx_L3:; + + /* "thinc/backends/numpy_ops.pyx":477 + * return output + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): # <<<<<<<<<<<<<< + * if table.dtype == 'float32' \ + * and indices.dtype == 'int32' \ +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.scatter_add", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":495 + * self.xp.add.at(table, indices, values) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def adam(self, np.ndarray[np.float32_t] weights, np.ndarray[np.float32_t] gradient, +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_51adam(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_50adam, "NumpyOps.adam(self, ndarray weights, ndarray gradient, ndarray mom1, ndarray mom2, float beta1, float beta2, float eps, float learn_rate, float mod_rate=1.)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_51adam = {"adam", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_51adam, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_50adam}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_51adam(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v_self = 0; + PyArrayObject *__pyx_v_weights = 0; + PyArrayObject *__pyx_v_gradient = 0; + PyArrayObject *__pyx_v_mom1 = 0; + PyArrayObject *__pyx_v_mom2 = 0; + float __pyx_v_beta1; + float __pyx_v_beta2; + float __pyx_v_eps; + float __pyx_v_learn_rate; + CYTHON_UNUSED float __pyx_v_mod_rate; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[10] = {0,0,0,0,0,0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("adam (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_weights,&__pyx_mstate_global->__pyx_n_u_gradient,&__pyx_mstate_global->__pyx_n_u_mom1,&__pyx_mstate_global->__pyx_n_u_mom2,&__pyx_mstate_global->__pyx_n_u_beta1,&__pyx_mstate_global->__pyx_n_u_beta2,&__pyx_mstate_global->__pyx_n_u_eps,&__pyx_mstate_global->__pyx_n_u_learn_rate,&__pyx_mstate_global->__pyx_n_u_mod_rate,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 495, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 10: + values[9] = __Pyx_ArgRef_FASTCALL(__pyx_args, 9); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[9])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 9: + values[8] = __Pyx_ArgRef_FASTCALL(__pyx_args, 8); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[8])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 8: + values[7] = __Pyx_ArgRef_FASTCALL(__pyx_args, 7); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[7])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 7: + values[6] = __Pyx_ArgRef_FASTCALL(__pyx_args, 6); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 6: + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "adam", 0) < (0)) __PYX_ERR(0, 495, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 9; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("adam", 0, 9, 10, i); __PYX_ERR(0, 495, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 10: + values[9] = __Pyx_ArgRef_FASTCALL(__pyx_args, 9); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[9])) __PYX_ERR(0, 495, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 9: + values[8] = __Pyx_ArgRef_FASTCALL(__pyx_args, 8); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[8])) __PYX_ERR(0, 495, __pyx_L3_error) + values[7] = __Pyx_ArgRef_FASTCALL(__pyx_args, 7); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[7])) __PYX_ERR(0, 495, __pyx_L3_error) + values[6] = __Pyx_ArgRef_FASTCALL(__pyx_args, 6); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[6])) __PYX_ERR(0, 495, __pyx_L3_error) + values[5] = __Pyx_ArgRef_FASTCALL(__pyx_args, 5); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[5])) __PYX_ERR(0, 495, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 495, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 495, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 495, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 495, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 495, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_self = values[0]; + __pyx_v_weights = ((PyArrayObject *)values[1]); + __pyx_v_gradient = ((PyArrayObject *)values[2]); + __pyx_v_mom1 = ((PyArrayObject *)values[3]); + __pyx_v_mom2 = ((PyArrayObject *)values[4]); + __pyx_v_beta1 = __Pyx_PyFloat_AsFloat(values[5]); if (unlikely((__pyx_v_beta1 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 499, __pyx_L3_error) + __pyx_v_beta2 = __Pyx_PyFloat_AsFloat(values[6]); if (unlikely((__pyx_v_beta2 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 499, __pyx_L3_error) + __pyx_v_eps = __Pyx_PyFloat_AsFloat(values[7]); if (unlikely((__pyx_v_eps == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 499, __pyx_L3_error) + __pyx_v_learn_rate = __Pyx_PyFloat_AsFloat(values[8]); if (unlikely((__pyx_v_learn_rate == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 500, __pyx_L3_error) + if (values[9]) { + __pyx_v_mod_rate = __Pyx_PyFloat_AsFloat(values[9]); if (unlikely((__pyx_v_mod_rate == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 500, __pyx_L3_error) + } else { + __pyx_v_mod_rate = ((float)((double)1.)); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("adam", 0, 9, 10, __pyx_nargs); __PYX_ERR(0, 495, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.adam", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_weights), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "weights", 0))) __PYX_ERR(0, 497, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_gradient), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "gradient", 0))) __PYX_ERR(0, 497, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_mom1), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "mom1", 0))) __PYX_ERR(0, 498, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_mom2), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "mom2", 0))) __PYX_ERR(0, 498, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_50adam(__pyx_self, __pyx_v_self, __pyx_v_weights, __pyx_v_gradient, __pyx_v_mom1, __pyx_v_mom2, __pyx_v_beta1, __pyx_v_beta2, __pyx_v_eps, __pyx_v_learn_rate, __pyx_v_mod_rate); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_50adam(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_self, PyArrayObject *__pyx_v_weights, PyArrayObject *__pyx_v_gradient, PyArrayObject *__pyx_v_mom1, PyArrayObject *__pyx_v_mom2, float __pyx_v_beta1, float __pyx_v_beta2, float __pyx_v_eps, float __pyx_v_learn_rate, CYTHON_UNUSED float __pyx_v_mod_rate) { + __Pyx_LocalBuf_ND __pyx_pybuffernd_gradient; + __Pyx_Buffer __pyx_pybuffer_gradient; + __Pyx_LocalBuf_ND __pyx_pybuffernd_mom1; + __Pyx_Buffer __pyx_pybuffer_mom1; + __Pyx_LocalBuf_ND __pyx_pybuffernd_mom2; + __Pyx_Buffer __pyx_pybuffer_mom2; + __Pyx_LocalBuf_ND __pyx_pybuffernd_weights; + __Pyx_Buffer __pyx_pybuffer_weights; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("adam", 0); + __pyx_pybuffer_weights.pybuffer.buf = NULL; + __pyx_pybuffer_weights.refcount = 0; + __pyx_pybuffernd_weights.data = NULL; + __pyx_pybuffernd_weights.rcbuffer = &__pyx_pybuffer_weights; + __pyx_pybuffer_gradient.pybuffer.buf = NULL; + __pyx_pybuffer_gradient.refcount = 0; + __pyx_pybuffernd_gradient.data = NULL; + __pyx_pybuffernd_gradient.rcbuffer = &__pyx_pybuffer_gradient; + __pyx_pybuffer_mom1.pybuffer.buf = NULL; + __pyx_pybuffer_mom1.refcount = 0; + __pyx_pybuffernd_mom1.data = NULL; + __pyx_pybuffernd_mom1.rcbuffer = &__pyx_pybuffer_mom1; + __pyx_pybuffer_mom2.pybuffer.buf = NULL; + __pyx_pybuffer_mom2.refcount = 0; + __pyx_pybuffernd_mom2.data = NULL; + __pyx_pybuffernd_mom2.rcbuffer = &__pyx_pybuffer_mom2; + { + __Pyx_BufFmt_StackElem __pyx_stack[1]; + if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_weights.rcbuffer->pybuffer, (PyObject*)__pyx_v_weights, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) __PYX_ERR(0, 495, __pyx_L1_error) + } + __pyx_pybuffernd_weights.diminfo[0].strides = __pyx_pybuffernd_weights.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_weights.diminfo[0].shape = __pyx_pybuffernd_weights.rcbuffer->pybuffer.shape[0]; + { + __Pyx_BufFmt_StackElem __pyx_stack[1]; + if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_gradient.rcbuffer->pybuffer, (PyObject*)__pyx_v_gradient, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) __PYX_ERR(0, 495, __pyx_L1_error) + } + __pyx_pybuffernd_gradient.diminfo[0].strides = __pyx_pybuffernd_gradient.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_gradient.diminfo[0].shape = __pyx_pybuffernd_gradient.rcbuffer->pybuffer.shape[0]; + { + __Pyx_BufFmt_StackElem __pyx_stack[1]; + if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_mom1.rcbuffer->pybuffer, (PyObject*)__pyx_v_mom1, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) __PYX_ERR(0, 495, __pyx_L1_error) + } + __pyx_pybuffernd_mom1.diminfo[0].strides = __pyx_pybuffernd_mom1.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_mom1.diminfo[0].shape = __pyx_pybuffernd_mom1.rcbuffer->pybuffer.shape[0]; + { + __Pyx_BufFmt_StackElem __pyx_stack[1]; + if (unlikely(__Pyx_GetBufferAndValidate(&__pyx_pybuffernd_mom2.rcbuffer->pybuffer, (PyObject*)__pyx_v_mom2, &__Pyx_TypeInfo_nn___pyx_t_5numpy_float32_t, PyBUF_FORMAT| PyBUF_STRIDES, 1, 0, __pyx_stack) == -1)) __PYX_ERR(0, 495, __pyx_L1_error) + } + __pyx_pybuffernd_mom2.diminfo[0].strides = __pyx_pybuffernd_mom2.rcbuffer->pybuffer.strides[0]; __pyx_pybuffernd_mom2.diminfo[0].shape = __pyx_pybuffernd_mom2.rcbuffer->pybuffer.shape[0]; + + /* "thinc/backends/numpy_ops.pyx":501 + * const float beta1, const float beta2, float eps, + * float learn_rate, float mod_rate=1.): + * _check_compatible_shape(weights, gradient) # <<<<<<<<<<<<<< + * _check_compatible_shape(weights, mom1) + * _check_compatible_shape(weights, mom2) +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_check_compatible_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 501, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, ((PyObject *)__pyx_v_weights), ((PyObject *)__pyx_v_gradient)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 501, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":502 + * float learn_rate, float mod_rate=1.): + * _check_compatible_shape(weights, gradient) + * _check_compatible_shape(weights, mom1) # <<<<<<<<<<<<<< + * _check_compatible_shape(weights, mom2) + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_check_compatible_shape); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 502, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, ((PyObject *)__pyx_v_weights), ((PyObject *)__pyx_v_mom1)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 502, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":503 + * _check_compatible_shape(weights, gradient) + * _check_compatible_shape(weights, mom1) + * _check_compatible_shape(weights, mom2) # <<<<<<<<<<<<<< + * + * _adam_momentum(gradient.data, mom1.data, mom2.data, +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_check_compatible_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 503, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, ((PyObject *)__pyx_v_weights), ((PyObject *)__pyx_v_mom2)}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 503, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":505 + * _check_compatible_shape(weights, mom2) + * + * _adam_momentum(gradient.data, mom1.data, mom2.data, # <<<<<<<<<<<<<< + * weights.shape[0], beta1, beta2, eps, learn_rate) + * VecVec.add_i(weights.data, +*/ + __pyx_f_5thinc_8backends_9numpy_ops__adam_momentum(((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_gradient))), ((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_mom1))), ((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_mom2))), (__pyx_f_5numpy_7ndarray_5shape_shape(((PyArrayObject *)__pyx_v_weights))[0]), __pyx_v_beta1, __pyx_v_beta2, __pyx_v_eps, __pyx_v_learn_rate); + + /* "thinc/backends/numpy_ops.pyx":507 + * _adam_momentum(gradient.data, mom1.data, mom2.data, + * weights.shape[0], beta1, beta2, eps, learn_rate) + * VecVec.add_i(weights.data, # <<<<<<<<<<<<<< + * gradient.data, -learn_rate, weights.shape[0]) + * memset(gradient.data, 0, gradient.size * sizeof(float)) +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_weights))), ((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_gradient))), (-__pyx_v_learn_rate), (__pyx_f_5numpy_7ndarray_5shape_shape(((PyArrayObject *)__pyx_v_weights))[0])); + + /* "thinc/backends/numpy_ops.pyx":509 + * VecVec.add_i(weights.data, + * gradient.data, -learn_rate, weights.shape[0]) + * memset(gradient.data, 0, gradient.size * sizeof(float)) # <<<<<<<<<<<<<< + * return weights, gradient, mom1, mom2 + * +*/ + (void)(memset(((float *)__pyx_f_5numpy_7ndarray_4data_data(((PyArrayObject *)__pyx_v_gradient))), 0, (__pyx_f_5numpy_7ndarray_4size_size(((PyArrayObject *)__pyx_v_gradient)) * (sizeof(float))))); + + /* "thinc/backends/numpy_ops.pyx":510 + * gradient.data, -learn_rate, weights.shape[0]) + * memset(gradient.data, 0, gradient.size * sizeof(float)) + * return weights, gradient, mom1, mom2 # <<<<<<<<<<<<<< + * + * def ngrams(self, int n, const uint64_t[::1] keys): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 510, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF((PyObject *)__pyx_v_weights); + __Pyx_GIVEREF((PyObject *)__pyx_v_weights); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_weights)) != (0)) __PYX_ERR(0, 510, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_gradient); + __Pyx_GIVEREF((PyObject *)__pyx_v_gradient); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_gradient)) != (0)) __PYX_ERR(0, 510, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_mom1); + __Pyx_GIVEREF((PyObject *)__pyx_v_mom1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, ((PyObject *)__pyx_v_mom1)) != (0)) __PYX_ERR(0, 510, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_mom2); + __Pyx_GIVEREF((PyObject *)__pyx_v_mom2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_v_mom2)) != (0)) __PYX_ERR(0, 510, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":495 + * self.xp.add.at(table, indices, values) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def adam(self, np.ndarray[np.float32_t] weights, np.ndarray[np.float32_t] gradient, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + { PyObject *__pyx_type, *__pyx_value, *__pyx_tb; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_gradient.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_mom1.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_mom2.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_weights.rcbuffer->pybuffer); + __Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);} + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.adam", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + goto __pyx_L2; + __pyx_L0:; + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_gradient.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_mom1.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_mom2.rcbuffer->pybuffer); + __Pyx_SafeReleaseBuffer(&__pyx_pybuffernd_weights.rcbuffer->pybuffer); + __pyx_L2:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":512 + * return weights, gradient, mom1, mom2 + * + * def ngrams(self, int n, const uint64_t[::1] keys): # <<<<<<<<<<<<<< + * if n < 1: + * return self.alloc((0,), dtype="uint64") +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_53ngrams(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_52ngrams, "NumpyOps.ngrams(self, int n, const uint64_t[::1] keys)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_53ngrams = {"ngrams", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_53ngrams, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_52ngrams}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_53ngrams(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + int __pyx_v_n; + __Pyx_memviewslice __pyx_v_keys = { 0, 0, { 0 }, { 0 }, { 0 } }; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("ngrams (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_n,&__pyx_mstate_global->__pyx_n_u_keys,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 512, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 512, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 512, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 512, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "ngrams", 0) < (0)) __PYX_ERR(0, 512, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("ngrams", 1, 3, 3, i); __PYX_ERR(0, 512, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 512, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 512, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 512, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_n = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_n == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 512, __pyx_L3_error) + __pyx_v_keys = __Pyx_PyObject_to_MemoryviewSlice_dc_nn_uint64_t__const__(values[2], 0); if (unlikely(!__pyx_v_keys.memview)) __PYX_ERR(0, 512, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("ngrams", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 512, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_keys, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_52ngrams(__pyx_self, __pyx_v_self, __pyx_v_n, __pyx_v_keys); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_keys, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_52ngrams(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, int __pyx_v_n, __Pyx_memviewslice __pyx_v_keys) { + uint64_t *__pyx_v_keys_; + Py_ssize_t __pyx_v_length; + PyArrayObject *__pyx_v_output_ = 0; + uint64_t *__pyx_v_output; + Py_ssize_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + Py_ssize_t __pyx_t_8; + long __pyx_t_9; + Py_ssize_t __pyx_t_10; + PyObject *__pyx_t_11 = NULL; + Py_ssize_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("ngrams", 0); + + /* "thinc/backends/numpy_ops.pyx":513 + * + * def ngrams(self, int n, const uint64_t[::1] keys): + * if n < 1: # <<<<<<<<<<<<<< + * return self.alloc((0,), dtype="uint64") + * keys_ = &keys[0] +*/ + __pyx_t_1 = (__pyx_v_n < 1); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":514 + * def ngrams(self, int n, const uint64_t[::1] keys): + * if n < 1: + * return self.alloc((0,), dtype="uint64") # <<<<<<<<<<<<<< + * keys_ = &keys[0] + * length = max(0, keys.shape[0]-(n-1)) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_mstate_global->__pyx_tuple[2]}; + __pyx_t_5 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 514, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_uint64, __pyx_t_5, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 514, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_5); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 514, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":513 + * + * def ngrams(self, int n, const uint64_t[::1] keys): + * if n < 1: # <<<<<<<<<<<<<< + * return self.alloc((0,), dtype="uint64") + * keys_ = &keys[0] +*/ + } + + /* "thinc/backends/numpy_ops.pyx":515 + * if n < 1: + * return self.alloc((0,), dtype="uint64") + * keys_ = &keys[0] # <<<<<<<<<<<<<< + * length = max(0, keys.shape[0]-(n-1)) + * cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") +*/ + __pyx_t_6 = 0; + __pyx_t_7 = -1; + if (__pyx_t_6 < 0) { + __pyx_t_6 += __pyx_v_keys.shape[0]; + if (unlikely(__pyx_t_6 < 0)) __pyx_t_7 = 0; + } else if (unlikely(__pyx_t_6 >= __pyx_v_keys.shape[0])) __pyx_t_7 = 0; + if (unlikely(__pyx_t_7 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_7); + __PYX_ERR(0, 515, __pyx_L1_error) + } + __pyx_v_keys_ = ((uint64_t *)(&(*((uint64_t const *) ( /* dim=0 */ ((char *) (((uint64_t const *) __pyx_v_keys.data) + __pyx_t_6)) ))))); + + /* "thinc/backends/numpy_ops.pyx":516 + * return self.alloc((0,), dtype="uint64") + * keys_ = &keys[0] + * length = max(0, keys.shape[0]-(n-1)) # <<<<<<<<<<<<<< + * cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") + * output = output_.data +*/ + __pyx_t_8 = ((__pyx_v_keys.shape[0]) - (__pyx_v_n - 1)); + __pyx_t_9 = 0; + __pyx_t_1 = (__pyx_t_8 > __pyx_t_9); + if (__pyx_t_1) { + __pyx_t_10 = __pyx_t_8; + } else { + __pyx_t_10 = __pyx_t_9; + } + __pyx_v_length = __pyx_t_10; + + /* "thinc/backends/numpy_ops.pyx":517 + * keys_ = &keys[0] + * length = max(0, keys.shape[0]-(n-1)) + * cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") # <<<<<<<<<<<<<< + * output = output_.data + * for i in range(keys.shape[0]-(n-1)): +*/ + __pyx_t_5 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_11 = PyTuple_New(1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 517, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_11}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_uint64, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 517, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 517, __pyx_L1_error) + __pyx_v_output_ = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":518 + * length = max(0, keys.shape[0]-(n-1)) + * cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") + * output = output_.data # <<<<<<<<<<<<<< + * for i in range(keys.shape[0]-(n-1)): + * output[i] = hash64(&keys_[i], n*sizeof(keys_[0]), 0) +*/ + __pyx_v_output = ((uint64_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_output_)); + + /* "thinc/backends/numpy_ops.pyx":519 + * cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") + * output = output_.data + * for i in range(keys.shape[0]-(n-1)): # <<<<<<<<<<<<<< + * output[i] = hash64(&keys_[i], n*sizeof(keys_[0]), 0) + * return output_ +*/ + __pyx_t_10 = ((__pyx_v_keys.shape[0]) - (__pyx_v_n - 1)); + __pyx_t_8 = __pyx_t_10; + for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_8; __pyx_t_12+=1) { + __pyx_v_i = __pyx_t_12; + + /* "thinc/backends/numpy_ops.pyx":520 + * output = output_.data + * for i in range(keys.shape[0]-(n-1)): + * output[i] = hash64(&keys_[i], n*sizeof(keys_[0]), 0) # <<<<<<<<<<<<<< + * return output_ + * +*/ + (__pyx_v_output[__pyx_v_i]) = __pyx_f_10murmurhash_4mrmr_hash64((&(__pyx_v_keys_[__pyx_v_i])), (__pyx_v_n * (sizeof((__pyx_v_keys_[0])))), 0); + } + + /* "thinc/backends/numpy_ops.pyx":521 + * for i in range(keys.shape[0]-(n-1)): + * output[i] = hash64(&keys_[i], n*sizeof(keys_[0]), 0) + * return output_ # <<<<<<<<<<<<<< + * + * def position_encode(self, int N, int D, int period=10000, out=None): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_output_); + __pyx_r = ((PyObject *)__pyx_v_output_); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":512 + * return weights, gradient, mom1, mom2 + * + * def ngrams(self, int n, const uint64_t[::1] keys): # <<<<<<<<<<<<<< + * if n < 1: + * return self.alloc((0,), dtype="uint64") +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.ngrams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_output_); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":523 + * return output_ + * + * def position_encode(self, int N, int D, int period=10000, out=None): # <<<<<<<<<<<<<< + * cdef np.ndarray out_ + * if out is None: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_55position_encode(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_54position_encode, "NumpyOps.position_encode(self, int N, int D, int period=10000, out=None)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_55position_encode = {"position_encode", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_55position_encode, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8NumpyOps_54position_encode}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_8NumpyOps_55position_encode(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + int __pyx_v_N; + int __pyx_v_D; + int __pyx_v_period; + PyObject *__pyx_v_out = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("position_encode (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_N,&__pyx_mstate_global->__pyx_n_u_D,&__pyx_mstate_global->__pyx_n_u_period,&__pyx_mstate_global->__pyx_n_u_out,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 523, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "position_encode", 0) < (0)) __PYX_ERR(0, 523, __pyx_L3_error) + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)Py_None)); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("position_encode", 0, 3, 5, i); __PYX_ERR(0, 523, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 523, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 523, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 523, __pyx_L3_error) + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 523, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[4]) values[4] = __Pyx_NewRef(((PyObject *)Py_None)); + } + __pyx_v_self = values[0]; + __pyx_v_N = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_N == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 523, __pyx_L3_error) + __pyx_v_D = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_D == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 523, __pyx_L3_error) + if (values[3]) { + __pyx_v_period = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_period == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 523, __pyx_L3_error) + } else { + __pyx_v_period = ((int)((int)0x2710)); + } + __pyx_v_out = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("position_encode", 0, 3, 5, __pyx_nargs); __PYX_ERR(0, 523, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.position_encode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_54position_encode(__pyx_self, __pyx_v_self, __pyx_v_N, __pyx_v_D, __pyx_v_period, __pyx_v_out); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8NumpyOps_54position_encode(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, int __pyx_v_N, int __pyx_v_D, int __pyx_v_period, PyObject *__pyx_v_out) { + PyArrayObject *__pyx_v_out_ = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("position_encode", 0); + + /* "thinc/backends/numpy_ops.pyx":525 + * def position_encode(self, int N, int D, int period=10000, out=None): + * cdef np.ndarray out_ + * if out is None: # <<<<<<<<<<<<<< + * out_ = self.alloc((N, D), zeros=False) + * else: +*/ + __pyx_t_1 = (__pyx_v_out == Py_None); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":526 + * cdef np.ndarray out_ + * if out is None: + * out_ = self.alloc((N, D), zeros=False) # <<<<<<<<<<<<<< + * else: + * out_ = out +*/ + __pyx_t_3 = __pyx_v_self; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 526, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_D); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 526, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 526, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 526, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 526, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_t_6}; + __pyx_t_5 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 526, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_zeros, Py_False, __pyx_t_5, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 526, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_5); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 526, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 526, __pyx_L1_error) + __pyx_v_out_ = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":525 + * def position_encode(self, int N, int D, int period=10000, out=None): + * cdef np.ndarray out_ + * if out is None: # <<<<<<<<<<<<<< + * out_ = self.alloc((N, D), zeros=False) + * else: +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":528 + * out_ = self.alloc((N, D), zeros=False) + * else: + * out_ = out # <<<<<<<<<<<<<< + * assert out_.shape[0] == N + * assert out_.shape[1] == D +*/ + /*else*/ { + __pyx_t_2 = __pyx_v_out; + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 528, __pyx_L1_error) + __pyx_v_out_ = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + } + __pyx_L3:; + + /* "thinc/backends/numpy_ops.pyx":529 + * else: + * out_ = out + * assert out_.shape[0] == N # <<<<<<<<<<<<<< + * assert out_.shape[1] == D + * cpu_position_encode(out_.data, period, N, D) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_out_)[0]) == __pyx_v_N); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 529, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 529, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":530 + * out_ = out + * assert out_.shape[0] == N + * assert out_.shape[1] == D # <<<<<<<<<<<<<< + * cpu_position_encode(out_.data, period, N, D) + * return out_ +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_out_)[1]) == __pyx_v_D); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 530, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 530, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":531 + * assert out_.shape[0] == N + * assert out_.shape[1] == D + * cpu_position_encode(out_.data, period, N, D) # <<<<<<<<<<<<<< + * return out_ + * +*/ + __pyx_f_5thinc_8backends_9numpy_ops_cpu_position_encode(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_out_)), __pyx_v_period, __pyx_v_N, __pyx_v_D); + + /* "thinc/backends/numpy_ops.pyx":532 + * assert out_.shape[1] == D + * cpu_position_encode(out_.data, period, N, D) + * return out_ # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_out_); + __pyx_r = ((PyObject *)__pyx_v_out_); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":523 + * return output_ + * + * def position_encode(self, int N, int D, int period=10000, out=None): # <<<<<<<<<<<<<< + * cdef np.ndarray out_ + * if out is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops.NumpyOps.position_encode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_out_); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":535 + * + * + * def check_seq2col_lengths(ops, lengths, B): # <<<<<<<<<<<<<< + * if lengths is None: + * lengths = ops.asarray1i([B]) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_1check_seq2col_lengths(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_check_seq2col_lengths, "check_seq2col_lengths(ops, lengths, B)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_1check_seq2col_lengths = {"check_seq2col_lengths", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_1check_seq2col_lengths, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_check_seq2col_lengths}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_1check_seq2col_lengths(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_ops = 0; + PyObject *__pyx_v_lengths = 0; + PyObject *__pyx_v_B = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("check_seq2col_lengths (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_ops,&__pyx_mstate_global->__pyx_n_u_lengths,&__pyx_mstate_global->__pyx_n_u_B,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 535, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 535, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 535, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 535, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "check_seq2col_lengths", 0) < (0)) __PYX_ERR(0, 535, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("check_seq2col_lengths", 1, 3, 3, i); __PYX_ERR(0, 535, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 535, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 535, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 535, __pyx_L3_error) + } + __pyx_v_ops = values[0]; + __pyx_v_lengths = values[1]; + __pyx_v_B = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("check_seq2col_lengths", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 535, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.check_seq2col_lengths", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_check_seq2col_lengths(__pyx_self, __pyx_v_ops, __pyx_v_lengths, __pyx_v_B); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_check_seq2col_lengths(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_ops, PyObject *__pyx_v_lengths, PyObject *__pyx_v_B) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("check_seq2col_lengths", 0); + __Pyx_INCREF(__pyx_v_lengths); + + /* "thinc/backends/numpy_ops.pyx":536 + * + * def check_seq2col_lengths(ops, lengths, B): + * if lengths is None: # <<<<<<<<<<<<<< + * lengths = ops.asarray1i([B]) + * else: +*/ + __pyx_t_1 = (__pyx_v_lengths == Py_None); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":537 + * def check_seq2col_lengths(ops, lengths, B): + * if lengths is None: + * lengths = ops.asarray1i([B]) # <<<<<<<<<<<<<< + * else: + * assert ops.xp.all(ops.xp.array(lengths) >= 0), "All sequence lengths must be >= 0" +*/ + __pyx_t_3 = __pyx_v_ops; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyList_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 537, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_B); + __Pyx_GIVEREF(__pyx_v_B); + if (__Pyx_PyList_SET_ITEM(__pyx_t_4, 0, __pyx_v_B) != (0)) __PYX_ERR(0, 537, __pyx_L1_error); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_asarray1i, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 537, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_lengths, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":536 + * + * def check_seq2col_lengths(ops, lengths, B): + * if lengths is None: # <<<<<<<<<<<<<< + * lengths = ops.asarray1i([B]) + * else: +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":539 + * lengths = ops.asarray1i([B]) + * else: + * assert ops.xp.all(ops.xp.array(lengths) >= 0), "All sequence lengths must be >= 0" # <<<<<<<<<<<<<< + * assert ops.xp.sum(lengths) == B, "The lengths must sum up to the batch length" + * +*/ + /*else*/ { + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_ops, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __pyx_t_3; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_ops, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_v_lengths}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_array, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_8 = PyObject_RichCompare(__pyx_t_6, __pyx_mstate_global->__pyx_int_0, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_t_8}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_all, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 539, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_mstate_global->__pyx_kp_u_All_sequence_lengths_must_be_0, 0, 0); + __PYX_ERR(0, 539, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 539, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":540 + * else: + * assert ops.xp.all(ops.xp.array(lengths) >= 0), "All sequence lengths must be >= 0" + * assert ops.xp.sum(lengths) == B, "The lengths must sum up to the batch length" # <<<<<<<<<<<<<< + * + * return lengths +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_ops, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 540, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_3 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v_lengths}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_sum, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 540, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_8 = PyObject_RichCompare(__pyx_t_2, __pyx_v_B, Py_EQ); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 540, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 540, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_mstate_global->__pyx_kp_u_The_lengths_must_sum_up_to_the_b, 0, 0); + __PYX_ERR(0, 540, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 540, __pyx_L1_error) + #endif + } + __pyx_L3:; + + /* "thinc/backends/numpy_ops.pyx":542 + * assert ops.xp.sum(lengths) == B, "The lengths must sum up to the batch length" + * + * return lengths # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_lengths); + __pyx_r = __pyx_v_lengths; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":535 + * + * + * def check_seq2col_lengths(ops, lengths, B): # <<<<<<<<<<<<<< + * if lengths is None: + * lengths = ops.asarray1i([B]) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.check_seq2col_lengths", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_lengths); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":545 + * + * + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): # <<<<<<<<<<<<<< + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_3cpu_clip_gradient(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_2cpu_clip_gradient, "cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_3cpu_clip_gradient = {"cpu_clip_gradient", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_3cpu_clip_gradient, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_2cpu_clip_gradient}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_3cpu_clip_gradient(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + __Pyx_memviewslice __pyx_v_gradient = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_threshold; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("cpu_clip_gradient (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_gradient,&__pyx_mstate_global->__pyx_n_u_threshold,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 545, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 545, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 545, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "cpu_clip_gradient", 0) < (0)) __PYX_ERR(0, 545, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("cpu_clip_gradient", 1, 2, 2, i); __PYX_ERR(0, 545, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 545, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 545, __pyx_L3_error) + } + __pyx_v_gradient = __Pyx_PyObject_to_MemoryviewSlice_dc_nn___pyx_t_5thinc_8backends_9numpy_ops_weight_t(values[0], PyBUF_WRITABLE); if (unlikely(!__pyx_v_gradient.memview)) __PYX_ERR(0, 545, __pyx_L3_error) + __pyx_v_threshold = __Pyx_PyFloat_AsFloat(values[1]); if (unlikely((__pyx_v_threshold == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 545, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("cpu_clip_gradient", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 545, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.cpu_clip_gradient", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_2cpu_clip_gradient(__pyx_self, __pyx_v_gradient, __pyx_v_threshold); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_2cpu_clip_gradient(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_memviewslice __pyx_v_gradient, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_threshold) { + __pyx_t_5thinc_8backends_6linalg_weight_t __pyx_v_grad_norm; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("cpu_clip_gradient", 0); + + /* "thinc/backends/numpy_ops.pyx":546 + * + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) # <<<<<<<<<<<<<< + * if grad_norm >= threshold: + * Vec.mul_i(&gradient[0], threshold / grad_norm, gradient.shape[0]) +*/ + __pyx_t_1 = 0; + __pyx_t_2 = -1; + if (__pyx_t_1 < 0) { + __pyx_t_1 += __pyx_v_gradient.shape[0]; + if (unlikely(__pyx_t_1 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_1 >= __pyx_v_gradient.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 546, __pyx_L1_error) + } + __pyx_v_grad_norm = __pyx_f_5thinc_8backends_6linalg_3Vec_norm((&(*((__pyx_t_5thinc_8backends_9numpy_ops_weight_t *) ( /* dim=0 */ ((char *) (((__pyx_t_5thinc_8backends_9numpy_ops_weight_t *) __pyx_v_gradient.data) + __pyx_t_1)) )))), (__pyx_v_gradient.shape[0])); + + /* "thinc/backends/numpy_ops.pyx":547 + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: # <<<<<<<<<<<<<< + * Vec.mul_i(&gradient[0], threshold / grad_norm, gradient.shape[0]) + * +*/ + __pyx_t_3 = (__pyx_v_grad_norm >= __pyx_v_threshold); + if (__pyx_t_3) { + + /* "thinc/backends/numpy_ops.pyx":548 + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: + * Vec.mul_i(&gradient[0], threshold / grad_norm, gradient.shape[0]) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = 0; + __pyx_t_2 = -1; + if (__pyx_t_1 < 0) { + __pyx_t_1 += __pyx_v_gradient.shape[0]; + if (unlikely(__pyx_t_1 < 0)) __pyx_t_2 = 0; + } else if (unlikely(__pyx_t_1 >= __pyx_v_gradient.shape[0])) __pyx_t_2 = 0; + if (unlikely(__pyx_t_2 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_2); + __PYX_ERR(0, 548, __pyx_L1_error) + } + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i((&(*((__pyx_t_5thinc_8backends_9numpy_ops_weight_t *) ( /* dim=0 */ ((char *) (((__pyx_t_5thinc_8backends_9numpy_ops_weight_t *) __pyx_v_gradient.data) + __pyx_t_1)) )))), (__pyx_v_threshold / __pyx_v_grad_norm), (__pyx_v_gradient.shape[0])); + + /* "thinc/backends/numpy_ops.pyx":547 + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: # <<<<<<<<<<<<<< + * Vec.mul_i(&gradient[0], threshold / grad_norm, gradient.shape[0]) + * +*/ + } + + /* "thinc/backends/numpy_ops.pyx":545 + * + * + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): # <<<<<<<<<<<<<< + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.backends.numpy_ops.cpu_clip_gradient", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":551 + * + * + * def add_gradient_noise(float[::1] gradient, weight_t noise_level, # <<<<<<<<<<<<<< + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_5add_gradient_noise(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_4add_gradient_noise, "add_gradient_noise(float[::1] gradient, weight_t noise_level, weight_t timestep)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_5add_gradient_noise = {"add_gradient_noise", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_5add_gradient_noise, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_4add_gradient_noise}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_5add_gradient_noise(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + __Pyx_memviewslice __pyx_v_gradient = { 0, 0, { 0 }, { 0 }, { 0 } }; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_noise_level; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_timestep; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("add_gradient_noise (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_gradient,&__pyx_mstate_global->__pyx_n_u_noise_level,&__pyx_mstate_global->__pyx_n_u_timestep,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 551, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 551, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 551, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 551, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "add_gradient_noise", 0) < (0)) __PYX_ERR(0, 551, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("add_gradient_noise", 1, 3, 3, i); __PYX_ERR(0, 551, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 551, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 551, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 551, __pyx_L3_error) + } + __pyx_v_gradient = __Pyx_PyObject_to_MemoryviewSlice_dc_float(values[0], PyBUF_WRITABLE); if (unlikely(!__pyx_v_gradient.memview)) __PYX_ERR(0, 551, __pyx_L3_error) + __pyx_v_noise_level = __Pyx_PyFloat_AsFloat(values[1]); if (unlikely((__pyx_v_noise_level == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 551, __pyx_L3_error) + __pyx_v_timestep = __Pyx_PyFloat_AsFloat(values[2]); if (unlikely((__pyx_v_timestep == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 552, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("add_gradient_noise", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 551, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.add_gradient_noise", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_4add_gradient_noise(__pyx_self, __pyx_v_gradient, __pyx_v_noise_level, __pyx_v_timestep); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_4add_gradient_noise(CYTHON_UNUSED PyObject *__pyx_self, __Pyx_memviewslice __pyx_v_gradient, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_noise_level, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_timestep) { + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_variance; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + double __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + Py_ssize_t __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + size_t __pyx_t_13; + PyObject *__pyx_t_14 = NULL; + __Pyx_memviewslice __pyx_t_15 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("add_gradient_noise", 0); + __PYX_INC_MEMVIEW(&__pyx_v_gradient, 1); + + /* "thinc/backends/numpy_ops.pyx":553 + * def add_gradient_noise(float[::1] gradient, weight_t noise_level, + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) # <<<<<<<<<<<<<< + * if variance >= 0.000001: + * gradient += numpy.asarray( +*/ + __pyx_t_1 = __Pyx_SoftComplexToDouble(__Pyx_c_quot_double(__pyx_t_double_complex_from_parts(__pyx_v_noise_level, 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts((1.0 + __pyx_v_timestep), 0), __pyx_t_double_complex_from_parts(0.55, 0))), 0); if (unlikely(__pyx_t_1 == ((double)-1) && PyErr_Occurred())) __PYX_ERR(0, 553, __pyx_L1_error) + __pyx_v_variance = __pyx_t_1; + + /* "thinc/backends/numpy_ops.pyx":554 + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) + * if variance >= 0.000001: # <<<<<<<<<<<<<< + * gradient += numpy.asarray( + * numpy.random.normal(scale=variance, loc=0., size=len(gradient)), +*/ + __pyx_t_2 = (__pyx_v_variance >= 0.000001); + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":555 + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) + * if variance >= 0.000001: + * gradient += numpy.asarray( # <<<<<<<<<<<<<< + * numpy.random.normal(scale=variance, loc=0., size=len(gradient)), + * dtype='float32') +*/ + __pyx_t_3 = __pyx_memoryview_fromslice(__pyx_v_gradient, 1, (PyObject *(*)(char *)) __pyx_memview_get_float, (int (*)(char *, PyObject *)) __pyx_memview_set_float, 0);; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_asarray); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":556 + * if variance >= 0.000001: + * gradient += numpy.asarray( + * numpy.random.normal(scale=variance, loc=0., size=len(gradient)), # <<<<<<<<<<<<<< + * dtype='float32') + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_random); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_8 = __pyx_t_10; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_9 = PyFloat_FromDouble(__pyx_v_variance); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_11 = __Pyx_MemoryView_Len(__pyx_v_gradient); + __pyx_t_12 = PyLong_FromSsize_t(__pyx_t_11); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_13 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 3 : 0)] = {__pyx_t_8, NULL}; + __pyx_t_14 = __Pyx_MakeVectorcallBuilderKwds(3); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_scale, __pyx_t_9, __pyx_t_14, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 556, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_loc, __pyx_mstate_global->__pyx_float_0_, __pyx_t_14, __pyx_callargs+1, 1) < (0)) __PYX_ERR(0, 556, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_size, __pyx_t_12, __pyx_t_14, __pyx_callargs+1, 2) < (0)) __PYX_ERR(0, 556, __pyx_L1_error) + __pyx_t_6 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_normal, __pyx_callargs+__pyx_t_13, (1-__pyx_t_13) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_14); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 556, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_13 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_7); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_7, __pyx__function); + __pyx_t_13 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_6}; + __pyx_t_10 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_float32, __pyx_t_10, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 555, __pyx_L1_error) + __pyx_t_4 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_13, (2-__pyx_t_13) | (__pyx_t_13*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_10); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + + /* "thinc/backends/numpy_ops.pyx":555 + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) + * if variance >= 0.000001: + * gradient += numpy.asarray( # <<<<<<<<<<<<<< + * numpy.random.normal(scale=variance, loc=0., size=len(gradient)), + * dtype='float32') +*/ + __pyx_t_7 = PyNumber_InPlaceAdd(__pyx_t_3, __pyx_t_4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_15 = __Pyx_PyObject_to_MemoryviewSlice_dc_float(__pyx_t_7, PyBUF_WRITABLE); if (unlikely(!__pyx_t_15.memview)) __PYX_ERR(0, 555, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __pyx_v_gradient = __pyx_t_15; + __pyx_t_15.memview = NULL; + __pyx_t_15.data = NULL; + + /* "thinc/backends/numpy_ops.pyx":554 + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) + * if variance >= 0.000001: # <<<<<<<<<<<<<< + * gradient += numpy.asarray( + * numpy.random.normal(scale=variance, loc=0., size=len(gradient)), +*/ + } + + /* "thinc/backends/numpy_ops.pyx":551 + * + * + * def add_gradient_noise(float[::1] gradient, weight_t noise_level, # <<<<<<<<<<<<<< + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_14); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_15, 1); + __Pyx_AddTraceback("thinc.backends.numpy_ops.add_gradient_noise", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __PYX_XCLEAR_MEMVIEW(&__pyx_v_gradient, 1); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":560 + * + * + * cdef void cpu_position_encode(float* output, float period, int N, int D) noexcept nogil: # <<<<<<<<<<<<<< + * cdef float pos, d + * cdef int j +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_position_encode(float *__pyx_v_output, float __pyx_v_period, int __pyx_v_N, int __pyx_v_D) { + float __pyx_v_pos; + float __pyx_v_d; + int __pyx_v_j; + float __pyx_v_dimensions; + int __pyx_v_i; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + double __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "thinc/backends/numpy_ops.pyx":563 + * cdef float pos, d + * cdef int j + * cdef float dimensions = D # <<<<<<<<<<<<<< + * for i in range(N): + * pos = i +*/ + __pyx_v_dimensions = __pyx_v_D; + + /* "thinc/backends/numpy_ops.pyx":564 + * cdef int j + * cdef float dimensions = D + * for i in range(N): # <<<<<<<<<<<<<< + * pos = i + * j = 0 +*/ + __pyx_t_1 = __pyx_v_N; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/numpy_ops.pyx":565 + * cdef float dimensions = D + * for i in range(N): + * pos = i # <<<<<<<<<<<<<< + * j = 0 + * d = 0 +*/ + __pyx_v_pos = __pyx_v_i; + + /* "thinc/backends/numpy_ops.pyx":566 + * for i in range(N): + * pos = i + * j = 0 # <<<<<<<<<<<<<< + * d = 0 + * while (j+1) < D: +*/ + __pyx_v_j = 0; + + /* "thinc/backends/numpy_ops.pyx":567 + * pos = i + * j = 0 + * d = 0 # <<<<<<<<<<<<<< + * while (j+1) < D: + * d = j +*/ + __pyx_v_d = 0.0; + + /* "thinc/backends/numpy_ops.pyx":568 + * j = 0 + * d = 0 + * while (j+1) < D: # <<<<<<<<<<<<<< + * d = j + * output[j] = sinf(pos / period ** (2 * d / dimensions)) +*/ + while (1) { + __pyx_t_4 = ((__pyx_v_j + 1) < __pyx_v_D); + if (!__pyx_t_4) break; + + /* "thinc/backends/numpy_ops.pyx":569 + * d = 0 + * while (j+1) < D: + * d = j # <<<<<<<<<<<<<< + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) +*/ + __pyx_v_d = __pyx_v_j; + + /* "thinc/backends/numpy_ops.pyx":570 + * while (j+1) < D: + * d = j + * output[j] = sinf(pos / period ** (2 * d / dimensions)) # <<<<<<<<<<<<<< + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) + * j += 2 +*/ + __pyx_t_5 = __Pyx_SoftComplexToDouble(__Pyx_c_quot_double(__pyx_t_double_complex_from_parts(__pyx_v_pos, 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts(__pyx_v_period, 0), __pyx_t_double_complex_from_parts(((2.0 * __pyx_v_d) / __pyx_v_dimensions), 0))), 2); if (unlikely(__pyx_t_5 == ((double)-1) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 570, __pyx_L1_error) + (__pyx_v_output[__pyx_v_j]) = sinf(__pyx_t_5); + + /* "thinc/backends/numpy_ops.pyx":571 + * d = j + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) # <<<<<<<<<<<<<< + * j += 2 + * if j < D: +*/ + __pyx_t_5 = __Pyx_SoftComplexToDouble(__Pyx_c_quot_double(__pyx_t_double_complex_from_parts(__pyx_v_pos, 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts(__pyx_v_period, 0), __pyx_t_double_complex_from_parts(((2.0 * __pyx_v_d) / __pyx_v_dimensions), 0))), 2); if (unlikely(__pyx_t_5 == ((double)-1) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 571, __pyx_L1_error) + (__pyx_v_output[(__pyx_v_j + 1)]) = cosf(__pyx_t_5); + + /* "thinc/backends/numpy_ops.pyx":572 + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) + * j += 2 # <<<<<<<<<<<<<< + * if j < D: + * output[j] = sinf(pos / period ** (2 * d / dimensions)) +*/ + __pyx_v_j = (__pyx_v_j + 2); + } + + /* "thinc/backends/numpy_ops.pyx":573 + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) + * j += 2 + * if j < D: # <<<<<<<<<<<<<< + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output += D +*/ + __pyx_t_4 = (__pyx_v_j < __pyx_v_D); + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":574 + * j += 2 + * if j < D: + * output[j] = sinf(pos / period ** (2 * d / dimensions)) # <<<<<<<<<<<<<< + * output += D + * +*/ + __pyx_t_5 = __Pyx_SoftComplexToDouble(__Pyx_c_quot_double(__pyx_t_double_complex_from_parts(__pyx_v_pos, 0), __Pyx_c_pow_double(__pyx_t_double_complex_from_parts(__pyx_v_period, 0), __pyx_t_double_complex_from_parts(((2.0 * __pyx_v_d) / __pyx_v_dimensions), 0))), 2); if (unlikely(__pyx_t_5 == ((double)-1) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 574, __pyx_L1_error) + (__pyx_v_output[__pyx_v_j]) = sinf(__pyx_t_5); + + /* "thinc/backends/numpy_ops.pyx":573 + * output[j+1] = cosf(pos / period ** (2 * d / dimensions)) + * j += 2 + * if j < D: # <<<<<<<<<<<<<< + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output += D +*/ + } + + /* "thinc/backends/numpy_ops.pyx":575 + * if j < D: + * output[j] = sinf(pos / period ** (2 * d / dimensions)) + * output += D # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_output = (__pyx_v_output + __pyx_v_D); + } + + /* "thinc/backends/numpy_ops.pyx":560 + * + * + * cdef void cpu_position_encode(float* output, float period, int N, int D) noexcept nogil: # <<<<<<<<<<<<<< + * cdef float pos, d + * cdef int j +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_WriteUnraisable("thinc.backends.numpy_ops.cpu_position_encode", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; +} + +/* "thinc/backends/numpy_ops.pyx":578 + * + * + * cdef void cpu_scatter_add(float* dest, # <<<<<<<<<<<<<< + * const int* indices, const float* src, + * int nr_id, int nr_col) noexcept nogil: +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_scatter_add(float *__pyx_v_dest, int const *__pyx_v_indices, float const *__pyx_v_src, int __pyx_v_nr_id, int __pyx_v_nr_col) { + int __pyx_v_i; + int __pyx_v_id_; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "thinc/backends/numpy_ops.pyx":582 + * int nr_id, int nr_col) noexcept nogil: + * cdef int i + * for i in range(nr_id): # <<<<<<<<<<<<<< + * id_ = indices[i] + * if id_ >= 0: +*/ + __pyx_t_1 = __pyx_v_nr_id; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/numpy_ops.pyx":583 + * cdef int i + * for i in range(nr_id): + * id_ = indices[i] # <<<<<<<<<<<<<< + * if id_ >= 0: + * VecVec.add_i(&dest[id_*nr_col], +*/ + __pyx_v_id_ = (__pyx_v_indices[__pyx_v_i]); + + /* "thinc/backends/numpy_ops.pyx":584 + * for i in range(nr_id): + * id_ = indices[i] + * if id_ >= 0: # <<<<<<<<<<<<<< + * VecVec.add_i(&dest[id_*nr_col], + * &src[i*nr_col], 1., nr_col) +*/ + __pyx_t_4 = (__pyx_v_id_ >= 0); + if (__pyx_t_4) { + + /* "thinc/backends/numpy_ops.pyx":585 + * id_ = indices[i] + * if id_ >= 0: + * VecVec.add_i(&dest[id_*nr_col], # <<<<<<<<<<<<<< + * &src[i*nr_col], 1., nr_col) + * +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i((&(__pyx_v_dest[(__pyx_v_id_ * __pyx_v_nr_col)])), (&(__pyx_v_src[(__pyx_v_i * __pyx_v_nr_col)])), 1., __pyx_v_nr_col); + + /* "thinc/backends/numpy_ops.pyx":584 + * for i in range(nr_id): + * id_ = indices[i] + * if id_ >= 0: # <<<<<<<<<<<<<< + * VecVec.add_i(&dest[id_*nr_col], + * &src[i*nr_col], 1., nr_col) +*/ + } + } + + /* "thinc/backends/numpy_ops.pyx":578 + * + * + * cdef void cpu_scatter_add(float* dest, # <<<<<<<<<<<<<< + * const int* indices, const float* src, + * int nr_id, int nr_col) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":589 + * + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _adam_momentum(weight_t* gradient, weight_t* mom1, weight_t* mom2, + * int nr_weight, weight_t beta1, weight_t beta2, weight_t eps, +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops__adam_momentum(__pyx_t_5thinc_8backends_9numpy_ops_weight_t *__pyx_v_gradient, __pyx_t_5thinc_8backends_9numpy_ops_weight_t *__pyx_v_mom1, __pyx_t_5thinc_8backends_9numpy_ops_weight_t *__pyx_v_mom2, int __pyx_v_nr_weight, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_beta1, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_beta2, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_eps, CYTHON_UNUSED __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_learn_rate) { + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_one_minus_beta1; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_one_minus_beta2; + CYTHON_UNUSED int __pyx_v_i; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_buff[64]; + int __pyx_v_steps; + long __pyx_v_idx; + long __pyx_v_step_size; + long __pyx_v_j; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + long __pyx_t_5; + long __pyx_t_6; + long __pyx_t_7; + long __pyx_t_8; + + /* "thinc/backends/numpy_ops.pyx":596 + * # Assumes the learning rate adjustment is calculated by the caller; + * # a_t = learn_rate * sqrt(1-beta2**timestep) / (1-beta1**timestep) + * cdef weight_t one_minus_beta1 = 1-beta1 # <<<<<<<<<<<<<< + * cdef weight_t one_minus_beta2 = 1-beta2 + * cdef weight_t m1, m2, g +*/ + __pyx_v_one_minus_beta1 = (1.0 - __pyx_v_beta1); + + /* "thinc/backends/numpy_ops.pyx":597 + * # a_t = learn_rate * sqrt(1-beta2**timestep) / (1-beta1**timestep) + * cdef weight_t one_minus_beta1 = 1-beta1 + * cdef weight_t one_minus_beta2 = 1-beta2 # <<<<<<<<<<<<<< + * cdef weight_t m1, m2, g + * cdef int i +*/ + __pyx_v_one_minus_beta2 = (1.0 - __pyx_v_beta2); + + /* "thinc/backends/numpy_ops.pyx":602 + * # Blockwise implementation is a bit faster. Adam is slooow :( + * cdef weight_t[64] buff + * cdef int steps = nr_weight // 64 # <<<<<<<<<<<<<< + * if steps * 64 < nr_weight: + * steps += 1 +*/ + __pyx_v_steps = (__pyx_v_nr_weight / 64); + + /* "thinc/backends/numpy_ops.pyx":603 + * cdef weight_t[64] buff + * cdef int steps = nr_weight // 64 + * if steps * 64 < nr_weight: # <<<<<<<<<<<<<< + * steps += 1 + * idx = 0 +*/ + __pyx_t_1 = ((__pyx_v_steps * 64) < __pyx_v_nr_weight); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":604 + * cdef int steps = nr_weight // 64 + * if steps * 64 < nr_weight: + * steps += 1 # <<<<<<<<<<<<<< + * idx = 0 + * for i in range(steps): +*/ + __pyx_v_steps = (__pyx_v_steps + 1); + + /* "thinc/backends/numpy_ops.pyx":603 + * cdef weight_t[64] buff + * cdef int steps = nr_weight // 64 + * if steps * 64 < nr_weight: # <<<<<<<<<<<<<< + * steps += 1 + * idx = 0 +*/ + } + + /* "thinc/backends/numpy_ops.pyx":605 + * if steps * 64 < nr_weight: + * steps += 1 + * idx = 0 # <<<<<<<<<<<<<< + * for i in range(steps): + * step_size = min(64, nr_weight-idx) +*/ + __pyx_v_idx = 0; + + /* "thinc/backends/numpy_ops.pyx":606 + * steps += 1 + * idx = 0 + * for i in range(steps): # <<<<<<<<<<<<<< + * step_size = min(64, nr_weight-idx) + * Vec.mul_i(mom1, beta1, step_size) +*/ + __pyx_t_2 = __pyx_v_steps; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/backends/numpy_ops.pyx":607 + * idx = 0 + * for i in range(steps): + * step_size = min(64, nr_weight-idx) # <<<<<<<<<<<<<< + * Vec.mul_i(mom1, beta1, step_size) + * VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) +*/ + __pyx_t_5 = (__pyx_v_nr_weight - __pyx_v_idx); + __pyx_t_6 = 64; + __pyx_t_1 = (__pyx_t_5 < __pyx_t_6); + if (__pyx_t_1) { + __pyx_t_7 = __pyx_t_5; + } else { + __pyx_t_7 = __pyx_t_6; + } + __pyx_v_step_size = __pyx_t_7; + + /* "thinc/backends/numpy_ops.pyx":608 + * for i in range(steps): + * step_size = min(64, nr_weight-idx) + * Vec.mul_i(mom1, beta1, step_size) # <<<<<<<<<<<<<< + * VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) + * Vec.mul_i(mom2, beta2, step_size) +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_mom1, __pyx_v_beta1, __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":609 + * step_size = min(64, nr_weight-idx) + * Vec.mul_i(mom1, beta1, step_size) + * VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) # <<<<<<<<<<<<<< + * Vec.mul_i(mom2, beta2, step_size) + * for j in range(step_size): +*/ + __pyx_f_5thinc_8backends_6linalg_6VecVec_add_i(__pyx_v_mom1, __pyx_v_gradient, __pyx_v_one_minus_beta1, __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":610 + * Vec.mul_i(mom1, beta1, step_size) + * VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) + * Vec.mul_i(mom2, beta2, step_size) # <<<<<<<<<<<<<< + * for j in range(step_size): + * mom2[j] += one_minus_beta2 * gradient[j] ** 2 +*/ + __pyx_f_5thinc_8backends_6linalg_3Vec_mul_i(__pyx_v_mom2, __pyx_v_beta2, __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":611 + * VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) + * Vec.mul_i(mom2, beta2, step_size) + * for j in range(step_size): # <<<<<<<<<<<<<< + * mom2[j] += one_minus_beta2 * gradient[j] ** 2 + * for j in range(step_size): +*/ + __pyx_t_7 = __pyx_v_step_size; + __pyx_t_5 = __pyx_t_7; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_j = __pyx_t_6; + + /* "thinc/backends/numpy_ops.pyx":612 + * Vec.mul_i(mom2, beta2, step_size) + * for j in range(step_size): + * mom2[j] += one_minus_beta2 * gradient[j] ** 2 # <<<<<<<<<<<<<< + * for j in range(step_size): + * buff[j] = sqrtf(mom2[j]) +*/ + __pyx_t_8 = __pyx_v_j; + (__pyx_v_mom2[__pyx_t_8]) = ((__pyx_v_mom2[__pyx_t_8]) + (__pyx_v_one_minus_beta2 * powf((__pyx_v_gradient[__pyx_v_j]), 2.0))); + } + + /* "thinc/backends/numpy_ops.pyx":613 + * for j in range(step_size): + * mom2[j] += one_minus_beta2 * gradient[j] ** 2 + * for j in range(step_size): # <<<<<<<<<<<<<< + * buff[j] = sqrtf(mom2[j]) + * for j in range(step_size): +*/ + __pyx_t_7 = __pyx_v_step_size; + __pyx_t_5 = __pyx_t_7; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_j = __pyx_t_6; + + /* "thinc/backends/numpy_ops.pyx":614 + * mom2[j] += one_minus_beta2 * gradient[j] ** 2 + * for j in range(step_size): + * buff[j] = sqrtf(mom2[j]) # <<<<<<<<<<<<<< + * for j in range(step_size): + * buff[j] += eps +*/ + (__pyx_v_buff[__pyx_v_j]) = sqrtf((__pyx_v_mom2[__pyx_v_j])); + } + + /* "thinc/backends/numpy_ops.pyx":615 + * for j in range(step_size): + * buff[j] = sqrtf(mom2[j]) + * for j in range(step_size): # <<<<<<<<<<<<<< + * buff[j] += eps + * for j in range(step_size): +*/ + __pyx_t_7 = __pyx_v_step_size; + __pyx_t_5 = __pyx_t_7; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_j = __pyx_t_6; + + /* "thinc/backends/numpy_ops.pyx":616 + * buff[j] = sqrtf(mom2[j]) + * for j in range(step_size): + * buff[j] += eps # <<<<<<<<<<<<<< + * for j in range(step_size): + * buff[j] = mom1[j] / buff[j] +*/ + __pyx_t_8 = __pyx_v_j; + (__pyx_v_buff[__pyx_t_8]) = ((__pyx_v_buff[__pyx_t_8]) + __pyx_v_eps); + } + + /* "thinc/backends/numpy_ops.pyx":617 + * for j in range(step_size): + * buff[j] += eps + * for j in range(step_size): # <<<<<<<<<<<<<< + * buff[j] = mom1[j] / buff[j] + * for j in range(step_size): +*/ + __pyx_t_7 = __pyx_v_step_size; + __pyx_t_5 = __pyx_t_7; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_j = __pyx_t_6; + + /* "thinc/backends/numpy_ops.pyx":618 + * buff[j] += eps + * for j in range(step_size): + * buff[j] = mom1[j] / buff[j] # <<<<<<<<<<<<<< + * for j in range(step_size): + * gradient[j] = buff[j] +*/ + (__pyx_v_buff[__pyx_v_j]) = ((__pyx_v_mom1[__pyx_v_j]) / (__pyx_v_buff[__pyx_v_j])); + } + + /* "thinc/backends/numpy_ops.pyx":619 + * for j in range(step_size): + * buff[j] = mom1[j] / buff[j] + * for j in range(step_size): # <<<<<<<<<<<<<< + * gradient[j] = buff[j] + * mom1 += step_size +*/ + __pyx_t_7 = __pyx_v_step_size; + __pyx_t_5 = __pyx_t_7; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_j = __pyx_t_6; + + /* "thinc/backends/numpy_ops.pyx":620 + * buff[j] = mom1[j] / buff[j] + * for j in range(step_size): + * gradient[j] = buff[j] # <<<<<<<<<<<<<< + * mom1 += step_size + * mom2 += step_size +*/ + (__pyx_v_gradient[__pyx_v_j]) = (__pyx_v_buff[__pyx_v_j]); + } + + /* "thinc/backends/numpy_ops.pyx":621 + * for j in range(step_size): + * gradient[j] = buff[j] + * mom1 += step_size # <<<<<<<<<<<<<< + * mom2 += step_size + * gradient += step_size +*/ + __pyx_v_mom1 = (__pyx_v_mom1 + __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":622 + * gradient[j] = buff[j] + * mom1 += step_size + * mom2 += step_size # <<<<<<<<<<<<<< + * gradient += step_size + * idx += step_size +*/ + __pyx_v_mom2 = (__pyx_v_mom2 + __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":623 + * mom1 += step_size + * mom2 += step_size + * gradient += step_size # <<<<<<<<<<<<<< + * idx += step_size + * +*/ + __pyx_v_gradient = (__pyx_v_gradient + __pyx_v_step_size); + + /* "thinc/backends/numpy_ops.pyx":624 + * mom2 += step_size + * gradient += step_size + * idx += step_size # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_idx = (__pyx_v_idx + __pyx_v_step_size); + } + + /* "thinc/backends/numpy_ops.pyx":589 + * + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _adam_momentum(weight_t* gradient, weight_t* mom1, weight_t* mom2, + * int nr_weight, weight_t beta1, weight_t beta2, weight_t eps, +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":627 + * + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void cpu_update_averages(weight_t* ema, + * const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_update_averages(__pyx_t_5thinc_8backends_9numpy_ops_weight_t *__pyx_v_ema, __pyx_t_5thinc_8backends_9numpy_ops_weight_t const *__pyx_v_weights, int __pyx_v_nr_weight, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_t, __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_max_decay) { + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_decay; + __pyx_t_5thinc_8backends_9numpy_ops_weight_t __pyx_v_one_minus_decay; + int __pyx_v_i; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + + /* "thinc/backends/numpy_ops.pyx":630 + * cdef void cpu_update_averages(weight_t* ema, + * const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: + * cdef weight_t decay = (1.0 + t) / (10.0 + t) # <<<<<<<<<<<<<< + * if decay > max_decay: + * decay = max_decay +*/ + __pyx_v_decay = ((1.0 + __pyx_v_t) / (10.0 + __pyx_v_t)); + + /* "thinc/backends/numpy_ops.pyx":631 + * const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: + * cdef weight_t decay = (1.0 + t) / (10.0 + t) + * if decay > max_decay: # <<<<<<<<<<<<<< + * decay = max_decay + * cdef weight_t one_minus_decay = 1-decay +*/ + __pyx_t_1 = (__pyx_v_decay > __pyx_v_max_decay); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":632 + * cdef weight_t decay = (1.0 + t) / (10.0 + t) + * if decay > max_decay: + * decay = max_decay # <<<<<<<<<<<<<< + * cdef weight_t one_minus_decay = 1-decay + * cdef int i +*/ + __pyx_v_decay = __pyx_v_max_decay; + + /* "thinc/backends/numpy_ops.pyx":631 + * const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: + * cdef weight_t decay = (1.0 + t) / (10.0 + t) + * if decay > max_decay: # <<<<<<<<<<<<<< + * decay = max_decay + * cdef weight_t one_minus_decay = 1-decay +*/ + } + + /* "thinc/backends/numpy_ops.pyx":633 + * if decay > max_decay: + * decay = max_decay + * cdef weight_t one_minus_decay = 1-decay # <<<<<<<<<<<<<< + * cdef int i + * for i in range(nr_weight): # num_threads=4, schedule='static'): +*/ + __pyx_v_one_minus_decay = (1.0 - __pyx_v_decay); + + /* "thinc/backends/numpy_ops.pyx":635 + * cdef weight_t one_minus_decay = 1-decay + * cdef int i + * for i in range(nr_weight): # num_threads=4, schedule='static'): # <<<<<<<<<<<<<< + * ema[i] -= one_minus_decay * (ema[i] - weights[i]) + * +*/ + __pyx_t_2 = __pyx_v_nr_weight; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/backends/numpy_ops.pyx":636 + * cdef int i + * for i in range(nr_weight): # num_threads=4, schedule='static'): + * ema[i] -= one_minus_decay * (ema[i] - weights[i]) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = __pyx_v_i; + (__pyx_v_ema[__pyx_t_5]) = ((__pyx_v_ema[__pyx_t_5]) - (__pyx_v_one_minus_decay * ((__pyx_v_ema[__pyx_v_i]) - (__pyx_v_weights[__pyx_v_i])))); + } + + /* "thinc/backends/numpy_ops.pyx":627 + * + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void cpu_update_averages(weight_t* ema, + * const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":639 + * + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * np.ndarray params, np.ndarray c_init, np.ndarray h_init, + * np.ndarray X, np.ndarray lengths +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_7lstm_forward_training(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_6lstm_forward_training, "lstm_forward_training(ndarray params, ndarray c_init, ndarray h_init, ndarray X, ndarray lengths)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_7lstm_forward_training = {"lstm_forward_training", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_7lstm_forward_training, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_6lstm_forward_training}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_7lstm_forward_training(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyArrayObject *__pyx_v_params = 0; + PyArrayObject *__pyx_v_c_init = 0; + PyArrayObject *__pyx_v_h_init = 0; + PyArrayObject *__pyx_v_X = 0; + PyArrayObject *__pyx_v_lengths = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lstm_forward_training (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_c_init,&__pyx_mstate_global->__pyx_n_u_h_init,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 639, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 639, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 639, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 639, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 639, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 639, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "lstm_forward_training", 0) < (0)) __PYX_ERR(0, 639, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 5; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("lstm_forward_training", 1, 5, 5, i); __PYX_ERR(0, 639, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 5)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 639, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 639, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 639, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 639, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 639, __pyx_L3_error) + } + __pyx_v_params = ((PyArrayObject *)values[0]); + __pyx_v_c_init = ((PyArrayObject *)values[1]); + __pyx_v_h_init = ((PyArrayObject *)values[2]); + __pyx_v_X = ((PyArrayObject *)values[3]); + __pyx_v_lengths = ((PyArrayObject *)values[4]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("lstm_forward_training", 1, 5, 5, __pyx_nargs); __PYX_ERR(0, 639, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.lstm_forward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 640, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_c_init), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "c_init", 0))) __PYX_ERR(0, 640, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_h_init), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "h_init", 0))) __PYX_ERR(0, 640, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "X", 0))) __PYX_ERR(0, 641, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lengths), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "lengths", 0))) __PYX_ERR(0, 641, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_6lstm_forward_training(__pyx_self, __pyx_v_params, __pyx_v_c_init, __pyx_v_h_init, __pyx_v_X, __pyx_v_lengths); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_6lstm_forward_training(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_params, PyArrayObject *__pyx_v_c_init, PyArrayObject *__pyx_v_h_init, PyArrayObject *__pyx_v_X, PyArrayObject *__pyx_v_lengths) { + PyObject *__pyx_v_xp = NULL; + npy_intp __pyx_v_depth; + npy_intp __pyx_v_dirs; + npy_intp __pyx_v_nO; + npy_intp __pyx_v_N; + npy_intp __pyx_v_nI; + npy_intp __pyx_v_nT; + int __pyx_v_batch_size; + PyArrayObject *__pyx_v_G = 0; + PyArrayObject *__pyx_v_Y = 0; + PyArrayObject *__pyx_v_C = 0; + PyArrayObject *__pyx_v_Yt2 = 0; + PyArrayObject *__pyx_v_Ct2 = 0; + int __pyx_v_params_i; + CYTHON_UNUSED int __pyx_v_seq_i; + PyArrayObject *__pyx_v_orig_X = NULL; + int __pyx_v_i; + PyArrayObject *__pyx_v_Yid = 0; + PyArrayObject *__pyx_v_Cid = 0; + PyArrayObject *__pyx_v_Gid = 0; + PyArrayObject *__pyx_v_Wx = 0; + PyArrayObject *__pyx_v_Wh = 0; + PyArrayObject *__pyx_v_bias = 0; + npy_intp __pyx_v_d; + PyObject *__pyx_v_layer_params = NULL; + PyObject *__pyx_v_H = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + npy_intp __pyx_t_10; + npy_intp __pyx_t_11; + npy_intp __pyx_t_12; + npy_intp __pyx_t_13; + npy_intp __pyx_t_14; + PyObject *(*__pyx_t_15)(PyObject *); + int __pyx_t_16; + int __pyx_t_17; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("lstm_forward_training", 0); + __Pyx_INCREF((PyObject *)__pyx_v_X); + + /* "thinc/backends/numpy_ops.pyx":643 + * np.ndarray X, np.ndarray lengths + * ): + * xp = numpy # <<<<<<<<<<<<<< + * depth = c_init.shape[0] + * dirs = c_init.shape[1] +*/ + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 643, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_xp = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":644 + * ): + * xp = numpy + * depth = c_init.shape[0] # <<<<<<<<<<<<<< + * dirs = c_init.shape[1] + * nO = c_init.shape[2] +*/ + __pyx_v_depth = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_c_init)[0]); + + /* "thinc/backends/numpy_ops.pyx":645 + * xp = numpy + * depth = c_init.shape[0] + * dirs = c_init.shape[1] # <<<<<<<<<<<<<< + * nO = c_init.shape[2] + * N = X.shape[0] +*/ + __pyx_v_dirs = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_c_init)[1]); + + /* "thinc/backends/numpy_ops.pyx":646 + * depth = c_init.shape[0] + * dirs = c_init.shape[1] + * nO = c_init.shape[2] # <<<<<<<<<<<<<< + * N = X.shape[0] + * nI = X.shape[1] +*/ + __pyx_v_nO = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_c_init)[2]); + + /* "thinc/backends/numpy_ops.pyx":647 + * dirs = c_init.shape[1] + * nO = c_init.shape[2] + * N = X.shape[0] # <<<<<<<<<<<<<< + * nI = X.shape[1] + * nT = lengths.shape[0] +*/ + __pyx_v_N = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[0]); + + /* "thinc/backends/numpy_ops.pyx":648 + * nO = c_init.shape[2] + * N = X.shape[0] + * nI = X.shape[1] # <<<<<<<<<<<<<< + * nT = lengths.shape[0] + * cdef int batch_size = lengths[0] +*/ + __pyx_v_nI = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1]); + + /* "thinc/backends/numpy_ops.pyx":649 + * N = X.shape[0] + * nI = X.shape[1] + * nT = lengths.shape[0] # <<<<<<<<<<<<<< + * cdef int batch_size = lengths[0] + * # Preallocate these so we can pass them through for loop. +*/ + __pyx_v_nT = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_lengths)[0]); + + /* "thinc/backends/numpy_ops.pyx":650 + * nI = X.shape[1] + * nT = lengths.shape[0] + * cdef int batch_size = lengths[0] # <<<<<<<<<<<<<< + * # Preallocate these so we can pass them through for loop. + * cdef np.ndarray G = xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f") +*/ + __pyx_t_1 = __Pyx_GetItemInt(((PyObject *)__pyx_v_lengths), 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 650, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyLong_As_int(__pyx_t_1); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 650, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_batch_size = __pyx_t_2; + + /* "thinc/backends/numpy_ops.pyx":652 + * cdef int batch_size = lengths[0] + * # Preallocate these so we can pass them through for loop. + * cdef np.ndarray G = xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f") # <<<<<<<<<<<<<< + * cdef np.ndarray Y = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") +*/ + __pyx_t_3 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyLong_From_npy_intp(__pyx_v_depth); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_dirs); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[0])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_From_npy_intp((__pyx_v_nO * 4)); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 652, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 652, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 652, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 3, __pyx_t_7) != (0)) __PYX_ERR(0, 652, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_7 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_t_8}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_7, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 652, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 652, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 652, __pyx_L1_error) + __pyx_v_G = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":653 + * # Preallocate these so we can pass them through for loop. + * cdef np.ndarray G = xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f") + * cdef np.ndarray Y = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") # <<<<<<<<<<<<<< + * cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray Yt2 = numpy.zeros((batch_size, nO), dtype="f") +*/ + __pyx_t_7 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyLong_From_npy_intp(__pyx_v_depth); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_3 = __Pyx_PyLong_From_npy_intp(__pyx_v_dirs); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[0])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = PyTuple_New(4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 653, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 653, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 653, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_t_5) != (0)) __PYX_ERR(0, 653, __pyx_L1_error); + __pyx_t_8 = 0; + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_5 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_7, __pyx_t_4}; + __pyx_t_5 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_5, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 653, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_5); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 653, __pyx_L1_error) + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":654 + * cdef np.ndarray G = xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f") + * cdef np.ndarray Y = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") # <<<<<<<<<<<<<< + * cdef np.ndarray Yt2 = numpy.zeros((batch_size, nO), dtype="f") + * cdef np.ndarray Ct2 = numpy.zeros((batch_size, nO), dtype="f") +*/ + __pyx_t_5 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyLong_From_npy_intp(__pyx_v_depth); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = __Pyx_PyLong_From_npy_intp(__pyx_v_dirs); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[0])); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyTuple_New(4); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 654, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 654, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 654, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 3, __pyx_t_3) != (0)) __PYX_ERR(0, 654, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_7 = 0; + __pyx_t_6 = 0; + __pyx_t_3 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_8}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 654, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 654, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 654, __pyx_L1_error) + __pyx_v_C = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":655 + * cdef np.ndarray Y = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray Yt2 = numpy.zeros((batch_size, nO), dtype="f") # <<<<<<<<<<<<<< + * cdef np.ndarray Ct2 = numpy.zeros((batch_size, nO), dtype="f") + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_batch_size); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 655, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 655, __pyx_L1_error); + __pyx_t_8 = 0; + __pyx_t_6 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_t_7}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 655, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 655, __pyx_L1_error) + __pyx_v_Yt2 = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":656 + * cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + * cdef np.ndarray Yt2 = numpy.zeros((batch_size, nO), dtype="f") + * cdef np.ndarray Ct2 = numpy.zeros((batch_size, nO), dtype="f") # <<<<<<<<<<<<<< + * + * cdef int params_i = 0 +*/ + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_v_batch_size); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 656, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 656, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_3 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_7); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_7, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_8}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 656, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_7, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 656, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 656, __pyx_L1_error) + __pyx_v_Ct2 = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":658 + * cdef np.ndarray Ct2 = numpy.zeros((batch_size, nO), dtype="f") + * + * cdef int params_i = 0 # <<<<<<<<<<<<<< + * cdef int seq_i = 0 + * orig_X = X +*/ + __pyx_v_params_i = 0; + + /* "thinc/backends/numpy_ops.pyx":659 + * + * cdef int params_i = 0 + * cdef int seq_i = 0 # <<<<<<<<<<<<<< + * orig_X = X + * cdef int i +*/ + __pyx_v_seq_i = 0; + + /* "thinc/backends/numpy_ops.pyx":660 + * cdef int params_i = 0 + * cdef int seq_i = 0 + * orig_X = X # <<<<<<<<<<<<<< + * cdef int i + * cdef np.ndarray Yid +*/ + __Pyx_INCREF((PyObject *)__pyx_v_X); + __pyx_v_orig_X = __pyx_v_X; + + /* "thinc/backends/numpy_ops.pyx":668 + * cdef np.ndarray Wh + * cdef np.ndarray bias + * for i in range(depth): # <<<<<<<<<<<<<< + * nI = X.shape[1] + * for d in range(dirs): +*/ + __pyx_t_10 = __pyx_v_depth; + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_11; __pyx_t_2+=1) { + __pyx_v_i = __pyx_t_2; + + /* "thinc/backends/numpy_ops.pyx":669 + * cdef np.ndarray bias + * for i in range(depth): + * nI = X.shape[1] # <<<<<<<<<<<<<< + * for d in range(dirs): + * # The inits are shaped (depth, dirs, nO). We add the internal dimension +*/ + __pyx_v_nI = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1]); + + /* "thinc/backends/numpy_ops.pyx":670 + * for i in range(depth): + * nI = X.shape[1] + * for d in range(dirs): # <<<<<<<<<<<<<< + * # The inits are shaped (depth, dirs, nO). We add the internal dimension + * # to make them set correctly. +*/ + __pyx_t_12 = __pyx_v_dirs; + __pyx_t_13 = __pyx_t_12; + for (__pyx_t_14 = 0; __pyx_t_14 < __pyx_t_13; __pyx_t_14+=1) { + __pyx_v_d = __pyx_t_14; + + /* "thinc/backends/numpy_ops.pyx":673 + * # The inits are shaped (depth, dirs, nO). We add the internal dimension + * # to make them set correctly. + * Yt2[:] = h_init[i, d].reshape((1, nO)) # <<<<<<<<<<<<<< + * Ct2[:] = c_init[i, d].reshape((1, nO)) + * layer_params, params_i = _split_weights(params, i, nO, nI, params_i) +*/ + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = __Pyx_PyLong_From_npy_intp(__pyx_v_d); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 673, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 673, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_8 = 0; + __pyx_t_8 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_h_init), __pyx_t_5); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_7 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_mstate_global->__pyx_int_1) != (0)) __PYX_ERR(0, 673, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 673, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (__Pyx_PyObject_SetSlice(((PyObject *)__pyx_v_Yt2), __pyx_t_1, 0, 0, NULL, NULL, &__pyx_mstate_global->__pyx_slice[0], 0, 0, 1) < (0)) __PYX_ERR(0, 673, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":674 + * # to make them set correctly. + * Yt2[:] = h_init[i, d].reshape((1, nO)) + * Ct2[:] = c_init[i, d].reshape((1, nO)) # <<<<<<<<<<<<<< + * layer_params, params_i = _split_weights(params, i, nO, nI, params_i) + * Wx, Wh, bias = _transpose_weights(layer_params) +*/ + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_7 = __Pyx_PyLong_From_npy_intp(__pyx_v_d); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 674, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_7) != (0)) __PYX_ERR(0, 674, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_c_init), __pyx_t_5); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_8 = __pyx_t_7; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_mstate_global->__pyx_int_1) != (0)) __PYX_ERR(0, 674, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 674, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (__Pyx_PyObject_SetSlice(((PyObject *)__pyx_v_Ct2), __pyx_t_1, 0, 0, NULL, NULL, &__pyx_mstate_global->__pyx_slice[0], 0, 0, 1) < (0)) __PYX_ERR(0, 674, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":675 + * Yt2[:] = h_init[i, d].reshape((1, nO)) + * Ct2[:] = c_init[i, d].reshape((1, nO)) + * layer_params, params_i = _split_weights(params, i, nO, nI, params_i) # <<<<<<<<<<<<<< + * Wx, Wh, bias = _transpose_weights(layer_params) + * Yid = Y[i, d] +*/ + __pyx_t_7 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_split_weights); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_nO); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp(__pyx_v_nI); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_params_i); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_7); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_7); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_7, ((PyObject *)__pyx_v_params), __pyx_t_8, __pyx_t_5, __pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (6-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 675, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_6 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_15 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); + index = 0; __pyx_t_3 = __pyx_t_15(__pyx_t_6); if (unlikely(!__pyx_t_3)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_4 = __pyx_t_15(__pyx_t_6); if (unlikely(!__pyx_t_4)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_15(__pyx_t_6), 2) < (0)) __PYX_ERR(0, 675, __pyx_L1_error) + __pyx_t_15 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L8_unpacking_done; + __pyx_L7_unpacking_failed:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_15 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 675, __pyx_L1_error) + __pyx_L8_unpacking_done:; + } + __pyx_t_16 = __Pyx_PyLong_As_int(__pyx_t_4); if (unlikely((__pyx_t_16 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 675, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_layer_params, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_v_params_i = __pyx_t_16; + + /* "thinc/backends/numpy_ops.pyx":676 + * Ct2[:] = c_init[i, d].reshape((1, nO)) + * layer_params, params_i = _split_weights(params, i, nO, nI, params_i) + * Wx, Wh, bias = _transpose_weights(layer_params) # <<<<<<<<<<<<<< + * Yid = Y[i, d] + * Cid = C[i, d] +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_transpose_weights); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_v_layer_params}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 3)) { + if (size > 3) __Pyx_RaiseTooManyValuesError(3); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 676, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_6 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_6); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_6 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_6); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = __Pyx_PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_15 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + index = 0; __pyx_t_3 = __pyx_t_15(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L9_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_4 = __pyx_t_15(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L9_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 2; __pyx_t_6 = __pyx_t_15(__pyx_t_5); if (unlikely(!__pyx_t_6)) goto __pyx_L9_unpacking_failed; + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_15(__pyx_t_5), 3) < (0)) __PYX_ERR(0, 676, __pyx_L1_error) + __pyx_t_15 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L10_unpacking_done; + __pyx_L9_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_15 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 676, __pyx_L1_error) + __pyx_L10_unpacking_done:; + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 676, __pyx_L1_error) + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 676, __pyx_L1_error) + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 676, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Wx, ((PyArrayObject *)__pyx_t_3)); + __pyx_t_3 = 0; + __Pyx_XDECREF_SET(__pyx_v_Wh, ((PyArrayObject *)__pyx_t_4)); + __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_bias, ((PyArrayObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":677 + * layer_params, params_i = _split_weights(params, i, nO, nI, params_i) + * Wx, Wh, bias = _transpose_weights(layer_params) + * Yid = Y[i, d] # <<<<<<<<<<<<<< + * Cid = C[i, d] + * Gid = G[i, d] +*/ + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 677, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyLong_From_npy_intp(__pyx_v_d); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 677, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 677, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 677, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 677, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_Y), __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 677, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (!(likely(((__pyx_t_6) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_6, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 677, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Yid, ((PyArrayObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":678 + * Wx, Wh, bias = _transpose_weights(layer_params) + * Yid = Y[i, d] + * Cid = C[i, d] # <<<<<<<<<<<<<< + * Gid = G[i, d] + * _lstm_forward_training( +*/ + __pyx_t_6 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_4 = __Pyx_PyLong_From_npy_intp(__pyx_v_d); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 678, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 678, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_C), __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 678, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Cid, ((PyArrayObject *)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":679 + * Yid = Y[i, d] + * Cid = C[i, d] + * Gid = G[i, d] # <<<<<<<<<<<<<< + * _lstm_forward_training( + * d, N, nO, nI, nT, +*/ + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_i); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 679, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyLong_From_npy_intp(__pyx_v_d); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 679, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 679, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 679, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 679, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_G), __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 679, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 679, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Gid, ((PyArrayObject *)__pyx_t_1)); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":680 + * Cid = C[i, d] + * Gid = G[i, d] + * _lstm_forward_training( # <<<<<<<<<<<<<< + * d, N, nO, nI, nT, + * Gid, +*/ + __pyx_t_16 = __pyx_f_5thinc_8backends_9numpy_ops__lstm_forward_training(__pyx_v_d, __pyx_v_N, __pyx_v_nO, __pyx_v_nI, __pyx_v_nT, __pyx_v_Gid, ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Yid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Cid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_X)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Wx)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Wh)), __pyx_v_bias, ((int *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_lengths)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Yt2)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Ct2))); if (unlikely(__pyx_t_16 == ((int)-1))) __PYX_ERR(0, 680, __pyx_L1_error) + } + + /* "thinc/backends/numpy_ops.pyx":693 + * Ct2.data + * ) + * H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) # <<<<<<<<<<<<<< + * if dirs == 2: + * H = xp.ascontiguousarray(H) +*/ + __pyx_t_5 = __Pyx_GetItemInt(((PyObject *)__pyx_v_Y), __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_6 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp(__pyx_v_N); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 693, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 693, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_XDECREF_SET(__pyx_v_H, __pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":694 + * ) + * H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) + * if dirs == 2: # <<<<<<<<<<<<<< + * H = xp.ascontiguousarray(H) + * X = H +*/ + __pyx_t_17 = (__pyx_v_dirs == 2); + if (__pyx_t_17) { + + /* "thinc/backends/numpy_ops.pyx":695 + * H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) + * if dirs == 2: + * H = xp.ascontiguousarray(H) # <<<<<<<<<<<<<< + * X = H + * return H, (Y, G, C, orig_X) +*/ + __pyx_t_4 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_v_H}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_ascontiguousarray, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 695, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF_SET(__pyx_v_H, __pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":694 + * ) + * H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) + * if dirs == 2: # <<<<<<<<<<<<<< + * H = xp.ascontiguousarray(H) + * X = H +*/ + } + + /* "thinc/backends/numpy_ops.pyx":696 + * if dirs == 2: + * H = xp.ascontiguousarray(H) + * X = H # <<<<<<<<<<<<<< + * return H, (Y, G, C, orig_X) + * +*/ + __pyx_t_1 = __pyx_v_H; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 696, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_X, ((PyArrayObject *)__pyx_t_1)); + __pyx_t_1 = 0; + } + + /* "thinc/backends/numpy_ops.pyx":697 + * H = xp.ascontiguousarray(H) + * X = H + * return H, (Y, G, C, orig_X) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(!__pyx_v_H)) { __Pyx_RaiseUnboundLocalError("H"); __PYX_ERR(0, 697, __pyx_L1_error) } + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 697, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF((PyObject *)__pyx_v_Y); + __Pyx_GIVEREF((PyObject *)__pyx_v_Y); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_Y)) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_G); + __Pyx_GIVEREF((PyObject *)__pyx_v_G); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, ((PyObject *)__pyx_v_G)) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_C); + __Pyx_GIVEREF((PyObject *)__pyx_v_C); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, ((PyObject *)__pyx_v_C)) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_orig_X); + __Pyx_GIVEREF((PyObject *)__pyx_v_orig_X); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_v_orig_X)) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 697, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_H); + __Pyx_GIVEREF(__pyx_v_H); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_H) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 697, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":639 + * + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * np.ndarray params, np.ndarray c_init, np.ndarray h_init, + * np.ndarray X, np.ndarray lengths +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops.lstm_forward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_xp); + __Pyx_XDECREF((PyObject *)__pyx_v_G); + __Pyx_XDECREF((PyObject *)__pyx_v_Y); + __Pyx_XDECREF((PyObject *)__pyx_v_C); + __Pyx_XDECREF((PyObject *)__pyx_v_Yt2); + __Pyx_XDECREF((PyObject *)__pyx_v_Ct2); + __Pyx_XDECREF((PyObject *)__pyx_v_orig_X); + __Pyx_XDECREF((PyObject *)__pyx_v_Yid); + __Pyx_XDECREF((PyObject *)__pyx_v_Cid); + __Pyx_XDECREF((PyObject *)__pyx_v_Gid); + __Pyx_XDECREF((PyObject *)__pyx_v_Wx); + __Pyx_XDECREF((PyObject *)__pyx_v_Wh); + __Pyx_XDECREF((PyObject *)__pyx_v_bias); + __Pyx_XDECREF(__pyx_v_layer_params); + __Pyx_XDECREF(__pyx_v_H); + __Pyx_XDECREF((PyObject *)__pyx_v_X); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":700 + * + * + * cdef int _lstm_forward_training( # <<<<<<<<<<<<<< + * int d, int N, int nO, int nI, int nT, + * np.ndarray G, +*/ + +static int __pyx_f_5thinc_8backends_9numpy_ops__lstm_forward_training(int __pyx_v_d, int __pyx_v_N, int __pyx_v_nO, int __pyx_v_nI, int __pyx_v_nT, PyArrayObject *__pyx_v_G, float *__pyx_v_Y, float *__pyx_v_C, float *__pyx_v_X, float *__pyx_v_Wx, float *__pyx_v_Wh, PyArrayObject *__pyx_v_bias, int *__pyx_v_lengths, float *__pyx_v_Yt2, float *__pyx_v_Ct2) { + double __pyx_v_one; + int __pyx_v_t; + int __pyx_v_batch_size; + int __pyx_v_seq_i; + PyArrayObject *__pyx_v_Gt3_ = 0; + float *__pyx_v_Yt3; + float *__pyx_v_Ct3; + float *__pyx_v_Gt3; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_lstm_forward_training", 0); + + /* "thinc/backends/numpy_ops.pyx":713 + * float* Ct2, + * ) except -1: + * cdef double one = 1.0 # <<<<<<<<<<<<<< + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, + * N, nO*4, nI, +*/ + __pyx_v_one = 1.0; + + /* "thinc/backends/numpy_ops.pyx":714 + * ) except -1: + * cdef double one = 1.0 + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, # <<<<<<<<<<<<<< + * N, nO*4, nI, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_e_4blis_2cy_TRANSPOSE, __pyx_v_N, (__pyx_v_nO * 4), __pyx_v_nI, __pyx_v_one, __pyx_v_X, __pyx_v_nI, 1, __pyx_v_Wx, __pyx_v_nI, 1, __pyx_v_one, ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_G)), (__pyx_v_nO * 4), 1); + + /* "thinc/backends/numpy_ops.pyx":723 + * ) + * cdef int t, batch_size + * cdef int seq_i = 0 if d == 0 else N # <<<<<<<<<<<<<< + * cdef int i, j + * cdef np.ndarray Gt3_ +*/ + __pyx_t_2 = (__pyx_v_d == 0); + if (__pyx_t_2) { + __pyx_t_1 = 0; + } else { + __pyx_t_1 = __pyx_v_N; + } + __pyx_v_seq_i = __pyx_t_1; + + /* "thinc/backends/numpy_ops.pyx":726 + * cdef int i, j + * cdef np.ndarray Gt3_ + * for t in range(nT): # <<<<<<<<<<<<<< + * if d == 0: + * batch_size = lengths[t] +*/ + __pyx_t_1 = __pyx_v_nT; + __pyx_t_3 = __pyx_t_1; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_t = __pyx_t_4; + + /* "thinc/backends/numpy_ops.pyx":727 + * cdef np.ndarray Gt3_ + * for t in range(nT): + * if d == 0: # <<<<<<<<<<<<<< + * batch_size = lengths[t] + * else: +*/ + __pyx_t_2 = (__pyx_v_d == 0); + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":728 + * for t in range(nT): + * if d == 0: + * batch_size = lengths[t] # <<<<<<<<<<<<<< + * else: + * batch_size = lengths[nT-(t+1)] +*/ + __pyx_v_batch_size = (__pyx_v_lengths[__pyx_v_t]); + + /* "thinc/backends/numpy_ops.pyx":727 + * cdef np.ndarray Gt3_ + * for t in range(nT): + * if d == 0: # <<<<<<<<<<<<<< + * batch_size = lengths[t] + * else: +*/ + goto __pyx_L5; + } + + /* "thinc/backends/numpy_ops.pyx":730 + * batch_size = lengths[t] + * else: + * batch_size = lengths[nT-(t+1)] # <<<<<<<<<<<<<< + * seq_i -= batch_size + * # Prepare the inputs +*/ + /*else*/ { + __pyx_v_batch_size = (__pyx_v_lengths[(__pyx_v_nT - (__pyx_v_t + 1))]); + + /* "thinc/backends/numpy_ops.pyx":731 + * else: + * batch_size = lengths[nT-(t+1)] + * seq_i -= batch_size # <<<<<<<<<<<<<< + * # Prepare the inputs + * Yt3 = &Y[seq_i*nO] +*/ + __pyx_v_seq_i = (__pyx_v_seq_i - __pyx_v_batch_size); + } + __pyx_L5:; + + /* "thinc/backends/numpy_ops.pyx":733 + * seq_i -= batch_size + * # Prepare the inputs + * Yt3 = &Y[seq_i*nO] # <<<<<<<<<<<<<< + * Ct3 = &C[seq_i*nO] + * Gt3_ = G[seq_i : seq_i+batch_size] +*/ + __pyx_v_Yt3 = (&(__pyx_v_Y[(__pyx_v_seq_i * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":734 + * # Prepare the inputs + * Yt3 = &Y[seq_i*nO] + * Ct3 = &C[seq_i*nO] # <<<<<<<<<<<<<< + * Gt3_ = G[seq_i : seq_i+batch_size] + * Gt3 = Gt3_.data +*/ + __pyx_v_Ct3 = (&(__pyx_v_C[(__pyx_v_seq_i * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":735 + * Yt3 = &Y[seq_i*nO] + * Ct3 = &C[seq_i*nO] + * Gt3_ = G[seq_i : seq_i+batch_size] # <<<<<<<<<<<<<< + * Gt3 = Gt3_.data + * # Now do the actual calculation +*/ + __pyx_t_5 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_G), __pyx_v_seq_i, (__pyx_v_seq_i + __pyx_v_batch_size), NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 735, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 735, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Gt3_, ((PyArrayObject *)__pyx_t_5)); + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":736 + * Ct3 = &C[seq_i*nO] + * Gt3_ = G[seq_i : seq_i+batch_size] + * Gt3 = Gt3_.data # <<<<<<<<<<<<<< + * # Now do the actual calculation + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, +*/ + __pyx_v_Gt3 = ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Gt3_)); + + /* "thinc/backends/numpy_ops.pyx":738 + * Gt3 = Gt3_.data + * # Now do the actual calculation + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, # <<<<<<<<<<<<<< + * batch_size, nO*4, nO, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_e_4blis_2cy_TRANSPOSE, __pyx_v_batch_size, (__pyx_v_nO * 4), __pyx_v_nO, __pyx_v_one, __pyx_v_Yt2, __pyx_v_nO, 1, __pyx_v_Wh, __pyx_v_nO, 1, __pyx_v_one, __pyx_v_Gt3, (__pyx_v_nO * 4), 1); + + /* "thinc/backends/numpy_ops.pyx":750 + * # It's annoying though --- it means I can't really refactor further, + * # because speed goes down if I remove this. + * Gt3_ += bias # <<<<<<<<<<<<<< + * #for i in range(batch_size): + * # for j in range(nO*4): +*/ + __pyx_t_5 = PyNumber_InPlaceAdd(((PyObject *)__pyx_v_Gt3_), ((PyObject *)__pyx_v_bias)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 750, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 750, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_Gt3_, ((PyArrayObject *)__pyx_t_5)); + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":754 + * # for j in range(nO*4): + * # Gt3[i*nO*4+j] += bias[j] + * cpu_lstm_activate_fwd(Gt3, # <<<<<<<<<<<<<< + * batch_size, nO) + * cpu_lstm_gates_fwd(Yt3, Ct3, +*/ + __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_activate_fwd(__pyx_v_Gt3, __pyx_v_batch_size, __pyx_v_nO); + + /* "thinc/backends/numpy_ops.pyx":756 + * cpu_lstm_activate_fwd(Gt3, + * batch_size, nO) + * cpu_lstm_gates_fwd(Yt3, Ct3, # <<<<<<<<<<<<<< + * Gt3, Ct2, batch_size, nO) + * if d == 0: +*/ + __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_fwd(__pyx_v_Yt3, __pyx_v_Ct3, __pyx_v_Gt3, __pyx_v_Ct2, __pyx_v_batch_size, __pyx_v_nO); + + /* "thinc/backends/numpy_ops.pyx":758 + * cpu_lstm_gates_fwd(Yt3, Ct3, + * Gt3, Ct2, batch_size, nO) + * if d == 0: # <<<<<<<<<<<<<< + * seq_i += batch_size + * # We need to keep a full-sized array here, padded with the sequence-start +*/ + __pyx_t_2 = (__pyx_v_d == 0); + if (__pyx_t_2) { + + /* "thinc/backends/numpy_ops.pyx":759 + * Gt3, Ct2, batch_size, nO) + * if d == 0: + * seq_i += batch_size # <<<<<<<<<<<<<< + * # We need to keep a full-sized array here, padded with the sequence-start + * # values. This isn't necessary for the l2r part, but for the r2l part +*/ + __pyx_v_seq_i = (__pyx_v_seq_i + __pyx_v_batch_size); + + /* "thinc/backends/numpy_ops.pyx":758 + * cpu_lstm_gates_fwd(Yt3, Ct3, + * Gt3, Ct2, batch_size, nO) + * if d == 0: # <<<<<<<<<<<<<< + * seq_i += batch_size + * # We need to keep a full-sized array here, padded with the sequence-start +*/ + } + + /* "thinc/backends/numpy_ops.pyx":764 + * # it's necessary, as we otherwise would have the previous step smaller + * # than the current. + * memcpy(Yt2, Yt3, sizeof(Yt3[0]) * batch_size * nO) # <<<<<<<<<<<<<< + * memcpy(Ct2, Ct3, sizeof(Ct3[0]) * batch_size * nO) + * +*/ + (void)(memcpy(__pyx_v_Yt2, __pyx_v_Yt3, (((sizeof((__pyx_v_Yt3[0]))) * __pyx_v_batch_size) * __pyx_v_nO))); + + /* "thinc/backends/numpy_ops.pyx":765 + * # than the current. + * memcpy(Yt2, Yt3, sizeof(Yt3[0]) * batch_size * nO) + * memcpy(Ct2, Ct3, sizeof(Ct3[0]) * batch_size * nO) # <<<<<<<<<<<<<< + * + * +*/ + (void)(memcpy(__pyx_v_Ct2, __pyx_v_Ct3, (((sizeof((__pyx_v_Ct3[0]))) * __pyx_v_batch_size) * __pyx_v_nO))); + } + + /* "thinc/backends/numpy_ops.pyx":700 + * + * + * cdef int _lstm_forward_training( # <<<<<<<<<<<<<< + * int d, int N, int nO, int nI, int nT, + * np.ndarray G, +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops._lstm_forward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_Gt3_); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":768 + * + * + * def backprop_lstm(np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state): # <<<<<<<<<<<<<< + * xp = numpy + * cdef np.ndarray Y +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_9backprop_lstm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_8backprop_lstm, "backprop_lstm(ndarray dY, ndarray lengths, ndarray params, fwd_state)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_9backprop_lstm = {"backprop_lstm", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_9backprop_lstm, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_8backprop_lstm}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_9backprop_lstm(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyArrayObject *__pyx_v_dY = 0; + PyArrayObject *__pyx_v_lengths = 0; + PyArrayObject *__pyx_v_params = 0; + PyObject *__pyx_v_fwd_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_lstm (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_dY,&__pyx_mstate_global->__pyx_n_u_lengths,&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_fwd_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 768, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 768, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 768, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 768, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 768, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_lstm", 0) < (0)) __PYX_ERR(0, 768, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_lstm", 1, 4, 4, i); __PYX_ERR(0, 768, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 768, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 768, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 768, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 768, __pyx_L3_error) + } + __pyx_v_dY = ((PyArrayObject *)values[0]); + __pyx_v_lengths = ((PyArrayObject *)values[1]); + __pyx_v_params = ((PyArrayObject *)values[2]); + __pyx_v_fwd_state = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_lstm", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 768, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops.backprop_lstm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_dY), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "dY", 0))) __PYX_ERR(0, 768, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lengths), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "lengths", 0))) __PYX_ERR(0, 768, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 768, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_8backprop_lstm(__pyx_self, __pyx_v_dY, __pyx_v_lengths, __pyx_v_params, __pyx_v_fwd_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_8backprop_lstm(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_dY, PyArrayObject *__pyx_v_lengths, PyArrayObject *__pyx_v_params, PyObject *__pyx_v_fwd_state) { + PyObject *__pyx_v_xp = NULL; + PyArrayObject *__pyx_v_Y = 0; + PyArrayObject *__pyx_v_G = 0; + PyArrayObject *__pyx_v_C = 0; + PyArrayObject *__pyx_v_X = 0; + PyArrayObject *__pyx_v_Yid = 0; + PyArrayObject *__pyx_v_Cid = 0; + PyArrayObject *__pyx_v_Gid = 0; + PyArrayObject *__pyx_v_Wx = 0; + PyArrayObject *__pyx_v_Wh = 0; + CYTHON_UNUSED PyArrayObject *__pyx_v_bias = 0; + PyArrayObject *__pyx_v_dWx = 0; + PyArrayObject *__pyx_v_dWh = 0; + PyArrayObject *__pyx_v_d_bias = 0; + PyArrayObject *__pyx_v_dYid = 0; + int __pyx_v_depth; + int __pyx_v_dirs; + int __pyx_v_N; + int __pyx_v_nO; + int __pyx_v_nI; + int __pyx_v_batch_size; + int __pyx_v_nT; + PyArrayObject *__pyx_v_dC = 0; + PyArrayObject *__pyx_v_dG = 0; + CYTHON_UNUSED PyArrayObject *__pyx_v_d_params = 0; + PyObject *__pyx_v_params_i = NULL; + PyObject *__pyx_v_all_layer_params = NULL; + long __pyx_v_i; + int __pyx_v_n_inputs; + int __pyx_v_d; + PyObject *__pyx_v_layer_params = NULL; + PyObject *__pyx_v_all_layer_grads = NULL; + PyObject *__pyx_v_layer_grads = NULL; + PyObject *__pyx_v_indices = NULL; + long __pyx_v_seq_i; + PyArrayObject *__pyx_v_dX = 0; + PyObject *__pyx_v_Xs = NULL; + PyObject *__pyx_v_dXs = NULL; + PyObject *__pyx_v_grad_parts = NULL; + PyObject *__pyx_v_dir_grads = NULL; + CYTHON_UNUSED PyObject *__pyx_v__ = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + int __pyx_t_7; + size_t __pyx_t_8; + int __pyx_t_9; + long __pyx_t_10; + int __pyx_t_11; + int __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_t_15; + PyObject *__pyx_t_16 = NULL; + Py_ssize_t __pyx_t_17; + PyObject *(*__pyx_t_18)(PyObject *); + long __pyx_t_19; + long __pyx_t_20; + PyObject *__pyx_t_21 = NULL; + PyObject *__pyx_t_22 = NULL; + npy_intp __pyx_t_23; + npy_intp __pyx_t_24; + Py_ssize_t __pyx_t_25; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_lstm", 0); + __Pyx_INCREF((PyObject *)__pyx_v_dY); + + /* "thinc/backends/numpy_ops.pyx":769 + * + * def backprop_lstm(np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state): + * xp = numpy # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * cdef np.ndarray G +*/ + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_xp = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":780 + * cdef np.ndarray dWx, dWh, d_bias + * cdef np.ndarray dYid + * Y, G, C, X = fwd_state # <<<<<<<<<<<<<< + * cdef int depth = C.shape[0] + * cdef int dirs = C.shape[1] +*/ + if ((likely(PyTuple_CheckExact(__pyx_v_fwd_state))) || (PyList_CheckExact(__pyx_v_fwd_state))) { + PyObject* sequence = __pyx_v_fwd_state; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 4)) { + if (size > 4) __Pyx_RaiseTooManyValuesError(4); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 780, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 3); + __Pyx_INCREF(__pyx_t_4); + } else { + __pyx_t_1 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 3, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + } + #else + { + Py_ssize_t i; + PyObject** temps[4] = {&__pyx_t_1,&__pyx_t_2,&__pyx_t_3,&__pyx_t_4}; + for (i=0; i < 4; i++) { + PyObject* item = __Pyx_PySequence_ITEM(sequence, i); if (unlikely(!item)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_GOTREF(item); + *(temps[i]) = item; + } + } + #endif + } else { + Py_ssize_t index = -1; + PyObject** temps[4] = {&__pyx_t_1,&__pyx_t_2,&__pyx_t_3,&__pyx_t_4}; + __pyx_t_5 = PyObject_GetIter(__pyx_v_fwd_state); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + for (index=0; index < 4; index++) { + PyObject* item = __pyx_t_6(__pyx_t_5); if (unlikely(!item)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(item); + *(temps[index]) = item; + } + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 4) < (0)) __PYX_ERR(0, 780, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 780, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 780, __pyx_L1_error) + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 780, __pyx_L1_error) + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 780, __pyx_L1_error) + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 780, __pyx_L1_error) + __pyx_v_Y = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_v_G = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + __pyx_v_C = ((PyArrayObject *)__pyx_t_3); + __pyx_t_3 = 0; + __pyx_v_X = ((PyArrayObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":781 + * cdef np.ndarray dYid + * Y, G, C, X = fwd_state + * cdef int depth = C.shape[0] # <<<<<<<<<<<<<< + * cdef int dirs = C.shape[1] + * cdef int N = C.shape[2] +*/ + __pyx_v_depth = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C)[0]); + + /* "thinc/backends/numpy_ops.pyx":782 + * Y, G, C, X = fwd_state + * cdef int depth = C.shape[0] + * cdef int dirs = C.shape[1] # <<<<<<<<<<<<<< + * cdef int N = C.shape[2] + * cdef int nO = C.shape[3] +*/ + __pyx_v_dirs = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C)[1]); + + /* "thinc/backends/numpy_ops.pyx":783 + * cdef int depth = C.shape[0] + * cdef int dirs = C.shape[1] + * cdef int N = C.shape[2] # <<<<<<<<<<<<<< + * cdef int nO = C.shape[3] + * cdef int nI = X.shape[1] +*/ + __pyx_v_N = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C)[2]); + + /* "thinc/backends/numpy_ops.pyx":784 + * cdef int dirs = C.shape[1] + * cdef int N = C.shape[2] + * cdef int nO = C.shape[3] # <<<<<<<<<<<<<< + * cdef int nI = X.shape[1] + * cdef int batch_size = lengths[0] +*/ + __pyx_v_nO = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_C)[3]); + + /* "thinc/backends/numpy_ops.pyx":785 + * cdef int N = C.shape[2] + * cdef int nO = C.shape[3] + * cdef int nI = X.shape[1] # <<<<<<<<<<<<<< + * cdef int batch_size = lengths[0] + * cdef int nT = lengths.shape[0] +*/ + __pyx_v_nI = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1]); + + /* "thinc/backends/numpy_ops.pyx":786 + * cdef int nO = C.shape[3] + * cdef int nI = X.shape[1] + * cdef int batch_size = lengths[0] # <<<<<<<<<<<<<< + * cdef int nT = lengths.shape[0] + * # We don't need to store all the cells for all the layers. +*/ + __pyx_t_4 = __Pyx_GetItemInt(((PyObject *)__pyx_v_lengths), 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 786, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_4); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 786, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_batch_size = __pyx_t_7; + + /* "thinc/backends/numpy_ops.pyx":787 + * cdef int nI = X.shape[1] + * cdef int batch_size = lengths[0] + * cdef int nT = lengths.shape[0] # <<<<<<<<<<<<<< + * # We don't need to store all the cells for all the layers. + * cdef np.ndarray dC = xp.zeros((N, nO), dtype=C.dtype) +*/ + __pyx_v_nT = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_lengths)[0]); + + /* "thinc/backends/numpy_ops.pyx":789 + * cdef int nT = lengths.shape[0] + * # We don't need to store all the cells for all the layers. + * cdef np.ndarray dC = xp.zeros((N, nO), dtype=C.dtype) # <<<<<<<<<<<<<< + * cdef np.ndarray dG = xp.zeros((N, nO*4), dtype=C.dtype) + * cdef np.ndarray d_params = xp.zeros((params.shape[0],), dtype=params.dtype) +*/ + __pyx_t_3 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 789, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(0, 789, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_C), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_t_5}; + __pyx_t_2 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_t_1, __pyx_t_2, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 789, __pyx_L1_error) + __pyx_t_4 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 789, __pyx_L1_error) + __pyx_v_dC = ((PyArrayObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":790 + * # We don't need to store all the cells for all the layers. + * cdef np.ndarray dC = xp.zeros((N, nO), dtype=C.dtype) + * cdef np.ndarray dG = xp.zeros((N, nO*4), dtype=C.dtype) # <<<<<<<<<<<<<< + * cdef np.ndarray d_params = xp.zeros((params.shape[0],), dtype=params.dtype) + * # Collect the params and slices. It makes it a bit easier to get the indexing +*/ + __pyx_t_2 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = __Pyx_PyLong_From_long((__pyx_v_nO * 4)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 790, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 790, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_C), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_t_3}; + __pyx_t_1 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_t_5, __pyx_t_1, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 790, __pyx_L1_error) + __pyx_t_4 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 790, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 790, __pyx_L1_error) + __pyx_v_dG = ((PyArrayObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":791 + * cdef np.ndarray dC = xp.zeros((N, nO), dtype=C.dtype) + * cdef np.ndarray dG = xp.zeros((N, nO*4), dtype=C.dtype) + * cdef np.ndarray d_params = xp.zeros((params.shape[0],), dtype=params.dtype) # <<<<<<<<<<<<<< + * # Collect the params and slices. It makes it a bit easier to get the indexing + * # right, when we're iterating backwards. +*/ + __pyx_t_1 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_params)[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 791, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_1, __pyx_t_3}; + __pyx_t_2 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_t_5, __pyx_t_2, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 791, __pyx_L1_error) + __pyx_t_4 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_2); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 791, __pyx_L1_error) + __pyx_v_d_params = ((PyArrayObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":794 + * # Collect the params and slices. It makes it a bit easier to get the indexing + * # right, when we're iterating backwards. + * params_i = 0 # <<<<<<<<<<<<<< + * all_layer_params = [] + * for i in range(depth): +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __pyx_v_params_i = __pyx_mstate_global->__pyx_int_0; + + /* "thinc/backends/numpy_ops.pyx":795 + * # right, when we're iterating backwards. + * params_i = 0 + * all_layer_params = [] # <<<<<<<<<<<<<< + * for i in range(depth): + * all_layer_params.append([]) +*/ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 795, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_v_all_layer_params = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":796 + * params_i = 0 + * all_layer_params = [] + * for i in range(depth): # <<<<<<<<<<<<<< + * all_layer_params.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) +*/ + __pyx_t_7 = __pyx_v_depth; + __pyx_t_9 = __pyx_t_7; + for (__pyx_t_10 = 0; __pyx_t_10 < __pyx_t_9; __pyx_t_10+=1) { + __pyx_v_i = __pyx_t_10; + + /* "thinc/backends/numpy_ops.pyx":797 + * all_layer_params = [] + * for i in range(depth): + * all_layer_params.append([]) # <<<<<<<<<<<<<< + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): +*/ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 797, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_11 = __Pyx_PyList_Append(__pyx_v_all_layer_params, __pyx_t_4); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 797, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":798 + * for i in range(depth): + * all_layer_params.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) # <<<<<<<<<<<<<< + * for d in range(dirs): + * layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) +*/ + __pyx_t_13 = (__pyx_v_i == 0); + if (__pyx_t_13) { + __pyx_t_12 = __pyx_v_nI; + } else { + __pyx_t_12 = (__pyx_v_nO * __pyx_v_dirs); + } + __pyx_v_n_inputs = __pyx_t_12; + + /* "thinc/backends/numpy_ops.pyx":799 + * all_layer_params.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): # <<<<<<<<<<<<<< + * layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_params = _transpose_weights(layer_params) +*/ + __pyx_t_12 = __pyx_v_dirs; + __pyx_t_14 = __pyx_t_12; + for (__pyx_t_15 = 0; __pyx_t_15 < __pyx_t_14; __pyx_t_15+=1) { + __pyx_v_d = __pyx_t_15; + + /* "thinc/backends/numpy_ops.pyx":800 + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): + * layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) # <<<<<<<<<<<<<< + * layer_params = _transpose_weights(layer_params) + * all_layer_params[-1].append((layer_params, params_i)) +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_split_weights); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = __Pyx_PyLong_From_long(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_16 = __Pyx_PyLong_From_int(__pyx_v_n_inputs); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_2, ((PyObject *)__pyx_v_params), __pyx_t_3, __pyx_t_1, __pyx_t_16, __pyx_v_params_i}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_8, (6-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if ((likely(PyTuple_CheckExact(__pyx_t_4))) || (PyList_CheckExact(__pyx_t_4))) { + PyObject* sequence = __pyx_t_4; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 800, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_5); + __pyx_t_16 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_16); + } else { + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_5); + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_16); + } + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_16 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + #endif + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_1 = PyObject_GetIter(__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 800, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); + index = 0; __pyx_t_5 = __pyx_t_6(__pyx_t_1); if (unlikely(!__pyx_t_5)) goto __pyx_L9_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + index = 1; __pyx_t_16 = __pyx_t_6(__pyx_t_1); if (unlikely(!__pyx_t_16)) goto __pyx_L9_unpacking_failed; + __Pyx_GOTREF(__pyx_t_16); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_1), 2) < (0)) __PYX_ERR(0, 800, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L10_unpacking_done; + __pyx_L9_unpacking_failed:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 800, __pyx_L1_error) + __pyx_L10_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_layer_params, __pyx_t_5); + __pyx_t_5 = 0; + __Pyx_DECREF_SET(__pyx_v_params_i, __pyx_t_16); + __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":801 + * for d in range(dirs): + * layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_params = _transpose_weights(layer_params) # <<<<<<<<<<<<<< + * all_layer_params[-1].append((layer_params, params_i)) + * params_i = 0 +*/ + __pyx_t_16 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_transpose_weights); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 801, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_16 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_16); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_16); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_16, __pyx_v_layer_params}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 801, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF_SET(__pyx_v_layer_params, __pyx_t_4); + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":802 + * layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_params = _transpose_weights(layer_params) + * all_layer_params[-1].append((layer_params, params_i)) # <<<<<<<<<<<<<< + * params_i = 0 + * all_layer_grads = [] +*/ + __pyx_t_4 = __Pyx_GetItemInt_List(__pyx_v_all_layer_params, -1L, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 802, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 802, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_v_layer_params); + __Pyx_GIVEREF(__pyx_v_layer_params); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_layer_params) != (0)) __PYX_ERR(0, 802, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_params_i); + __Pyx_GIVEREF(__pyx_v_params_i); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_params_i) != (0)) __PYX_ERR(0, 802, __pyx_L1_error); + __pyx_t_11 = __Pyx_PyObject_Append(__pyx_t_4, __pyx_t_5); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 802, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + + /* "thinc/backends/numpy_ops.pyx":803 + * layer_params = _transpose_weights(layer_params) + * all_layer_params[-1].append((layer_params, params_i)) + * params_i = 0 # <<<<<<<<<<<<<< + * all_layer_grads = [] + * for i in range(depth): +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_DECREF_SET(__pyx_v_params_i, __pyx_mstate_global->__pyx_int_0); + + /* "thinc/backends/numpy_ops.pyx":804 + * all_layer_params[-1].append((layer_params, params_i)) + * params_i = 0 + * all_layer_grads = [] # <<<<<<<<<<<<<< + * for i in range(depth): + * all_layer_grads.append([]) +*/ + __pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 804, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_v_all_layer_grads = ((PyObject*)__pyx_t_5); + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":805 + * params_i = 0 + * all_layer_grads = [] + * for i in range(depth): # <<<<<<<<<<<<<< + * all_layer_grads.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) +*/ + __pyx_t_7 = __pyx_v_depth; + __pyx_t_9 = __pyx_t_7; + for (__pyx_t_10 = 0; __pyx_t_10 < __pyx_t_9; __pyx_t_10+=1) { + __pyx_v_i = __pyx_t_10; + + /* "thinc/backends/numpy_ops.pyx":806 + * all_layer_grads = [] + * for i in range(depth): + * all_layer_grads.append([]) # <<<<<<<<<<<<<< + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): +*/ + __pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 806, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_11 = __Pyx_PyList_Append(__pyx_v_all_layer_grads, __pyx_t_5); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 806, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":807 + * for i in range(depth): + * all_layer_grads.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) # <<<<<<<<<<<<<< + * for d in range(dirs): + * layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) +*/ + __pyx_t_13 = (__pyx_v_i == 0); + if (__pyx_t_13) { + __pyx_t_12 = __pyx_v_nI; + } else { + __pyx_t_12 = (__pyx_v_nO * __pyx_v_dirs); + } + __pyx_v_n_inputs = __pyx_t_12; + + /* "thinc/backends/numpy_ops.pyx":808 + * all_layer_grads.append([]) + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): # <<<<<<<<<<<<<< + * layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_grads = _transpose_weights(layer_grads) +*/ + __pyx_t_12 = __pyx_v_dirs; + __pyx_t_14 = __pyx_t_12; + for (__pyx_t_15 = 0; __pyx_t_15 < __pyx_t_14; __pyx_t_15+=1) { + __pyx_v_d = __pyx_t_15; + + /* "thinc/backends/numpy_ops.pyx":809 + * n_inputs = nI if i == 0 else (nO * dirs) + * for d in range(dirs): + * layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) # <<<<<<<<<<<<<< + * layer_grads = _transpose_weights(layer_grads) + * all_layer_grads[-1].append((layer_grads, params_i)) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_split_weights); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_1 = __Pyx_PyLong_From_long(__pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_n_inputs); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_16))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_16); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_16); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_16, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[6] = {__pyx_t_4, ((PyObject *)__pyx_v_params), __pyx_t_1, __pyx_t_3, __pyx_t_2, __pyx_v_params_i}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_16, __pyx_callargs+__pyx_t_8, (6-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { + PyObject* sequence = __pyx_t_5; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 809, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_16 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_16); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_16); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_16 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_3 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); + index = 0; __pyx_t_16 = __pyx_t_6(__pyx_t_3); if (unlikely(!__pyx_t_16)) goto __pyx_L15_unpacking_failed; + __Pyx_GOTREF(__pyx_t_16); + index = 1; __pyx_t_2 = __pyx_t_6(__pyx_t_3); if (unlikely(!__pyx_t_2)) goto __pyx_L15_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_3), 2) < (0)) __PYX_ERR(0, 809, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L16_unpacking_done; + __pyx_L15_unpacking_failed:; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 809, __pyx_L1_error) + __pyx_L16_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_layer_grads, __pyx_t_16); + __pyx_t_16 = 0; + __Pyx_DECREF_SET(__pyx_v_params_i, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":810 + * for d in range(dirs): + * layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_grads = _transpose_weights(layer_grads) # <<<<<<<<<<<<<< + * all_layer_grads[-1].append((layer_grads, params_i)) + * # Similarly, we want to compute the indices first +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_transpose_weights); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 810, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_16))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_16); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_16); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_16, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_layer_grads}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_16, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 810, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __Pyx_DECREF_SET(__pyx_v_layer_grads, __pyx_t_5); + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":811 + * layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) + * layer_grads = _transpose_weights(layer_grads) + * all_layer_grads[-1].append((layer_grads, params_i)) # <<<<<<<<<<<<<< + * # Similarly, we want to compute the indices first + * indices = [] +*/ + __pyx_t_5 = __Pyx_GetItemInt_List(__pyx_v_all_layer_grads, -1L, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 811, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 811, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF(__pyx_v_layer_grads); + __Pyx_GIVEREF(__pyx_v_layer_grads); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_v_layer_grads) != (0)) __PYX_ERR(0, 811, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_params_i); + __Pyx_GIVEREF(__pyx_v_params_i); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_v_params_i) != (0)) __PYX_ERR(0, 811, __pyx_L1_error); + __pyx_t_11 = __Pyx_PyObject_Append(__pyx_t_5, __pyx_t_16); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 811, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + } + } + + /* "thinc/backends/numpy_ops.pyx":813 + * all_layer_grads[-1].append((layer_grads, params_i)) + * # Similarly, we want to compute the indices first + * indices = [] # <<<<<<<<<<<<<< + * seq_i = 0 + * for batch_size in lengths: +*/ + __pyx_t_16 = PyList_New(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 813, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_v_indices = ((PyObject*)__pyx_t_16); + __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":814 + * # Similarly, we want to compute the indices first + * indices = [] + * seq_i = 0 # <<<<<<<<<<<<<< + * for batch_size in lengths: + * indices.append((seq_i, batch_size)) +*/ + __pyx_v_seq_i = 0; + + /* "thinc/backends/numpy_ops.pyx":815 + * indices = [] + * seq_i = 0 + * for batch_size in lengths: # <<<<<<<<<<<<<< + * indices.append((seq_i, batch_size)) + * seq_i += batch_size +*/ + if (likely(PyList_CheckExact(((PyObject *)__pyx_v_lengths))) || PyTuple_CheckExact(((PyObject *)__pyx_v_lengths))) { + __pyx_t_16 = ((PyObject *)__pyx_v_lengths); __Pyx_INCREF(__pyx_t_16); + __pyx_t_17 = 0; + __pyx_t_18 = NULL; + } else { + __pyx_t_17 = -1; __pyx_t_16 = PyObject_GetIter(((PyObject *)__pyx_v_lengths)); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 815, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_18 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_16); if (unlikely(!__pyx_t_18)) __PYX_ERR(0, 815, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_18)) { + if (likely(PyList_CheckExact(__pyx_t_16))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_16); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 815, __pyx_L1_error) + #endif + if (__pyx_t_17 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_16, __pyx_t_17, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_17; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_16); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 815, __pyx_L1_error) + #endif + if (__pyx_t_17 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_16, __pyx_t_17)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_16, __pyx_t_17); + #endif + ++__pyx_t_17; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 815, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_18(__pyx_t_16); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 815, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_5); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 815, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_batch_size = __pyx_t_7; + + /* "thinc/backends/numpy_ops.pyx":816 + * seq_i = 0 + * for batch_size in lengths: + * indices.append((seq_i, batch_size)) # <<<<<<<<<<<<<< + * seq_i += batch_size + * +*/ + __pyx_t_5 = __Pyx_PyLong_From_long(__pyx_v_seq_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 816, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_batch_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 816, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 816, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 816, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 816, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_2 = 0; + __pyx_t_11 = __Pyx_PyList_Append(__pyx_v_indices, __pyx_t_3); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 816, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":817 + * for batch_size in lengths: + * indices.append((seq_i, batch_size)) + * seq_i += batch_size # <<<<<<<<<<<<<< + * + * cdef np.ndarray dX +*/ + __pyx_v_seq_i = (__pyx_v_seq_i + __pyx_v_batch_size); + + /* "thinc/backends/numpy_ops.pyx":815 + * indices = [] + * seq_i = 0 + * for batch_size in lengths: # <<<<<<<<<<<<<< + * indices.append((seq_i, batch_size)) + * seq_i += batch_size +*/ + } + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":820 + * + * cdef np.ndarray dX + * Xs = [X] + [Y[i].transpose(1, 0, 2).reshape((N, -1)) for i in range(depth-1)] # <<<<<<<<<<<<<< + * dXs = [xp.zeros((X.shape[0], X.shape[1]), dtype=X.dtype) for X in Xs] + * # Okay, now do the actual looping +*/ + __pyx_t_16 = PyList_New(1); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF((PyObject *)__pyx_v_X); + __Pyx_GIVEREF((PyObject *)__pyx_v_X); + if (__Pyx_PyList_SET_ITEM(__pyx_t_16, 0, ((PyObject *)__pyx_v_X)) != (0)) __PYX_ERR(0, 820, __pyx_L1_error); + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_10 = (__pyx_v_depth - 1); + __pyx_t_19 = __pyx_t_10; + for (__pyx_t_20 = 0; __pyx_t_20 < __pyx_t_19; __pyx_t_20+=1) { + __pyx_v_i = __pyx_t_20; + __pyx_t_1 = __Pyx_GetItemInt(((PyObject *)__pyx_v_Y), __pyx_v_i, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_transpose); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_mstate_global->__pyx_tuple[3], NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_5 = __pyx_t_1; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_21 = PyTuple_New(2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 820, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 820, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_t_21}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_2))) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } + __pyx_t_2 = PyNumber_Add(__pyx_t_16, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 820, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_Xs = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":821 + * cdef np.ndarray dX + * Xs = [X] + [Y[i].transpose(1, 0, 2).reshape((N, -1)) for i in range(depth-1)] + * dXs = [xp.zeros((X.shape[0], X.shape[1]), dtype=X.dtype) for X in Xs] # <<<<<<<<<<<<<< + * # Okay, now do the actual looping + * for i in reversed(range(depth)): +*/ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __pyx_v_Xs; __Pyx_INCREF(__pyx_t_3); + __pyx_t_17 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 821, __pyx_L1_error) + #endif + if (__pyx_t_17 >= __pyx_temp) break; + } + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(__pyx_t_3, __pyx_t_17, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_17; + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + if (!(likely(((__pyx_t_16) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_16, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_X, ((PyArrayObject *)__pyx_t_16)); + __pyx_t_16 = 0; + __pyx_t_1 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_21 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[0])); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_5 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1])); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_21) != (0)) __PYX_ERR(0, 821, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 821, __pyx_L1_error); + __pyx_t_21 = 0; + __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_X), __pyx_mstate_global->__pyx_n_u_dtype); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_1, __pyx_t_4}; + __pyx_t_21 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_t_5, __pyx_t_21, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 821, __pyx_L1_error) + __pyx_t_16 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_zeros, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_21); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_16))) __PYX_ERR(0, 821, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_dXs = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":823 + * dXs = [xp.zeros((X.shape[0], X.shape[1]), dtype=X.dtype) for X in Xs] + * # Okay, now do the actual looping + * for i in reversed(range(depth)): # <<<<<<<<<<<<<< + * dY = dY.reshape((N, dirs, nO)).transpose((1, 0, 2)) + * dX = dXs[i] +*/ + for (__pyx_t_10 = __pyx_v_depth-1; __pyx_t_10 >= 0; __pyx_t_10-=1) { + __pyx_v_i = __pyx_t_10; + + /* "thinc/backends/numpy_ops.pyx":824 + * # Okay, now do the actual looping + * for i in reversed(range(depth)): + * dY = dY.reshape((N, dirs, nO)).transpose((1, 0, 2)) # <<<<<<<<<<<<<< + * dX = dXs[i] + * X = Xs[i] +*/ + __pyx_t_21 = ((PyObject *)__pyx_v_dY); + __Pyx_INCREF(__pyx_t_21); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_dirs); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_22 = PyTuple_New(3); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 824, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 824, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 2, __pyx_t_1) != (0)) __PYX_ERR(0, 824, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_21, __pyx_t_22}; + __pyx_t_16 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_21); __pyx_t_21 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + } + __pyx_t_3 = __pyx_t_16; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 824, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_dY, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":825 + * for i in reversed(range(depth)): + * dY = dY.reshape((N, dirs, nO)).transpose((1, 0, 2)) + * dX = dXs[i] # <<<<<<<<<<<<<< + * X = Xs[i] + * if dirs >= 2: +*/ + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_dXs, __pyx_v_i, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 825, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 825, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_dX, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":826 + * dY = dY.reshape((N, dirs, nO)).transpose((1, 0, 2)) + * dX = dXs[i] + * X = Xs[i] # <<<<<<<<<<<<<< + * if dirs >= 2: + * dY = numpy.ascontiguousarray(dY) +*/ + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_Xs, __pyx_v_i, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 826, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 826, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_X, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":827 + * dX = dXs[i] + * X = Xs[i] + * if dirs >= 2: # <<<<<<<<<<<<<< + * dY = numpy.ascontiguousarray(dY) + * for d in range(dirs): +*/ + __pyx_t_13 = (__pyx_v_dirs >= 2); + if (__pyx_t_13) { + + /* "thinc/backends/numpy_ops.pyx":828 + * X = Xs[i] + * if dirs >= 2: + * dY = numpy.ascontiguousarray(dY) # <<<<<<<<<<<<<< + * for d in range(dirs): + * Wx, Wh, bias = all_layer_params[i][d][0] +*/ + __pyx_t_16 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 828, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_ascontiguousarray); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 828, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_22))) { + __pyx_t_16 = PyMethod_GET_SELF(__pyx_t_22); + assert(__pyx_t_16); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_22); + __Pyx_INCREF(__pyx_t_16); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_22, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_16, ((PyObject *)__pyx_v_dY)}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_22, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 828, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 828, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_dY, ((PyArrayObject *)__pyx_t_2)); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":827 + * dX = dXs[i] + * X = Xs[i] + * if dirs >= 2: # <<<<<<<<<<<<<< + * dY = numpy.ascontiguousarray(dY) + * for d in range(dirs): +*/ + } + + /* "thinc/backends/numpy_ops.pyx":829 + * if dirs >= 2: + * dY = numpy.ascontiguousarray(dY) + * for d in range(dirs): # <<<<<<<<<<<<<< + * Wx, Wh, bias = all_layer_params[i][d][0] + * dWx, dWh, d_bias = all_layer_grads[i][d][0] +*/ + __pyx_t_7 = __pyx_v_dirs; + __pyx_t_9 = __pyx_t_7; + for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_9; __pyx_t_12+=1) { + __pyx_v_d = __pyx_t_12; + + /* "thinc/backends/numpy_ops.pyx":830 + * dY = numpy.ascontiguousarray(dY) + * for d in range(dirs): + * Wx, Wh, bias = all_layer_params[i][d][0] # <<<<<<<<<<<<<< + * dWx, dWh, d_bias = all_layer_grads[i][d][0] + * assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) +*/ + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_all_layer_params, __pyx_v_i, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_22 = __Pyx_GetItemInt(__pyx_t_2, __pyx_v_d, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_22, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 3)) { + if (size > 3) __Pyx_RaiseTooManyValuesError(3); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 830, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_22 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_22); + __pyx_t_16 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_16); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_22 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_22); + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_16); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_22 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_16 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_21 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_21); + index = 0; __pyx_t_22 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_22)) goto __pyx_L30_unpacking_failed; + __Pyx_GOTREF(__pyx_t_22); + index = 1; __pyx_t_16 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_16)) goto __pyx_L30_unpacking_failed; + __Pyx_GOTREF(__pyx_t_16); + index = 2; __pyx_t_3 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_3)) goto __pyx_L30_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_21), 3) < (0)) __PYX_ERR(0, 830, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + goto __pyx_L31_unpacking_done; + __pyx_L30_unpacking_failed:; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 830, __pyx_L1_error) + __pyx_L31_unpacking_done:; + } + if (!(likely(((__pyx_t_22) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_22, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 830, __pyx_L1_error) + if (!(likely(((__pyx_t_16) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_16, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 830, __pyx_L1_error) + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 830, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Wx, ((PyArrayObject *)__pyx_t_22)); + __pyx_t_22 = 0; + __Pyx_XDECREF_SET(__pyx_v_Wh, ((PyArrayObject *)__pyx_t_16)); + __pyx_t_16 = 0; + __Pyx_XDECREF_SET(__pyx_v_bias, ((PyArrayObject *)__pyx_t_3)); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":831 + * for d in range(dirs): + * Wx, Wh, bias = all_layer_params[i][d][0] + * dWx, dWh, d_bias = all_layer_grads[i][d][0] # <<<<<<<<<<<<<< + * assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) + * dYid = dY[d] +*/ + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_all_layer_grads, __pyx_v_i, long, 1, __Pyx_PyLong_From_long, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_GetItemInt(__pyx_t_2, __pyx_v_d, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_3, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 3)) { + if (size > 3) __Pyx_RaiseTooManyValuesError(3); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 831, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_16 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_16); + __pyx_t_22 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_22); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_16); + __pyx_t_22 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_22); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_16 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_22 = __Pyx_PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_21 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_21); + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_3)) goto __pyx_L32_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_16 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_16)) goto __pyx_L32_unpacking_failed; + __Pyx_GOTREF(__pyx_t_16); + index = 2; __pyx_t_22 = __pyx_t_6(__pyx_t_21); if (unlikely(!__pyx_t_22)) goto __pyx_L32_unpacking_failed; + __Pyx_GOTREF(__pyx_t_22); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_21), 3) < (0)) __PYX_ERR(0, 831, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + goto __pyx_L33_unpacking_done; + __pyx_L32_unpacking_failed:; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 831, __pyx_L1_error) + __pyx_L33_unpacking_done:; + } + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 831, __pyx_L1_error) + if (!(likely(((__pyx_t_16) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_16, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 831, __pyx_L1_error) + if (!(likely(((__pyx_t_22) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_22, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 831, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_dWx, ((PyArrayObject *)__pyx_t_3)); + __pyx_t_3 = 0; + __Pyx_XDECREF_SET(__pyx_v_dWh, ((PyArrayObject *)__pyx_t_16)); + __pyx_t_16 = 0; + __Pyx_XDECREF_SET(__pyx_v_d_bias, ((PyArrayObject *)__pyx_t_22)); + __pyx_t_22 = 0; + + /* "thinc/backends/numpy_ops.pyx":832 + * Wx, Wh, bias = all_layer_params[i][d][0] + * dWx, dWh, d_bias = all_layer_grads[i][d][0] + * assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) # <<<<<<<<<<<<<< + * dYid = dY[d] + * dC.fill(0.) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_23 = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dWx)[1]); + __pyx_t_13 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Wx)[1]) == __pyx_t_23); + if (__pyx_t_13) { + __pyx_t_24 = (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1]); + __pyx_t_13 = (__pyx_t_23 == __pyx_t_24); + if (__pyx_t_13) { + __pyx_t_13 = (__pyx_t_24 == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dX)[1])); + } + } + if (unlikely(!__pyx_t_13)) { + __pyx_t_2 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Wx)[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_22 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dWx)[1])); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_16 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1])); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dX)[1])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = PyTuple_New(4); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 832, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_t_22) != (0)) __PYX_ERR(0, 832, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_16); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 2, __pyx_t_16) != (0)) __PYX_ERR(0, 832, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 3, __pyx_t_3) != (0)) __PYX_ERR(0, 832, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_22 = 0; + __pyx_t_16 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyTuple_Pack(1, __pyx_t_21); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 832, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 832, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 832, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":833 + * dWx, dWh, d_bias = all_layer_grads[i][d][0] + * assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) + * dYid = dY[d] # <<<<<<<<<<<<<< + * dC.fill(0.) + * dG.fill(0.) +*/ + __pyx_t_3 = __Pyx_GetItemInt(((PyObject *)__pyx_v_dY), __pyx_v_d, int, 1, __Pyx_PyLong_From_int, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 833, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 833, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_dYid, ((PyArrayObject *)__pyx_t_3)); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":834 + * assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) + * dYid = dY[d] + * dC.fill(0.) # <<<<<<<<<<<<<< + * dG.fill(0.) + * Cid = C[i, d] +*/ + __pyx_t_21 = ((PyObject *)__pyx_v_dC); + __Pyx_INCREF(__pyx_t_21); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_21, __pyx_mstate_global->__pyx_float_0_}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_fill, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_21); __pyx_t_21 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 834, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":835 + * dYid = dY[d] + * dC.fill(0.) + * dG.fill(0.) # <<<<<<<<<<<<<< + * Cid = C[i, d] + * Gid = G[i, d] +*/ + __pyx_t_21 = ((PyObject *)__pyx_v_dG); + __Pyx_INCREF(__pyx_t_21); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_21, __pyx_mstate_global->__pyx_float_0_}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_fill, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_21); __pyx_t_21 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 835, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":836 + * dC.fill(0.) + * dG.fill(0.) + * Cid = C[i, d] # <<<<<<<<<<<<<< + * Gid = G[i, d] + * Yid = Y[i, d] +*/ + __pyx_t_3 = __Pyx_PyLong_From_long(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 836, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = __Pyx_PyLong_From_int(__pyx_v_d); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 836, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 836, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 836, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_21) != (0)) __PYX_ERR(0, 836, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_21 = 0; + __pyx_t_21 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_C), __pyx_t_16); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 836, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (!(likely(((__pyx_t_21) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_21, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 836, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Cid, ((PyArrayObject *)__pyx_t_21)); + __pyx_t_21 = 0; + + /* "thinc/backends/numpy_ops.pyx":837 + * dG.fill(0.) + * Cid = C[i, d] + * Gid = G[i, d] # <<<<<<<<<<<<<< + * Yid = Y[i, d] + * assert (Cid.shape[0], Cid.shape[1]) == (N, nO) +*/ + __pyx_t_21 = __Pyx_PyLong_From_long(__pyx_v_i); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 837, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_16 = __Pyx_PyLong_From_int(__pyx_v_d); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 837, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 837, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_21) != (0)) __PYX_ERR(0, 837, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_16); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_16) != (0)) __PYX_ERR(0, 837, __pyx_L1_error); + __pyx_t_21 = 0; + __pyx_t_16 = 0; + __pyx_t_16 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_G), __pyx_t_3); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 837, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(((__pyx_t_16) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_16, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 837, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Gid, ((PyArrayObject *)__pyx_t_16)); + __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":838 + * Cid = C[i, d] + * Gid = G[i, d] + * Yid = Y[i, d] # <<<<<<<<<<<<<< + * assert (Cid.shape[0], Cid.shape[1]) == (N, nO) + * assert (Yid.shape[0], Yid.shape[1]) == (N, nO) +*/ + __pyx_t_16 = __Pyx_PyLong_From_long(__pyx_v_i); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 838, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_d); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 838, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = PyTuple_New(2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 838, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_16); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_16) != (0)) __PYX_ERR(0, 838, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 838, __pyx_L1_error); + __pyx_t_16 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_Y), __pyx_t_21); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 838, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 838, __pyx_L1_error) + __Pyx_XDECREF_SET(__pyx_v_Yid, ((PyArrayObject *)__pyx_t_3)); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":839 + * Gid = G[i, d] + * Yid = Y[i, d] + * assert (Cid.shape[0], Cid.shape[1]) == (N, nO) # <<<<<<<<<<<<<< + * assert (Yid.shape[0], Yid.shape[1]) == (N, nO) + * assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Cid)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Cid)[1])); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 839, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_21) != (0)) __PYX_ERR(0, 839, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_21 = 0; + __pyx_t_21 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = PyTuple_New(2); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 0, __pyx_t_21) != (0)) __PYX_ERR(0, 839, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 839, __pyx_L1_error); + __pyx_t_21 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_22, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 839, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 839, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 839, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":840 + * Yid = Y[i, d] + * assert (Cid.shape[0], Cid.shape[1]) == (N, nO) + * assert (Yid.shape[0], Yid.shape[1]) == (N, nO) # <<<<<<<<<<<<<< + * assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) + * assert (dYid.shape[0], dYid.shape[1]) == (N, nO) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Yid)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Yid)[1])); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 840, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_22) != (0)) __PYX_ERR(0, 840, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_22 = 0; + __pyx_t_22 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = PyTuple_New(2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_22) != (0)) __PYX_ERR(0, 840, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 840, __pyx_L1_error); + __pyx_t_22 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_21, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 840, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 840, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 840, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":841 + * assert (Cid.shape[0], Cid.shape[1]) == (N, nO) + * assert (Yid.shape[0], Yid.shape[1]) == (N, nO) + * assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) # <<<<<<<<<<<<<< + * assert (dYid.shape[0], dYid.shape[1]) == (N, nO) + * assert (dC.shape[0], dC.shape[1]) == (N, nO) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Gid)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_Gid)[1])); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 841, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_21) != (0)) __PYX_ERR(0, 841, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_21 = 0; + __pyx_t_21 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_3 = __Pyx_PyLong_From_long((__pyx_v_nO * 4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = PyTuple_New(2); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 0, __pyx_t_21) != (0)) __PYX_ERR(0, 841, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 841, __pyx_L1_error); + __pyx_t_21 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_22, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 841, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 841, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 841, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":842 + * assert (Yid.shape[0], Yid.shape[1]) == (N, nO) + * assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) + * assert (dYid.shape[0], dYid.shape[1]) == (N, nO) # <<<<<<<<<<<<<< + * assert (dC.shape[0], dC.shape[1]) == (N, nO) + * assert (dG.shape[0], dG.shape[1]) == (N, nO*4) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dYid)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dYid)[1])); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 842, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_22) != (0)) __PYX_ERR(0, 842, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_22 = 0; + __pyx_t_22 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = PyTuple_New(2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_22) != (0)) __PYX_ERR(0, 842, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 842, __pyx_L1_error); + __pyx_t_22 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_21, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 842, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 842, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 842, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":843 + * assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) + * assert (dYid.shape[0], dYid.shape[1]) == (N, nO) + * assert (dC.shape[0], dC.shape[1]) == (N, nO) # <<<<<<<<<<<<<< + * assert (dG.shape[0], dG.shape[1]) == (N, nO*4) + * _lstm_backward_training(d, N, nO, dX.shape[1], nT, +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dC)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dC)[1])); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 843, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_21) != (0)) __PYX_ERR(0, 843, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_21 = 0; + __pyx_t_21 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = PyTuple_New(2); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_GIVEREF(__pyx_t_21); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 0, __pyx_t_21) != (0)) __PYX_ERR(0, 843, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 843, __pyx_L1_error); + __pyx_t_21 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_22, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 843, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 843, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 843, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":844 + * assert (dYid.shape[0], dYid.shape[1]) == (N, nO) + * assert (dC.shape[0], dC.shape[1]) == (N, nO) + * assert (dG.shape[0], dG.shape[1]) == (N, nO*4) # <<<<<<<<<<<<<< + * _lstm_backward_training(d, N, nO, dX.shape[1], nT, + * dX.data, +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_3 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dG)[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_22 = __Pyx_PyLong_From_npy_intp((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dG)[1])); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_16 = PyTuple_New(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 844, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_16, 1, __pyx_t_22) != (0)) __PYX_ERR(0, 844, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_22 = 0; + __pyx_t_22 = __Pyx_PyLong_From_int(__pyx_v_N); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_3 = __Pyx_PyLong_From_long((__pyx_v_nO * 4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_21 = PyTuple_New(2); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_GIVEREF(__pyx_t_22); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 0, __pyx_t_22) != (0)) __PYX_ERR(0, 844, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_21, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 844, __pyx_L1_error); + __pyx_t_22 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_16, __pyx_t_21, Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_13 < 0))) __PYX_ERR(0, 844, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 844, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 844, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":859 + * Wx.data, + * Wh.data, + * list(indices) # <<<<<<<<<<<<<< + * ) + * dY = dX +*/ + __pyx_t_3 = PySequence_List(__pyx_v_indices); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 859, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + + /* "thinc/backends/numpy_ops.pyx":845 + * assert (dC.shape[0], dC.shape[1]) == (N, nO) + * assert (dG.shape[0], dG.shape[1]) == (N, nO*4) + * _lstm_backward_training(d, N, nO, dX.shape[1], nT, # <<<<<<<<<<<<<< + * dX.data, + * dYid.data, +*/ + __pyx_t_14 = __pyx_f_5thinc_8backends_9numpy_ops__lstm_backward_training(__pyx_v_d, __pyx_v_N, __pyx_v_nO, (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dX)[1]), __pyx_v_nT, ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dX)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dYid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dC)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dG)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dWx)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_dWh)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_d_bias)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Cid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Gid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Yid)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_X)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Wx)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_Wh)), __pyx_t_3); if (unlikely(__pyx_t_14 == ((int)-1))) __PYX_ERR(0, 845, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + + /* "thinc/backends/numpy_ops.pyx":861 + * list(indices) + * ) + * dY = dX # <<<<<<<<<<<<<< + * assert dX.shape[1] == X.shape[1] + * grad_parts = [] +*/ + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __Pyx_DECREF_SET(__pyx_v_dY, __pyx_v_dX); + } + + /* "thinc/backends/numpy_ops.pyx":862 + * ) + * dY = dX + * assert dX.shape[1] == X.shape[1] # <<<<<<<<<<<<<< + * grad_parts = [] + * for layer_grads in all_layer_grads: +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + if (unlikely(!__pyx_v_dX)) { __Pyx_RaiseUnboundLocalError("dX"); __PYX_ERR(0, 862, __pyx_L1_error) } + __pyx_t_13 = ((__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_dX)[1]) == (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_X)[1])); + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 862, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 862, __pyx_L1_error) + #endif + + /* "thinc/backends/numpy_ops.pyx":863 + * dY = dX + * assert dX.shape[1] == X.shape[1] + * grad_parts = [] # <<<<<<<<<<<<<< + * for layer_grads in all_layer_grads: + * for dir_grads, _ in layer_grads: +*/ + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 863, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_grad_parts = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":864 + * assert dX.shape[1] == X.shape[1] + * grad_parts = [] + * for layer_grads in all_layer_grads: # <<<<<<<<<<<<<< + * for dir_grads, _ in layer_grads: + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) +*/ + __pyx_t_3 = __pyx_v_all_layer_grads; __Pyx_INCREF(__pyx_t_3); + __pyx_t_17 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 864, __pyx_L1_error) + #endif + if (__pyx_t_17 >= __pyx_temp) break; + } + __pyx_t_21 = __Pyx_PyList_GetItemRefFast(__pyx_t_3, __pyx_t_17, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_17; + if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 864, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __Pyx_XDECREF_SET(__pyx_v_layer_grads, __pyx_t_21); + __pyx_t_21 = 0; + + /* "thinc/backends/numpy_ops.pyx":865 + * grad_parts = [] + * for layer_grads in all_layer_grads: + * for dir_grads, _ in layer_grads: # <<<<<<<<<<<<<< + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) + * return dX, numpy.concatenate(grad_parts) +*/ + if (likely(PyList_CheckExact(__pyx_v_layer_grads)) || PyTuple_CheckExact(__pyx_v_layer_grads)) { + __pyx_t_21 = __pyx_v_layer_grads; __Pyx_INCREF(__pyx_t_21); + __pyx_t_25 = 0; + __pyx_t_18 = NULL; + } else { + __pyx_t_25 = -1; __pyx_t_21 = PyObject_GetIter(__pyx_v_layer_grads); if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_18 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_21); if (unlikely(!__pyx_t_18)) __PYX_ERR(0, 865, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_18)) { + if (likely(PyList_CheckExact(__pyx_t_21))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_21); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 865, __pyx_L1_error) + #endif + if (__pyx_t_25 >= __pyx_temp) break; + } + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(__pyx_t_21, __pyx_t_25, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_25; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_21); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 865, __pyx_L1_error) + #endif + if (__pyx_t_25 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_16 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_21, __pyx_t_25)); + #else + __pyx_t_16 = __Pyx_PySequence_ITEM(__pyx_t_21, __pyx_t_25); + #endif + ++__pyx_t_25; + } + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 865, __pyx_L1_error) + } else { + __pyx_t_16 = __pyx_t_18(__pyx_t_21); + if (unlikely(!__pyx_t_16)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 865, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_16); + if ((likely(PyTuple_CheckExact(__pyx_t_16))) || (PyList_CheckExact(__pyx_t_16))) { + PyObject* sequence = __pyx_t_16; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 865, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_22 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_22); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_22 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_22); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_22 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_1 = PyObject_GetIter(__pyx_t_16); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 865, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); + index = 0; __pyx_t_22 = __pyx_t_6(__pyx_t_1); if (unlikely(!__pyx_t_22)) goto __pyx_L38_unpacking_failed; + __Pyx_GOTREF(__pyx_t_22); + index = 1; __pyx_t_2 = __pyx_t_6(__pyx_t_1); if (unlikely(!__pyx_t_2)) goto __pyx_L38_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_1), 2) < (0)) __PYX_ERR(0, 865, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L39_unpacking_done; + __pyx_L38_unpacking_failed:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 865, __pyx_L1_error) + __pyx_L39_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_dir_grads, __pyx_t_22); + __pyx_t_22 = 0; + __Pyx_XDECREF_SET(__pyx_v__, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":866 + * for layer_grads in all_layer_grads: + * for dir_grads, _ in layer_grads: + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) # <<<<<<<<<<<<<< + * return dX, numpy.concatenate(grad_parts) + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_22, __pyx_mstate_global->__pyx_n_u_untranspose_unsplit_weights); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 866, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_22))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_22); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_22); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_22, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_dir_grads}; + __pyx_t_16 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_22, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 866, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + } + __pyx_t_11 = __Pyx_PyList_Append(__pyx_v_grad_parts, __pyx_t_16); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(0, 866, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":865 + * grad_parts = [] + * for layer_grads in all_layer_grads: + * for dir_grads, _ in layer_grads: # <<<<<<<<<<<<<< + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) + * return dX, numpy.concatenate(grad_parts) +*/ + } + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + + /* "thinc/backends/numpy_ops.pyx":864 + * assert dX.shape[1] == X.shape[1] + * grad_parts = [] + * for layer_grads in all_layer_grads: # <<<<<<<<<<<<<< + * for dir_grads, _ in layer_grads: + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":867 + * for dir_grads, _ in layer_grads: + * grad_parts.append(_untranspose_unsplit_weights(dir_grads)) + * return dX, numpy.concatenate(grad_parts) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(!__pyx_v_dX)) { __Pyx_RaiseUnboundLocalError("dX"); __PYX_ERR(0, 867, __pyx_L1_error) } + __pyx_t_21 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 867, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_22 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_concatenate); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 867, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_8 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_22))) { + __pyx_t_21 = PyMethod_GET_SELF(__pyx_t_22); + assert(__pyx_t_21); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_22); + __Pyx_INCREF(__pyx_t_21); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_22, __pyx__function); + __pyx_t_8 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_21, __pyx_v_grad_parts}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_22, __pyx_callargs+__pyx_t_8, (2-__pyx_t_8) | (__pyx_t_8*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_21); __pyx_t_21 = 0; + __Pyx_DECREF(__pyx_t_22); __pyx_t_22 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 867, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_t_22 = PyTuple_New(2); if (unlikely(!__pyx_t_22)) __PYX_ERR(0, 867, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_22); + __Pyx_INCREF((PyObject *)__pyx_v_dX); + __Pyx_GIVEREF((PyObject *)__pyx_v_dX); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 0, ((PyObject *)__pyx_v_dX)) != (0)) __PYX_ERR(0, 867, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_22, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 867, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_r = __pyx_t_22; + __pyx_t_22 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":768 + * + * + * def backprop_lstm(np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state): # <<<<<<<<<<<<<< + * xp = numpy + * cdef np.ndarray Y +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_16); + __Pyx_XDECREF(__pyx_t_21); + __Pyx_XDECREF(__pyx_t_22); + __Pyx_AddTraceback("thinc.backends.numpy_ops.backprop_lstm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_xp); + __Pyx_XDECREF((PyObject *)__pyx_v_Y); + __Pyx_XDECREF((PyObject *)__pyx_v_G); + __Pyx_XDECREF((PyObject *)__pyx_v_C); + __Pyx_XDECREF((PyObject *)__pyx_v_X); + __Pyx_XDECREF((PyObject *)__pyx_v_Yid); + __Pyx_XDECREF((PyObject *)__pyx_v_Cid); + __Pyx_XDECREF((PyObject *)__pyx_v_Gid); + __Pyx_XDECREF((PyObject *)__pyx_v_Wx); + __Pyx_XDECREF((PyObject *)__pyx_v_Wh); + __Pyx_XDECREF((PyObject *)__pyx_v_bias); + __Pyx_XDECREF((PyObject *)__pyx_v_dWx); + __Pyx_XDECREF((PyObject *)__pyx_v_dWh); + __Pyx_XDECREF((PyObject *)__pyx_v_d_bias); + __Pyx_XDECREF((PyObject *)__pyx_v_dYid); + __Pyx_XDECREF((PyObject *)__pyx_v_dC); + __Pyx_XDECREF((PyObject *)__pyx_v_dG); + __Pyx_XDECREF((PyObject *)__pyx_v_d_params); + __Pyx_XDECREF(__pyx_v_params_i); + __Pyx_XDECREF(__pyx_v_all_layer_params); + __Pyx_XDECREF(__pyx_v_layer_params); + __Pyx_XDECREF(__pyx_v_all_layer_grads); + __Pyx_XDECREF(__pyx_v_layer_grads); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XDECREF((PyObject *)__pyx_v_dX); + __Pyx_XDECREF(__pyx_v_Xs); + __Pyx_XDECREF(__pyx_v_dXs); + __Pyx_XDECREF(__pyx_v_grad_parts); + __Pyx_XDECREF(__pyx_v_dir_grads); + __Pyx_XDECREF(__pyx_v__); + __Pyx_XDECREF((PyObject *)__pyx_v_dY); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":870 + * + * + * def _split_directions(X, dirs): # <<<<<<<<<<<<<< + * if dirs == 1: + * return [X] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_11_split_directions(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_10_split_directions, "_split_directions(X, dirs)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_11_split_directions = {"_split_directions", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_11_split_directions, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_10_split_directions}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_11_split_directions(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_X = 0; + PyObject *__pyx_v_dirs = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_split_directions (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_dirs,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 870, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 870, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 870, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_split_directions", 0) < (0)) __PYX_ERR(0, 870, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_split_directions", 1, 2, 2, i); __PYX_ERR(0, 870, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 870, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 870, __pyx_L3_error) + } + __pyx_v_X = values[0]; + __pyx_v_dirs = values[1]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_split_directions", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 870, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops._split_directions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_10_split_directions(__pyx_self, __pyx_v_X, __pyx_v_dirs); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_10_split_directions(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_X, PyObject *__pyx_v_dirs) { + PyObject *__pyx_v_X_ = NULL; + PyObject *__pyx_v_Xs = NULL; + PyObject *__pyx_v_d = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_split_directions", 0); + + /* "thinc/backends/numpy_ops.pyx":871 + * + * def _split_directions(X, dirs): + * if dirs == 1: # <<<<<<<<<<<<<< + * return [X] + * else: +*/ + __pyx_t_1 = (__Pyx_PyLong_BoolEqObjC(__pyx_v_dirs, __pyx_mstate_global->__pyx_int_1, 1, 0)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 871, __pyx_L1_error) + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":872 + * def _split_directions(X, dirs): + * if dirs == 1: + * return [X] # <<<<<<<<<<<<<< + * else: + * X_ = X.reshape((X.shape[0], -1, dirs)) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 872, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_v_X); + __Pyx_GIVEREF(__pyx_v_X); + if (__Pyx_PyList_SET_ITEM(__pyx_t_2, 0, __pyx_v_X) != (0)) __PYX_ERR(0, 872, __pyx_L1_error); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":871 + * + * def _split_directions(X, dirs): + * if dirs == 1: # <<<<<<<<<<<<<< + * return [X] + * else: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":874 + * return [X] + * else: + * X_ = X.reshape((X.shape[0], -1, dirs)) # <<<<<<<<<<<<<< + * Xs = [] + * for d in range(dirs): +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_X; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_X, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 874, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_GetItemInt(__pyx_t_4, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 874, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = PyTuple_New(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 874, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(0, 874, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 874, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_dirs); + __Pyx_GIVEREF(__pyx_v_dirs); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_v_dirs) != (0)) __PYX_ERR(0, 874, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 874, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_X_ = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":875 + * else: + * X_ = X.reshape((X.shape[0], -1, dirs)) + * Xs = [] # <<<<<<<<<<<<<< + * for d in range(dirs): + * Xs.append(numpy.ascontiguousarray(X_[:, d])) +*/ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 875, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_Xs = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":876 + * X_ = X.reshape((X.shape[0], -1, dirs)) + * Xs = [] + * for d in range(dirs): # <<<<<<<<<<<<<< + * Xs.append(numpy.ascontiguousarray(X_[:, d])) + * return Xs +*/ + __pyx_t_4 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_v_dirs}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)(&PyRange_Type), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 876, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_4 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 876, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 876, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + for (;;) { + { + __pyx_t_2 = __pyx_t_7(__pyx_t_4); + if (unlikely(!__pyx_t_2)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 876, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_d, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":877 + * Xs = [] + * for d in range(dirs): + * Xs.append(numpy.ascontiguousarray(X_[:, d])) # <<<<<<<<<<<<<< + * return Xs + * +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ascontiguousarray); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(0, 877, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_d); + __Pyx_GIVEREF(__pyx_v_d); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_d) != (0)) __PYX_ERR(0, 877, __pyx_L1_error); + __pyx_t_9 = __Pyx_PyObject_GetItem(__pyx_v_X_, __pyx_t_5); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_9}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_10 = __Pyx_PyList_Append(__pyx_v_Xs, __pyx_t_2); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(0, 877, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":876 + * X_ = X.reshape((X.shape[0], -1, dirs)) + * Xs = [] + * for d in range(dirs): # <<<<<<<<<<<<<< + * Xs.append(numpy.ascontiguousarray(X_[:, d])) + * return Xs +*/ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":878 + * for d in range(dirs): + * Xs.append(numpy.ascontiguousarray(X_[:, d])) + * return Xs # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_Xs); + __pyx_r = __pyx_v_Xs; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":870 + * + * + * def _split_directions(X, dirs): # <<<<<<<<<<<<<< + * if dirs == 1: + * return [X] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.backends.numpy_ops._split_directions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_X_); + __Pyx_XDECREF(__pyx_v_Xs); + __Pyx_XDECREF(__pyx_v_d); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":881 + * + * + * cdef int _lstm_backward_training( # <<<<<<<<<<<<<< + * int d, int N, int nO, int nI, int nT, + * float* dX, +*/ + +static int __pyx_f_5thinc_8backends_9numpy_ops__lstm_backward_training(int __pyx_v_d, int __pyx_v_N, int __pyx_v_nO, int __pyx_v_nI, CYTHON_UNUSED int __pyx_v_nT, float *__pyx_v_dX, float *__pyx_v_dY, float *__pyx_v_dC, float *__pyx_v_dG, float *__pyx_v_dWx, float *__pyx_v_dWh, float *__pyx_v_d_bias, float const *__pyx_v_C, float const *__pyx_v_G, float const *__pyx_v_Y, float const *__pyx_v_X, float const *__pyx_v_Wx, float const *__pyx_v_Wh, PyObject *__pyx_v_indices) { + int __pyx_v_seq_t2; + int __pyx_v_seq_t3; + double __pyx_v_one; + PyObject *__pyx_v_size_t3 = NULL; + int __pyx_v_batch_size; + PyObject *__pyx_v_size_t2 = NULL; + float *__pyx_v_dGt3; + CYTHON_UNUSED float *__pyx_v_dXt3; + float *__pyx_v_dYt3; + float *__pyx_v_dCt3; + float *__pyx_v_dYt2; + float *__pyx_v_dCt2; + float const *__pyx_v_Ct3; + float const *__pyx_v_Gt3; + float const *__pyx_v_Ct2; + int __pyx_v_i; + long __pyx_v_j; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *(*__pyx_t_6)(PyObject *); + int __pyx_t_7; + size_t __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *(*__pyx_t_10)(PyObject *); + PyObject *__pyx_t_11 = NULL; + int __pyx_t_12; + int __pyx_t_13; + long __pyx_t_14; + long __pyx_t_15; + long __pyx_t_16; + long __pyx_t_17; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_lstm_backward_training", 0); + __Pyx_INCREF(__pyx_v_indices); + + /* "thinc/backends/numpy_ops.pyx":900 + * cdef int seq_t2 + * cdef int seq_t3 + * cdef double one = 1.0 # <<<<<<<<<<<<<< + * if d == 0: + * seq_t3, size_t3 = indices[-1] +*/ + __pyx_v_one = 1.0; + + /* "thinc/backends/numpy_ops.pyx":901 + * cdef int seq_t3 + * cdef double one = 1.0 + * if d == 0: # <<<<<<<<<<<<<< + * seq_t3, size_t3 = indices[-1] + * indices = indices[:-1] +*/ + __pyx_t_1 = (__pyx_v_d == 0); + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":902 + * cdef double one = 1.0 + * if d == 0: + * seq_t3, size_t3 = indices[-1] # <<<<<<<<<<<<<< + * indices = indices[:-1] + * indices.reverse() +*/ + __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_indices, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 902, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + index = 0; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L4_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_4 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L4_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < (0)) __PYX_ERR(0, 902, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L5_unpacking_done; + __pyx_L4_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 902, __pyx_L1_error) + __pyx_L5_unpacking_done:; + } + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_3); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 902, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_seq_t3 = __pyx_t_7; + __pyx_v_size_t3 = __pyx_t_4; + __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":903 + * if d == 0: + * seq_t3, size_t3 = indices[-1] + * indices = indices[:-1] # <<<<<<<<<<<<<< + * indices.reverse() + * else: +*/ + __pyx_t_2 = __Pyx_PyObject_GetSlice(__pyx_v_indices, 0, -1L, NULL, NULL, &__pyx_mstate_global->__pyx_slice[1], 0, 1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 903, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF_SET(__pyx_v_indices, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":904 + * seq_t3, size_t3 = indices[-1] + * indices = indices[:-1] + * indices.reverse() # <<<<<<<<<<<<<< + * else: + * seq_t3, size_t3 = indices[0] +*/ + __pyx_t_4 = __pyx_v_indices; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_8 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reverse, __pyx_callargs+__pyx_t_8, (1-__pyx_t_8) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 904, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":901 + * cdef int seq_t3 + * cdef double one = 1.0 + * if d == 0: # <<<<<<<<<<<<<< + * seq_t3, size_t3 = indices[-1] + * indices = indices[:-1] +*/ + goto __pyx_L3; + } + + /* "thinc/backends/numpy_ops.pyx":906 + * indices.reverse() + * else: + * seq_t3, size_t3 = indices[0] # <<<<<<<<<<<<<< + * indices = indices[1:] + * cdef int batch_size +*/ + /*else*/ { + __pyx_t_2 = __Pyx_GetItemInt(__pyx_v_indices, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 906, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_5 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_5); + index = 0; __pyx_t_4 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_4)) goto __pyx_L6_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_3 = __pyx_t_6(__pyx_t_5); if (unlikely(!__pyx_t_3)) goto __pyx_L6_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_5), 2) < (0)) __PYX_ERR(0, 906, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L7_unpacking_done; + __pyx_L6_unpacking_failed:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 906, __pyx_L1_error) + __pyx_L7_unpacking_done:; + } + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_4); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 906, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_seq_t3 = __pyx_t_7; + __pyx_v_size_t3 = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":907 + * else: + * seq_t3, size_t3 = indices[0] + * indices = indices[1:] # <<<<<<<<<<<<<< + * cdef int batch_size + * for seq_t2, size_t2 in indices: +*/ + __pyx_t_2 = __Pyx_PyObject_GetSlice(__pyx_v_indices, 1, 0, NULL, NULL, &__pyx_mstate_global->__pyx_slice[2], 1, 0, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 907, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF_SET(__pyx_v_indices, __pyx_t_2); + __pyx_t_2 = 0; + } + __pyx_L3:; + + /* "thinc/backends/numpy_ops.pyx":909 + * indices = indices[1:] + * cdef int batch_size + * for seq_t2, size_t2 in indices: # <<<<<<<<<<<<<< + * dGt3 = &dG[seq_t3*nO*4] + * dXt3 = &dX[seq_t3*nI] +*/ + if (likely(PyList_CheckExact(__pyx_v_indices)) || PyTuple_CheckExact(__pyx_v_indices)) { + __pyx_t_2 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_2); + __pyx_t_9 = 0; + __pyx_t_10 = NULL; + } else { + __pyx_t_9 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_10 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 909, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_10)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 909, __pyx_L1_error) + #endif + if (__pyx_t_9 >= __pyx_temp) break; + } + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_9, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_9; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 909, __pyx_L1_error) + #endif + if (__pyx_t_9 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_9)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_9); + #endif + ++__pyx_t_9; + } + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 909, __pyx_L1_error) + } else { + __pyx_t_3 = __pyx_t_10(__pyx_t_2); + if (unlikely(!__pyx_t_3)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 909, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_3); + if ((likely(PyTuple_CheckExact(__pyx_t_3))) || (PyList_CheckExact(__pyx_t_3))) { + PyObject* sequence = __pyx_t_3; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 909, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_5); + } else { + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_5); + } + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + #endif + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_11 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_6 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_11); + index = 0; __pyx_t_4 = __pyx_t_6(__pyx_t_11); if (unlikely(!__pyx_t_4)) goto __pyx_L10_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_5 = __pyx_t_6(__pyx_t_11); if (unlikely(!__pyx_t_5)) goto __pyx_L10_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_6(__pyx_t_11), 2) < (0)) __PYX_ERR(0, 909, __pyx_L1_error) + __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + goto __pyx_L11_unpacking_done; + __pyx_L10_unpacking_failed:; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_6 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 909, __pyx_L1_error) + __pyx_L11_unpacking_done:; + } + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_4); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 909, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_seq_t2 = __pyx_t_7; + __Pyx_XDECREF_SET(__pyx_v_size_t2, __pyx_t_5); + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":910 + * cdef int batch_size + * for seq_t2, size_t2 in indices: + * dGt3 = &dG[seq_t3*nO*4] # <<<<<<<<<<<<<< + * dXt3 = &dX[seq_t3*nI] + * dYt3 = &dY[seq_t3*nO] +*/ + __pyx_v_dGt3 = (&(__pyx_v_dG[((__pyx_v_seq_t3 * __pyx_v_nO) * 4)])); + + /* "thinc/backends/numpy_ops.pyx":911 + * for seq_t2, size_t2 in indices: + * dGt3 = &dG[seq_t3*nO*4] + * dXt3 = &dX[seq_t3*nI] # <<<<<<<<<<<<<< + * dYt3 = &dY[seq_t3*nO] + * dCt3 = &dC[seq_t3*nO] +*/ + __pyx_v_dXt3 = (&(__pyx_v_dX[(__pyx_v_seq_t3 * __pyx_v_nI)])); + + /* "thinc/backends/numpy_ops.pyx":912 + * dGt3 = &dG[seq_t3*nO*4] + * dXt3 = &dX[seq_t3*nI] + * dYt3 = &dY[seq_t3*nO] # <<<<<<<<<<<<<< + * dCt3 = &dC[seq_t3*nO] + * dYt2 = &dY[seq_t2*nO] +*/ + __pyx_v_dYt3 = (&(__pyx_v_dY[(__pyx_v_seq_t3 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":913 + * dXt3 = &dX[seq_t3*nI] + * dYt3 = &dY[seq_t3*nO] + * dCt3 = &dC[seq_t3*nO] # <<<<<<<<<<<<<< + * dYt2 = &dY[seq_t2*nO] + * dCt2 = &dC[seq_t2*nO] +*/ + __pyx_v_dCt3 = (&(__pyx_v_dC[(__pyx_v_seq_t3 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":914 + * dYt3 = &dY[seq_t3*nO] + * dCt3 = &dC[seq_t3*nO] + * dYt2 = &dY[seq_t2*nO] # <<<<<<<<<<<<<< + * dCt2 = &dC[seq_t2*nO] + * Ct3 = &C[seq_t3*nO] +*/ + __pyx_v_dYt2 = (&(__pyx_v_dY[(__pyx_v_seq_t2 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":915 + * dCt3 = &dC[seq_t3*nO] + * dYt2 = &dY[seq_t2*nO] + * dCt2 = &dC[seq_t2*nO] # <<<<<<<<<<<<<< + * Ct3 = &C[seq_t3*nO] + * Gt3 = &G[seq_t3*nO*4] +*/ + __pyx_v_dCt2 = (&(__pyx_v_dC[(__pyx_v_seq_t2 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":916 + * dYt2 = &dY[seq_t2*nO] + * dCt2 = &dC[seq_t2*nO] + * Ct3 = &C[seq_t3*nO] # <<<<<<<<<<<<<< + * Gt3 = &G[seq_t3*nO*4] + * Ct2 = &C[seq_t2*nO] +*/ + __pyx_v_Ct3 = (&(__pyx_v_C[(__pyx_v_seq_t3 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":917 + * dCt2 = &dC[seq_t2*nO] + * Ct3 = &C[seq_t3*nO] + * Gt3 = &G[seq_t3*nO*4] # <<<<<<<<<<<<<< + * Ct2 = &C[seq_t2*nO] + * +*/ + __pyx_v_Gt3 = (&(__pyx_v_G[((__pyx_v_seq_t3 * __pyx_v_nO) * 4)])); + + /* "thinc/backends/numpy_ops.pyx":918 + * Ct3 = &C[seq_t3*nO] + * Gt3 = &G[seq_t3*nO*4] + * Ct2 = &C[seq_t2*nO] # <<<<<<<<<<<<<< + * + * batch_size = min(size_t2, size_t3) +*/ + __pyx_v_Ct2 = (&(__pyx_v_C[(__pyx_v_seq_t2 * __pyx_v_nO)])); + + /* "thinc/backends/numpy_ops.pyx":920 + * Ct2 = &C[seq_t2*nO] + * + * batch_size = min(size_t2, size_t3) # <<<<<<<<<<<<<< + * cpu_lstm_gates_bwd(dGt3, dCt2, + * dYt3, dCt3, Gt3, Ct3, Ct2, batch_size * nO +*/ + __Pyx_INCREF(__pyx_v_size_t3); + __pyx_t_3 = __pyx_v_size_t3; + __Pyx_INCREF(__pyx_v_size_t2); + __pyx_t_5 = __pyx_v_size_t2; + __pyx_t_11 = PyObject_RichCompare(__pyx_t_3, __pyx_t_5, Py_LT); __Pyx_XGOTREF(__pyx_t_11); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 920, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_11); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 920, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (__pyx_t_1) { + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = __pyx_t_3; + } else { + __Pyx_INCREF(__pyx_t_5); + __pyx_t_4 = __pyx_t_5; + } + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_4); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 920, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_batch_size = __pyx_t_7; + + /* "thinc/backends/numpy_ops.pyx":921 + * + * batch_size = min(size_t2, size_t3) + * cpu_lstm_gates_bwd(dGt3, dCt2, # <<<<<<<<<<<<<< + * dYt3, dCt3, Gt3, Ct3, Ct2, batch_size * nO + * ) +*/ + __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_bwd(__pyx_v_dGt3, __pyx_v_dCt2, __pyx_v_dYt3, __pyx_v_dCt3, __pyx_v_Gt3, __pyx_v_Ct3, __pyx_v_Ct2, (__pyx_v_batch_size * __pyx_v_nO)); + + /* "thinc/backends/numpy_ops.pyx":926 + * # Backprop hidden-to-hidden w.r.t. hidden. + * # dYt2 += dGt3 @ Wh + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.NO_TRANSPOSE, # <<<<<<<<<<<<<< + * batch_size, nO, nO*4, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_v_batch_size, __pyx_v_nO, (__pyx_v_nO * 4), __pyx_v_one, ((float *)__pyx_v_dGt3), (__pyx_v_nO * 4), 1, ((float *)__pyx_v_Wh), __pyx_v_nO, 1, __pyx_v_one, __pyx_v_dYt2, __pyx_v_nO, 1); + + /* "thinc/backends/numpy_ops.pyx":934 + * dYt2, nO, 1 + * ) + * seq_t3 = seq_t2 # <<<<<<<<<<<<<< + * size_t3 = size_t2 + * +*/ + __pyx_v_seq_t3 = __pyx_v_seq_t2; + + /* "thinc/backends/numpy_ops.pyx":935 + * ) + * seq_t3 = seq_t2 + * size_t3 = size_t2 # <<<<<<<<<<<<<< + * + * # Backprop input-to-hidden w.r.t. weights. +*/ + __Pyx_INCREF(__pyx_v_size_t2); + __Pyx_DECREF_SET(__pyx_v_size_t3, __pyx_v_size_t2); + + /* "thinc/backends/numpy_ops.pyx":909 + * indices = indices[1:] + * cdef int batch_size + * for seq_t2, size_t2 in indices: # <<<<<<<<<<<<<< + * dGt3 = &dG[seq_t3*nO*4] + * dXt3 = &dX[seq_t3*nI] +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":939 + * # Backprop input-to-hidden w.r.t. weights. + * # dWx += dG @ X + * blis.cy.gemm(blis.cy.TRANSPOSE, blis.cy.NO_TRANSPOSE, # <<<<<<<<<<<<<< + * nO*4, nI, N, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_TRANSPOSE, __pyx_e_4blis_2cy_NO_TRANSPOSE, (__pyx_v_nO * 4), __pyx_v_nI, __pyx_v_N, __pyx_v_one, ((float *)__pyx_v_dG), (__pyx_v_nO * 4), 1, ((float *)__pyx_v_X), __pyx_v_nI, 1, __pyx_v_one, __pyx_v_dWx, __pyx_v_nI, 1); + + /* "thinc/backends/numpy_ops.pyx":949 + * # Backprop hidden-to-hidden w.r.t weights. + * # dWh += dG @ Y + * blis.cy.gemm(blis.cy.TRANSPOSE, blis.cy.NO_TRANSPOSE, # <<<<<<<<<<<<<< + * nO*4, nO, N, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_TRANSPOSE, __pyx_e_4blis_2cy_NO_TRANSPOSE, (__pyx_v_nO * 4), __pyx_v_nO, __pyx_v_N, __pyx_v_one, ((float *)__pyx_v_dG), (__pyx_v_nO * 4), 1, ((float *)__pyx_v_Y), __pyx_v_nO, 1, __pyx_v_one, __pyx_v_dWh, __pyx_v_nO, 1); + + /* "thinc/backends/numpy_ops.pyx":958 + * ) + * # Backprop bias + * for i in range(N): # <<<<<<<<<<<<<< + * for j in range(nO*4): + * d_bias[j] += dG[i*nO*4+j] +*/ + __pyx_t_7 = __pyx_v_N; + __pyx_t_12 = __pyx_t_7; + for (__pyx_t_13 = 0; __pyx_t_13 < __pyx_t_12; __pyx_t_13+=1) { + __pyx_v_i = __pyx_t_13; + + /* "thinc/backends/numpy_ops.pyx":959 + * # Backprop bias + * for i in range(N): + * for j in range(nO*4): # <<<<<<<<<<<<<< + * d_bias[j] += dG[i*nO*4+j] + * +*/ + __pyx_t_14 = (__pyx_v_nO * 4); + __pyx_t_15 = __pyx_t_14; + for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_15; __pyx_t_16+=1) { + __pyx_v_j = __pyx_t_16; + + /* "thinc/backends/numpy_ops.pyx":960 + * for i in range(N): + * for j in range(nO*4): + * d_bias[j] += dG[i*nO*4+j] # <<<<<<<<<<<<<< + * + * # Backprop input-to-hidden w.r.t. input +*/ + __pyx_t_17 = __pyx_v_j; + (__pyx_v_d_bias[__pyx_t_17]) = ((__pyx_v_d_bias[__pyx_t_17]) + (__pyx_v_dG[(((__pyx_v_i * __pyx_v_nO) * 4) + __pyx_v_j)])); + } + } + + /* "thinc/backends/numpy_ops.pyx":963 + * + * # Backprop input-to-hidden w.r.t. input + * blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.NO_TRANSPOSE, # <<<<<<<<<<<<<< + * N, nI, nO*4, + * one, +*/ + __pyx_fuse_0__pyx_f_4blis_2cy_gemm(__pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_e_4blis_2cy_NO_TRANSPOSE, __pyx_v_N, __pyx_v_nI, (__pyx_v_nO * 4), __pyx_v_one, ((float *)__pyx_v_dG), (__pyx_v_nO * 4), 1, ((float *)__pyx_v_Wx), __pyx_v_nI, 1, __pyx_v_one, __pyx_v_dX, __pyx_v_nI, 1); + + /* "thinc/backends/numpy_ops.pyx":881 + * + * + * cdef int _lstm_backward_training( # <<<<<<<<<<<<<< + * int d, int N, int nO, int nI, int nT, + * float* dX, +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("thinc.backends.numpy_ops._lstm_backward_training", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_size_t3); + __Pyx_XDECREF(__pyx_v_size_t2); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":973 + * + * + * def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): # <<<<<<<<<<<<<< + * Wx_size = 4 * nO * nI + * bx_size = 4 * nO +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_13_split_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_12_split_weights, "_split_weights(ndarray params, int i, int nO, int nI, int params_i)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_13_split_weights = {"_split_weights", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_13_split_weights, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_12_split_weights}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_13_split_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyArrayObject *__pyx_v_params = 0; + CYTHON_UNUSED int __pyx_v_i; + int __pyx_v_nO; + int __pyx_v_nI; + int __pyx_v_params_i; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_split_weights (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_params,&__pyx_mstate_global->__pyx_n_u_i,&__pyx_mstate_global->__pyx_n_u_nO,&__pyx_mstate_global->__pyx_n_u_nI,&__pyx_mstate_global->__pyx_n_u_params_i,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 973, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 973, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 973, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 973, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 973, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 973, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_split_weights", 0) < (0)) __PYX_ERR(0, 973, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 5; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_split_weights", 1, 5, 5, i); __PYX_ERR(0, 973, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 5)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 973, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 973, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 973, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 973, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 973, __pyx_L3_error) + } + __pyx_v_params = ((PyArrayObject *)values[0]); + __pyx_v_i = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_i == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 973, __pyx_L3_error) + __pyx_v_nO = __Pyx_PyLong_As_int(values[2]); if (unlikely((__pyx_v_nO == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 973, __pyx_L3_error) + __pyx_v_nI = __Pyx_PyLong_As_int(values[3]); if (unlikely((__pyx_v_nI == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 973, __pyx_L3_error) + __pyx_v_params_i = __Pyx_PyLong_As_int(values[4]); if (unlikely((__pyx_v_params_i == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 973, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_split_weights", 1, 5, 5, __pyx_nargs); __PYX_ERR(0, 973, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops._split_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_params), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "params", 0))) __PYX_ERR(0, 973, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_12_split_weights(__pyx_self, __pyx_v_params, __pyx_v_i, __pyx_v_nO, __pyx_v_nI, __pyx_v_params_i); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_12_split_weights(CYTHON_UNUSED PyObject *__pyx_self, PyArrayObject *__pyx_v_params, CYTHON_UNUSED int __pyx_v_i, int __pyx_v_nO, int __pyx_v_nI, int __pyx_v_params_i) { + long __pyx_v_Wx_size; + long __pyx_v_bx_size; + long __pyx_v_Wh_size; + long __pyx_v_bh_size; + PyObject *__pyx_v_Wx = NULL; + PyObject *__pyx_v_bx = NULL; + PyObject *__pyx_v_Wh = NULL; + PyObject *__pyx_v_bh = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_split_weights", 0); + + /* "thinc/backends/numpy_ops.pyx":974 + * + * def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): + * Wx_size = 4 * nO * nI # <<<<<<<<<<<<<< + * bx_size = 4 * nO + * Wh_size = 4 * nO * nO +*/ + __pyx_v_Wx_size = ((4 * __pyx_v_nO) * __pyx_v_nI); + + /* "thinc/backends/numpy_ops.pyx":975 + * def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): + * Wx_size = 4 * nO * nI + * bx_size = 4 * nO # <<<<<<<<<<<<<< + * Wh_size = 4 * nO * nO + * bh_size = 4 * nO +*/ + __pyx_v_bx_size = (4 * __pyx_v_nO); + + /* "thinc/backends/numpy_ops.pyx":976 + * Wx_size = 4 * nO * nI + * bx_size = 4 * nO + * Wh_size = 4 * nO * nO # <<<<<<<<<<<<<< + * bh_size = 4 * nO + * Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) +*/ + __pyx_v_Wh_size = ((4 * __pyx_v_nO) * __pyx_v_nO); + + /* "thinc/backends/numpy_ops.pyx":977 + * bx_size = 4 * nO + * Wh_size = 4 * nO * nO + * bh_size = 4 * nO # <<<<<<<<<<<<<< + * Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) + * params_i += Wx_size +*/ + __pyx_v_bh_size = (4 * __pyx_v_nO); + + /* "thinc/backends/numpy_ops.pyx":978 + * Wh_size = 4 * nO * nO + * bh_size = 4 * nO + * Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) # <<<<<<<<<<<<<< + * params_i += Wx_size + * bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) +*/ + __pyx_t_3 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_params), __pyx_v_params_i, (__pyx_v_params_i + __pyx_v_Wx_size), NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 978, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __pyx_t_3; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyLong_From_long((4 * __pyx_v_nO)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 978, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyLong_From_int(__pyx_v_nI); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 978, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 978, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4) != (0)) __PYX_ERR(0, 978, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 978, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 978, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_Wx = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":979 + * bh_size = 4 * nO + * Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) + * params_i += Wx_size # <<<<<<<<<<<<<< + * bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) + * params_i += bx_size +*/ + __pyx_v_params_i = (__pyx_v_params_i + __pyx_v_Wx_size); + + /* "thinc/backends/numpy_ops.pyx":980 + * Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) + * params_i += Wx_size + * bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) # <<<<<<<<<<<<<< + * params_i += bx_size + * Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) +*/ + __pyx_t_6 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_params), __pyx_v_params_i, (__pyx_v_params_i + __pyx_v_bx_size), NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 980, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyLong_From_long((4 * __pyx_v_nO)); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 980, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 980, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 980, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_5}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 980, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_bx = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":981 + * params_i += Wx_size + * bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) + * params_i += bx_size # <<<<<<<<<<<<<< + * Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) + * params_i += Wh_size +*/ + __pyx_v_params_i = (__pyx_v_params_i + __pyx_v_bx_size); + + /* "thinc/backends/numpy_ops.pyx":982 + * bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) + * params_i += bx_size + * Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) # <<<<<<<<<<<<<< + * params_i += Wh_size + * bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) +*/ + __pyx_t_5 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_params), __pyx_v_params_i, (__pyx_v_params_i + __pyx_v_Wh_size), NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 982, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_3 = __Pyx_PyLong_From_long((4 * __pyx_v_nO)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 982, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 982, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 982, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 982, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 982, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_2 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 982, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_Wh = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":983 + * params_i += bx_size + * Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) + * params_i += Wh_size # <<<<<<<<<<<<<< + * bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) + * params_i += bh_size +*/ + __pyx_v_params_i = (__pyx_v_params_i + __pyx_v_Wh_size); + + /* "thinc/backends/numpy_ops.pyx":984 + * Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) + * params_i += Wh_size + * bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) # <<<<<<<<<<<<<< + * params_i += bh_size + * return ((Wx, bx), (Wh, bh)), params_i +*/ + __pyx_t_4 = __Pyx_PyObject_GetSlice(((PyObject *)__pyx_v_params), __pyx_v_params_i, (__pyx_v_params_i + __pyx_v_bh_size), NULL, NULL, NULL, 1, 1, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 984, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_long((4 * __pyx_v_nO)); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 984, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 984, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 984, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_t_2}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 984, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_bh = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/backends/numpy_ops.pyx":985 + * params_i += Wh_size + * bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) + * params_i += bh_size # <<<<<<<<<<<<<< + * return ((Wx, bx), (Wh, bh)), params_i + * +*/ + __pyx_v_params_i = (__pyx_v_params_i + __pyx_v_bh_size); + + /* "thinc/backends/numpy_ops.pyx":986 + * bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) + * params_i += bh_size + * return ((Wx, bx), (Wh, bh)), params_i # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 986, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_Wx); + __Pyx_GIVEREF(__pyx_v_Wx); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_Wx) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_bx); + __Pyx_GIVEREF(__pyx_v_bx); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_bx) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 986, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_Wh); + __Pyx_GIVEREF(__pyx_v_Wh); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_Wh) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_bh); + __Pyx_GIVEREF(__pyx_v_bh); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_bh) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 986, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_params_i); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 986, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 986, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4) != (0)) __PYX_ERR(0, 986, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_4 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":973 + * + * + * def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): # <<<<<<<<<<<<<< + * Wx_size = 4 * nO * nI + * bx_size = 4 * nO +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.backends.numpy_ops._split_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_Wx); + __Pyx_XDECREF(__pyx_v_bx); + __Pyx_XDECREF(__pyx_v_Wh); + __Pyx_XDECREF(__pyx_v_bh); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":989 + * + * + * def _transpose_weights(params): # <<<<<<<<<<<<<< + * # Transpose the parameters so that the gates are the last dimension. This + * # makes it easier to fuse. +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_15_transpose_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_14_transpose_weights, "_transpose_weights(params)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_15_transpose_weights = {"_transpose_weights", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_15_transpose_weights, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_14_transpose_weights}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_15_transpose_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_params = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_transpose_weights (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_params,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 989, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 989, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_transpose_weights", 0) < (0)) __PYX_ERR(0, 989, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_transpose_weights", 1, 1, 1, i); __PYX_ERR(0, 989, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 989, __pyx_L3_error) + } + __pyx_v_params = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_transpose_weights", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 989, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops._transpose_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_14_transpose_weights(__pyx_self, __pyx_v_params); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_14_transpose_weights(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_params) { + PyObject *__pyx_v_Wx = NULL; + PyObject *__pyx_v_bx = NULL; + PyObject *__pyx_v_Wh = NULL; + PyObject *__pyx_v_bh = NULL; + PyObject *__pyx_v_ascontig = NULL; + PyObject *__pyx_v_bias = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_transpose_weights", 0); + + /* "thinc/backends/numpy_ops.pyx":992 + * # Transpose the parameters so that the gates are the last dimension. This + * # makes it easier to fuse. + * (Wx, bx), (Wh, bh) = params # <<<<<<<<<<<<<< + * Wx = Wx.reshape((4, -1, Wx.shape[-1])) + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) +*/ + if ((likely(PyTuple_CheckExact(__pyx_v_params))) || (PyList_CheckExact(__pyx_v_params))) { + PyObject* sequence = __pyx_v_params; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 992, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_1 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_1 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + } else { + Py_ssize_t index = -1; + __pyx_t_3 = PyObject_GetIter(__pyx_v_params); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); + index = 0; __pyx_t_1 = __pyx_t_4(__pyx_t_3); if (unlikely(!__pyx_t_1)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_1); + index = 1; __pyx_t_2 = __pyx_t_4(__pyx_t_3); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_4(__pyx_t_3), 2) < (0)) __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_t_4 = NULL; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_4 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 992, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_5); + } else { + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_5); + } + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_6 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); + index = 0; __pyx_t_3 = __pyx_t_4(__pyx_t_6); if (unlikely(!__pyx_t_3)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + index = 1; __pyx_t_5 = __pyx_t_4(__pyx_t_6); if (unlikely(!__pyx_t_5)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_4(__pyx_t_6), 2) < (0)) __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_t_4 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_4 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __pyx_v_Wx = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_bx = __pyx_t_5; + __pyx_t_5 = 0; + if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 992, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_5); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_5); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_6 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 992, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); + index = 0; __pyx_t_5 = __pyx_t_4(__pyx_t_6); if (unlikely(!__pyx_t_5)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_5); + index = 1; __pyx_t_3 = __pyx_t_4(__pyx_t_6); if (unlikely(!__pyx_t_3)) goto __pyx_L7_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_4(__pyx_t_6), 2) < (0)) __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_t_4 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L8_unpacking_done; + __pyx_L7_unpacking_failed:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_4 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_L8_unpacking_done:; + } + __pyx_v_Wh = __pyx_t_5; + __pyx_t_5 = 0; + __pyx_v_bh = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":993 + * # makes it easier to fuse. + * (Wx, bx), (Wh, bh) = params + * Wx = Wx.reshape((4, -1, Wx.shape[-1])) # <<<<<<<<<<<<<< + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) +*/ + __pyx_t_1 = __pyx_v_Wx; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wx, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 993, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_GetItemInt(__pyx_t_3, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 993, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 993, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_4); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_mstate_global->__pyx_int_4) != (0)) __PYX_ERR(0, 993, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 993, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_5) != (0)) __PYX_ERR(0, 993, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_t_3}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 993, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_Wx, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":994 + * (Wx, bx), (Wh, bh) = params + * Wx = Wx.reshape((4, -1, Wx.shape[-1])) + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) # <<<<<<<<<<<<<< + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) +*/ + __pyx_t_5 = __pyx_v_Wx; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 994, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_3 = __pyx_t_1; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wx, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 994, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_GetItemInt(__pyx_t_5, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 994, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 994, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 994, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_6) != (0)) __PYX_ERR(0, 994, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_5}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 994, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_Wx, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":995 + * Wx = Wx.reshape((4, -1, Wx.shape[-1])) + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) # <<<<<<<<<<<<<< + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) +*/ + __pyx_t_8 = __pyx_v_bx; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_mstate_global->__pyx_tuple[4]}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 995, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_3 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_tuple[5]}; + __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 995, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __pyx_t_1 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_tuple[1]}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 995, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_bx, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":996 + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) # <<<<<<<<<<<<<< + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) + * bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) +*/ + __pyx_t_5 = __pyx_v_Wh; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wh, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_GetItemInt(__pyx_t_1, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_4); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_mstate_global->__pyx_int_4) != (0)) __PYX_ERR(0, 996, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 996, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_6) != (0)) __PYX_ERR(0, 996, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_t_1}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_Wh, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":997 + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) # <<<<<<<<<<<<<< + * bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * ascontig = numpy.ascontiguousarray +*/ + __pyx_t_6 = __pyx_v_Wh; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 997, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __pyx_t_1 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wh, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 997, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = __Pyx_GetItemInt(__pyx_t_6, -1L, long, 1, __Pyx_PyLong_From_long, 0, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 997, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 997, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 997, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_3) != (0)) __PYX_ERR(0, 997, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_t_6}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 997, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_Wh, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":998 + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) + * bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) # <<<<<<<<<<<<<< + * ascontig = numpy.ascontiguousarray + * Wx = ascontig(Wx) +*/ + __pyx_t_8 = __pyx_v_bh; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_mstate_global->__pyx_tuple[4]}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 998, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_t_1 = __pyx_t_3; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_tuple[5]}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 998, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_5 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_tuple[1]}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 998, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF_SET(__pyx_v_bh, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":999 + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) + * bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * ascontig = numpy.ascontiguousarray # <<<<<<<<<<<<<< + * Wx = ascontig(Wx) + * Wh = ascontig(Wh) +*/ + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 999, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_ascontiguousarray); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 999, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_ascontig = __pyx_t_6; + __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":1000 + * bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + * ascontig = numpy.ascontiguousarray + * Wx = ascontig(Wx) # <<<<<<<<<<<<<< + * Wh = ascontig(Wh) + * bias = ascontig(bx) + bh +*/ + __pyx_t_2 = NULL; + __Pyx_INCREF(__pyx_v_ascontig); + __pyx_t_5 = __pyx_v_ascontig; + __pyx_t_7 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_7 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_Wx}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1000, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_Wx, __pyx_t_6); + __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":1001 + * ascontig = numpy.ascontiguousarray + * Wx = ascontig(Wx) + * Wh = ascontig(Wh) # <<<<<<<<<<<<<< + * bias = ascontig(bx) + bh + * return Wx, Wh, bias +*/ + __pyx_t_5 = NULL; + __Pyx_INCREF(__pyx_v_ascontig); + __pyx_t_2 = __pyx_v_ascontig; + __pyx_t_7 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_7 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_v_Wh}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1001, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_Wh, __pyx_t_6); + __pyx_t_6 = 0; + + /* "thinc/backends/numpy_ops.pyx":1002 + * Wx = ascontig(Wx) + * Wh = ascontig(Wh) + * bias = ascontig(bx) + bh # <<<<<<<<<<<<<< + * return Wx, Wh, bias + * +*/ + __pyx_t_2 = NULL; + __Pyx_INCREF(__pyx_v_ascontig); + __pyx_t_5 = __pyx_v_ascontig; + __pyx_t_7 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_7 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_bx}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1002, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_5 = PyNumber_Add(__pyx_t_6, __pyx_v_bh); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1002, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_v_bias = __pyx_t_5; + __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":1003 + * Wh = ascontig(Wh) + * bias = ascontig(bx) + bh + * return Wx, Wh, bias # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1003, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_v_Wx); + __Pyx_GIVEREF(__pyx_v_Wx); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_Wx) != (0)) __PYX_ERR(0, 1003, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_Wh); + __Pyx_GIVEREF(__pyx_v_Wh); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_Wh) != (0)) __PYX_ERR(0, 1003, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_bias); + __Pyx_GIVEREF(__pyx_v_bias); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_bias) != (0)) __PYX_ERR(0, 1003, __pyx_L1_error); + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":989 + * + * + * def _transpose_weights(params): # <<<<<<<<<<<<<< + * # Transpose the parameters so that the gates are the last dimension. This + * # makes it easier to fuse. +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops._transpose_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_Wx); + __Pyx_XDECREF(__pyx_v_bx); + __Pyx_XDECREF(__pyx_v_Wh); + __Pyx_XDECREF(__pyx_v_bh); + __Pyx_XDECREF(__pyx_v_ascontig); + __Pyx_XDECREF(__pyx_v_bias); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1006 + * + * + * def _untranspose_unsplit_weights(params): # <<<<<<<<<<<<<< + * Wx, Wh, bias = params + * nO = Wh.shape[1] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_17_untranspose_unsplit_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_16_untranspose_unsplit_weights, "_untranspose_unsplit_weights(params)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_17_untranspose_unsplit_weights = {"_untranspose_unsplit_weights", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_17_untranspose_unsplit_weights, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_16_untranspose_unsplit_weights}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_17_untranspose_unsplit_weights(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_params = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_untranspose_unsplit_weights (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_params,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 1006, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 1006, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_untranspose_unsplit_weights", 0) < (0)) __PYX_ERR(0, 1006, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_untranspose_unsplit_weights", 1, 1, 1, i); __PYX_ERR(0, 1006, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 1006, __pyx_L3_error) + } + __pyx_v_params = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_untranspose_unsplit_weights", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 1006, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops._untranspose_unsplit_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_16_untranspose_unsplit_weights(__pyx_self, __pyx_v_params); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_16_untranspose_unsplit_weights(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_params) { + PyObject *__pyx_v_Wx = NULL; + PyObject *__pyx_v_Wh = NULL; + PyObject *__pyx_v_bias = NULL; + PyObject *__pyx_v_nO = NULL; + PyObject *__pyx_v_nI = NULL; + PyObject *__pyx_v_zeros = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *(*__pyx_t_5)(PyObject *); + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_untranspose_unsplit_weights", 0); + + /* "thinc/backends/numpy_ops.pyx":1007 + * + * def _untranspose_unsplit_weights(params): + * Wx, Wh, bias = params # <<<<<<<<<<<<<< + * nO = Wh.shape[1] + * nI = Wx.shape[1] +*/ + if ((likely(PyTuple_CheckExact(__pyx_v_params))) || (PyList_CheckExact(__pyx_v_params))) { + PyObject* sequence = __pyx_v_params; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 3)) { + if (size > 3) __Pyx_RaiseTooManyValuesError(3); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 1007, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_1 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_1 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } else { + Py_ssize_t index = -1; + __pyx_t_4 = PyObject_GetIter(__pyx_v_params); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1007, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_4); + index = 0; __pyx_t_1 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_1)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_1); + index = 1; __pyx_t_2 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + index = 2; __pyx_t_3 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_5(__pyx_t_4), 3) < (0)) __PYX_ERR(0, 1007, __pyx_L1_error) + __pyx_t_5 = NULL; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_5 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 1007, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_Wx = __pyx_t_1; + __pyx_t_1 = 0; + __pyx_v_Wh = __pyx_t_2; + __pyx_t_2 = 0; + __pyx_v_bias = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1008 + * def _untranspose_unsplit_weights(params): + * Wx, Wh, bias = params + * nO = Wh.shape[1] # <<<<<<<<<<<<<< + * nI = Wx.shape[1] + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wh, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1008, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_GetItemInt(__pyx_t_3, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1008, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_nO = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/backends/numpy_ops.pyx":1009 + * Wx, Wh, bias = params + * nO = Wh.shape[1] + * nI = Wx.shape[1] # <<<<<<<<<<<<<< + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_Wx, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1009, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_GetItemInt(__pyx_t_2, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1009, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_nI = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1010 + * nO = Wh.shape[1] + * nI = Wx.shape[1] + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) # <<<<<<<<<<<<<< + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) +*/ + __pyx_t_7 = __pyx_v_Wx; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 1010, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 1010, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_4); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_mstate_global->__pyx_int_4) != (0)) __PYX_ERR(0, 1010, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_nI); + __Pyx_GIVEREF(__pyx_v_nI); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_v_nI) != (0)) __PYX_ERR(0, 1010, __pyx_L1_error); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_t_8}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1010, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_4 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1010, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_2 = __pyx_t_1; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1010, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 1010, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_nI); + __Pyx_GIVEREF(__pyx_v_nI); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_v_nI) != (0)) __PYX_ERR(0, 1010, __pyx_L1_error); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_6}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1010, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF_SET(__pyx_v_Wx, __pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1011 + * nI = Wx.shape[1] + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) # <<<<<<<<<<<<<< + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) + * zeros = numpy.zeros(bias.shape, dtype="f") +*/ + __pyx_t_8 = __pyx_v_Wh; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_7 = PyTuple_New(3); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 1011, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 1011, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_4); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_mstate_global->__pyx_int_4) != (0)) __PYX_ERR(0, 1011, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_nO); + __Pyx_GIVEREF(__pyx_v_nO); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_v_nO) != (0)) __PYX_ERR(0, 1011, __pyx_L1_error); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_t_7}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1011, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_tuple[3]}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1011, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_1 = __pyx_t_6; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1011, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(0, 1011, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_nO); + __Pyx_GIVEREF(__pyx_v_nO); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_nO) != (0)) __PYX_ERR(0, 1011, __pyx_L1_error); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_t_4}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1011, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF_SET(__pyx_v_Wh, __pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1012 + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) # <<<<<<<<<<<<<< + * zeros = numpy.zeros(bias.shape, dtype="f") + * return numpy.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) +*/ + __pyx_t_7 = __pyx_v_bias; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, __pyx_mstate_global->__pyx_tuple[6]}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1012, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_1 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_mstate_global->__pyx_tuple[5]}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_transpose, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1012, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_6 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, __pyx_mstate_global->__pyx_tuple[1]}; + __pyx_t_3 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1012, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_DECREF_SET(__pyx_v_bias, __pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1013 + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) + * zeros = numpy.zeros(bias.shape, dtype="f") # <<<<<<<<<<<<<< + * return numpy.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) + * +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1013, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_zeros); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1013, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_bias, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1013, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_4, __pyx_t_6}; + __pyx_t_1 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1013, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_1, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 1013, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1013, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_v_zeros = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1014 + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) + * zeros = numpy.zeros(bias.shape, dtype="f") + * return numpy.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_concatenate); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4 = __pyx_v_Wx; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_ravel, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_7 = __pyx_v_Wh; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_9 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, NULL}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_ravel, __pyx_callargs+__pyx_t_9, (1-__pyx_t_9) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_7 = PyTuple_New(4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1) != (0)) __PYX_ERR(0, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_bias); + __Pyx_GIVEREF(__pyx_v_bias); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_v_bias) != (0)) __PYX_ERR(0, 1014, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 2, __pyx_t_4) != (0)) __PYX_ERR(0, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_zeros); + __Pyx_GIVEREF(__pyx_v_zeros); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 3, __pyx_v_zeros) != (0)) __PYX_ERR(0, 1014, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_4 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_7}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":1006 + * + * + * def _untranspose_unsplit_weights(params): # <<<<<<<<<<<<<< + * Wx, Wh, bias = params + * nO = Wh.shape[1] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.backends.numpy_ops._untranspose_unsplit_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_Wx); + __Pyx_XDECREF(__pyx_v_Wh); + __Pyx_XDECREF(__pyx_v_bias); + __Pyx_XDECREF(__pyx_v_nO); + __Pyx_XDECREF(__pyx_v_nI); + __Pyx_XDECREF(__pyx_v_zeros); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1017 + * + * + * cdef inline float sigmoid(float X) noexcept nogil: # <<<<<<<<<<<<<< + * return 1./(1. + expf(-X)) + * +*/ + +static CYTHON_INLINE float __pyx_f_5thinc_8backends_9numpy_ops_sigmoid(float __pyx_v_X) { + float __pyx_r; + + /* "thinc/backends/numpy_ops.pyx":1018 + * + * cdef inline float sigmoid(float X) noexcept nogil: + * return 1./(1. + expf(-X)) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = (1. / (1. + expf((-__pyx_v_X)))); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":1017 + * + * + * cdef inline float sigmoid(float X) noexcept nogil: # <<<<<<<<<<<<<< + * return 1./(1. + expf(-X)) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1021 + * + * + * cdef inline float dsigmoid(float y) noexcept nogil: # <<<<<<<<<<<<<< + * return y*(1-y) + * +*/ + +static CYTHON_INLINE float __pyx_f_5thinc_8backends_9numpy_ops_dsigmoid(float __pyx_v_y) { + float __pyx_r; + + /* "thinc/backends/numpy_ops.pyx":1022 + * + * cdef inline float dsigmoid(float y) noexcept nogil: + * return y*(1-y) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = (__pyx_v_y * (1.0 - __pyx_v_y)); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":1021 + * + * + * cdef inline float dsigmoid(float y) noexcept nogil: # <<<<<<<<<<<<<< + * return y*(1-y) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1025 + * + * + * cdef inline float dtanh(float y) noexcept nogil: # <<<<<<<<<<<<<< + * return 1-y**2 + * +*/ + +static CYTHON_INLINE float __pyx_f_5thinc_8backends_9numpy_ops_dtanh(float __pyx_v_y) { + float __pyx_r; + + /* "thinc/backends/numpy_ops.pyx":1026 + * + * cdef inline float dtanh(float y) noexcept nogil: + * return 1-y**2 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = (1.0 - powf(__pyx_v_y, 2.0)); + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":1025 + * + * + * cdef inline float dtanh(float y) noexcept nogil: # <<<<<<<<<<<<<< + * return 1-y**2 + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1029 + * + * + * cdef void cpu_lstm_activate_fwd(float* gates, int B, int N) noexcept nogil: # <<<<<<<<<<<<<< + * """Apply sigmoid activation in-place to columns 0, 1, 2 and tanh to column 3. + * The data is assumed to have the gates in the last dimension. +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_activate_fwd(float *__pyx_v_gates, int __pyx_v_B, int __pyx_v_N) { + int __pyx_v_b; + int __pyx_v_g; + long __pyx_v_end; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + long __pyx_t_5; + + /* "thinc/backends/numpy_ops.pyx":1048 + * # stride here? Surely something to do with cache lines would make more sense? + * cdef int i, b, g + * g = 0 # <<<<<<<<<<<<<< + * for b in range(B): + * g = b * N * 4 +*/ + __pyx_v_g = 0; + + /* "thinc/backends/numpy_ops.pyx":1049 + * cdef int i, b, g + * g = 0 + * for b in range(B): # <<<<<<<<<<<<<< + * g = b * N * 4 + * end = g + N*4 +*/ + __pyx_t_1 = __pyx_v_B; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_b = __pyx_t_3; + + /* "thinc/backends/numpy_ops.pyx":1050 + * g = 0 + * for b in range(B): + * g = b * N * 4 # <<<<<<<<<<<<<< + * end = g + N*4 + * while g < end: +*/ + __pyx_v_g = ((__pyx_v_b * __pyx_v_N) * 4); + + /* "thinc/backends/numpy_ops.pyx":1051 + * for b in range(B): + * g = b * N * 4 + * end = g + N*4 # <<<<<<<<<<<<<< + * while g < end: + * gates[g+0] = expf(-gates[g+0]) +*/ + __pyx_v_end = (__pyx_v_g + (__pyx_v_N * 4)); + + /* "thinc/backends/numpy_ops.pyx":1052 + * g = b * N * 4 + * end = g + N*4 + * while g < end: # <<<<<<<<<<<<<< + * gates[g+0] = expf(-gates[g+0]) + * gates[g+1] = expf(-gates[g+1]) +*/ + while (1) { + __pyx_t_4 = (__pyx_v_g < __pyx_v_end); + if (!__pyx_t_4) break; + + /* "thinc/backends/numpy_ops.pyx":1053 + * end = g + N*4 + * while g < end: + * gates[g+0] = expf(-gates[g+0]) # <<<<<<<<<<<<<< + * gates[g+1] = expf(-gates[g+1]) + * gates[g+2] = expf(-gates[g+2]) +*/ + (__pyx_v_gates[(__pyx_v_g + 0)]) = expf((-(__pyx_v_gates[(__pyx_v_g + 0)]))); + + /* "thinc/backends/numpy_ops.pyx":1054 + * while g < end: + * gates[g+0] = expf(-gates[g+0]) + * gates[g+1] = expf(-gates[g+1]) # <<<<<<<<<<<<<< + * gates[g+2] = expf(-gates[g+2]) + * g += 4 +*/ + (__pyx_v_gates[(__pyx_v_g + 1)]) = expf((-(__pyx_v_gates[(__pyx_v_g + 1)]))); + + /* "thinc/backends/numpy_ops.pyx":1055 + * gates[g+0] = expf(-gates[g+0]) + * gates[g+1] = expf(-gates[g+1]) + * gates[g+2] = expf(-gates[g+2]) # <<<<<<<<<<<<<< + * g += 4 + * g = b * N * 4 +*/ + (__pyx_v_gates[(__pyx_v_g + 2)]) = expf((-(__pyx_v_gates[(__pyx_v_g + 2)]))); + + /* "thinc/backends/numpy_ops.pyx":1056 + * gates[g+1] = expf(-gates[g+1]) + * gates[g+2] = expf(-gates[g+2]) + * g += 4 # <<<<<<<<<<<<<< + * g = b * N * 4 + * while g < end: +*/ + __pyx_v_g = (__pyx_v_g + 4); + } + + /* "thinc/backends/numpy_ops.pyx":1057 + * gates[g+2] = expf(-gates[g+2]) + * g += 4 + * g = b * N * 4 # <<<<<<<<<<<<<< + * while g < end: + * gates[g+0] += 1 +*/ + __pyx_v_g = ((__pyx_v_b * __pyx_v_N) * 4); + + /* "thinc/backends/numpy_ops.pyx":1058 + * g += 4 + * g = b * N * 4 + * while g < end: # <<<<<<<<<<<<<< + * gates[g+0] += 1 + * gates[g+1] += 1 +*/ + while (1) { + __pyx_t_4 = (__pyx_v_g < __pyx_v_end); + if (!__pyx_t_4) break; + + /* "thinc/backends/numpy_ops.pyx":1059 + * g = b * N * 4 + * while g < end: + * gates[g+0] += 1 # <<<<<<<<<<<<<< + * gates[g+1] += 1 + * gates[g+2] += 1 +*/ + __pyx_t_5 = (__pyx_v_g + 0); + (__pyx_v_gates[__pyx_t_5]) = ((__pyx_v_gates[__pyx_t_5]) + 1.0); + + /* "thinc/backends/numpy_ops.pyx":1060 + * while g < end: + * gates[g+0] += 1 + * gates[g+1] += 1 # <<<<<<<<<<<<<< + * gates[g+2] += 1 + * g += 4 +*/ + __pyx_t_5 = (__pyx_v_g + 1); + (__pyx_v_gates[__pyx_t_5]) = ((__pyx_v_gates[__pyx_t_5]) + 1.0); + + /* "thinc/backends/numpy_ops.pyx":1061 + * gates[g+0] += 1 + * gates[g+1] += 1 + * gates[g+2] += 1 # <<<<<<<<<<<<<< + * g += 4 + * g = b * N * 4 +*/ + __pyx_t_5 = (__pyx_v_g + 2); + (__pyx_v_gates[__pyx_t_5]) = ((__pyx_v_gates[__pyx_t_5]) + 1.0); + + /* "thinc/backends/numpy_ops.pyx":1062 + * gates[g+1] += 1 + * gates[g+2] += 1 + * g += 4 # <<<<<<<<<<<<<< + * g = b * N * 4 + * while g < end: +*/ + __pyx_v_g = (__pyx_v_g + 4); + } + + /* "thinc/backends/numpy_ops.pyx":1063 + * gates[g+2] += 1 + * g += 4 + * g = b * N * 4 # <<<<<<<<<<<<<< + * while g < end: + * gates[g+0] = 1.0 / gates[g+0] +*/ + __pyx_v_g = ((__pyx_v_b * __pyx_v_N) * 4); + + /* "thinc/backends/numpy_ops.pyx":1064 + * g += 4 + * g = b * N * 4 + * while g < end: # <<<<<<<<<<<<<< + * gates[g+0] = 1.0 / gates[g+0] + * gates[g+1] = 1.0 / gates[g+1] +*/ + while (1) { + __pyx_t_4 = (__pyx_v_g < __pyx_v_end); + if (!__pyx_t_4) break; + + /* "thinc/backends/numpy_ops.pyx":1065 + * g = b * N * 4 + * while g < end: + * gates[g+0] = 1.0 / gates[g+0] # <<<<<<<<<<<<<< + * gates[g+1] = 1.0 / gates[g+1] + * gates[g+2] = 1.0 / gates[g+2] +*/ + (__pyx_v_gates[(__pyx_v_g + 0)]) = (1.0 / (__pyx_v_gates[(__pyx_v_g + 0)])); + + /* "thinc/backends/numpy_ops.pyx":1066 + * while g < end: + * gates[g+0] = 1.0 / gates[g+0] + * gates[g+1] = 1.0 / gates[g+1] # <<<<<<<<<<<<<< + * gates[g+2] = 1.0 / gates[g+2] + * g += 4 +*/ + (__pyx_v_gates[(__pyx_v_g + 1)]) = (1.0 / (__pyx_v_gates[(__pyx_v_g + 1)])); + + /* "thinc/backends/numpy_ops.pyx":1067 + * gates[g+0] = 1.0 / gates[g+0] + * gates[g+1] = 1.0 / gates[g+1] + * gates[g+2] = 1.0 / gates[g+2] # <<<<<<<<<<<<<< + * g += 4 + * g = b * N * 4 +*/ + (__pyx_v_gates[(__pyx_v_g + 2)]) = (1.0 / (__pyx_v_gates[(__pyx_v_g + 2)])); + + /* "thinc/backends/numpy_ops.pyx":1068 + * gates[g+1] = 1.0 / gates[g+1] + * gates[g+2] = 1.0 / gates[g+2] + * g += 4 # <<<<<<<<<<<<<< + * g = b * N * 4 + * while g < end: +*/ + __pyx_v_g = (__pyx_v_g + 4); + } + + /* "thinc/backends/numpy_ops.pyx":1069 + * gates[g+2] = 1.0 / gates[g+2] + * g += 4 + * g = b * N * 4 # <<<<<<<<<<<<<< + * while g < end: + * gates[g+3] = tanhf(gates[g+3]) +*/ + __pyx_v_g = ((__pyx_v_b * __pyx_v_N) * 4); + + /* "thinc/backends/numpy_ops.pyx":1070 + * g += 4 + * g = b * N * 4 + * while g < end: # <<<<<<<<<<<<<< + * gates[g+3] = tanhf(gates[g+3]) + * g += 4 +*/ + while (1) { + __pyx_t_4 = (__pyx_v_g < __pyx_v_end); + if (!__pyx_t_4) break; + + /* "thinc/backends/numpy_ops.pyx":1071 + * g = b * N * 4 + * while g < end: + * gates[g+3] = tanhf(gates[g+3]) # <<<<<<<<<<<<<< + * g += 4 + * +*/ + (__pyx_v_gates[(__pyx_v_g + 3)]) = tanhf((__pyx_v_gates[(__pyx_v_g + 3)])); + + /* "thinc/backends/numpy_ops.pyx":1072 + * while g < end: + * gates[g+3] = tanhf(gates[g+3]) + * g += 4 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_g = (__pyx_v_g + 4); + } + } + + /* "thinc/backends/numpy_ops.pyx":1029 + * + * + * cdef void cpu_lstm_activate_fwd(float* gates, int B, int N) noexcept nogil: # <<<<<<<<<<<<<< + * """Apply sigmoid activation in-place to columns 0, 1, 2 and tanh to column 3. + * The data is assumed to have the gates in the last dimension. +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":1075 + * + * + * cdef void cpu_lstm_gates_fwd(float* hiddens, float* cells, # <<<<<<<<<<<<<< + * const float* gates, const float* prevcells, int B, int N) noexcept nogil: + * cdef float hf, hi, ho, hc, ct2, ct3 +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_fwd(float *__pyx_v_hiddens, float *__pyx_v_cells, float const *__pyx_v_gates, float const *__pyx_v_prevcells, int __pyx_v_B, int __pyx_v_N) { + float __pyx_v_hf; + float __pyx_v_hi; + float __pyx_v_ho; + float __pyx_v_hc; + float __pyx_v_ct2; + float __pyx_v_ct3; + int __pyx_v_g; + int __pyx_v_c; + int __pyx_v_h; + int __pyx_t_1; + + /* "thinc/backends/numpy_ops.pyx":1079 + * cdef float hf, hi, ho, hc, ct2, ct3 + * cdef int i, b, g, c, h + * g = 0 # <<<<<<<<<<<<<< + * c = 0 + * h = 0 +*/ + __pyx_v_g = 0; + + /* "thinc/backends/numpy_ops.pyx":1080 + * cdef int i, b, g, c, h + * g = 0 + * c = 0 # <<<<<<<<<<<<<< + * h = 0 + * while g < B*N*4: +*/ + __pyx_v_c = 0; + + /* "thinc/backends/numpy_ops.pyx":1081 + * g = 0 + * c = 0 + * h = 0 # <<<<<<<<<<<<<< + * while g < B*N*4: + * hf = gates[g+0] +*/ + __pyx_v_h = 0; + + /* "thinc/backends/numpy_ops.pyx":1082 + * c = 0 + * h = 0 + * while g < B*N*4: # <<<<<<<<<<<<<< + * hf = gates[g+0] + * hi = gates[g+1] +*/ + while (1) { + __pyx_t_1 = (__pyx_v_g < ((__pyx_v_B * __pyx_v_N) * 4)); + if (!__pyx_t_1) break; + + /* "thinc/backends/numpy_ops.pyx":1083 + * h = 0 + * while g < B*N*4: + * hf = gates[g+0] # <<<<<<<<<<<<<< + * hi = gates[g+1] + * ho = gates[g+2] +*/ + __pyx_v_hf = (__pyx_v_gates[(__pyx_v_g + 0)]); + + /* "thinc/backends/numpy_ops.pyx":1084 + * while g < B*N*4: + * hf = gates[g+0] + * hi = gates[g+1] # <<<<<<<<<<<<<< + * ho = gates[g+2] + * hc = gates[g+3] +*/ + __pyx_v_hi = (__pyx_v_gates[(__pyx_v_g + 1)]); + + /* "thinc/backends/numpy_ops.pyx":1085 + * hf = gates[g+0] + * hi = gates[g+1] + * ho = gates[g+2] # <<<<<<<<<<<<<< + * hc = gates[g+3] + * ct2 = prevcells[c] +*/ + __pyx_v_ho = (__pyx_v_gates[(__pyx_v_g + 2)]); + + /* "thinc/backends/numpy_ops.pyx":1086 + * hi = gates[g+1] + * ho = gates[g+2] + * hc = gates[g+3] # <<<<<<<<<<<<<< + * ct2 = prevcells[c] + * ct3 = hf * ct2 + hi * hc +*/ + __pyx_v_hc = (__pyx_v_gates[(__pyx_v_g + 3)]); + + /* "thinc/backends/numpy_ops.pyx":1087 + * ho = gates[g+2] + * hc = gates[g+3] + * ct2 = prevcells[c] # <<<<<<<<<<<<<< + * ct3 = hf * ct2 + hi * hc + * hiddens[h] = tanhf(ct3) * ho +*/ + __pyx_v_ct2 = (__pyx_v_prevcells[__pyx_v_c]); + + /* "thinc/backends/numpy_ops.pyx":1088 + * hc = gates[g+3] + * ct2 = prevcells[c] + * ct3 = hf * ct2 + hi * hc # <<<<<<<<<<<<<< + * hiddens[h] = tanhf(ct3) * ho + * cells[c] = ct3 +*/ + __pyx_v_ct3 = ((__pyx_v_hf * __pyx_v_ct2) + (__pyx_v_hi * __pyx_v_hc)); + + /* "thinc/backends/numpy_ops.pyx":1089 + * ct2 = prevcells[c] + * ct3 = hf * ct2 + hi * hc + * hiddens[h] = tanhf(ct3) * ho # <<<<<<<<<<<<<< + * cells[c] = ct3 + * g += 4 +*/ + (__pyx_v_hiddens[__pyx_v_h]) = (tanhf(__pyx_v_ct3) * __pyx_v_ho); + + /* "thinc/backends/numpy_ops.pyx":1090 + * ct3 = hf * ct2 + hi * hc + * hiddens[h] = tanhf(ct3) * ho + * cells[c] = ct3 # <<<<<<<<<<<<<< + * g += 4 + * c += 1 +*/ + (__pyx_v_cells[__pyx_v_c]) = __pyx_v_ct3; + + /* "thinc/backends/numpy_ops.pyx":1091 + * hiddens[h] = tanhf(ct3) * ho + * cells[c] = ct3 + * g += 4 # <<<<<<<<<<<<<< + * c += 1 + * h += 1 +*/ + __pyx_v_g = (__pyx_v_g + 4); + + /* "thinc/backends/numpy_ops.pyx":1092 + * cells[c] = ct3 + * g += 4 + * c += 1 # <<<<<<<<<<<<<< + * h += 1 + * +*/ + __pyx_v_c = (__pyx_v_c + 1); + + /* "thinc/backends/numpy_ops.pyx":1093 + * g += 4 + * c += 1 + * h += 1 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_h = (__pyx_v_h + 1); + } + + /* "thinc/backends/numpy_ops.pyx":1075 + * + * + * cdef void cpu_lstm_gates_fwd(float* hiddens, float* cells, # <<<<<<<<<<<<<< + * const float* gates, const float* prevcells, int B, int N) noexcept nogil: + * cdef float hf, hi, ho, hc, ct2, ct3 +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":1096 + * + * + * cdef void cpu_lstm_gates_bwd( # <<<<<<<<<<<<<< + * float* dGt3, + * float* dCt2, +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_cpu_lstm_gates_bwd(float *__pyx_v_dGt3, float *__pyx_v_dCt2, float const *__pyx_v_dYt3, float const *__pyx_v_dCt3, float const *__pyx_v_Gt3, float const *__pyx_v_Ct3, float const *__pyx_v_Ct2, int __pyx_v_N) { + int __pyx_v_i; + float __pyx_v_ct2; + float __pyx_v_ct3; + float __pyx_v_hf; + float __pyx_v_hi; + float __pyx_v_ho; + float __pyx_v_hc; + float __pyx_v_tanh_ct3; + float __pyx_v_d_ho; + float __pyx_v_d_tanh_ct3; + float __pyx_v_dct3; + float __pyx_v_d_hi; + float __pyx_v_d_hc; + float __pyx_v_d_hf; + float __pyx_v_dyt3; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + + /* "thinc/backends/numpy_ops.pyx":1109 + * cdef float ct2, ct3, hf, hi, ho, hc, tanh_ct3 + * cdef float d_ho, d_tanh_ct3, dct3, d_hi, d_hc, d_hf + * for i in range(N): # <<<<<<<<<<<<<< + * ct2 = Ct2[i] + * ct3 = Ct3[i] +*/ + __pyx_t_1 = __pyx_v_N; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/backends/numpy_ops.pyx":1110 + * cdef float d_ho, d_tanh_ct3, dct3, d_hi, d_hc, d_hf + * for i in range(N): + * ct2 = Ct2[i] # <<<<<<<<<<<<<< + * ct3 = Ct3[i] + * dct3 = dCt3[i] +*/ + __pyx_v_ct2 = (__pyx_v_Ct2[__pyx_v_i]); + + /* "thinc/backends/numpy_ops.pyx":1111 + * for i in range(N): + * ct2 = Ct2[i] + * ct3 = Ct3[i] # <<<<<<<<<<<<<< + * dct3 = dCt3[i] + * dyt3 = dYt3[i] +*/ + __pyx_v_ct3 = (__pyx_v_Ct3[__pyx_v_i]); + + /* "thinc/backends/numpy_ops.pyx":1112 + * ct2 = Ct2[i] + * ct3 = Ct3[i] + * dct3 = dCt3[i] # <<<<<<<<<<<<<< + * dyt3 = dYt3[i] + * hf = Gt3[i*4+0] +*/ + __pyx_v_dct3 = (__pyx_v_dCt3[__pyx_v_i]); + + /* "thinc/backends/numpy_ops.pyx":1113 + * ct3 = Ct3[i] + * dct3 = dCt3[i] + * dyt3 = dYt3[i] # <<<<<<<<<<<<<< + * hf = Gt3[i*4+0] + * hi = Gt3[i*4+1] +*/ + __pyx_v_dyt3 = (__pyx_v_dYt3[__pyx_v_i]); + + /* "thinc/backends/numpy_ops.pyx":1114 + * dct3 = dCt3[i] + * dyt3 = dYt3[i] + * hf = Gt3[i*4+0] # <<<<<<<<<<<<<< + * hi = Gt3[i*4+1] + * ho = Gt3[i*4+2] +*/ + __pyx_v_hf = (__pyx_v_Gt3[((__pyx_v_i * 4) + 0)]); + + /* "thinc/backends/numpy_ops.pyx":1115 + * dyt3 = dYt3[i] + * hf = Gt3[i*4+0] + * hi = Gt3[i*4+1] # <<<<<<<<<<<<<< + * ho = Gt3[i*4+2] + * hc = Gt3[i*4+3] +*/ + __pyx_v_hi = (__pyx_v_Gt3[((__pyx_v_i * 4) + 1)]); + + /* "thinc/backends/numpy_ops.pyx":1116 + * hf = Gt3[i*4+0] + * hi = Gt3[i*4+1] + * ho = Gt3[i*4+2] # <<<<<<<<<<<<<< + * hc = Gt3[i*4+3] + * +*/ + __pyx_v_ho = (__pyx_v_Gt3[((__pyx_v_i * 4) + 2)]); + + /* "thinc/backends/numpy_ops.pyx":1117 + * hi = Gt3[i*4+1] + * ho = Gt3[i*4+2] + * hc = Gt3[i*4+3] # <<<<<<<<<<<<<< + * + * tanh_ct3 = tanhf(ct3) +*/ + __pyx_v_hc = (__pyx_v_Gt3[((__pyx_v_i * 4) + 3)]); + + /* "thinc/backends/numpy_ops.pyx":1119 + * hc = Gt3[i*4+3] + * + * tanh_ct3 = tanhf(ct3) # <<<<<<<<<<<<<< + * # 3b: Yt3 = tanhCt3 * ho + * d_ho = dyt3 * tanh_ct3 +*/ + __pyx_v_tanh_ct3 = tanhf(__pyx_v_ct3); + + /* "thinc/backends/numpy_ops.pyx":1121 + * tanh_ct3 = tanhf(ct3) + * # 3b: Yt3 = tanhCt3 * ho + * d_ho = dyt3 * tanh_ct3 # <<<<<<<<<<<<<< + * d_tanh_ct3 = dyt3 * ho + * # 3a: tanhCt3 = tanh(Ct3) +*/ + __pyx_v_d_ho = (__pyx_v_dyt3 * __pyx_v_tanh_ct3); + + /* "thinc/backends/numpy_ops.pyx":1122 + * # 3b: Yt3 = tanhCt3 * ho + * d_ho = dyt3 * tanh_ct3 + * d_tanh_ct3 = dyt3 * ho # <<<<<<<<<<<<<< + * # 3a: tanhCt3 = tanh(Ct3) + * dct3 += d_tanh_ct3 * dtanh(tanh_ct3) +*/ + __pyx_v_d_tanh_ct3 = (__pyx_v_dyt3 * __pyx_v_ho); + + /* "thinc/backends/numpy_ops.pyx":1124 + * d_tanh_ct3 = dyt3 * ho + * # 3a: tanhCt3 = tanh(Ct3) + * dct3 += d_tanh_ct3 * dtanh(tanh_ct3) # <<<<<<<<<<<<<< + * # 2b: Ct3 += hi * hc + * d_hi = dct3 * hc +*/ + __pyx_v_dct3 = (__pyx_v_dct3 + (__pyx_v_d_tanh_ct3 * __pyx_f_5thinc_8backends_9numpy_ops_dtanh(__pyx_v_tanh_ct3))); + + /* "thinc/backends/numpy_ops.pyx":1126 + * dct3 += d_tanh_ct3 * dtanh(tanh_ct3) + * # 2b: Ct3 += hi * hc + * d_hi = dct3 * hc # <<<<<<<<<<<<<< + * d_hc = dct3 * hi + * # 2a: Ct3 = hf * Ct2 +*/ + __pyx_v_d_hi = (__pyx_v_dct3 * __pyx_v_hc); + + /* "thinc/backends/numpy_ops.pyx":1127 + * # 2b: Ct3 += hi * hc + * d_hi = dct3 * hc + * d_hc = dct3 * hi # <<<<<<<<<<<<<< + * # 2a: Ct3 = hf * Ct2 + * d_hf = dct3 * ct2 +*/ + __pyx_v_d_hc = (__pyx_v_dct3 * __pyx_v_hi); + + /* "thinc/backends/numpy_ops.pyx":1129 + * d_hc = dct3 * hi + * # 2a: Ct3 = hf * Ct2 + * d_hf = dct3 * ct2 # <<<<<<<<<<<<<< + * dCt2[i] = dct3 * hf + * dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a +*/ + __pyx_v_d_hf = (__pyx_v_dct3 * __pyx_v_ct2); + + /* "thinc/backends/numpy_ops.pyx":1130 + * # 2a: Ct3 = hf * Ct2 + * d_hf = dct3 * ct2 + * dCt2[i] = dct3 * hf # <<<<<<<<<<<<<< + * dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a + * dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b +*/ + (__pyx_v_dCt2[__pyx_v_i]) = (__pyx_v_dct3 * __pyx_v_hf); + + /* "thinc/backends/numpy_ops.pyx":1131 + * d_hf = dct3 * ct2 + * dCt2[i] = dct3 * hf + * dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a # <<<<<<<<<<<<<< + * dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b + * dGt3[i*4+2] = d_ho * dsigmoid(ho) # 1c +*/ + (__pyx_v_dGt3[((__pyx_v_i * 4) + 0)]) = (__pyx_v_d_hf * __pyx_f_5thinc_8backends_9numpy_ops_dsigmoid(__pyx_v_hf)); + + /* "thinc/backends/numpy_ops.pyx":1132 + * dCt2[i] = dct3 * hf + * dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a + * dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b # <<<<<<<<<<<<<< + * dGt3[i*4+2] = d_ho * dsigmoid(ho) # 1c + * dGt3[i*4+3] = d_hc * dtanh(hc) # 1d +*/ + (__pyx_v_dGt3[((__pyx_v_i * 4) + 1)]) = (__pyx_v_d_hi * __pyx_f_5thinc_8backends_9numpy_ops_dsigmoid(__pyx_v_hi)); + + /* "thinc/backends/numpy_ops.pyx":1133 + * dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a + * dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b + * dGt3[i*4+2] = d_ho * dsigmoid(ho) # 1c # <<<<<<<<<<<<<< + * dGt3[i*4+3] = d_hc * dtanh(hc) # 1d + * +*/ + (__pyx_v_dGt3[((__pyx_v_i * 4) + 2)]) = (__pyx_v_d_ho * __pyx_f_5thinc_8backends_9numpy_ops_dsigmoid(__pyx_v_ho)); + + /* "thinc/backends/numpy_ops.pyx":1134 + * dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b + * dGt3[i*4+2] = d_ho * dsigmoid(ho) # 1c + * dGt3[i*4+3] = d_hc * dtanh(hc) # 1d # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_dGt3[((__pyx_v_i * 4) + 3)]) = (__pyx_v_d_hc * __pyx_f_5thinc_8backends_9numpy_ops_dtanh(__pyx_v_hc)); + } + + /* "thinc/backends/numpy_ops.pyx":1096 + * + * + * cdef void cpu_lstm_gates_bwd( # <<<<<<<<<<<<<< + * float* dGt3, + * float* dCt2, +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":1137 + * + * + * cdef void MurmurHash3_x86_128_uint64( # <<<<<<<<<<<<<< + * const uint64_t val, + * const uint32_t seed, +*/ + +static void __pyx_f_5thinc_8backends_9numpy_ops_MurmurHash3_x86_128_uint64(uint64_t const __pyx_v_val, uint32_t const __pyx_v_seed, uint32_t *__pyx_v_out) { + uint64_t __pyx_v_h1; + uint64_t __pyx_v_h2; + + /* "thinc/backends/numpy_ops.pyx":1144 + * cdef uint64_t h1, h2 + * + * h1 = val # <<<<<<<<<<<<<< + * h1 *= 0x87c37b91114253d5ull + * h1 = (h1 << 31) | (h1 >> 33) +*/ + __pyx_v_h1 = __pyx_v_val; + + /* "thinc/backends/numpy_ops.pyx":1145 + * + * h1 = val + * h1 *= 0x87c37b91114253d5ull # <<<<<<<<<<<<<< + * h1 = (h1 << 31) | (h1 >> 33) + * h1 *= 0x4cf5ad432745937full +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0x87c37b91114253d5ULL); + + /* "thinc/backends/numpy_ops.pyx":1146 + * h1 = val + * h1 *= 0x87c37b91114253d5ull + * h1 = (h1 << 31) | (h1 >> 33) # <<<<<<<<<<<<<< + * h1 *= 0x4cf5ad432745937full + * h1 ^= seed +*/ + __pyx_v_h1 = ((__pyx_v_h1 << 31) | (__pyx_v_h1 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1147 + * h1 *= 0x87c37b91114253d5ull + * h1 = (h1 << 31) | (h1 >> 33) + * h1 *= 0x4cf5ad432745937full # <<<<<<<<<<<<<< + * h1 ^= seed + * h1 ^= 8 +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0x4cf5ad432745937fULL); + + /* "thinc/backends/numpy_ops.pyx":1148 + * h1 = (h1 << 31) | (h1 >> 33) + * h1 *= 0x4cf5ad432745937full + * h1 ^= seed # <<<<<<<<<<<<<< + * h1 ^= 8 + * h2 = seed +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ __pyx_v_seed); + + /* "thinc/backends/numpy_ops.pyx":1149 + * h1 *= 0x4cf5ad432745937full + * h1 ^= seed + * h1 ^= 8 # <<<<<<<<<<<<<< + * h2 = seed + * h2 ^= 8 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ 8); + + /* "thinc/backends/numpy_ops.pyx":1150 + * h1 ^= seed + * h1 ^= 8 + * h2 = seed # <<<<<<<<<<<<<< + * h2 ^= 8 + * h1 += h2 +*/ + __pyx_v_h2 = __pyx_v_seed; + + /* "thinc/backends/numpy_ops.pyx":1151 + * h1 ^= 8 + * h2 = seed + * h2 ^= 8 # <<<<<<<<<<<<<< + * h1 += h2 + * h2 += h1 +*/ + __pyx_v_h2 = (__pyx_v_h2 ^ 8); + + /* "thinc/backends/numpy_ops.pyx":1152 + * h2 = seed + * h2 ^= 8 + * h1 += h2 # <<<<<<<<<<<<<< + * h2 += h1 + * h1 ^= h1 >> 33 +*/ + __pyx_v_h1 = (__pyx_v_h1 + __pyx_v_h2); + + /* "thinc/backends/numpy_ops.pyx":1153 + * h2 ^= 8 + * h1 += h2 + * h2 += h1 # <<<<<<<<<<<<<< + * h1 ^= h1 >> 33 + * h1 *= 0xff51afd7ed558ccdull +*/ + __pyx_v_h2 = (__pyx_v_h2 + __pyx_v_h1); + + /* "thinc/backends/numpy_ops.pyx":1154 + * h1 += h2 + * h2 += h1 + * h1 ^= h1 >> 33 # <<<<<<<<<<<<<< + * h1 *= 0xff51afd7ed558ccdull + * h1 ^= h1 >> 33 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1155 + * h2 += h1 + * h1 ^= h1 >> 33 + * h1 *= 0xff51afd7ed558ccdull # <<<<<<<<<<<<<< + * h1 ^= h1 >> 33 + * h1 *= 0xc4ceb9fe1a85ec53ull +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0xff51afd7ed558ccdULL); + + /* "thinc/backends/numpy_ops.pyx":1156 + * h1 ^= h1 >> 33 + * h1 *= 0xff51afd7ed558ccdull + * h1 ^= h1 >> 33 # <<<<<<<<<<<<<< + * h1 *= 0xc4ceb9fe1a85ec53ull + * h1 ^= h1 >> 33 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1157 + * h1 *= 0xff51afd7ed558ccdull + * h1 ^= h1 >> 33 + * h1 *= 0xc4ceb9fe1a85ec53ull # <<<<<<<<<<<<<< + * h1 ^= h1 >> 33 + * h2 ^= h2 >> 33 +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0xc4ceb9fe1a85ec53ULL); + + /* "thinc/backends/numpy_ops.pyx":1158 + * h1 ^= h1 >> 33 + * h1 *= 0xc4ceb9fe1a85ec53ull + * h1 ^= h1 >> 33 # <<<<<<<<<<<<<< + * h2 ^= h2 >> 33 + * h2 *= 0xff51afd7ed558ccdull +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1159 + * h1 *= 0xc4ceb9fe1a85ec53ull + * h1 ^= h1 >> 33 + * h2 ^= h2 >> 33 # <<<<<<<<<<<<<< + * h2 *= 0xff51afd7ed558ccdull + * h2 ^= h2 >> 33 +*/ + __pyx_v_h2 = (__pyx_v_h2 ^ (__pyx_v_h2 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1160 + * h1 ^= h1 >> 33 + * h2 ^= h2 >> 33 + * h2 *= 0xff51afd7ed558ccdull # <<<<<<<<<<<<<< + * h2 ^= h2 >> 33 + * h2 *= 0xc4ceb9fe1a85ec53ull +*/ + __pyx_v_h2 = (__pyx_v_h2 * 0xff51afd7ed558ccdULL); + + /* "thinc/backends/numpy_ops.pyx":1161 + * h2 ^= h2 >> 33 + * h2 *= 0xff51afd7ed558ccdull + * h2 ^= h2 >> 33 # <<<<<<<<<<<<<< + * h2 *= 0xc4ceb9fe1a85ec53ull + * h2 ^= h2 >> 33 +*/ + __pyx_v_h2 = (__pyx_v_h2 ^ (__pyx_v_h2 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1162 + * h2 *= 0xff51afd7ed558ccdull + * h2 ^= h2 >> 33 + * h2 *= 0xc4ceb9fe1a85ec53ull # <<<<<<<<<<<<<< + * h2 ^= h2 >> 33 + * h1 += h2 +*/ + __pyx_v_h2 = (__pyx_v_h2 * 0xc4ceb9fe1a85ec53ULL); + + /* "thinc/backends/numpy_ops.pyx":1163 + * h2 ^= h2 >> 33 + * h2 *= 0xc4ceb9fe1a85ec53ull + * h2 ^= h2 >> 33 # <<<<<<<<<<<<<< + * h1 += h2 + * h2 += h1 +*/ + __pyx_v_h2 = (__pyx_v_h2 ^ (__pyx_v_h2 >> 33)); + + /* "thinc/backends/numpy_ops.pyx":1164 + * h2 *= 0xc4ceb9fe1a85ec53ull + * h2 ^= h2 >> 33 + * h1 += h2 # <<<<<<<<<<<<<< + * h2 += h1 + * +*/ + __pyx_v_h1 = (__pyx_v_h1 + __pyx_v_h2); + + /* "thinc/backends/numpy_ops.pyx":1165 + * h2 ^= h2 >> 33 + * h1 += h2 + * h2 += h1 # <<<<<<<<<<<<<< + * + * out[0] = h1 & 0xffffffffu +*/ + __pyx_v_h2 = (__pyx_v_h2 + __pyx_v_h1); + + /* "thinc/backends/numpy_ops.pyx":1167 + * h2 += h1 + * + * out[0] = h1 & 0xffffffffu # <<<<<<<<<<<<<< + * out[1] = h1 >> 32 + * out[2] = h2 & 0xffffffffu +*/ + (__pyx_v_out[0]) = (__pyx_v_h1 & 0xffffffffU); + + /* "thinc/backends/numpy_ops.pyx":1168 + * + * out[0] = h1 & 0xffffffffu + * out[1] = h1 >> 32 # <<<<<<<<<<<<<< + * out[2] = h2 & 0xffffffffu + * out[3] = h2 >> 32 +*/ + (__pyx_v_out[1]) = (__pyx_v_h1 >> 32); + + /* "thinc/backends/numpy_ops.pyx":1169 + * out[0] = h1 & 0xffffffffu + * out[1] = h1 >> 32 + * out[2] = h2 & 0xffffffffu # <<<<<<<<<<<<<< + * out[3] = h2 >> 32 + * +*/ + (__pyx_v_out[2]) = (__pyx_v_h2 & 0xffffffffU); + + /* "thinc/backends/numpy_ops.pyx":1170 + * out[1] = h1 >> 32 + * out[2] = h2 & 0xffffffffu + * out[3] = h2 >> 32 # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_out[3]) = (__pyx_v_h2 >> 32); + + /* "thinc/backends/numpy_ops.pyx":1137 + * + * + * cdef void MurmurHash3_x86_128_uint64( # <<<<<<<<<<<<<< + * const uint64_t val, + * const uint32_t seed, +*/ + + /* function exit code */ +} + +/* "thinc/backends/numpy_ops.pyx":1173 + * + * + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): # <<<<<<<<<<<<<< + * if u.shape != v.shape: + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_19_check_compatible_shape(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_8backends_9numpy_ops_18_check_compatible_shape, "_check_compatible_shape(u: np.ndarray, v: np.ndarray)"); +static PyMethodDef __pyx_mdef_5thinc_8backends_9numpy_ops_19_check_compatible_shape = {"_check_compatible_shape", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_8backends_9numpy_ops_19_check_compatible_shape, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_8backends_9numpy_ops_18_check_compatible_shape}; +static PyObject *__pyx_pw_5thinc_8backends_9numpy_ops_19_check_compatible_shape(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_u = 0; + PyObject *__pyx_v_v = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_check_compatible_shape (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_u,&__pyx_mstate_global->__pyx_n_u_v,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 1173, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 1173, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 1173, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_check_compatible_shape", 0) < (0)) __PYX_ERR(0, 1173, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_check_compatible_shape", 1, 2, 2, i); __PYX_ERR(0, 1173, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 1173, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 1173, __pyx_L3_error) + } + __pyx_v_u = values[0]; + __pyx_v_v = values[1]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_check_compatible_shape", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 1173, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.backends.numpy_ops._check_compatible_shape", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_8backends_9numpy_ops_18_check_compatible_shape(__pyx_self, __pyx_v_u, __pyx_v_v); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_8backends_9numpy_ops_18_check_compatible_shape(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_u, PyObject *__pyx_v_v) { + PyObject *__pyx_v_msg = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5[4]; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_check_compatible_shape", 0); + + /* "thinc/backends/numpy_ops.pyx":1174 + * + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): + * if u.shape != v.shape: # <<<<<<<<<<<<<< + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + * raise ValueError(msg) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_u, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1174, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_v, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1174, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_t_2, Py_NE); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1174, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(0, 1174, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(__pyx_t_4)) { + + /* "thinc/backends/numpy_ops.pyx":1175 + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): + * if u.shape != v.shape: + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" # <<<<<<<<<<<<<< + * raise ValueError(msg) + * +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_u, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyObject_FormatSimple(__pyx_t_3, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_v, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = __Pyx_PyObject_FormatSimple(__pyx_t_3, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5[0] = __pyx_mstate_global->__pyx_kp_u_arrays_have_incompatible_shapes; + __pyx_t_5[1] = __pyx_t_2; + __pyx_t_5[2] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_5[3] = __pyx_t_1; + __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_5, 4, 33 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_1), 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_1)); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_msg = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/backends/numpy_ops.pyx":1176 + * if u.shape != v.shape: + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + * raise ValueError(msg) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_msg}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1176, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 1176, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":1174 + * + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): + * if u.shape != v.shape: # <<<<<<<<<<<<<< + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + * raise ValueError(msg) +*/ + } + + /* "thinc/backends/numpy_ops.pyx":1173 + * + * + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): # <<<<<<<<<<<<<< + * if u.shape != v.shape: + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("thinc.backends.numpy_ops._check_compatible_shape", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/backends/numpy_ops.pyx":1179 + * + * + * cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): # <<<<<<<<<<<<<< + * if inplace: + * return X +*/ + +static CYTHON_INLINE PyArrayObject *__pyx_f_5thinc_8backends_9numpy_ops__inplace_or_copy(PyArrayObject *__pyx_v_X, PyObject *__pyx_v_inplace) { + PyArrayObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inplace_or_copy", 0); + + /* "thinc/backends/numpy_ops.pyx":1180 + * + * cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): + * if inplace: # <<<<<<<<<<<<<< + * return X + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_inplace); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 1180, __pyx_L1_error) + if (__pyx_t_1) { + + /* "thinc/backends/numpy_ops.pyx":1181 + * cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): + * if inplace: + * return X # <<<<<<<<<<<<<< + * else: + * return numpy.array(X) +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_X); + __pyx_r = __pyx_v_X; + goto __pyx_L0; + + /* "thinc/backends/numpy_ops.pyx":1180 + * + * cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): + * if inplace: # <<<<<<<<<<<<<< + * return X + * else: +*/ + } + + /* "thinc/backends/numpy_ops.pyx":1183 + * return X + * else: + * return numpy.array(X) # <<<<<<<<<<<<<< +*/ + /*else*/ { + __Pyx_XDECREF((PyObject *)__pyx_r); + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_array); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 1183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, ((PyObject *)__pyx_v_X)}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 1183, __pyx_L1_error) + __pyx_r = ((PyArrayObject *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "thinc/backends/numpy_ops.pyx":1179 + * + * + * cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): # <<<<<<<<<<<<<< + * if inplace: + * return X +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.backends.numpy_ops._inplace_or_copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults(PyObject *o) { + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->arg0); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_5thinc_8backends_9numpy_ops___pyx_defaults(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->arg0) { + e = (*v)(p->arg0, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_8backends_9numpy_ops___pyx_defaults(PyObject *o) { + PyObject* tmp; + struct __pyx_defaults *p = (struct __pyx_defaults *)o; + tmp = ((PyObject*)p->arg0); + p->arg0 = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_8backends_9numpy_ops___pyx_defaults}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_8backends_9numpy_ops___pyx_defaults}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults_spec = { + "thinc.backends.numpy_ops.__pyx_defaults", + sizeof(struct __pyx_defaults), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""__pyx_defaults", /*tp_name*/ + sizeof(struct __pyx_defaults), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5thinc_8backends_9numpy_ops___pyx_defaults, /*tp_traverse*/ + __pyx_tp_clear_5thinc_8backends_9numpy_ops___pyx_defaults, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults1(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_defaults1 *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + p = ((struct __pyx_defaults1 *)o); + p->arg0.data = NULL; + p->arg0.memview = NULL; + return o; +} + +static void __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults1(PyObject *o) { + struct __pyx_defaults1 *p = (struct __pyx_defaults1 *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults1) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + __PYX_XCLEAR_MEMVIEW(&p->arg0, 1); + p->arg0.memview = NULL; p->arg0.data = NULL; + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults1}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults1}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1_spec = { + "thinc.backends.numpy_ops.__pyx_defaults1", + sizeof(struct __pyx_defaults1), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, + __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1 = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""__pyx_defaults1", /*tp_name*/ + sizeof(struct __pyx_defaults1), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_8backends_9numpy_ops___pyx_defaults1, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_8backends_9numpy_ops___pyx_defaults1, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_array __pyx_vtable_array; + +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_array_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_array_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_array; + p->mode = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_format = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (unlikely(__pyx_array___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_array(PyObject *o) { + struct __pyx_array_obj *p = (struct __pyx_array_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_array) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_array___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->mode); + Py_CLEAR(p->_format); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static PyObject *__pyx_sq_item_array(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_array(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_array___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_tp_getattro_array(PyObject *o, PyObject *n) { + PyObject *v = PyObject_GenericGetAttr(o, n); + if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + v = __pyx_array___getattr__(o, n); + } + return v; +} + +static PyObject *__pyx_getprop___pyx_array_memview(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(o); +} + +static PyMethodDef __pyx_methods_array[] = { + {"__getattr__", (PyCFunction)__pyx_array___getattr__, METH_O|METH_COEXIST, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_array[] = { + {"memview", __pyx_getprop___pyx_array_memview, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_array_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_array}, + {Py_sq_length, (void *)__pyx_array___len__}, + {Py_sq_item, (void *)__pyx_sq_item_array}, + {Py_mp_length, (void *)__pyx_array___len__}, + {Py_mp_subscript, (void *)__pyx_array___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_array}, + {Py_tp_getattro, (void *)__pyx_tp_getattro_array}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_array_getbuffer}, + #endif + {Py_tp_methods, (void *)__pyx_methods_array}, + {Py_tp_getset, (void *)__pyx_getsets_array}, + {Py_tp_new, (void *)__pyx_tp_new_array}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_array_spec = { + "thinc.backends.numpy_ops.array", + sizeof(struct __pyx_array_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_array_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_array = { + __pyx_array___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_array, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_array = { + __pyx_array___len__, /*mp_length*/ + __pyx_array___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_array, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_array = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""array", /*tp_name*/ + sizeof(struct __pyx_array_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_array, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_array, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_array, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + __pyx_tp_getattro_array, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_array, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_array, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_array, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_array, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_MemviewEnum_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_MemviewEnum_obj *)o); + p->name = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_Enum(PyObject *o) { + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_Enum) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->name); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_Enum(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->name) { + e = (*v)(p->name, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_Enum(PyObject *o) { + PyObject* tmp; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + tmp = ((PyObject*)p->name); + p->name = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_Enum[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_MemviewEnum_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_Enum}, + {Py_tp_repr, (void *)__pyx_MemviewEnum___repr__}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_Enum}, + {Py_tp_clear, (void *)__pyx_tp_clear_Enum}, + {Py_tp_methods, (void *)__pyx_methods_Enum}, + {Py_tp_init, (void *)__pyx_MemviewEnum___init__}, + {Py_tp_new, (void *)__pyx_tp_new_Enum}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_MemviewEnum_spec = { + "thinc.backends.numpy_ops.Enum", + sizeof(struct __pyx_MemviewEnum_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_MemviewEnum_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_MemviewEnum = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""Enum", /*tp_name*/ + sizeof(struct __pyx_MemviewEnum_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_Enum, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_MemviewEnum___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_Enum, /*tp_traverse*/ + __pyx_tp_clear_Enum, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_Enum, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_MemviewEnum___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_Enum, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_memoryview __pyx_vtable_memoryview; + +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryview_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryview_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_memoryview; + p->obj = Py_None; Py_INCREF(Py_None); + p->_size = Py_None; Py_INCREF(Py_None); + p->view.obj = NULL; + if (unlikely(__pyx_memoryview___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_memoryview(PyObject *o) { + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_memoryview) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryview___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->obj); + Py_CLEAR(p->_size); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_memoryview(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->obj) { + e = (*v)(p->obj, a); if (e) return e; + } + if (p->_size) { + e = (*v)(p->_size, a); if (e) return e; + } + if (p->view.obj) { + e = (*v)(p->view.obj, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_memoryview(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + tmp = ((PyObject*)p->obj); + p->obj = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_size); + p->_size = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + Py_CLEAR(p->view.obj); + return 0; +} + +static PyObject *__pyx_sq_item_memoryview(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_memoryview(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_memoryview___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_getprop___pyx_memoryview_T(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_base(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_shape(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_strides(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_suboffsets(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_ndim(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_itemsize(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_nbytes(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_size(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(o); +} + +static PyMethodDef __pyx_methods_memoryview[] = { + {"is_c_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_c_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"is_f_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_f_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy_fortran", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy_fortran, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_memoryview[] = { + {"T", __pyx_getprop___pyx_memoryview_T, 0, 0, 0}, + {"base", __pyx_getprop___pyx_memoryview_base, 0, 0, 0}, + {"shape", __pyx_getprop___pyx_memoryview_shape, 0, 0, 0}, + {"strides", __pyx_getprop___pyx_memoryview_strides, 0, 0, 0}, + {"suboffsets", __pyx_getprop___pyx_memoryview_suboffsets, 0, 0, 0}, + {"ndim", __pyx_getprop___pyx_memoryview_ndim, 0, 0, 0}, + {"itemsize", __pyx_getprop___pyx_memoryview_itemsize, 0, 0, 0}, + {"nbytes", __pyx_getprop___pyx_memoryview_nbytes, 0, 0, 0}, + {"size", __pyx_getprop___pyx_memoryview_size, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_memoryview_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_memoryview}, + {Py_tp_repr, (void *)__pyx_memoryview___repr__}, + {Py_sq_length, (void *)__pyx_memoryview___len__}, + {Py_sq_item, (void *)__pyx_sq_item_memoryview}, + {Py_mp_length, (void *)__pyx_memoryview___len__}, + {Py_mp_subscript, (void *)__pyx_memoryview___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_memoryview}, + {Py_tp_str, (void *)__pyx_memoryview___str__}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_memoryview_getbuffer}, + #endif + {Py_tp_traverse, (void *)__pyx_tp_traverse_memoryview}, + {Py_tp_clear, (void *)__pyx_tp_clear_memoryview}, + {Py_tp_methods, (void *)__pyx_methods_memoryview}, + {Py_tp_getset, (void *)__pyx_getsets_memoryview}, + {Py_tp_new, (void *)__pyx_tp_new_memoryview}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryview_spec = { + "thinc.backends.numpy_ops.memoryview", + sizeof(struct __pyx_memoryview_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_memoryview_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_memoryview = { + __pyx_memoryview___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_memoryview, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_memoryview = { + __pyx_memoryview___len__, /*mp_length*/ + __pyx_memoryview___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_memoryview, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_memoryview = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""memoryview", /*tp_name*/ + sizeof(struct __pyx_memoryview_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_memoryview, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_memoryview___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_memoryview, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_memoryview, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_memoryview___str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_memoryview, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_memoryview, /*tp_traverse*/ + __pyx_tp_clear_memoryview, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_memoryview, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_memoryview, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_memoryview, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct__memoryviewslice __pyx_vtable__memoryviewslice; + +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryviewslice_obj *p; + PyObject *o = __pyx_tp_new_memoryview(t, a, k); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryviewslice_obj *)o); + p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_memoryview*)__pyx_vtabptr__memoryviewslice; + __Pyx_default_placement_construct(&(p->from_slice)); + p->from_object = Py_None; Py_INCREF(Py_None); + p->from_slice.memview = NULL; + return o; +} + +static void __pyx_tp_dealloc__memoryviewslice(PyObject *o) { + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc__memoryviewslice) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryviewslice___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + __Pyx_call_destructor(p->from_slice); + Py_CLEAR(p->from_object); + PyObject_GC_Track(o); + __pyx_tp_dealloc_memoryview(o); +} + +static int __pyx_tp_traverse__memoryviewslice(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + e = __pyx_tp_traverse_memoryview(o, v, a); if (e) return e; + if (p->from_object) { + e = (*v)(p->from_object, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear__memoryviewslice(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + __pyx_tp_clear_memoryview(o); + tmp = ((PyObject*)p->from_object); + p->from_object = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + __PYX_XCLEAR_MEMVIEW(&p->from_slice, 1); + return 0; +} + +static PyMethodDef __pyx_methods__memoryviewslice[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_memoryviewslice_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc__memoryviewslice}, + {Py_tp_doc, (void *)PyDoc_STR("Internal class for passing memoryview slices to Python")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse__memoryviewslice}, + {Py_tp_clear, (void *)__pyx_tp_clear__memoryviewslice}, + {Py_tp_methods, (void *)__pyx_methods__memoryviewslice}, + {Py_tp_new, (void *)__pyx_tp_new__memoryviewslice}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryviewslice_spec = { + "thinc.backends.numpy_ops._memoryviewslice", + sizeof(struct __pyx_memoryviewslice_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_memoryviewslice_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_memoryviewslice = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.backends.numpy_ops.""_memoryviewslice", /*tp_name*/ + sizeof(struct __pyx_memoryviewslice_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc__memoryviewslice, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___repr__, /*tp_repr*/ + #else + 0, /*tp_repr*/ + #endif + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___str__, /*tp_str*/ + #else + 0, /*tp_str*/ + #endif + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + PyDoc_STR("Internal class for passing memoryview slices to Python"), /*tp_doc*/ + __pyx_tp_traverse__memoryviewslice, /*tp_traverse*/ + __pyx_tp_clear__memoryviewslice, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods__memoryviewslice, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new__memoryviewslice, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_collections_abc_Sequence = Py_None; Py_INCREF(Py_None); + generic = Py_None; Py_INCREF(Py_None); + strided = Py_None; Py_INCREF(Py_None); + indirect = Py_None; Py_INCREF(Py_None); + contiguous = Py_None; Py_INCREF(Py_None); + indirect_contiguous = Py_None; Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 165, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults) < (0)) __PYX_ERR(0, 165, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults = &__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults) < (0)) __PYX_ERR(0, 165, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults->tp_getattro = PyObject_GenericGetAttr; + } + #endif + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1 = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1)) __PYX_ERR(0, 228, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1_spec, __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1) < (0)) __PYX_ERR(0, 228, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1 = &__pyx_type_5thinc_8backends_9numpy_ops___pyx_defaults1; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1) < (0)) __PYX_ERR(0, 228, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1->tp_getattro = PyObject_GenericGetAttr; + } + #endif + __pyx_vtabptr_array = &__pyx_vtable_array; + __pyx_vtable_array.get_memview = (PyObject *(*)(struct __pyx_array_obj *))__pyx_array_get_memview; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_array_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_array_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_array_type)) __PYX_ERR(1, 110, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_array_type->tp_as_buffer = &__pyx_tp_as_buffer_array; + if (!__pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_array_spec, __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #else + __pyx_mstate->__pyx_array_type = &__pyx_type___pyx_array; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_array_type); + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_array_type, __pyx_vtabptr_array) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_MemviewEnum_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_MemviewEnum_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_MemviewEnum_type)) __PYX_ERR(1, 299, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_MemviewEnum_spec, __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #else + __pyx_mstate->__pyx_MemviewEnum_type = &__pyx_type___pyx_MemviewEnum; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_MemviewEnum_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_MemviewEnum_type->tp_dictoffset && __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + __pyx_vtabptr_memoryview = &__pyx_vtable_memoryview; + __pyx_vtable_memoryview.get_item_pointer = (char *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_get_item_pointer; + __pyx_vtable_memoryview.is_slice = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_is_slice; + __pyx_vtable_memoryview.setitem_slice_assignment = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_slice_assignment; + __pyx_vtable_memoryview.setitem_slice_assign_scalar = (PyObject *(*)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_setitem_slice_assign_scalar; + __pyx_vtable_memoryview.setitem_indexed = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_indexed; + __pyx_vtable_memoryview.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryview_convert_item_to_object; + __pyx_vtable_memoryview.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryview_assign_item_from_object; + __pyx_vtable_memoryview._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryview__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_memoryview_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryview_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_memoryview_type)) __PYX_ERR(1, 334, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer = &__pyx_tp_as_buffer_memoryview; + if (!__pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryview_spec, __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryview_type = &__pyx_type___pyx_memoryview; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryview_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryview_type->tp_dictoffset && __pyx_mstate->__pyx_memoryview_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryview_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryview_type, __pyx_vtabptr_memoryview) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + __pyx_vtabptr__memoryviewslice = &__pyx_vtable__memoryviewslice; + __pyx_vtable__memoryviewslice.__pyx_base = *__pyx_vtabptr_memoryview; + __pyx_vtable__memoryviewslice.__pyx_base.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryviewslice_convert_item_to_object; + __pyx_vtable__memoryviewslice.__pyx_base.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryviewslice_assign_item_from_object; + __pyx_vtable__memoryviewslice.__pyx_base._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryviewslice__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_memoryview_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_memoryviewslice_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryviewslice_spec, __pyx_t_1); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_mstate->__pyx_memoryviewslice_type)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryviewslice_spec, __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryviewslice_type = &__pyx_type___pyx_memoryviewslice; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate_global->__pyx_memoryviewslice_type->tp_base = __pyx_mstate_global->__pyx_memoryview_type; + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryviewslice_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryviewslice_type->tp_dictoffset && __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryviewslice_type, __pyx_vtabptr__memoryviewslice) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule("thinc.backends.cblas"); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 22, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas = __Pyx_ImportType_3_2_1(__pyx_t_1, "thinc.backends.cblas", "CBlas", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), + #else + sizeof(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_5cblas_CBlas), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5thinc_8backends_5cblas_CBlas) __PYX_ERR(4, 22, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(5, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7cpython_4type_type = __Pyx_ImportType_3_2_1(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyTypeObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + 0, 0, + #else + sizeof(PyHeapTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyHeapTypeObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7cpython_4type_type) __PYX_ERR(5, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5numpy_dtype = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "dtype", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #else + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_dtype) __PYX_ERR(2, 228, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_flatiter = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "flatiter", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #else + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_flatiter) __PYX_ERR(2, 273, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_broadcast = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "broadcast", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #else + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_broadcast) __PYX_ERR(2, 277, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_ndarray = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "ndarray", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #else + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_ndarray) __PYX_ERR(2, 316, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_generic = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "generic", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_generic) __PYX_ERR(2, 825, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_number = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "number", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_number) __PYX_ERR(2, 827, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_integer = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "integer", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_integer) __PYX_ERR(2, 829, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_signedinteger = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "signedinteger", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_signedinteger) __PYX_ERR(2, 831, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_unsignedinteger = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "unsignedinteger", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_unsignedinteger) __PYX_ERR(2, 833, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_inexact = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "inexact", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_inexact) __PYX_ERR(2, 835, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_floating = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "floating", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_floating) __PYX_ERR(2, 837, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_complexfloating = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "complexfloating", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_complexfloating) __PYX_ERR(2, 839, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_flexible = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "flexible", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_flexible) __PYX_ERR(2, 841, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_character = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "character", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_character) __PYX_ERR(2, 843, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_ufunc = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "ufunc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #else + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_ufunc) __PYX_ERR(2, 907, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("cymem.cymem"); if (unlikely(!__pyx_t_1)) __PYX_ERR(6, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyMalloc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc) __PYX_ERR(6, 4, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyMalloc = (struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyMalloc)) __PYX_ERR(6, 4, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyFree", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree) __PYX_ERR(6, 10, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyFree = (struct __pyx_vtabstruct_5cymem_5cymem_PyFree*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyFree)) __PYX_ERR(6, 10, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Pool", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool) __PYX_ERR(6, 16, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_Pool = (struct __pyx_vtabstruct_5cymem_5cymem_Pool*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_Pool)) __PYX_ERR(6, 16, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Address = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Address", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Address) __PYX_ERR(6, 28, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("preshed.maps"); if (unlikely(!__pyx_t_1)) __PYX_ERR(7, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMap", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap) __PYX_ERR(7, 45, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMap = (struct __pyx_vtabstruct_7preshed_4maps_PreshMap*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMap)) __PYX_ERR(7, 45, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMapArray", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray) __PYX_ERR(7, 53, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMapArray = (struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMapArray)) __PYX_ERR(7, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("thinc.backends.linalg"); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix = __Pyx_ImportType_3_2_1(__pyx_t_1, "thinc.backends.linalg", "Matrix", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), + #else + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Matrix), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Matrix) __PYX_ERR(3, 23, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec = __Pyx_ImportType_3_2_1(__pyx_t_1, "thinc.backends.linalg", "Vec", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Vec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Vec), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Vec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Vec), + #else + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Vec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Vec), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec) __PYX_ERR(3, 30, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_Vec = (struct __pyx_vtabstruct_5thinc_8backends_6linalg_Vec*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Vec); if (unlikely(!__pyx_vtabptr_5thinc_8backends_6linalg_Vec)) __PYX_ERR(3, 30, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec = __Pyx_ImportType_3_2_1(__pyx_t_1, "thinc.backends.linalg", "VecVec", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), + #else + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_VecVec), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec) __PYX_ERR(3, 159, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_VecVec = (struct __pyx_vtabstruct_5thinc_8backends_6linalg_VecVec*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_VecVec); if (unlikely(!__pyx_vtabptr_5thinc_8backends_6linalg_VecVec)) __PYX_ERR(3, 159, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat = __Pyx_ImportType_3_2_1(__pyx_t_1, "thinc.backends.linalg", "Mat", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Mat), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Mat), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Mat), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Mat), + #else + sizeof(struct __pyx_obj_5thinc_8backends_6linalg_Mat), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5thinc_8backends_6linalg_Mat), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat) __PYX_ERR(3, 250, __pyx_L1_error) + __pyx_vtabptr_5thinc_8backends_6linalg_Mat = (struct __pyx_vtabstruct_5thinc_8backends_6linalg_Mat*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5thinc_8backends_6linalg_Mat); if (unlikely(!__pyx_vtabptr_5thinc_8backends_6linalg_Mat)) __PYX_ERR(3, 250, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + { + __pyx_t_1 = PyImport_ImportModule("thinc.backends.cblas"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_pyx_t_5thinc_8backends_5cblas); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + const char * __pyx_import_name = __pyx_import_signature + 186; + void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_5thinc_8backends_5cblas_daxpy, (void (**)(void))&__pyx_f_5thinc_8backends_5cblas_saxpy, (void (**)(void)) NULL}; + void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers; + const char *__pyx_import_current_signature = __pyx_import_signature; + while (*__pyx_import_pointer) { + if (__Pyx_ImportFunction_3_2_1(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + ++__pyx_import_pointer; + __pyx_import_name = strchr(__pyx_import_name, '\0') + 1; + __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1; + if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + { + __pyx_t_1 = PyImport_ImportModule("blis.cy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_void_enum___pyx_t_4blis_2cy_tran); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + const char * __pyx_import_name = __pyx_import_signature + 1592; + void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_fuse_3__pyx_f_4blis_2cy_gemm, (void (**)(void))&__pyx_fuse_1__pyx_f_4blis_2cy_gemm, (void (**)(void))&__pyx_fuse_2__pyx_f_4blis_2cy_gemm, (void (**)(void))&__pyx_fuse_0__pyx_f_4blis_2cy_gemm, (void (**)(void)) NULL}; + void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers; + const char *__pyx_import_current_signature = __pyx_import_signature; + while (*__pyx_import_pointer) { + if (__Pyx_ImportFunction_3_2_1(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + ++__pyx_import_pointer; + __pyx_import_name = strchr(__pyx_import_name, '\0') + 1; + __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1; + if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + { + __pyx_t_1 = PyImport_ImportModule("murmurhash.mrmr"); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + const char * __pyx_import_signature = __Pyx_PyBytes_AsString(__pyx_mstate_global->__pyx_kp_b_uint64_t_void_int_uint64_t_hash6); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!__pyx_import_signature)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + const char * __pyx_import_name = __pyx_import_signature + 33; + void (**const __pyx_import_pointers[])(void) = {(void (**)(void))&__pyx_f_10murmurhash_4mrmr_hash64, (void (**)(void)) NULL}; + void (**const *__pyx_import_pointer)(void) = __pyx_import_pointers; + const char *__pyx_import_current_signature = __pyx_import_signature; + while (*__pyx_import_pointer) { + if (__Pyx_ImportFunction_3_2_1(__pyx_t_1, __pyx_import_name, *__pyx_import_pointer, __pyx_import_current_signature) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + ++__pyx_import_pointer; + __pyx_import_name = strchr(__pyx_import_name, '\0') + 1; + __pyx_import_signature = strchr(__pyx_import_signature, '\0') + 1; + if (*__pyx_import_signature != '\0') __pyx_import_current_signature = __pyx_import_signature; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_numpy_ops(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_numpy_ops}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "numpy_ops", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_numpy_ops(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_numpy_ops(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_numpy_ops(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + static PyThread_type_lock __pyx_t_7[8]; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + PyObject *__pyx_t_15 = NULL; + PyObject *__pyx_t_16 = NULL; + __Pyx_memviewslice __pyx_t_17 = { 0, 0, { 0 }, { 0 }, { 0 } }; + PyObject *__pyx_t_18 = NULL; + PyObject *__pyx_t_19 = NULL; + PyObject *__pyx_t_20 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'numpy_ops' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "numpy_ops" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_numpy_ops", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__backends__numpy_ops) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.backends.numpy_ops")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.backends.numpy_ops", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Execution code ---*/ + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":101 + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence # <<<<<<<<<<<<<< + * except: + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_collections_abc}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin___import__, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_abc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L7_try_end; + __pyx_L2_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":102 + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: # <<<<<<<<<<<<<< + * + * __pyx_collections_abc_Sequence = None +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":104 + * except: + * + * __pyx_collections_abc_Sequence = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, Py_None); + __Pyx_GIVEREF(Py_None); + goto __pyx_L3_exception_handled; + } + __pyx_L3_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L7_try_end:; + } + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":240 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":241 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L15_try_end; + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":242 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L11_exception_handled; + } + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L15_try_end:; + } + + /* "View.MemoryView":307 + * return self.name + * + * cdef generic = Enum("") # <<<<<<<<<<<<<< + * cdef strided = Enum("") # default + * cdef indirect = Enum("") +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct_or_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 307, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(generic); + __Pyx_DECREF_SET(generic, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":308 + * + * cdef generic = Enum("") + * cdef strided = Enum("") # default # <<<<<<<<<<<<<< + * cdef indirect = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 308, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(strided); + __Pyx_DECREF_SET(strided, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":309 + * cdef generic = Enum("") + * cdef strided = Enum("") # default + * cdef indirect = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 309, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect); + __Pyx_DECREF_SET(indirect, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":312 + * + * + * cdef contiguous = Enum("") # <<<<<<<<<<<<<< + * cdef indirect_contiguous = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 312, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(contiguous); + __Pyx_DECREF_SET(contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":313 + * + * cdef contiguous = Enum("") + * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 313, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect_contiguous); + __Pyx_DECREF_SET(indirect_contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":321 + * + * + * cdef int __pyx_memoryview_thread_locks_used = 0 # <<<<<<<<<<<<<< + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ + * PyThread_allocate_lock(), +*/ + __pyx_memoryview_thread_locks_used = 0; + + /* "View.MemoryView":322 + * + * cdef int __pyx_memoryview_thread_locks_used = 0 + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ # <<<<<<<<<<<<<< + * PyThread_allocate_lock(), + * PyThread_allocate_lock(), +*/ + __pyx_t_7[0] = PyThread_allocate_lock(); + __pyx_t_7[1] = PyThread_allocate_lock(); + __pyx_t_7[2] = PyThread_allocate_lock(); + __pyx_t_7[3] = PyThread_allocate_lock(); + __pyx_t_7[4] = PyThread_allocate_lock(); + __pyx_t_7[5] = PyThread_allocate_lock(); + __pyx_t_7[6] = PyThread_allocate_lock(); + __pyx_t_7[7] = PyThread_allocate_lock(); + memcpy(&(__pyx_memoryview_thread_locks[0]), __pyx_t_7, sizeof(__pyx_memoryview_thread_locks[0]) * (8)); + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":984 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":985 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L23_try_end; + __pyx_L18_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":986 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L19_exception_handled; + } + __pyx_L19_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L23_try_end:; + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_collections_abc_Sequence); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(1, 990, __pyx_L26_error) + if (__pyx_t_8) { + + /* "View.MemoryView":994 + * + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence.register(array) + * except: +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_memoryviewslice_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 994, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":995 + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) # <<<<<<<<<<<<<< + * except: + * pass # ignore failure, it's a minor issue +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_array_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 995, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L31_try_end; + __pyx_L26_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":996 + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) + * except: # <<<<<<<<<<<<<< + * pass # ignore failure, it's a minor issue + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L27_exception_handled; + } + __pyx_L27_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L31_try_end:; + } + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + __pyx_t_4 = PyCFunction_NewEx(&__pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum, NULL, __pyx_mstate_global->__pyx_n_u_View_MemoryView); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum, __pyx_t_4) < (0)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":3 + * # cython: cdivision=True + * # cython: infer_types=True + * from collections.abc import Sized # <<<<<<<<<<<<<< + * from typing import Optional + * +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Sized}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_collections_abc, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_backends_collections_abc, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 3, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Sized}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 3, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":4 + * # cython: infer_types=True + * from collections.abc import Sized + * from typing import Optional # <<<<<<<<<<<<<< + * + * import numpy +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Optional}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_typing, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_backends_typing, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Optional}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":6 + * from typing import Optional + * + * import numpy # <<<<<<<<<<<<<< + * + * cimport blis.cy +*/ + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_numpy, 0, 0, __pyx_mstate_global->__pyx_kp_u_thinc_backends_numpy, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_numpy, __pyx_t_4) < (0)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":19 + * from preshed.maps cimport PreshMap + * + * from .. import registry # <<<<<<<<<<<<<< + * from ..types import ArrayXd, DeviceTypes, DTypes, Shape + * from ..util import copy_array, get_array_module +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u__8, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 19, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":20 + * + * from .. import registry + * from ..types import ArrayXd, DeviceTypes, DTypes, Shape # <<<<<<<<<<<<<< + * from ..util import copy_array, get_array_module + * +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_ArrayXd,__pyx_mstate_global->__pyx_n_u_DeviceTypes,__pyx_mstate_global->__pyx_n_u_DTypes,__pyx_mstate_global->__pyx_n_u_Shape}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_types, __pyx_imported_names, 4, __pyx_mstate_global->__pyx_kp_u_thinc_types, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 20, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_ArrayXd,__pyx_mstate_global->__pyx_n_u_DeviceTypes,__pyx_mstate_global->__pyx_n_u_DTypes,__pyx_mstate_global->__pyx_n_u_Shape}; + for (__pyx_t_9=0; __pyx_t_9 < 4; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 20, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":21 + * from .. import registry + * from ..types import ArrayXd, DeviceTypes, DTypes, Shape + * from ..util import copy_array, get_array_module # <<<<<<<<<<<<<< + * + * from .cblas cimport CBlas, daxpy, saxpy +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_copy_array,__pyx_mstate_global->__pyx_n_u_get_array_module}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_util, __pyx_imported_names, 2, __pyx_mstate_global->__pyx_kp_u_thinc_util, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 21, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_copy_array,__pyx_mstate_global->__pyx_n_u_get_array_module}; + for (__pyx_t_9=0; __pyx_t_9 < 2; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":26 + * from .linalg cimport Vec, VecVec + * + * from .ops import Ops # <<<<<<<<<<<<<< + * + * try: +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Ops}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_ops, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_backends_ops, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 26, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Ops}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":28 + * from .ops import Ops + * + * try: # <<<<<<<<<<<<<< + * import blis.py + * has_blis = True +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "thinc/backends/numpy_ops.pyx":29 + * + * try: + * import blis.py # <<<<<<<<<<<<<< + * has_blis = True + * except ImportError: +*/ + __pyx_t_10 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_blis_py, 0, 0, __pyx_mstate_global->__pyx_kp_u_thinc_backends_blis_py, -1); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 29, __pyx_L35_error) + __pyx_t_4 = __pyx_t_10; + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_blis, __pyx_t_4) < (0)) __PYX_ERR(0, 29, __pyx_L35_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":30 + * try: + * import blis.py + * has_blis = True # <<<<<<<<<<<<<< + * except ImportError: + * has_blis = False +*/ + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_has_blis, Py_True) < (0)) __PYX_ERR(0, 30, __pyx_L35_error) + + /* "thinc/backends/numpy_ops.pyx":28 + * from .ops import Ops + * + * try: # <<<<<<<<<<<<<< + * import blis.py + * has_blis = True +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L40_try_end; + __pyx_L35_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "thinc/backends/numpy_ops.pyx":31 + * import blis.py + * has_blis = True + * except ImportError: # <<<<<<<<<<<<<< + * has_blis = False + * +*/ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_ImportError)))); + if (__pyx_t_11) { + __Pyx_AddTraceback("thinc.backends.numpy_ops", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_5, &__pyx_t_12) < 0) __PYX_ERR(0, 31, __pyx_L37_except_error) + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_12); + + /* "thinc/backends/numpy_ops.pyx":32 + * has_blis = True + * except ImportError: + * has_blis = False # <<<<<<<<<<<<<< + * + * +*/ + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_has_blis, Py_False) < (0)) __PYX_ERR(0, 32, __pyx_L37_except_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + goto __pyx_L36_exception_handled; + } + goto __pyx_L37_except_error; + + /* "thinc/backends/numpy_ops.pyx":28 + * from .ops import Ops + * + * try: # <<<<<<<<<<<<<< + * import blis.py + * has_blis = True +*/ + __pyx_L37_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L36_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L40_try_end:; + } + + /* "thinc/backends/numpy_ops.pyx":48 + * + * @registry.ops("NumpyOps") + * class NumpyOps(Ops): # <<<<<<<<<<<<<< + * name = "numpy" + * xp = numpy +*/ + __Pyx_GetModuleGlobalName(__pyx_t_12, __pyx_mstate_global->__pyx_n_u_Ops); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_5 = PyTuple_Pack(1, __pyx_t_12); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_t_12 = __Pyx_PEP560_update_bases(__pyx_t_5); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_4 = __Pyx_CalculateMetaclass(NULL, __pyx_t_12); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_13 = __Pyx_Py3MetaclassPrepare(__pyx_t_4, __pyx_t_12, __pyx_mstate_global->__pyx_n_u_NumpyOps, __pyx_mstate_global->__pyx_n_u_NumpyOps, (PyObject *) NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, (PyObject *) NULL); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (__pyx_t_12 != __pyx_t_5) { + if (unlikely((PyDict_SetItemString(__pyx_t_13, "__orig_bases__", __pyx_t_5) < 0))) __PYX_ERR(0, 48, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyList_New(0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + + /* "thinc/backends/numpy_ops.pyx":49 + * @registry.ops("NumpyOps") + * class NumpyOps(Ops): + * name = "numpy" # <<<<<<<<<<<<<< + * xp = numpy + * +*/ + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_numpy) < (0)) __PYX_ERR(0, 49, __pyx_L1_error) + + /* "thinc/backends/numpy_ops.pyx":50 + * class NumpyOps(Ops): + * name = "numpy" + * xp = numpy # <<<<<<<<<<<<<< + * + * def __init__( +*/ + __Pyx_GetModuleGlobalName(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_xp, __pyx_t_14) < (0)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":52 + * xp = numpy + * + * def __init__( # <<<<<<<<<<<<<< + * self, + * device_type: DeviceTypes = "cpu", +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_use_blis, ((PyObject*)Py_True)) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + __pyx_t_15 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_device_type, __pyx_mstate_global->__pyx_n_u_DeviceTypes) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_device_id, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_use_blis, __pyx_mstate_global->__pyx_n_u_bool) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_None) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + __pyx_t_16 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_1__init__, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps___init, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_16); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_16, __pyx_mstate_global->__pyx_tuple[7]); + __Pyx_CyFunction_SetDefaultsKwDict(__pyx_t_16, __pyx_t_14); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_16, __pyx_t_15); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_init, __pyx_t_16) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":65 + * raise ValueError("BLIS support requires blis: pip install blis") + * + * def asarray(self, data, dtype=None): # <<<<<<<<<<<<<< + * if isinstance(data, self.xp.ndarray): + * array = data +*/ + __pyx_t_16 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_3asarray, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_asarray, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_16); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_16, __pyx_mstate_global->__pyx_tuple[8]); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_asarray, __pyx_t_16) < (0)) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + + /* "thinc/backends/numpy_ops.pyx":82 + * + * + * def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: # <<<<<<<<<<<<<< + * if zeros: + * return self.xp.zeros(shape, dtype=dtype) +*/ + __pyx_t_16 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + if (PyDict_SetItem(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_dtype, ((PyObject*)__pyx_mstate_global->__pyx_n_u_float32)) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_16, __pyx_mstate_global->__pyx_n_u_zeros, ((PyObject*)Py_True)) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + __pyx_t_15 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_shape, __pyx_mstate_global->__pyx_n_u_Shape) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_kp_u_Optional_DTypes) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_zeros, __pyx_mstate_global->__pyx_n_u_bool) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_ArrayXd) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_5alloc, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_alloc, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsKwDict(__pyx_t_14, __pyx_t_16); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_14, __pyx_t_15); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_alloc, __pyx_t_14) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":88 + * return self.xp.empty(shape, dtype=dtype) + * + * def cblas(self) -> CBlas: # <<<<<<<<<<<<<< + * return CBlas() + * +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_CBlas) < (0)) __PYX_ERR(0, 88, __pyx_L1_error) + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_7cblas, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_cblas, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[3])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_15, __pyx_t_14); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_cblas, __pyx_t_15) < (0)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":91 + * return CBlas() + * + * def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): # <<<<<<<<<<<<<< + * if x.ndim != 2: + * raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(3); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_out, Py_None) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_trans1, ((PyObject*)Py_False)) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_trans2, ((PyObject*)Py_False)) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_9gemm, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gemm, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[4])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + PyList_Append(__pyx_t_5, __pyx_t_14); + __Pyx_CyFunction_SetDefaultsKwDict(__pyx_t_14, __pyx_t_15); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_gemm, __pyx_t_14) < (0)) __PYX_ERR(0, 91, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":104 + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + * + * def relu(self, np.ndarray X, inplace=False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_11relu, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_relu, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[5])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 104, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + PyList_Append(__pyx_t_5, __pyx_t_14); + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_tuple[9]); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_relu, __pyx_t_14) < (0)) __PYX_ERR(0, 104, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":118 + * return super().relu(X, inplace=inplace) + * + * def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, Y) + * +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_13backprop_relu, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_relu, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[6])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 118, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + PyList_Append(__pyx_t_5, __pyx_t_14); + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_tuple[9]); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_relu, __pyx_t_14) < (0)) __PYX_ERR(0, 118, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":135 + * return super().backprop_relu(dY, Y, inplace) + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_15lstm_forward_training, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_lstm_forward_training, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[7])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_lstm_forward_training, __pyx_t_14) < (0)) __PYX_ERR(0, 135, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":148 + * return Y, fwd_state + * + * def lstm_forward_inference( # <<<<<<<<<<<<<< + * self, + * np.ndarray params, +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_17lstm_forward_inference, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_lstm_forward_inference, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[8])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 148, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_lstm_forward_inference, __pyx_t_14) < (0)) __PYX_ERR(0, 148, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":159 + * return Y + * + * def backprop_lstm( # <<<<<<<<<<<<<< + * self, np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state + * ): +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_19backprop_lstm, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_lstm, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[9])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_lstm, __pyx_t_14) < (0)) __PYX_ERR(0, 159, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":165 + * return dX, d_params + * + * def maxout(self, reals3d_ft X): # <<<<<<<<<<<<<< + * cdef int B = X.shape[0] + * cdef int O = X.shape[1] +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_57maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_maxout_float3d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[10])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_float3d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_59maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_maxout_double3d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[11])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_double3d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_21maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_maxout, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[12])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 165, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_15)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_15)->__signatures__ = __pyx_t_14; + __Pyx_GIVEREF(__pyx_t_14); + __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_maxout, __pyx_t_15) < (0)) __PYX_ERR(0, 165, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":184 + * return best, which + * + * def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): # <<<<<<<<<<<<<< + * cdef int B = dY.shape[0] + * cdef int O = dY.shape[1] +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_63backprop_maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_maxout_float2d, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[13])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_65backprop_maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_maxout_double2, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[14])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_23backprop_maxout, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_maxout, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[15])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_14, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 184, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_14)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_14)->__signatures__ = __pyx_t_15; + __Pyx_GIVEREF(__pyx_t_15); + __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_maxout, __pyx_t_14) < (0)) __PYX_ERR(0, 184, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":198 + * return dX + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_inplace, __pyx_mstate_global->__pyx_n_u_bool) < (0)) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_25mish, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_mish, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[16])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + PyList_Append(__pyx_t_5, __pyx_t_15); + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_tuple[10]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_15, __pyx_t_14); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_mish, __pyx_t_15) < (0)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":212 + * return super().mish(X, inplace=inplace) + * + * def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): # <<<<<<<<<<<<<< + * _check_compatible_shape(dY, X) + * +*/ + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_27backprop_mish, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_mish, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[17])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 212, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + PyList_Append(__pyx_t_5, __pyx_t_15); + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_tuple[10]); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_mish, __pyx_t_15) < (0)) __PYX_ERR(0, 212, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":228 + * return super().backprop_mish(dY, X, threshold, inplace) + * + * def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + * sequence. The new sequence is constructed by concatenating nW preceding +*/ + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_29seq2col, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_seq2col, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[18])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + PyList_Append(__pyx_t_5, __pyx_t_15); + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1)) __PYX_ERR(0, 228, __pyx_L1_error) + __pyx_t_17 = __Pyx_PyObject_to_MemoryviewSlice_dc_int(Py_None, PyBUF_WRITABLE); if (unlikely(!__pyx_t_17.memview)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_t_15)->arg0 = __pyx_t_17; + __pyx_t_17.memview = NULL; + __pyx_t_17.data = NULL; + __Pyx_CyFunction_SetDefaultsGetter(__pyx_t_15, __pyx_pf_5thinc_8backends_9numpy_ops_20__defaults__); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_seq2col, __pyx_t_15) < (0)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":268 + * return super().seq2col(seq, nW, lengths=lengths) + * + * def backprop_seq2col(self, np.ndarray dY, int nW, *, int[::1] lengths=None): # <<<<<<<<<<<<<< + * # Note: the type of dY should be changed to reals2d_ft once + * # cython/cython#4697 is fixed. The following checks can then be +*/ + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_31backprop_seq2col, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_seq2col, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[19])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + PyList_Append(__pyx_t_5, __pyx_t_15); + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults1)) __PYX_ERR(0, 268, __pyx_L1_error) + __pyx_t_17 = __Pyx_PyObject_to_MemoryviewSlice_dc_int(Py_None, PyBUF_WRITABLE); if (unlikely(!__pyx_t_17.memview)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_CyFunction_Defaults(struct __pyx_defaults1, __pyx_t_15)->arg0 = __pyx_t_17; + __pyx_t_17.memview = NULL; + __pyx_t_17.data = NULL; + __Pyx_CyFunction_SetDefaultsGetter(__pyx_t_15, __pyx_pf_5thinc_8backends_9numpy_ops_22__defaults__); + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_seq2col, __pyx_t_15) < (0)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":302 + * return super().backprop_seq2col(dY, nW, lengths=lengths) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def hash(self, const uint64_t[::1] ids, uint32_t seed): +*/ + __pyx_t_15 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_33hash, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_hash, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[20])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_hash, __pyx_t_15) < (0)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":314 + * return keys + * + * def reduce_mean(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_69reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_mean_float2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[21])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_71reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_mean_double2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[22])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_35reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_mean, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[23])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_14, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_14)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_14)->__signatures__ = __pyx_t_15; + __Pyx_GIVEREF(__pyx_t_15); + __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_reduce_mean, __pyx_t_14) < (0)) __PYX_ERR(0, 314, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":335 + * return means + * + * def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_means.shape[1] +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_75backprop_reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_mean_fl, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[24])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_77backprop_reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_mean_do, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[25])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_37backprop_reduce_mean, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_mean, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[26])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_15)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_15)->__signatures__ = __pyx_t_14; + __Pyx_GIVEREF(__pyx_t_14); + __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_reduce_mean, __pyx_t_15) < (0)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":361 + * return dX + * + * def reduce_sum(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_81reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_sum_float2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[27])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_83reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_sum_double2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[28])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_39reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_sum, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[29])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_14, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 361, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_14)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_14)->__signatures__ = __pyx_t_15; + __Pyx_GIVEREF(__pyx_t_15); + __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_reduce_sum, __pyx_t_14) < (0)) __PYX_ERR(0, 361, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":382 + * return sums + * + * def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_sums.shape[1] +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_87backprop_reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_sum_flo, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[30])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_89backprop_reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_sum_dou, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[31])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_41backprop_reduce_sum, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_sum, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[32])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 382, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_15)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_15)->__signatures__ = __pyx_t_14; + __Pyx_GIVEREF(__pyx_t_14); + __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_reduce_sum, __pyx_t_15) < (0)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":408 + * return dX + * + * def reduce_max(self, reals2d_ft X, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = X.shape[1] +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_93reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_max_float2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[33])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_95reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_max_double2d_t, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[34])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_43reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_reduce_max, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[35])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_14, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_14)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_14)->__signatures__ = __pyx_t_15; + __Pyx_GIVEREF(__pyx_t_15); + __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_reduce_max, __pyx_t_14) < (0)) __PYX_ERR(0, 408, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":432 + * return maxes, which + * + * def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): # <<<<<<<<<<<<<< + * cdef int B = lengths.shape[0] + * cdef int O = d_maxes.shape[1] +*/ + __pyx_t_14 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_99backprop_reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_max_flo, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[36])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_float2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_fuse_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_101backprop_reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_max_dou, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[37])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_14, __pyx_mstate_global->__pyx_n_u_double2d_t, __pyx_t_15) < (0)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_15 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_45backprop_reduce_max, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_backprop_reduce_max, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[38])); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_15); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_15, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_15)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_15, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_15)->__signatures__ = __pyx_t_14; + __Pyx_GIVEREF(__pyx_t_14); + __pyx_t_14 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_backprop_reduce_max, __pyx_t_15) < (0)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + + /* "thinc/backends/numpy_ops.pyx":460 + * return dX + * + * def gather_add(self, reals2d_ft table, ints2d_ft indices): # <<<<<<<<<<<<<< + * cdef CBlas cblas = self.cblas() + * rows = indices.shape[0] +*/ + __pyx_t_15 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_105gather_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gather_add_float2d_t_in, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[39])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_kp_u_float2d_t_int2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_0_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_107gather_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gather_add_float2d_t_ui, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[40])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_kp_u_float2d_t_uint2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1_0__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_109gather_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gather_add_double2d_t_i, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[41])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_kp_u_double2d_t_int2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_fuse_1_1__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_111gather_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gather_add_double2d_t_u, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[42])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + if (PyDict_SetItem(__pyx_t_15, __pyx_mstate_global->__pyx_kp_u_double2d_t_uint2d_t, __pyx_t_14) < (0)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __pyx_FusedFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_47gather_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_gather_add, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[43])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (!__Pyx_CyFunction_InitDefaults(__pyx_t_14, __pyx_mstate_global->__pyx_ptype_5thinc_8backends_9numpy_ops___pyx_defaults)) __PYX_ERR(0, 460, __pyx_L1_error) + __pyx_t_16 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_CyFunction_Defaults(struct __pyx_defaults, __pyx_t_14)->arg0 = __pyx_t_16; + __Pyx_GIVEREF(__pyx_t_16); + __pyx_t_16 = 0; + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_mstate_global->__pyx_empty_tuple); + ((__pyx_FusedFunctionObject *) __pyx_t_14)->__signatures__ = __pyx_t_15; + __Pyx_GIVEREF(__pyx_t_15); + __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_gather_add, __pyx_t_14) < (0)) __PYX_ERR(0, 460, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":477 + * return output + * + * def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): # <<<<<<<<<<<<<< + * if table.dtype == 'float32' \ + * and indices.dtype == 'int32' \ +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_49scatter_add, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_scatter_add, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[44])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_scatter_add, __pyx_t_14) < (0)) __PYX_ERR(0, 477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":500 + * np.ndarray[np.float32_t] mom1, np.ndarray[np.float32_t] mom2, + * const float beta1, const float beta2, float eps, + * float learn_rate, float mod_rate=1.): # <<<<<<<<<<<<<< + * _check_compatible_shape(weights, gradient) + * _check_compatible_shape(weights, mom1) +*/ + __pyx_t_14 = PyFloat_FromDouble(((double)1.)); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 500, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + + /* "thinc/backends/numpy_ops.pyx":495 + * self.xp.add.at(table, indices, values) + * + * @cython.boundscheck(False) # <<<<<<<<<<<<<< + * @cython.wraparound(False) + * def adam(self, np.ndarray[np.float32_t] weights, np.ndarray[np.float32_t] gradient, +*/ + __pyx_t_15 = PyTuple_Pack(1, __pyx_t_14); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 495, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_51adam, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_adam, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[45])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 495, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_t_15); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_adam, __pyx_t_14) < (0)) __PYX_ERR(0, 495, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":512 + * return weights, gradient, mom1, mom2 + * + * def ngrams(self, int n, const uint64_t[::1] keys): # <<<<<<<<<<<<<< + * if n < 1: + * return self.alloc((0,), dtype="uint64") +*/ + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_53ngrams, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_ngrams, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[46])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 512, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_ngrams, __pyx_t_14) < (0)) __PYX_ERR(0, 512, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":523 + * return output_ + * + * def position_encode(self, int N, int D, int period=10000, out=None): # <<<<<<<<<<<<<< + * cdef np.ndarray out_ + * if out is None: +*/ + __pyx_t_14 = __Pyx_PyLong_From_int(((int)0x2710)); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 523, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_15 = PyTuple_Pack(2, __pyx_t_14, Py_None); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 523, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_8NumpyOps_55position_encode, 0, __pyx_mstate_global->__pyx_n_u_NumpyOps_position_encode, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[47])); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 523, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_14); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_14, __pyx_t_15); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__Pyx_SetNameInClass(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_position_encode, __pyx_t_14) < (0)) __PYX_ERR(0, 523, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + + /* "thinc/backends/numpy_ops.pyx":47 + * + * + * @registry.ops("NumpyOps") # <<<<<<<<<<<<<< + * class NumpyOps(Ops): + * name = "numpy" +*/ + __pyx_t_15 = NULL; + __pyx_t_18 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_19, __pyx_mstate_global->__pyx_n_u_registry); if (unlikely(!__pyx_t_19)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_19); + __pyx_t_20 = __Pyx_PyObject_GetAttrStr(__pyx_t_19, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_20)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_20); + __Pyx_DECREF(__pyx_t_19); __pyx_t_19 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_18, __pyx_mstate_global->__pyx_n_u_NumpyOps}; + __pyx_t_16 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_20, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_DECREF(__pyx_t_20); __pyx_t_20 = 0; + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + } + + /* "thinc/backends/numpy_ops.pyx":48 + * + * @registry.ops("NumpyOps") + * class NumpyOps(Ops): # <<<<<<<<<<<<<< + * name = "numpy" + * xp = numpy +*/ + __pyx_t_20 = __Pyx_Py3ClassCreate(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_NumpyOps, __pyx_t_12, __pyx_t_13, NULL, 0, 1); if (unlikely(!__pyx_t_20)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_20); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_20); + #endif + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_15, __pyx_t_20}; + __pyx_t_14 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_16, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_15); __pyx_t_15 = 0; + __Pyx_DECREF(__pyx_t_20); __pyx_t_20 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_14); + } + if (__Pyx_CyFunction_InitClassCell(__pyx_t_5, __pyx_t_14) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NumpyOps, __pyx_t_14) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":535 + * + * + * def check_seq2col_lengths(ops, lengths, B): # <<<<<<<<<<<<<< + * if lengths is None: + * lengths = ops.asarray1i([B]) +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_1check_seq2col_lengths, 0, __pyx_mstate_global->__pyx_n_u_check_seq2col_lengths, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[48])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 535, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_check_seq2col_lengths, __pyx_t_12) < (0)) __PYX_ERR(0, 535, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":545 + * + * + * def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): # <<<<<<<<<<<<<< + * grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + * if grad_norm >= threshold: +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_3cpu_clip_gradient, 0, __pyx_mstate_global->__pyx_n_u_cpu_clip_gradient, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[49])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 545, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_cpu_clip_gradient, __pyx_t_12) < (0)) __PYX_ERR(0, 545, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":551 + * + * + * def add_gradient_noise(float[::1] gradient, weight_t noise_level, # <<<<<<<<<<<<<< + * weight_t timestep): + * cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_5add_gradient_noise, 0, __pyx_mstate_global->__pyx_n_u_add_gradient_noise, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[50])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 551, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_add_gradient_noise, __pyx_t_12) < (0)) __PYX_ERR(0, 551, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":639 + * + * + * def lstm_forward_training( # <<<<<<<<<<<<<< + * np.ndarray params, np.ndarray c_init, np.ndarray h_init, + * np.ndarray X, np.ndarray lengths +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_7lstm_forward_training, 0, __pyx_mstate_global->__pyx_n_u_lstm_forward_training, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[51])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 639, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_lstm_forward_training, __pyx_t_12) < (0)) __PYX_ERR(0, 639, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":768 + * + * + * def backprop_lstm(np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state): # <<<<<<<<<<<<<< + * xp = numpy + * cdef np.ndarray Y +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_9backprop_lstm, 0, __pyx_mstate_global->__pyx_n_u_backprop_lstm, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[52])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 768, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_backprop_lstm, __pyx_t_12) < (0)) __PYX_ERR(0, 768, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":870 + * + * + * def _split_directions(X, dirs): # <<<<<<<<<<<<<< + * if dirs == 1: + * return [X] +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_11_split_directions, 0, __pyx_mstate_global->__pyx_n_u_split_directions, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[53])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 870, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_split_directions, __pyx_t_12) < (0)) __PYX_ERR(0, 870, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":973 + * + * + * def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): # <<<<<<<<<<<<<< + * Wx_size = 4 * nO * nI + * bx_size = 4 * nO +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_13_split_weights, 0, __pyx_mstate_global->__pyx_n_u_split_weights, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[54])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 973, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_split_weights, __pyx_t_12) < (0)) __PYX_ERR(0, 973, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":989 + * + * + * def _transpose_weights(params): # <<<<<<<<<<<<<< + * # Transpose the parameters so that the gates are the last dimension. This + * # makes it easier to fuse. +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_15_transpose_weights, 0, __pyx_mstate_global->__pyx_n_u_transpose_weights, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[55])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 989, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_transpose_weights, __pyx_t_12) < (0)) __PYX_ERR(0, 989, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":1006 + * + * + * def _untranspose_unsplit_weights(params): # <<<<<<<<<<<<<< + * Wx, Wh, bias = params + * nO = Wh.shape[1] +*/ + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_17_untranspose_unsplit_weights, 0, __pyx_mstate_global->__pyx_n_u_untranspose_unsplit_weights, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[56])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 1006, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_untranspose_unsplit_weights, __pyx_t_12) < (0)) __PYX_ERR(0, 1006, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + + /* "thinc/backends/numpy_ops.pyx":1173 + * + * + * def _check_compatible_shape(u: np.ndarray, v: np.ndarray): # <<<<<<<<<<<<<< + * if u.shape != v.shape: + * msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" +*/ + __pyx_t_12 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 1173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + if (PyDict_SetItem(__pyx_t_12, __pyx_mstate_global->__pyx_n_u_u, __pyx_mstate_global->__pyx_kp_u_np_ndarray) < (0)) __PYX_ERR(0, 1173, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_12, __pyx_mstate_global->__pyx_n_u_v, __pyx_mstate_global->__pyx_kp_u_np_ndarray) < (0)) __PYX_ERR(0, 1173, __pyx_L1_error) + __pyx_t_4 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_8backends_9numpy_ops_19_check_compatible_shape, 0, __pyx_mstate_global->__pyx_n_u_check_compatible_shape, NULL, __pyx_mstate_global->__pyx_n_u_thinc_backends_numpy_ops, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[57])); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_4); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_4, __pyx_t_12); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_check_compatible_shape, __pyx_t_4) < (0)) __PYX_ERR(0, 1173, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/backends/numpy_ops.pyx":1 + * # cython: cdivision=True # <<<<<<<<<<<<<< + * # cython: infer_types=True + * from collections.abc import Sized +*/ + __pyx_t_4 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_4) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_XDECREF(__pyx_t_16); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_17, 1); + __Pyx_XDECREF(__pyx_t_18); + __Pyx_XDECREF(__pyx_t_19); + __Pyx_XDECREF(__pyx_t_20); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.backends.numpy_ops", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.backends.numpy_ops"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_super = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_super); if (!__pyx_builtin_super) __PYX_ERR(0, 97, __pyx_L1_error) + __pyx_builtin_reversed = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_reversed); if (!__pyx_builtin_reversed) __PYX_ERR(0, 823, __pyx_L1_error) + __pyx_builtin_zip = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_zip); if (!__pyx_builtin_zip) __PYX_ERR(1, 21, __pyx_L1_error) + __pyx_builtin___import__ = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_import); if (!__pyx_builtin___import__) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 409, __pyx_L1_error) + __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_id); if (!__pyx_builtin_id) __PYX_ERR(1, 619, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_get.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_get.method_name = &__pyx_mstate->__pyx_n_u_get; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_self_name_is_not_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_New(1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_mstate_global->__pyx_tuple[1], 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(1, 583, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_mstate_global->__pyx_slice[0] = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[0])) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + + /* "thinc/backends/numpy_ops.pyx":514 + * def ngrams(self, int n, const uint64_t[::1] keys): + * if n < 1: + * return self.alloc((0,), dtype="uint64") # <<<<<<<<<<<<<< + * keys_ = &keys[0] + * length = max(0, keys.shape[0]-(n-1)) +*/ + __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_int_0); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 514, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]); + + /* "thinc/backends/numpy_ops.pyx":693 + * Ct2.data + * ) + * H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) # <<<<<<<<<<<<<< + * if dirs == 2: + * H = xp.ascontiguousarray(H) +*/ + __pyx_mstate_global->__pyx_tuple[3] = PyTuple_Pack(3, __pyx_mstate_global->__pyx_int_1, __pyx_mstate_global->__pyx_int_0, __pyx_mstate_global->__pyx_int_2); if (unlikely(!__pyx_mstate_global->__pyx_tuple[3])) __PYX_ERR(0, 693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[3]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[3]); + + /* "thinc/backends/numpy_ops.pyx":903 + * if d == 0: + * seq_t3, size_t3 = indices[-1] + * indices = indices[:-1] # <<<<<<<<<<<<<< + * indices.reverse() + * else: +*/ + __pyx_mstate_global->__pyx_slice[1] = PySlice_New(Py_None, __pyx_mstate_global->__pyx_int_neg_1, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[1])) __PYX_ERR(0, 903, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[1]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[1]); + + /* "thinc/backends/numpy_ops.pyx":907 + * else: + * seq_t3, size_t3 = indices[0] + * indices = indices[1:] # <<<<<<<<<<<<<< + * cdef int batch_size + * for seq_t2, size_t2 in indices: +*/ + __pyx_mstate_global->__pyx_slice[2] = PySlice_New(__pyx_mstate_global->__pyx_int_1, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[2])) __PYX_ERR(0, 907, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[2]); + + /* "thinc/backends/numpy_ops.pyx":995 + * Wx = Wx.reshape((4, -1, Wx.shape[-1])) + * Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + * bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) # <<<<<<<<<<<<<< + * Wh = Wh.reshape((4, -1, Wh.shape[-1])) + * Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) +*/ + __pyx_mstate_global->__pyx_tuple[4] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_int_4, __pyx_mstate_global->__pyx_int_neg_1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[4])) __PYX_ERR(0, 995, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[4]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[4]); + __pyx_mstate_global->__pyx_tuple[5] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_int_1, __pyx_mstate_global->__pyx_int_0); if (unlikely(!__pyx_mstate_global->__pyx_tuple[5])) __PYX_ERR(0, 995, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[5]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[5]); + + /* "thinc/backends/numpy_ops.pyx":1012 + * Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + * Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + * bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) # <<<<<<<<<<<<<< + * zeros = numpy.zeros(bias.shape, dtype="f") + * return numpy.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) +*/ + __pyx_mstate_global->__pyx_tuple[6] = PyTuple_Pack(2, __pyx_mstate_global->__pyx_int_neg_1, __pyx_mstate_global->__pyx_int_4); if (unlikely(!__pyx_mstate_global->__pyx_tuple[6])) __PYX_ERR(0, 1012, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[6]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[6]); + + /* "thinc/backends/numpy_ops.pyx":52 + * xp = numpy + * + * def __init__( # <<<<<<<<<<<<<< + * self, + * device_type: DeviceTypes = "cpu", +*/ + __pyx_mstate_global->__pyx_tuple[7] = PyTuple_Pack(2, ((PyObject*)__pyx_mstate_global->__pyx_n_u_cpu), ((PyObject*)__pyx_mstate_global->__pyx_int_neg_1)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[7])) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[7]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[7]); + + /* "thinc/backends/numpy_ops.pyx":65 + * raise ValueError("BLIS support requires blis: pip install blis") + * + * def asarray(self, data, dtype=None): # <<<<<<<<<<<<<< + * if isinstance(data, self.xp.ndarray): + * array = data +*/ + __pyx_mstate_global->__pyx_tuple[8] = PyTuple_Pack(1, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[8])) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[8]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[8]); + + /* "thinc/backends/numpy_ops.pyx":104 + * return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + * + * def relu(self, np.ndarray X, inplace=False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + __pyx_mstate_global->__pyx_tuple[9] = PyTuple_Pack(1, ((PyObject*)Py_False)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[9])) __PYX_ERR(0, 104, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[9]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[9]); + + /* "thinc/backends/numpy_ops.pyx":198 + * return dX + * + * def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): # <<<<<<<<<<<<<< + * cdef np.ndarray Y + * +*/ + __pyx_mstate_global->__pyx_tuple[10] = PyTuple_Pack(2, ((PyObject*)__pyx_mstate_global->__pyx_float_20_0), ((PyObject*)Py_False)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[10])) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[10]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[10]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<11; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_slice; + for (Py_ssize_t i=0; i<3; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 11; } index[] = {{2},{68},{33},{44},{35},{54},{37},{60},{24},{52},{26},{34},{18},{43},{29},{33},{45},{22},{15},{27},{179},{16},{37},{54},{54},{30},{43},{32},{1},{2},{1},{1},{1},{1},{1},{8},{39},{38},{5},{16},{33},{6},{66},{74},{15},{23},{25},{14},{7},{18},{19},{6},{17},{18},{2},{6},{35},{9},{30},{50},{10},{39},{34},{8},{21},{57},{65},{20},{32},{22},{6},{22},{30},{20},{28},{18},{21},{11},{10},{30},{37},{5},{7},{1},{1},{2},{5},{3},{3},{1},{6},{11},{8},{1},{3},{1},{2},{1},{1},{4},{8},{17},{13},{14},{16},{22},{24},{36},{35},{22},{28},{40},{39},{29},{41},{40},{28},{40},{39},{22},{25},{14},{19},{39},{40},{38},{39},{13},{13},{31},{30},{15},{27},{26},{13},{15},{24},{19},{31},{30},{20},{32},{31},{19},{31},{30},{13},{20},{16},{1},{3},{8},{1},{20},{8},{5},{5},{1},{15},{2},{7},{2},{7},{1},{2},{2},{1},{5},{3},{3},{1},{0},{3},{4},{3},{18},{3},{15},{16},{5},{15},{6},{4},{5},{9},{7},{9},{8},{17},{6},{18},{2},{13},{15},{27},{26},{13},{19},{31},{30},{20},{32},{31},{19},{31},{30},{13},{16},{4},{10},{4},{4},{5},{5},{2},{7},{4},{4},{7},{4},{2},{7},{1},{12},{6},{5},{23},{21},{9},{17},{18},{15},{4},{11},{4},{10},{5},{3},{17},{1},{2},{2},{3},{3},{2},{6},{3},{2},{4},{6},{7},{7},{8},{6},{4},{8},{5},{4},{9},{11},{8},{4},{9},{4},{7},{10},{10},{5},{15},{5},{6},{9},{3},{5},{1},{4},{5},{9},{7},{9},{7},{6},{7},{8},{15},{9},{10},{30},{31},{29},{30},{4},{3},{16},{12},{9},{10},{8},{6},{8},{4},{1},{2},{3},{10},{5},{7},{8},{7},{3},{7},{5},{13},{5},{8},{4},{5},{4},{6},{11},{12},{10},{6},{7},{3},{22},{21},{8},{5},{6},{18},{17},{5},{7},{13},{4},{8},{4},{10},{4},{4},{15},{3},{1},{2},{2},{2},{2},{2},{2},{8},{4},{8},{7},{4},{7},{6},{11},{6},{5},{3},{3},{6},{3},{4},{6},{7},{4},{6},{8},{6},{3},{15},{11},{2},{14},{11},{10},{19},{14},{12},{6},{5},{10},{17},{13},{10},{22},{21},{11},{23},{22},{10},{22},{21},{8},{8},{4},{7},{6},{7},{8},{4},{5},{11},{4},{4},{3},{7},{5},{12},{10},{12},{19},{5},{10},{4},{9},{17},{14},{5},{4},{4},{5},{6},{3},{4},{5},{5},{8},{24},{9},{8},{6},{6},{9},{18},{5},{6},{1},{8},{6},{6},{6},{28},{6},{8},{4},{1},{6},{8},{7},{5},{1},{2},{1},{5},{3},{112},{125},{180},{61},{168},{128},{149},{82},{29},{120},{122},{195},{28},{63},{141},{141},{109},{93},{94},{134},{134},{134},{134},{178},{178},{168},{168},{168},{169},{176},{177},{136},{73},{336},{507},{180},{333},{67},{226},{57},{91},{47},{187},{12},{1117},{80},{75},{197},{39},{1659},{1}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (5712 bytes) */ +const char* const cstring = "BZh91AY&SY\031~\321\244\000\004\362\377\377\377\377\377\377\377\377\377\377\277\377\377\377\377\377\377\367\300@@@@@@@@@@@@\000@\000`\030\375\357}\363\316\272\3408\207lcs\216\202\212]\246\327-K\273\033\261\255\226\245\025u\223\252wh\236\356\273op\347v\344\007oa\340}!@\000\000u_p\000\224i\t%C@\0314\310\000~\251\275H=4\307\243&\020\364\300\210\244z\207\247\244\323SFCOP\310\032hi\351\243Dz\200I\020\001\023 \211\246\251\372\t\224\364S\332\247\352z\236\246\247\350\203(4i\345\0006\223@\001\246 \320\032h1\000\000\tM\001$MMCDyOQ\246\312bzj\032\017)\351\222<\247\250\001\345=@\321\352\r\000\017P\320\0004mO)\3524\000I\251&\246\221\202\232O\325=O\"yA\240\000\032\000\0004\000\000\000\r\000\000\r4\000\000\014\222\024d\000h\0004\006\233P\323C@4\321\240\032\000\323@\000\000\000\003@\003F\200H\24154\323#@\023\023Q\351\220\0204\322\237\224\320\215\017$\323L&\201\243G\250\000\000\000\001\240\r\001\223oj\266Y\245\255\312\301\303L\0335h-\366\260\033\325\n\253k$\277\367\361\026F\262\0040\321G\336$O\217\222\271\006!%pX\037@}\230\254\003\024\264F\002\213D\213\0246\250\233i4S\302)\r$^-'\000\246\002C\030\262\340R\2476\r%1L\235\005'\374A\377ZHm\t\264\"\330\020L\300\343\273\201\233\227\013\240\272\2720a`\354\025\202\325%/\376\022\360\313\002&\030H\302X\013+\036\024b\305\213%aX,\213[\0025\rP\212\310\242\302\t\030##\",\026\002\310\250\311\005\001I\005\202\305#\243D\231\263K\351\201/p\205\3507\350\n+\203]\2077j=\210\002L\222I$\010I\026\306\333\020\274\0033$\005\204U\202FAE\200\030\201\223\025!\261\261\005\311\005\346\322\030\305D\202\206`\245)\341\225\010\322\267\021\306\335$\273pb0\246R\203z\032\227\306\030\3003\221\222\025\006.\226\314U\256\027D.\341\221Up\333C\311b\202\253\007h\373\245A\025H\334\342\227Dt\016\204\256\253Y\273\351\007\201g+I3VI\311E\355\211H\200\304#)TN\356\366\325\014#`2\226\246\2563;%\200\242,-fH\006`2\311Q\334\010\030\250\225@\324\344\260\253K@\371\027IE\256]]\006Pl\266550\036\321\246D\014\300\006r\001{YX\217_\036)t\320\306[<\\d\254#p\334\340\033\244\266\016F\272\330\310""\311c\013\030\014,`\341\223\007\230X2\3112d\214\343\010\3068\2248JEq\222rw8\221\200\215\263\010\300\3348\"\374L\230]\013\023\234\322g6\3459i\225G\207\367v\373\307\204\376$\370Q\036\004\341\352\311\251']\317\250\212\363\215\315\211\267\223\275vW\363S{\317\004\235\240\006\363\300}\334o\274\357|I6\334\222=-E>\227dgP\236\223\224\t\"\032jO\357?\002\022\241\256\366\324xJ?\341Nw#\226\272n\037\361\240\036\266Do\313]\3769\017Z\0048-\206$\303A\253\235\333\375k>,RP\201H\205\250\362l\224\272f\377z\330\232A\341\365 \205>\367TA\023\034\274q\3229V\035\237\207\317\217v\233\272\033n\213z\352\333\315u\020\021}\013p) \362\374\315\350\350^ht\003l%\345\326\326\250\331\313\007\030t3\224,\005\271Ma\215+\0103+&\027\301Ib\262\225\301\016\007\023Uz\232\377\210\032\300\223\225\272\252\252\252\301UU\025\"+\024\212\267\266\272\332\2204\240\3125\365\357\345\272\30333b\305\222\260X9\013\221u\314\372\337\267\373\346\000I\226\004a\006\014\0061\216\265\232\334\006\013\002\262\261UR\272\007\301P\250\212\201S\007\355\\.NsB\230\022\221\364r\026J\265*V\275\221K\224\0337\373wU\007\233\r\303Y\305\347\024\271S\367\"O/\250\324\006y9\344\373\377\223.\212\353\"\221\205\tu%\324\211)\207\024v\203\314\255Z\004'\364\307\007q\004\225edN\305\252R0\254\341Vr\r\027\014{\276u\323K\226\005d\256hvs-\332c\371>\035:|\303\347o\026\276\373\211n\006\353u\3148\235:\036\277\265\376:\372\233CD""\264KD\264\014\231\311\363m\340\357g\"2\0301\266\376\363L\t\234\000\n$P8pZr\376\213\n\252\265Y,s\276\016'\2570\272\275\356\036\036\356\357\267\307\227/\036\221\316L\372\317\265\320[ML\220qo\231N\353\361\273O&\351yr\354\267\365?\001\366\232\222\265\252\322F\3310\003Z\006t\215\026\372\031\250\272.\264\321<\033P\315\360\221#\351\240rw\261\317z9\233\341|BC\312\367\001\032F\366\370]V\023=\240\275|\261\317*`\274+0\222\367\004\330\217\3645\000x\356\304\312\213\215\356:q\365$c\002\244\2161\265;\261\336\036\210w\243\207~M\250\007\303$|\022$\346\356\310\033`\206;\234\336\356\375e#\326\243\205\313(\0165\266\303F<\213K\341\205$\353\225\311W~\254\024\232,\322\346\250\204F\3354\331\003\327\250\300\203\005\2218\032\340>K\324z\007A\316\216~~i]\322/k\026\332\306\266\006\021\351\016]!NK\320 \357[\274@\365<\226O\177\337\214B!g\027\213\274s@\036\223\261p\013\203\300fFV\317\t\245\374\006\333\313\207'_d\375\313\223\343\310^ECZs\243\275D\342@\223X\016\036\267\004\014\244\010h\r\t\032\026\203>u\304u\342\\\217\000?U\270\364FK\245\204\332\304*\221z\266\3515\216\354\300F\327!\224i\251ZIGF\013\370\342\270BTb\2614#4\222YQ_\033\014\301\252\317\324\200\256\333\200\026F)\026\317\247F\2546Xh*A\333\271?\211H\013A\334\237\202\022f`%\216H85k\311\361\325\300\315\275\013\034~\223\362\026U\350ia\256\230\275\231\211\246\226.\367c\002\351g\202\374\027KR\313\2240\222\024Zz\203\316\022QQ7l\304\341\205\247\022\023\307:\"C\305\354\374\031>\267s\243\000b\323\267K\223\021\306}\036A\001m\325\307\276\322\342f\342W`\025\244\004\177\031\324e\026\030CB\227\336h\243\272)B\211\025\2176\036\316\005\236Pd\230\010\253\325\027\315-\227~l\314(\224g6L\272\263B\204`g\220n\205\355\020\203d\302\004c\024\214\256\275>\305\267\336\022\252\346\027\331\344h\262T\200\030\013[\2569\"\375|\210Qs\351\307\227\036\014\324T\340\357\002-\030\221\263\303\232F-\314j\022d\364\352\317(\337\256\270\326Yl\021D\265\017a\223\322p\353Dg\254\312\263Dv\332\356\334\304\332~za\373\343O5\254\325\204z\031k\"\272\nnJ\234\235\033\201n\020\267#pn\r\333\272\013\270o\026\304\3473\303@\311\003R\031\3002k\266\334IBT\022n\243\351\324j\276\177\245\214UV\036\307\321H%\267\370\267\303\016\362\216a\235\215\212\026#2\241W\265\251[2\016L\266\2673\373\371u\210=\260X%\272\257(p\375\216\377t\267\300\236o\353?\221\317\201s\323\263}\364|fo4\0073\256H\2222IT7\336\350!\016\320\350\357!<2\216\341#44\221)\241!@\036\217\305<\361\277\017\251=\331;\375\367\0210\303Qh\036\310\262*\335\275d\303K\210\222\025\270\210\350\372V0\365\321!\276s\205&\206\201\335\274\002B,\351\217\375\343\323\241t\233\273\366e\230#\275Q\324]Q\014\3563I\003=\264\373\331\223\010\021\325\255\020\347\021\203\003\266!\376A\360\334n\250\267:\352\277\242Z\237\247\001\002\t\035'q\016\314\031\312\200T^^_\354W*\345\260C\310E\270ZT\231""\025\016\357*'\260\344\254~\335\220\345\314\275\257\202\2150\177)\256\277yOzO\026\335a}\260\005\355\334y\222\006\236\003\225\035\314\2050\356u6\263\275\261\201\201\022\031\010\274y\273\270B|/\263\374\273\327\016\366\262\013\036\353\204IIw\313]\353\345g\351\254\264\213J\ns\006LP\327\377\254{+w\206wz\340\260\351\231\"v\372\320)\025\027\204A\361,\261\253,\351\262w\262i]T\311\027\242\315\233\020~[\177E\226\203\321\234\251\333\245d\346\257\314\273\220\215\233\r\222\3642\0201c\222\215\345\275]g'}6\310[\n\325\002d\245i\022j\032\371_\340;PsKf\312\374\004\252:e\017\220\\\334\236=oZ\035\222e\\\034\215\247_y\316H\2569\244\202\202D\376\236`\316k\320@\341\2525D\242'\303\225\263\376n?\310i4\232\334a=\024\005N~\271\231\216\316df8\320-Q=n\210\222\354qy<]\235$\373s4\352\250\243\203\313?\242\371\275\034\372{:e\032\201)9Tu\3739\277L.,\353JG\031\252\2324f~1\310\217)\020h\222\342:8L\346u\262#\361\222\313\233ne\237r=\350\221\377EM^\034\306\261\314\355\307\251\343\214\322\346]\006\3434\031\334ZD\264\013nQ\370N1\326g8z\356\347\221\227\211\343\205c\305\324:\246\006S\003\265\226\216uq\312v\273\004\210\201u\375>\307Xs\317c\253\325\301\227R\005\013\020\314\032\322\351\322\0218\310\252\253e\221\004&9\301e \n\314\300*\2349\010\026d9S\227\362\365.\002N\276\311\277\366\375}\377Q\336\021-\376\366\371\367\014\177\027\000y\232\3708<\335O8\223\311\311\375\336\003\203\315z\226\355\232CS\010\022zO\221\257(\rM\225\210\022\327ka\322\215\272y\001+$\310/5\251 b\274\262\t:\376\304\353j\363\233\346\025~\007\370\324c\254p{9\3404\266\373\226\273d\003\020\356\207t=s\325;\247\203\273\223\213\351\375.\235~\226/\275\232\255\343M\317\227\277#\253\230q\"Ld\231\032#)s\312S\034\325\324,'\036\\\306\230DT=\023i\t\357\227\312\362\343b7l\262\205[\245\307q\305#\274m\025p\363_\211V-\36533\254\322\224\371\2234+r\256e\346*\257\0045\014[\244\320`\370\210\031\002\032 \203$\303\324M\221\213 \302\212\221\241\023\006\023\t\224\323\304\217wa)=\373/\245\206\343/S\314\347yY^\327\211\177\216\355q\356\265\305\023\240~8\226\212\273\340""\251\225%\021\020\213%\211\252\006\212\313\274\245j\203\001\017\002\034\307\206\205Q\220:\260z\305\2348\366\235\2374KI\0168\237TmCZ\324Tsg+_\371\314\337\252\177`\267\244X\316\2646\373I\345\261\215TfX\333\346|\216\036T\226f\314%\210I\326\352\0367\337\377\027rE8P\220\031~\321\244"; + PyObject *data = __Pyx_DecompressString(cstring, 5712, 2); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (5929 bytes) */ +const char* const cstring = "x\332\335ZKW\033W\2666\266\014\370\2150\006bw:%c\033;M\350\0100v|ss\227\000\203q\022\333\002\0148\257\272\245\252B*[RI\365\000\221\225\356\366P\303\032\326\260\2065\324PC\r5\254\241\206\374\204\374\204\373\355sJ\017@<\322\261\007}\327\002\251t\236\373\354\375\355o\357}\244\373\017\022\331\254\240h95ojz\336\024\n\206*\253\212\226O\267\033\205\273\212\220\263MKH\251\202\226W\324\222\252\010R^\021\362\272%\230Y\r\303i\rS-\332j^V\205\254\232O[\031\2635\345\233\377\026\276\234\373nyU0\355BA7,\301\300H\315PM!\225\325\314'BA+`Y\323\222\260\010\265\314\331[[\252!lk\352\216\240\350\030F\373\250\245\202n\252\202i\031\232\242\232\363R^\320\363\331]A6T\311R\005IH\361IVF\262\004\315\024d=oii[\267M,-\344\324\234n\354Nb\026-%\231\246\226\316\013\226\016A$\345\013\266\016\037A[\206\203\302\205w\014\315\222RY5\034\300\205\3322\364\334qs\231\216\204\035\315\312\010\326nA\025\306\303v\313\220\362&;F{\n\037\206\031P\210lu\030b\241S\373\032W\002\037\3644W\260v\0053#ai\313.@\270-\335\020\344]+\243\347'%\303\220v\237\226\n\030GV\262`\r\tfX\264\363\262E\213\311\244e\266\247\224K\205\032\222\214\264\215\315,&\255\271\314\244\327mK\320\267\204\224n\347\025S\270/\225 \302]\345\301\362aA9\014\270iUe9\277-e5\340EW\324\t\262\032\027d\\\036\027 \3448$%-\214O\010i\314j\016\346g\201\241\3306_\177\317\264\263N\332\201\010/t!'Yr\206 I\206\223,\333\240#C\260\027:L\304L>\317NOzR\324\254\226R\r\030\017\246!\270`\177\206\213\274\360\352\351\253/f\036\3170\354\032\352[HfB\360\224\234\005\"\0003:\256\255e-\310\301\0241),o\t\273\272-\344U\234\000p)`\\\347\004+\243\346\001{\213\036\204qfc\211\224,b:\244\035\017\355\245m\2534{Q\312\232\352\344\313\002\215\220\262?.\254\321\036?\277\334\247g\034!\004\262$\313\252\331T\374+C\337\006\354\241\306\322\270\300,\014\225\351vV!\367\232\372\242e\013);\201\371\026WN\307\254\335?0k\325R\013P\370.3+\006\376\252\032z\333\376k\231\003\376m\3329\301.\320\371H\t)2T8\340u\2369\016z\340\214\333*\374^\263\324\034}\326S\244\373\337\236\010\223\337\214?\370\037IQD\354\245J'q\010""\307\314\361\303\232\243\230\215[\3006\303\251\244\003S\310H\333\0046Y\317\025`.\222\221\341\017LD\376\362e)%\311\357\n\206^\020\261\315\224\254g\333|\2256$E#G\341\332\204\335:\224\250\301)\247&\204\035\t\274v\332\025\232t\005\310\315\303?\0242<\004o\263\027&gU\346\271\346\244\224\222\277\356\3405\032\310\001\366\315\301\346&\235|\323\226\357\276\371`R\321L2\211\242\333x\235RD\3537-o\321{G\213\0356\251\314z[Y]\262:G\266\033\232\003\323\262p\237\224\233f\374D\330%GUK\026\351\235\016\326\016#\232\311WU\010\010\246\366\253*|\r\263\036b\257\274\016'\336\222\354\254%\210\242\241*\266\254\212\242\240\330\014Ky=\377\005\234z[\223\262\350\225\265\274f\211b\2760\231W\370\\;W\330\235\024e\335P'sXA\343z\336\222\264,\367a-G<\3259\314\006\273d\016\215\010A\372\215\251f\267&\363RNmr\360\013=\257\0362k\013\220'\000\343\244\211'\342\341k\036\000\225}\326?\334F\\\333\002\301\276\376V\253\005B\225'\371+\341U\005\377LR\370\235,\354\036h=\000\302\003\275L\227\254\355\357\315\266\277\2636Q/\320b\245\003\343\321z\240\205\323e(\020\261\"\177\264--k\267H\004~\257\313,\3263M)\222%Mv\351\345\241\204N\032\246\n\223\211\325\371\345\345\004M\332T\346\346\347\277\234\237\003}\317k\312\2745\265\300IxA\335F\022\303\036\237f\263Z\301\324\314\245%My\366\354\313\345\027d\356\027t\232\227\005\263\371>)\212\034w\255\006I\221r\355\017$K\373\223\311\004n}nqC\326\264r\207[sR\t\241\367\250\366\037\333\236\372\363\221cZ>\332m\210ff\016\267\206N\206\371\307\365\035\277y\307\270\343\004h\016S\245\374\261\235\247\333\214\006\236b7\004\251\343\372N\265\027\215;~\253\254}\2705t\370V\207\234\002\370Z\237\322`\036\325\020\021\001\2734uH5\021r\355\317'\014\263\217\033\327\022~\342t\243\272,\246\346\332z\314H\035H\",\213\340\361\035\311P\340\033\010\002Dj\335\273\221\377\301{\362\351V\357\001\310\357C\372\364>\001:\001~\240\247\023\327yD\331\\[\313\310\2705\226\226A&d\245\255\366.\250?\001\354\307c\274\033\264OB\364\t@\356\202\337\023`{\251\306N\275\351R\345Q5v\372\2157*\317\252\211\2436\237\n\246\226j\377\014^\2577\372/a\357Eg\314\201\240\203\316\347n\"\370\3645\332\203\365\215`c\363\344\336\231\262\005y\366m0\355)\376]\177\247\222\251\246k\311F\377\345\362\202s\321\215\271\361}\2174s\207\316;\350\334q\336\342\004\321F\3775\2545\355\250n\334\235wM\357\016\016\375\250r\277:^\033\255\337\250K\357{\032\021\332\245\307\215\272\367\275\2307\343\225\374\355J\262\"5\"}\357\177u\256;\211\306\245\250\023ub\316c,\260\340]\364o\373\337VJ\325\355Z\262&\005\221\257\202\257\226\353\330\3433\210\030\361\022\277\367\223\322\247\312\033\316\274\363\2537\344%\275\264\277Z\201>\243A\024@\300\346\020\355S7\351f\274\224\337\347\227*\305j_\325\256%j\033\365\245`u-\264\3025\034AvoB\226\"\006\231\225\330\341\371\232g\370#\225\013\325(\316q\243&""\325v\352\031(o\237Y&\375\036\177\010\273_\252\3020_U\333B\356\365\337\302Q\360v7\270;[\221*\305=\250\355\004\231o8?zg=\0102\340\234w^3]\017p\265]\207V\213\215\241\233h{\350\365t\225\240\347}\242\301\266X)\027i\300\214\263\355\256\270\266\267\350\307\032\375\003\316%\246Xf*\202\332\252\033q\027a\211)o\005\347\357\361\321|\033Bl\371s\376\333joUj\364G\235[\356\222\307\014\377\324\271\341H\216\345\316B\320\333\336\234'\321\024\262~\014\022\364\002\343E\357\361\373\3053\027\246*C\225\365\352Tu\265\326\0135Z\365\231\3726\200\270\327\377W\367\035dO\266\3670\331\241/\337\303\026&V\233a(Q\252\261j\274\232\250&\233+\016\220\375g\202kw\274\204\267\351\257a\361\325jO\330\331\200\336g\334m\217\r\336q\266\334\204\313\036mg\221\326\306\334+\303\316\032\024\272\210\243,\372\343\225\341\n\324\025*\326\033\365oVf\253#\265\036fo\263~\247.\325w\230\303\374\020\374\360cc\364Sw\335\213{\013~\257\277UY\252\316\326\242\265\330\336\351\366\355?a\337O*\367\200\322b\355< /\327\207\352\311z\232\371\352\233\340\315\017]\367\375\375\362\231\013\003|\347K\345\207N\037\351\216\024\271\344_/vh\372\203\350b(\214\034;\236\214\010\260^\211s.\344\320-Vz\240\017B?E\336(l/C\212\304\307\320\305\021r<\001\r\2150)b\225i\026\200\036\325b\265\031D\204\261z\362\264\272\350\346\314vm\261\036;\205\222~\253\3059d\026\366\211\367_\325\255Z\202\213\370\330{\030z9\251(\006\310\314#\262\306Y\212\023\254\001\301?\006?\376\022\374\"r\255!]\373\240\022wS(\311|\365X\231\277Bt\030\251\364VT\246\3168\340\265Z\217\300\2517\2025x\035B\335O\301Ob \376\357\036\213\241\207%^g\214\023c\022\357\201\367\237#\314\254 1\272L\303\031\377\rS>\032\304\236\303\231aJ\344\224\247\005d\233)\206\274u\212fc\010\270\313\376\nra\251\262]MVS\214%6 \355Z\220\344\024\264y\024\"?\204\234GB\266\253\244\317\375T\245\017\211.R\267\252Q\033B\022\221F\n\321f\263\356\230\355\004\303\307\326'%koB))\3219\313(}\007\t\226\025\254\206\211\316Q\372\374\020r\236^\237$\351\017\276Q\031A\036K\251\340 \034\277X\357\253\027\217\010=\247\025Tv\243-AQ\317\324c\037\016\240;-\263\253\240\000\360uc J\341\030\351\320*E\366\216d\361\203\313\333]\261\003'\000\265\r\2010\252'\017\210\274\327*\037>\023xUu\301\217\036~\330\353\037F~SrKH72`j\2528\221=S\005\204D\327\275\203li\303\177Z\2116\256R\022\263\211\254\224\025\333\317\250\266\361\276\203(\344\333H\033\256\226\245r\t\353\020k\215\342\004F\225\025\265\273\356Ed8=\274\350\030(\033\224F#\361\231vS^/\325A4\246K\343\245\362\327\301\310""\004r\345Mp\350\0202\014V\236 O\013\"sH\335G\251\200\231.o\241\030\204\256\007\203\301\357\341\254r\220\\i%\256<\2355Y\216N\205V\253\203(z\030\345\033+\273\310n\341#\222\343`p\034\205Y\257\257\241\262\350\247\014\234j\206\004\277\242\230n\33694\013C\213\372\220H\236\205\305V|\013UO\224\247\222\224\366O!Y\034\301J2\364\0064\026\335~\004\205%\3771ba\032\352J\003g\nj#\231\021\341\353\260\240\036\010k\220\177s\213\013@\311\206\277\214\324\203\240\321\007\217\213\324\027\202W\253\214\034\366\027\335t\205\360\020)\360\263J\342}\017\340w\376b9\332\2104k\022\336FH\033\270\216RC\201_I\360\327\306\300(+\022\333o\244\376\001RB\254\365\026\226\203\311\375\203\211m\0228S\312\271\350\336fwFc\250\345\t&q^\320\236\243\253\251\001\347l0H|\314\216\235 \313\222W\214\021\226\351R\251\277\034A\375\032'+\016\302\346Q8\312\301\307\307\316\024\314/\243T\345\367J\363\276\211\314f\263\272\002\224\235\253\305O9\210\366-9\006\030e\014\364\324\034r\237\325\017I\302\036+\260Q\033=c\027.\021:\301ep\0371\202\342\337\253\234Ey\217\224:\210 B\021\031\323\375\325:\2774\204\337\362;9V\352\357'\000F-\373\t`\257\263\330\031\246\350\007\241\330\265\\\263]\366\006\211>\217\345\003~\023U$F\037\002T\246p(\226\366\021v\014v\013p\033\252\350C\305N\267T\354\232r\224Udtd\252O\226\020gZ\356\311.G:\257\325\242\177d\341O\240\315\245\312W(\364\322\007\363\026\344\203?w\272~p\363o~\334_\250D\350~\364}\317^\344S\367'$\231+\210\3267\341\241\354>q\227A\247\237\352\320Q/\332\030\371\013\250\376\021\022\332Y\014\231\001\362\037\242\262\212\327\023\215\221O\335$\350\243\367\314""\371\336\367k \006\213\335\214\264\340@w5s\356\026]w\361\306',\334\221\371\251|\214\301\251Rp\360T\265g\377\234\347^\312?\207\316\014uU?\306\202t\251\224\2401\243M\224\207\017\2270'\216\354\233\334.\022^\357\006\221[\250f6\202\211\347\365\361 )\005R\212\335\013s\2266\301U=\214\206\303\333\216\216\206 2\021L|Se\245=\371\\\007\261\233\354\2460\261\307,\316\356ifC\307\3509\364\221_\n-\260{X\223_\353\024\2033\237\271\377\362\231\261\212\345>v}\201u\256\006W\343\254\340\372\032N\327S\213RL~K\350&G\277\n\242\340\031G\261\371!vT\353\005\306\362\227Pm\314y\031\246\266\263\000\355\002bpd\300\3519\252?ql\357ig\323\225{\242\274\006\240\263\353\322 2\342\024\3715`\342=\217;\277\017\2379\337\377\336,\217aT\224\321}X\206E\302[\026\027k\321ex\030\313\210\177I\237\261}\217a\250\242\266qV\314\303\027>q\007\261\347\2407\215B~\2264\331\350hJ\261k\342u\252\356\033\221\317\020\027\206=\0031\212\356W\267\252\213\265{u\346\t\310\021\033Ld0k+\364\214\272}\264\001\335\301\315b\275i7\203\034i\320\357\270\032.\036\214.O(\003\273\325\212,\177aL\330\207\035\377\n.\002\200\3612\314Crk\227\021g\207\2474\307\355\362\211{\313{\001\210<\206\263+\265\317\353TW\0067'\370W5\267\3006+H\340R\376\225Jr\017x\220\210\230\023\264\313UG\342\367\262):\013]n\323\367E\2247\300\016+\224\023@\341\267\235\347\220\200\002\321\212W\002\243\014\261\334\355u\355.r\221\363`%,w\261\034c\304=\025Zm\215]\200\365\3707\350\032\005\361b\215\356\215\367\330\26161De\326D\354^a\301\203.\177\357xo\021\014\306\370\215\365\025\344\321\224\204\340\210s\310?\213\035\221/\352\334\010\206\357\363\233\371\016\005\204\021\233Xp\316\227 d\224}q\201n\215\\\2125\334\206(,\001\236\n\223\000:\022\022p\272\215\245$\254z\017\2317\014^\277QO\005\257x\354\341\227\005J\240\250\201\252\005\332\333\340m!(\024\203\242\035\330;\301\316?\202\177\3743\370\347\277\370\232\224K]\276ZVA5\211\216\207\023\263\207!\274\361\213`.\r\261\370|\345\217w 3\247\257\322\302\313Id\2313\354^Y\251\334\251\310\224a\0169s\241m\006\275Y?\352\337A\327\030)\244{\307\n\212""\221\036^\233\314\263\013\261T\230\266\305;\022\231\010\314x \2559\230\371\320E\351\241\306\243\263\"rv\304r\244uD\032\204)R\023\335\211\363\253\003\242\000B\3565~m\377-\n,(\231%R\301_\237\3207\"\234\342m\347;\372V\207\310\245\357\275]\236/3\272\001\240\331\332\345\014K\020\206\33174\253>\203\\r\037\240\350\313H\000\225\273j\n\nN\360/R\330z%\306\373\375\203\240X\372n\022\373\354\2612f\014\201\2113<\325;\273\225s\225\231J\261\321\321\365\000\214#\2438#>\014\177p >d\3378\213\217\233_9\213\017\371\257\244\024\251T\330\245\337\037\t\367\371W\326\002\237\241\247\336\0369\207\375\272Z\370\374\301\231\223\0267\377\314\342L\2623l\t\376\035\266h\t\367\267uM\021>\237\020\3200!4\233\037\234\241\337\327\314\316\260\316\373\364\013%\241)\331\014}\231-N\311\273\374+s\021\263N\352?\334E?%\371\267:\370/ \272\016`=q\245\373tR\314\037\353\370s\353}\0349\037\234\371\2173\210\371a\355a~\034s\230\377\357\255\301~$\364\341\274\343O-\367Q\244\374\317\263\206\371A\215a~\024[\374\233\216\301;\350\207\250\3424\375\352\263\263!~\260a\352`\303\227\324\360\362\377\000\317Z/v"; + PyObject *data = __Pyx_DecompressString(cstring, 5929, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (15716 bytes) */ +const char* const bytes = "()All dimensions preceding dimension %d must be indexed and not slicedAll sequence lengths must be >= 0BLIS support requires blis: pip install blisBuffer view does not expose stridesCan only create a buffer that is contiguous in memory.Cannot assign to read-only memoryviewCannot create writable memory view from read-only memoryviewCannot index with type 'Cannot transpose memoryview with indirect dimensionsDimension %d is not directEmpty shape tuple for cython.arrayExpected at least Function call with ambiguous argument typesIndex out of bounds (axis %d)Indirect dimensions not supportedInvalid mode, expected 'c' or 'fortran', got Invalid shape in axis ')?add_noteall sequence lengths must be >= 0, got all sequence lengths must be > 0, got and arguments, got arrays have incompatible shapes: at 0xbackprop_seq2col requires gradient array of dimensionality 2, was backprop_seq2col requires gradient array that is in C order and contiguouscollections.abc dimension(s).disabledouble2d_t|int2d_tdouble2d_t|uint2d_tenablefloat2d_t|int2d_tfloat2d_t|uint2d_tgc (got got differing extents in dimension isenableditemsize <= 0 for cython.arrayno default __reduce__ due to non-trivial __cinit__np.ndarraynumpy._core.multiarray failed to importnumpy._core.umath failed to import object>self.name is not Noneseq2col requires sequence array of dimensionality 2, was seq2col requires sequence array that is in C order and contiguousthinc.thinc.backends.blis.pythinc.backends.collections.abcthinc.backends.numpythinc/backends/numpy_ops.pyxthinc.backends.opsthinc.backends.typingthinc.typesthinc.utilunable to allocate array data.unable to allocate shape and strides.ASCIIArrayXdBCC0CBlasCidCt2DDTypesDeviceTypesEllipsisGGidHH0INNoneNumpyOpsNumpyOps.__init__NumpyOps.adamNumpyOps.allocNumpyOps.asarrayNumpyOps.backprop_lstmNumpyOps.backprop_maxoutNumpyOps.backprop_maxout[double2d_t]NumpyOps.backprop_maxout[float2d_t]NumpyOps.backprop_mishNumpyOps.backprop_reduce_maxNumpyOps.backprop_reduce_max[double2d_t]NumpyOps.backprop_reduce_max[float2d_t]NumpyOps.backprop_reduce_meanNumpyOps.backprop_reduce_mean[double2d_t]NumpyOps.backprop_reduce_mean[float2d_t]NumpyOps.backprop_reduce_sumNumpyOps.backprop_reduce_sum[double2d_t]NumpyOps.backprop_reduce_sum[float2d_t]NumpyOps.backprop_reluNumpyOps.backprop_seq2colNumpyOps.cblasNumpyOps.gather_addNumpyOps.gather_add[double2d_t,int2d_t]NumpyOps.gather_add[double2d_t,uint2d_t]NumpyOps.gather_add[float2d_t,int2d_t]NumpyOps.gather_add[float2d_t,uint2d_t]NumpyOps.gemmNumpyOps.hashNumpyOps.lstm_forward_inferenceNumpyOps.lstm_forward_trainingNumpyOps.maxoutNumpyOps.maxout[double3d_t]NumpyOps.maxout[float3d_t]NumpyOps.mishNumpyOps.ngramsNumpyOps.position_encodeNumpyOps.reduce_maxNumpyOps.reduce_max[double2d_t]NumpyOps.reduce_max[float2d_t]NumpyOps.reduce_meanNumpyOps.reduce_mean[double2d_t]NumpyOps.reduce_mean[float2d_t]NumpyOps.reduce_sumNumpyOps.reduce_sum[double2d_t]NumpyOps.reduce_sum[float2d_t]NumpyOps.reluNumpyOps.scatter_addNumpyOps.seq2colOOpsOptionalP__Pyx_PyDict_NextRefSequenceShapeSizedTView.MemoryViewWhWh_sizeWxWx_sizeXX_XsYY_ptrYidYt2_abcadamaddadd_gradient_noiseallall_layer_gradsall_layer_paramsallocallocate_bufferappendargsarrayas_contigasarrayasarray1iascontigascontiguousarrayastypeasyncio.coroutinesatbackprop_lstmbackprop_maxoutbackprop_maxout[double2d_t]backprop_maxout[float2d_t]backprop_mishbackprop_reduce_maxbackp""rop_reduce_max[double2d_t]backprop_reduce_max[float2d_t]backprop_reduce_meanbackprop_reduce_mean[double2d_t]backprop_reduce_mean[float2d_t]backprop_reduce_sumbackprop_reduce_sum[double2d_t]backprop_reduce_sum[float2d_t]backprop_relubackprop_seq2colbasebatch_sizebestbetabeta1beta2bhbh_sizebiasblisblis.pyboolbxbx_sizecc_contiguousc_initcblas_check_compatible_shapecheck_seq2col_lengths__class____class_getitem__cline_in_tracebackcollections.abccolsconcatenatecopycopy_arraycountcpucpu_clip_gradientddCdGdWhdWxdXdX_ptrdXsdYdYidd_biasd_maxesd_meansd_paramsd_sumsdatadefaultsdepthdestdevice_iddevice_type__dict__dimsdir_gradsdirs__doc__double2d_tdouble3d_tdtypedtype_is_objectemptyencodeenumerateepserrorffillflagsfloat2d_tfloat32float3d_tfloat64formatfortran__func___fused_sigindexfwd_stategather_addgather_add[double2d_t,int2d_t]gather_add[double2d_t,uint2d_t]gather_add[float2d_t,int2d_t]gather_add[float2d_t,uint2d_t]gemmgetget_array_module__getstate__grad_normgrad_partsgradienth_inithas_blishashiidids__import__indexindices__init__inplaceintint2d_tint32_is_coroutineitemsitemsizekeyskeys_kindkwargslayer_gradslayer_paramslearn_ratelengthlengthsloclstm_forward_inferencelstm_forward_training__main__maxesmaxoutmaxout[double3d_t]maxout[float3d_t]meansmemview__metaclass__mishmod_ratemode__module__mom1mom2__mro_entries__msgnnFnInLnOnTnWn_inputsname__name__ndarrayndim__new__ngramsnoise_levelnormalnumpyobjopsorig_Xoutout_outputoutput_packparamsparams_iperiodpopposition_encode__prepare__py__pyx_checksum__pyx_state__pyx_type__pyx_unpickle_Enum__pyx_vtable____qualname__randomravel__reduce____reduce_cython____reduce_ex__reduce_maxreduce_max[double2d_t]reduce_max[float2d_t]reduce_meanreduce_mean[double2d_t]reduce_mean[float2d_t]reduce_sumreduce_sum[double2d_t]reduce_sum[float2d_t]registerregistryrelureshapereturnreversereversedrowsscalescatter_addseedselfseqseq2colseq_i__set_name__setdefault__setstate____setstate_cython__shapesignaturessizesize_at_t_split_directions_split_weightsstartstepstops""tripstructsumsumssupertable__test__thinc.backends.numpy_opsthresholdtimesteptrans1trans2transpose_transpose_weightstypestypinguuint2d_tuint32uint64unpack_untranspose_unsplit_weightsupdateuse_blisutilvvaluesvarianceweightswhichxxpyzeroszip\320\004!\240\021\360\006\000\t\014\2101\210G\2203\220a\330\014\020\320\020 \240\001\240\023\240A\330\014\024\220A\220Y\230a\230w\240e\2501\250A\330\014\023\2201\330\r\016\210g\220S\230\001\330\014\020\320\020 \240\001\240\023\240A\330\014\024\220A\220Z\230q\240\007\240u\250A\250Q\330\014\023\2201\340\014\023\2205\230\002\230%\230q\240\003\2408\2501\320\004!\320!1\260\031\270!\360\006\000\t\014\2101\210G\2203\220a\330\014\020\320\020 \240\001\240\023\240A\330\014\024\220A\220Y\230a\230w\240e\2501\250G\2607\270!\330\014\023\2201\330\r\016\210g\220S\230\001\330\014\020\320\020 \240\001\240\023\240A\330\014\024\220A\220Z\230q\240\007\240u\250A\250W\260H\270A\330\014\023\2201\340\014\023\2205\230\002\230%\230q\240\003\2408\2501\320\0042\3202G\300~\320UV\330\010\013\2101\210F\220#\220Q\330\014\022\220*\230A\320\035U\320UV\320VW\320WX\330\010\013\2101\210F\220#\220Q\330\014\022\220*\230A\320\035U\320UV\320VW\320WX\330\010\013\2104\210t\2201\330\014\023\2205\230\002\230%\230q\240\003\2403\240d\250%\250w\260h\270g\300Q\330\010\014\210D\220\n\230!\2301\330\010\014\210D\220\n\230!\2301\330\010\013\2104\210w\220a\330\014\022\220$\220j\240\001\240\021\330\010\017\210t\2203\220e\2301\230C\230s\240$\240e\2507\260(\270'\300\030\310\025\310a\200\001\330\004\023\2205\230\001\230\021\230(\240!\2404\240x\250v\260Q\260a\330\004\007\200z\220\023\220A\330\013\021\220\021\220!\2208\2301\230D\240\n\250\"\250K\260x\270v\300Q\300a\320\0049\3209I\310\021\330\010\037\230q\240\004\240A\360\010\000\t\014\2102\210W\220C\220z\240\024\240Q\240g\250S\260\001\330\014\021\320\021!\240\021\240$\240a\330\014\035\230Q\230h\240b\250\007\250x\260q\270\007\270u\300A\300W\310G\320ST\330\014\023\2201\330\r\017\210w\220c\230\032\2404\240q\250\007\250s\260!\330\014\021\320\021!""\240\021\240$\240a\330\014\035\230Q\230i\240r\250\027\260\t\270\021\270'\300\025\300a\300w\310h\320VW\330\014\023\2201\340\014\023\2205\230\002\230.\250\001\250\024\250S\260\013\2701\320\0049\270\021\330\010\037\230q\240\004\240A\340\010\033\2301\230A\340\010%\320%6\260a\260q\340\010\013\2102\210W\220C\220z\240\024\240Q\240g\250S\260\001\330\014\021\320\021!\240\021\240$\240a\330\014\025\220[\240\002\240!\330\014\020\220\005\220U\230!\2301\330\020\023\2205\230\001\230\023\230C\230q\330\024\032\230!\2305\240\001\330\014\023\2201\340\014\023\2205\230\002\230.\250\001\250\024\250S\260\001\200A\330\010\013\2102\210R\210q\330\014\023\2204\220v\230R\230u\240F\250!\330\010\020\220\013\2301\230D\240\001\240\021\330\010\024\220A\220S\230\004\230F\240!\2402\240R\240q\250\001\250\021\330\010\"\240$\240f\250B\250j\270\006\270a\330\010\021\220\033\230G\2401\330\010\014\210E\220\025\220a\220t\2306\240\021\240\"\240B\240a\240q\250\001\330\014\022\220!\2205\230\006\230a\230q\240\005\240Q\240d\250!\2501\250G\2605\270\001\270\025\270a\330\010\017\210q\200A\360\n\000\t2\260\024\260V\2702\270S\300\006\300a\300t\3104\310v\320UV\340\010\036\230k\250\024\250Q\330\010\014\210E\220\025\220a\220s\230!\2301\330\014&\240a\240s\250!\2504\250v\260Q\260d\270!\2701\270A\270Q\330\010\017\210q\200A\360\020\000\t\014\2104\320\017$\240A\240X\250T\260\024\260S\270\001\330\010\017\210q\200A\330\010\033\2304\230v\240Q\330\010\017\210w\220f\230A\230Q\330\010\017\210u\220F\230!\2301\360\020\000\r\026\220T\230\023\230F\240\"\240F\250'\260\026\260q\330\014\032\230!\2305\240\001\240\030\250\032\2606\270\027\300\001\300\025\300a\300s\310$\310a\310w\320VW\320WZ\320Z[\330\030\035\230V\2401\240D\250\006\250f\260G\2706\300\021\300!\340\010\017\210q\200A\330\010\033\2304\230v\240Q\330\010\017\210w\220f\230A\230Q\330\010\017\210u\220F\230!\2301\360\010\000\r\026\220T\230\023\230F\240\"\240F\250'\260\026\260q\330\014\032\230!\2305\240\001\240\030\250\031\260&\270\007\270q\300\005\300Q\300c\310\024\310Q\310g\320UV""\320VY\320YZ\330\030\035\230V\2401\240D\250\006\250f\260G\2706\300\021\300!\360\014\000\t\020\210q\200A\330\010\013\2105\220\007\220s\230!\330\010\014\210G\2207\230#\230Q\330\010\014\210F\220'\230\023\230A\330\010\014\210E\220\026\220q\330\010\014\210G\2206\230\021\330\010\014\210F\220&\230\001\330\010\014\210G\2206\230\023\230A\330\010\014\210E\220\026\220s\230!\330\010\014\210F\220&\230\003\2301\330\010\014\210F\220&\230\001\230\023\230C\230w\240f\250A\250Q\330\010\014\210F\220&\230\001\230\023\230C\230u\240F\250!\2501\330\014\033\2301\230H\240E\250\021\330\020\026\220g\230W\240H\250F\260!\330\020\027\220v\230Q\230d\240%\240v\250Q\250a\340\014\020\220\003\2204\220s\230!\2307\240)\2501\200A\360\006\000\t\r\210K\220}\240A\240T\250\031\260(\270!\330\010\017\210t\2201\200A\340\010\025\220Q\330\010\023\2201\340\010\022\220!\330\t\n\330\010\014\210O\2301\330\010\014\210M\230\021\330\010\014\210L\230\001\330\010\013\2104\210z\230\024\230T\240\021\330\014\022\220*\230A\230Q\200A\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\360\006\000\t!\240\004\240F\250!\2507\260#\260T\270\026\270y\310\006\310a\340\014\023\2204\220v\230Q\230g\240S\250\004\250F\260+\270V\3001\330\014\017\210s\220!\2203\220b\230\001\330\020\032\230!\2308\2404\240w\250f\260E\270\021\330\024\025\220Q\220a\220s\230#\230T\240\023\240C\240q\360\014\000\t\020\210v\220Q\200A\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\360\006\000\t!\240\004\240F\250!\2507\260#\260T\270\026\270y\310\006\310a\360\016\000\r\024\2204\220v\230Q\230g\240S\250\004\250F\260+\270V\3001\330\014\017\210s\220!\2203\220b\230\001\330\020\032\230!\2309\240D\250\007\250v\260U\270!\330\024\025\220Q\220a\220s\230#\230T\240\023\240C\240q\330\010\017\210v\220Q\320\004\034\230A\330\010\013\210:\220Q\220f\230D\240\003\2401\330\014\024\220A\330\r\024\220A\220V\2301\340\014\024\220D\230\006\230a\330\r\024\220A\220V\2301\330\014\024\220D\230\004\230A""\340\014\024\220D\230\003\2306\240\021\240&\250\006\250a\340\010\013\2106\220\027\230\001\330\014\024\220E\230\027\240\001\240\026\240w\250e\2601\340\010\017\210q\200A\330\010\025\220R\220v\230Q\230a\330\010\025\220R\220v\230Q\230a\360\016\000\r\022\220\025\220f\230B\230c\240\023\240D\250\006\250a\330\014\037\230q\240\t\250\022\2507\260)\2701\270B\270a\270s\300$\300a\300u\310A\310S\320PT\320TW\320WZ\320Z[\340\010\017\210q\200A\330\010\025\220R\220v\230Q\230a\330\010\025\220R\220v\230Q\230a\360\010\000\r\022\220\025\220f\230B\230c\240\023\240D\250\006\250a\330\014\037\230q\240\010\250\002\250'\260\030\270\021\270\"\270A\270S\300\004\300A\300U\310!\3103\310d\320RU\320UX\320XY\360\n\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260q\360\014\000\r\025\220E\230\026\230q\240\007\240s\250$\250f\260A\330\014\033\2301\230I\240U\250'\260\021\260!\2601\260C\260t\2701\270G\3001\300D\310\003\3103\310a\360\n\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\340\010\013\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260q\360\016\000\r\025\220E\230\026\230q\240\007\240s\250$\250f\260A\330\014\033\2301\230J\240e\2507\260!\2601\260A\260S\270\004\270A\270W\300A\300T\310\023\310C\310q\340\010\017\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260q\360\014\000\r\024\2205\230\006\230a\230w\240c\250\024\250V\2601\330\014\032\230!\2309\240D\250\007\250q\260\001\260\021\260#\260T\270\021\270'\300\021\300$\300c\310\023\310A\360\n\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\340\010\013""\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260q\360\016\000\r\024\2205\230\006\230a\230w\240c\250\024\250V\2601\330\014\032\230!\230:\240T\250\027\260\001\260\021\260!\2603\260d\270!\2707\300!\3004\300s\310#\310Q\340\010\017\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\360\006\000\t!\240\004\240F\250!\2507\260#\260T\270\026\270u\300F\310!\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260|\3001\360\014\000\r\025\220D\230\006\230a\230w\240c\250\024\250V\260;\270f\300A\330\014\032\230!\2308\2405\250\007\250v\260U\270'\300\021\300!\3001\300C\300t\3101\310G\320ST\320TX\320X[\320[^\320^_\360\n\000\t\020\210w\220a\200A\330\010\025\220W\230F\240!\2401\330\010\025\220Q\220f\230A\230Q\330\010\025\220Q\220f\230A\230Q\360\006\000\t!\240\004\240F\250!\2507\260#\260T\270\026\270u\300F\310!\340\010\013\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260|\3001\360\016\000\r\025\220D\230\006\230a\230w\240c\250\024\250V\260;\270f\300A\330\014\032\230!\2309\240E\250\027\260\006\260e\2707\300!\3001\300A\300S\310\004\310A\310W\320TU\320UY\320Y\\\320\\_\320_`\340\010\017\210w\220a\200A\330\010\025\220W\230F\240!\2401\330\010\025\220V\2306\240\021\240!\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220b\230\001\330\020\026\220j\240\001\320!J\310!\3101\330\014\021\220\021\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260q\360\014\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014#\2401\240I\250R\250w\260a\260v\270Q\270b\300\004\300A\300W\310A\310T\320QT\320TW\320WX\360\n\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220V\2306\240\021\240!\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220b\230\001\330\020\026\220j\240\001\320!J\310!\3101\330\014\021\220\021""\340\010\013\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260q\360\016\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014#\2401\240J\250b\260\007\260q\270\006\270a\270r\300\024\300Q\300g\310Q\310d\320RU\320UX\320XY\340\010\017\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220W\230F\240!\2401\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220b\230\001\330\020\026\220j\240\001\320!J\310!\3101\330\014\021\220\021\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260q\360\014\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014$\240A\240Y\250b\260\007\260q\270\007\270q\300\002\300$\300a\300w\310a\310t\320SV\320VY\320YZ\360\n\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220W\230F\240!\2401\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220b\230\001\330\020\026\220j\240\001\320!J\310!\3101\330\014\021\220\021\340\010\013\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260q\360\016\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014$\240A\240Z\250r\260\027\270\001\270\027\300\001\300\022\3004\300q\310\007\310q\320PT\320TW\320WZ\320Z[\340\010\017\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220W\230F\240!\2401\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220c\230\021\330\020\026\220j\240\001\320!I\310\021\310!\330\014\021\220\021\340\010\013\2102\210S\220\002\220#\220R\220s\230!\340\020\027\220u\230F\240!\2407\250#\250T\260\026\260q\360\014\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014#\2401\240I\250R\250w\260a\260w\270a\270r\300\024\300Q\300e\3101\310C\310q\330\020\021\220\027\230\001\230\024\230S\240\003\2401\360\014\000\t\020\210q\200A\330\010\025\220W\230F\240!\2401\330\010\025\220W\230F\240!\2401\330\010\025\220Q\340\010\014\210J\220g\230R\230q\330\014\017\210w\220c\230\021\330\020\026\220j\240""\001\320!I\310\021\310!\330\014\021\220\021\340\010\013\2102\210S\220\002\220#\220R\220s\230!\360\010\000\021\030\220u\230F\240!\2407\250#\250T\260\026\260q\360\020\000\r\022\220\025\220f\230B\230c\240\024\240V\2501\330\014#\2401\240J\250b\260\007\260q\270\007\270q\300\002\300$\300a\300u\310A\310S\320PQ\330\020\021\220\027\230\001\230\024\230S\240\003\2401\340\010\017\210q\200A\360\n\000\037 \330\010\037\230q\240\t\250\021\330\010\037\230q\240\t\250\021\330\010\037\230q\240\t\250\021\340\010\026\220a\220x\230x\240w\250h\260d\270'\300\030\310\024\310Q\330\014\023\2206\230\021\230$\230g\240W\250E\260\021\330\016\024\220A\220X\230W\240A\330\014\024\220H\230G\2401\240L\260\007\260v\270Q\270a\330\010\016\210a\210x\220x\230w\240c\250\030\260\026\260r\270\021\330\010\017\210y\230\n\240&\250\001\200A\360\020\000\t\020\210r\220\026\220q\230\003\2303\230b\240\006\240a\240q\330\010\017\210r\220\026\220q\230\003\2303\230b\240\006\240a\240q\330\010\013\210<\320\027,\250A\250X\260T\270\024\270S\300\001\330\010\017\210s\220!\320\004B\300!\360\030\000\t\014\2103\210f\220C\220q\330\014\022\320\022M\310Q\310c\320QR\330\014\022\220*\230A\230Q\330\010\013\2104\210s\220&\230\001\330\014\022\220!\330\014\022\220*\230A\230Q\360\006\000\t\026\220S\230\006\230a\230q\330\010\025\220S\230\006\230a\230q\340\010\022\320\022'\240q\250\006\250i\260q\330\010\026\220g\230V\2401\240A\360\006\000\t\014\2103\210g\220S\230\001\330\014\023\2204\220v\230R\230t\2401\240A\240S\250\002\250#\250R\250t\2606\270\021\330\014\017\210s\220&\230\003\2302\230T\240\027\250\006\250c\260\021\330\020\027\220q\230\010\240\004\240G\2508\2603\260g\270Q\270g\300Q\300d\310$\310c\320QT\320TU\330\014\023\2201\330\r\020\220\007\220s\230!\330\014\023\2204\220v\230R\230t\2401\240A\240S\250\002\250#\250R\250t\2606\270\021\330\014\017\210s\220&\230\003\2302\230T\240\027\250\006\250c\260\021\330\020\027\220q\230\t\240\024\240W\250I\260S\270\007\270q\300\007\300q\310\004\310D\320PS\320SV\320VW\330\014\023\2201\340\014\023\2205\230""\002\230(\240!\2405\250\004\250H\260A\200\001\360\010\000\005\n\210\021\330\004\014\210F\220&\230\001\230\021\330\004\013\2106\220\026\220q\230\001\330\004\t\210\026\210v\220Q\220a\330\004\010\210\001\210\026\210q\220\001\330\004\t\210\021\210&\220\001\220\021\330\004\t\210\027\220\006\220a\220q\330\004\032\230'\240\021\240!\340\004\030\230\002\230&\240\002\240'\250\026\250q\260\006\260a\260t\2703\270b\300\004\300F\310!\330\004\030\230\002\230&\240\002\240'\250\026\250q\260\006\260a\260t\2705\300\006\300a\330\004\030\230\002\230&\240\002\240'\250\026\250q\260\006\260a\260t\2705\300\006\300a\330\004\032\230%\230v\240R\240|\2605\270\006\270a\330\004\032\230%\230v\240R\240|\2605\270\006\270a\340\004\030\230\001\330\004\025\220Q\330\004\r\210Q\360\020\000\005\t\210\005\210U\220!\2201\330\010\r\210Q\210f\220A\220Q\330\010\014\210E\220\025\220a\220q\360\006\000\r\020\210v\220V\2301\230C\230r\240\030\250\022\2503\250a\330\014\017\210v\220V\2301\230C\230r\240\030\250\022\2503\250a\330\014\032\230+\240^\2601\260H\270C\270t\3004\300q\330\014\020\220\004\220G\320\033-\250Q\250a\330\014\022\220!\2201\220C\220q\330\014\022\220!\2201\220C\220q\330\014\022\220!\2201\220C\220q\330\014\"\240!\330\020\023\2203\220d\230$\230a\330\020\021\330\020\030\230\003\2301\330\020\030\230\003\2301\330\020\030\230\001\230\021\330\020\030\230\002\230!\330\020\030\230\002\230!\330\020\021\330\020\026\220g\230Q\330\020\030\230\003\2301\330\020\030\230\003\2301\340\010\014\210A\210Q\210b\220\n\230\"\230C\230s\240#\240X\250R\250t\2601\330\010\013\2105\220\003\2201\330\014\020\220\002\320\022$\240A\240Q\330\010\014\210A\330\004\013\2104\210s\220#\220S\230\001\200\001\330\004\010\210\004\210G\2201\330\004\t\210\022\2106\220\021\220!\330\004\t\210\022\2106\220\021\220!\330\004\t\210\022\2108\2202\220Q\220c\230\023\230D\240\n\250\"\250C\250s\260#\260X\270R\270q\300\003\3001\330\004\t\210\022\2108\2202\220Q\220c\230\023\230D\240\n\250\"\250C\250s\260#\260X\270R\270q\300\003\3001\330\004\013\2104\210x\220r""\230\021\230#\230S\240\n\250\"\250C\250s\260(\270\"\270A\270Q\330\004\014\210E\220\026\220q\230\004\230H\240F\250!\330\004\013\2105\220\014\230B\230b\240\006\240d\250&\260\002\260&\270\004\270A\320\004J\310!\360\014\000\t\014\2102\210V\2203\220a\330\014\022\320\022V\320VW\320WY\320YZ\330\014\022\220*\230A\230Q\330\010\013\2104\210r\220\026\220q\330\014\022\220!\330\014\022\220*\230A\230Q\340\010\025\220R\220v\230Q\230a\330\010\026\220b\230\001\230\021\230!\2301\330\010\025\220R\220v\230Q\230c\240\022\2401\340\010\022\320\022'\240q\250\006\250i\260q\330\010\026\220g\230V\2401\240A\360\006\000\t\014\2102\210W\220C\220q\330\014\021\220\024\220V\2302\230S\240\004\240F\250!\330\014\017\210r\220\026\220s\230\"\230D\240\007\240v\250S\260\001\330\020 \240\001\240\030\250\022\2507\260(\270\"\270G\3001\300G\3101\310D\320PS\320SV\320VZ\320Z[\330\014\023\2201\330\r\017\210w\220c\230\021\330\014\021\220\024\220V\2302\230S\240\004\240F\250!\330\014\017\210r\220\026\220s\230\"\230D\240\007\240v\250S\260\001\330\020 \240\001\240\031\250\"\250G\2609\270B\270g\300Q\300g\310Q\310d\320RU\320UX\320X\\\320\\]\330\014\023\2201\340\014\023\2205\230\002\320\032+\2501\250D\260\004\260H\270A\200\001\340\004\035\230\\\250\024\250R\250r\260\032\2703\270a\330\004\007\200y\220\003\2201\330\010\024\220E\230\030\240\021\330\027\034\230G\2407\250!\2506\260\032\2704\270t\3005\310\003\3101\310A\330\027\035\230Q\200\001\360\006\000\005\006\200T\210\026\210t\2206\230\021\330\004\t\210\022\2108\2202\220T\230\023\230B\230f\240B\240a\330\004\t\210\022\210:\220R\220s\230#\230S\240\010\250\002\250!\2503\250b\260\006\260b\270\001\330\004\t\210\022\2108\2202\220T\230\023\230J\240b\250\003\2503\250h\260b\270\001\270\021\330\004\t\210\022\2108\2202\220T\230\023\230B\230f\240B\240a\330\004\t\210\022\210:\220R\220s\230#\230S\240\010\250\002\250!\2503\250b\260\006\260b\270\001\330\004\t\210\022\2108\2202\220T\230\023\230J\240b\250\003\2503\250h\260b\270\001\270\021\330\004\017\210u\220A\330\004\t\210\030\220\021""\220!\330\004\t\210\030\220\021\220!\330\004\013\2108\2201\220D\230\002\230!\330\004\013\2104\210t\2201\320\004\033\230:\240W\320,J\310'\320Qa\320ab\330\010\013\2101\330\014\023\2204\220s\230&\240\001\240\027\250\006\250a\340\014\023\2204\220s\230&\240\001\240\027\250\006\250a\320\004,\320,>\270a\340\010\013\2104\210s\220!\330\014\023\2204\220v\230R\230s\240$\240f\250A\340\014\023\2201\330\010\017\210t\2206\230\021\230#\230S\240\001\330\010\017\210t\2206\230\021\230#\230S\240\001\330\010\033\2301\230H\240D\250\007\250x\260s\270!\330\010\017\210q\320\000\037\230\177\250a\330\004\007\200q\210\007\210s\220!\2201\330\010\016\320\0161\260\021\260!\260<\270q\300\001\300\021\330\010\016\210j\230\001\230\021\200\001\330\004\016\210b\220\002\220#\220R\220q\330\004\016\210b\220\002\220!\330\004\016\210b\220\002\220#\220R\220q\330\004\016\210b\220\002\220!\330\004\t\210\026\210q\220\013\2309\240B\240h\250h\260b\270\002\270\"\270D\300\001\330\004\020\220\001\330\004\t\210\026\210q\220\013\2309\240B\240h\250h\260b\270\002\270\"\270A\330\004\020\220\001\330\004\t\210\026\210q\220\013\2309\240B\240h\250h\260b\270\002\270\"\270D\300\001\330\004\020\220\001\330\004\t\210\026\210q\220\013\2309\240B\240h\250h\260b\270\002\270\"\270A\330\004\020\220\001\330\004\013\2101\210A\210T\220\026\220t\2306\240\021\320\004\027\220q\330\010\017\210u\220A\200\001\330\004\t\210\021\360\026\000\005\010\200s\210#\210T\220\021\330\004\025\220Q\220f\230A\230Q\330\004\024\220A\220V\2301\230A\330\004\021\220\021\220&\230\001\230\021\330\004\022\220!\2206\230\021\230!\330\004\022\220!\2206\230\021\230!\330\004\032\230'\240\021\240!\330\004\022\220'\230\026\230q\240\001\340\004\031\230\022\2306\240\022\2403\240e\2506\260\021\260!\330\004\031\230\022\2306\240\022\2403\240b\250\001\250\024\250V\2601\260A\330\004\037\230r\240\026\240r\250\026\250v\260Q\260f\270F\300&\310\001\360\006\000\005\020\210q\330\004\027\220q\330\004\010\210\005\210U\220!\2201\330\010\030\230\007\230q\240\001\330\010\023\2206\230\022""\2303\230h\240c\250\022\2501\330\010\014\210E\220\025\220a\220q\330\014\032\230+\240^\2601\260H\270C\270t\300:\310Q\330\014\033\320\033-\250Q\250a\330\014\034\230B\230b\240\007\240r\250\036\260q\330\004\017\210q\330\004\026\220a\330\004\010\210\005\210U\220!\2201\330\010\027\220w\230a\230q\330\010\023\2206\230\022\2303\230h\240c\250\022\2501\330\010\014\210E\220\025\220a\220q\330\014\031\230\033\240N\260!\2608\2703\270d\300*\310A\330\014\032\320\032,\250A\250Q\330\014\033\2302\230R\230w\240b\250\r\260Q\340\004\016\210a\330\004\014\210A\330\004\010\210\016\220a\330\010\017\210w\220b\230\007\230q\330\010\021\220\021\360\006\000\005\n\210\021\210#\210R\210q\220\001\220\021\220\"\220J\230a\230s\240#\240R\240x\250r\260\024\260T\270\024\270U\300%\300q\310\005\310Q\310a\330\004\n\210!\2102\210V\2202\220Q\220f\230A\230T\240\021\240&\250\001\250\025\250f\260A\260X\270T\300\025\300a\340\004\010\210\005\210X\220Q\220e\2301\230A\330\010\r\210R\210x\220r\230\023\230F\240$\240j\260\002\260#\260S\270\001\330\010\r\210S\220\001\220\021\330\010\014\210B\210a\210q\330\010\013\2105\220\003\2201\330\014\021\220\025\320\026(\250\001\250\021\330\010\014\210E\220\025\220a\220q\330\014\020\220\004\220G\320\033+\2501\250B\250a\250r\260\021\260!\330\014\021\220\025\220i\230\177\250a\250r\260\021\260\"\260A\260Q\330\014\023\2202\220V\2301\230C\230s\240#\240V\2501\250F\260!\2606\270\021\270&\300\002\300&\310\001\310\025\310b\320PV\320VW\320W[\320[^\320^d\320de\320ei\320ij\320jp\320pq\320qu\320uw\320w}\320}~\320~\177\330\014\023\2202\220Q\220a\330\014\016\210e\2201\220A\330\014\016\210e\2201\220A\330\014\022\220!\2201\220C\220q\330\014\022\220!\2201\220C\220q\330\014\022\220!\2201\220C\220q\330\014\024\220C\220v\230Q\230d\240#\240V\2501\250D\260\004\260C\260q\330\014\024\220C\220v\230Q\230d\240#\240V\2501\250D\260\004\260C\260q\330\014\024\220C\220v\230Q\230d\240#\240V\2501\250D\260\004\260C\260r\270\021\270!\330\014\024\220D\230\006\230a\230t\2404\240v\250Q\250d\260$\260c\270\021\330\014\024""\220B\220f\230A\230T\240\022\2406\250\021\250$\250d\260#\260Q\330\014\024\220B\220f\230A\230T\240\022\2406\250\021\250$\250d\260#\260R\260q\270\001\330\014#\2401\240C\240s\250$\250b\260\006\260a\260t\2701\330\020\030\230\002\230!\330\020\030\230\004\230A\330\020\030\230\002\230!\330\020\030\230\002\230!\330\020\030\230\003\2301\330\020\030\230\003\2301\330\020\030\230\006\230a\330\020\030\230\003\2301\330\020\030\230\003\2301\330\020\030\230\003\2301\330\020\030\230\001\230\021\330\020\030\230\002\230!\330\020\030\230\002\230!\330\020\024\220A\220Q\340\010\r\210Q\330\004\013\2102\210V\2201\220C\220s\230!\2306\240\021\240!\330\004\021\220\021\330\004\010\210\017\220q\330\010\014\210K\220u\230A\330\014\026\220g\230Q\320\036:\270!\2701\330\004\013\2104\210u\220L\240\001\240\021\200\001\330\004\007\200u\210C\210q\330\010\017\210q\220\001\340\010\r\210Q\210h\220b\230\001\230\026\230q\240\005\240S\250\001\330\010\r\210Q\330\010\014\210E\220\025\220a\220q\330\014\016\210g\220Q\220e\320\033-\250Q\250b\260\004\260A\330\010\017\210q\200\001\330\004\007\200x\210s\220!\330\010\022\220#\220Z\230q\240\001\240\021\340\010\017\210s\220#\220T\230\021\230#\230S\240\006\240a\240y\260\003\2604\260q\330\010\017\210s\220#\220T\230\021\230)\2403\240c\250\021\340\004\013\2101__pyx_t_5thinc_8backends_5cblas_daxpy_ptr (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)\000__pyx_t_5thinc_8backends_5cblas_saxpy_ptr (struct __pyx_obj_5thinc_8backends_5cblas_CBlas *)\000daxpy\000saxpyuint64_t (void *, int, uint64_t)\000hash64void (enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_double1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t)\000void (enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_""t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_doubles_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t)\000void (enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_float1d_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t)\000void (enum __pyx_t_4blis_2cy_trans_t, enum __pyx_t_4blis_2cy_trans_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, __pyx_t_4blis_2cy_dim_t, double, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t, double, __pyx_t_4blis_2cy_floats_t, __pyx_t_4blis_2cy_inc_t, __pyx_t_4blis_2cy_inc_t)\000__pyx_fuse_3gemm\000__pyx_fuse_1gemm\000__pyx_fuse_2gemm\000__pyx_fuse_0gemmO"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 458; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 81) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 458; i < 509; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 509; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 458; + for (Py_ssize_t i=0; i<51; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab; + double const c_constants[] = {0.,20.0}; + for (int i = 0; i < 2; i++) { + numbertab[i] = PyFloat_FromDouble(c_constants[i]); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 2; + int8_t const cint_constants_1[] = {0,-1,1,2,4}; + int32_t const cint_constants_4[] = {136983863L}; + for (int i = 0; i < 6; i++) { + numbertab[i] = PyLong_FromLong((i < 5 ? cint_constants_1[i - 0] : cint_constants_4[i - 5])); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<8; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 4; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 2; + unsigned int nlocals : 6; + unsigned int flags : 10; + unsigned int first_line : 11; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 1, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 52}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_device_type, __pyx_mstate->__pyx_n_u_device_id, __pyx_mstate->__pyx_n_u_use_blis}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_init, __pyx_mstate->__pyx_kp_b_iso88591_A_Q_1_O1_M_L_4z_T_AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 65}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_data, __pyx_mstate->__pyx_n_u_dtype, __pyx_mstate->__pyx_n_u_array}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_asarray, __pyx_mstate->__pyx_kp_b_iso88591_A_QfD_1_A_AV1_D_a_AV1_D_A_D_6_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 2, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 82}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_shape, __pyx_mstate->__pyx_n_u_dtype, __pyx_mstate->__pyx_n_u_zeros}; + __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_alloc, __pyx_mstate->__pyx_kp_b_iso88591_W_J_Qaab_1_4s_a_4s_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 88}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self}; + __pyx_mstate_global->__pyx_codeobj_tab[3] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_cblas, __pyx_mstate->__pyx_kp_b_iso88591_q_uA, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[3])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 3, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 91}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_x, __pyx_mstate->__pyx_n_u_y, __pyx_mstate->__pyx_n_u_out, __pyx_mstate->__pyx_n_u_trans1, __pyx_mstate->__pyx_n_u_trans2}; + __pyx_mstate_global->__pyx_codeobj_tab[4] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gemm, __pyx_mstate->__pyx_kp_b_iso88591_22G_UV_1F_Q_A_UUVVWWX_1F_Q_A_UU, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[4])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 104}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_inplace, __pyx_mstate->__pyx_n_u_Y}; + __pyx_mstate_global->__pyx_codeobj_tab[5] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_relu, __pyx_mstate->__pyx_kp_b_iso88591_1G3a_A_AYawe1A_1_gS_A_AZq_uAQ_1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[5])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 9, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 118}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u_inplace, __pyx_mstate->__pyx_n_u_size, __pyx_mstate->__pyx_n_u_dX_ptr, __pyx_mstate->__pyx_n_u_Y_ptr, __pyx_mstate->__pyx_n_u_dX, __pyx_mstate->__pyx_n_u_i}; + __pyx_mstate_global->__pyx_codeobj_tab[6] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_relu, __pyx_mstate->__pyx_kp_b_iso88591_9_q_A_1A_6aq_2WCz_QgS_a_U_1_5_C, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[6])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {6, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 135}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_H0, __pyx_mstate->__pyx_n_u_C0, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_size_at_t, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u_fwd_state}; + __pyx_mstate_global->__pyx_codeobj_tab[7] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_lstm_forward_training, __pyx_mstate->__pyx_kp_b_iso88591_A_r_q_3b_aq_r_q_3b_aq_AXT_S_s, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[7])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {6, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 148}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_H0, __pyx_mstate->__pyx_n_u_C0, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_size_at_t, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u__10}; + __pyx_mstate_global->__pyx_codeobj_tab[8] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_lstm_forward_inference, __pyx_mstate->__pyx_kp_b_iso88591_A_4_AXT_S_q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[8])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 159}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_fwd_state, __pyx_mstate->__pyx_n_u_dX, __pyx_mstate->__pyx_n_u_d_params}; + __pyx_mstate_global->__pyx_codeobj_tab[9] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_lstm, __pyx_mstate->__pyx_kp_b_iso88591_A_K_AT_t1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[9])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 165}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X}; + __pyx_mstate_global->__pyx_codeobj_tab[10] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_maxout_float3d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_QfAQ_QfAQ_QfAQ_F_7_T_y_a_4vQgS, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[10])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 165}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X}; + __pyx_mstate_global->__pyx_codeobj_tab[11] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_maxout_double3d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_QfAQ_QfAQ_QfAQ_F_7_T_y_a_4vQgS_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[11])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 165}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_P, __pyx_mstate->__pyx_n_u_best, __pyx_mstate->__pyx_n_u_which}; + __pyx_mstate_global->__pyx_codeobj_tab[12] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_maxout_float3d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_QfAQ_QfAQ_QfAQ_F_7_T_y_a_4vQgS, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[12])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 184}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_P}; + __pyx_mstate_global->__pyx_codeobj_tab[13] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_maxout_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_RvQa_RvQa_fBc_D_a_q_AS_AU_3dRU, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[13])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 184}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_P}; + __pyx_mstate_global->__pyx_codeobj_tab[14] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_maxout_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_RvQa_RvQa_fBc_D_a_q_7_1Bas_auA, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[14])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 184}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_P, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[15] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_maxout_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_RvQa_RvQa_fBc_D_a_q_AS_AU_3dRU, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[15])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 5, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 198}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_threshold, __pyx_mstate->__pyx_n_u_inplace, __pyx_mstate->__pyx_n_u_Y}; + __pyx_mstate_global->__pyx_codeobj_tab[16] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_mish, __pyx_mstate->__pyx_kp_b_iso88591_1_1G3a_A_AYawe1G7_1_gS_A_AZq_uA, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[16])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 212}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_threshold, __pyx_mstate->__pyx_n_u_inplace, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[17] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_mish, __pyx_mstate->__pyx_kp_b_iso88591_99I_q_A_2WCz_QgS_a_Qhb_xq_uAWGS, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[17])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 1, 9, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 228}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_seq, __pyx_mstate->__pyx_n_u_nW, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_msg, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_I, __pyx_mstate->__pyx_n_u_nL, __pyx_mstate->__pyx_n_u_cols}; + __pyx_mstate_global->__pyx_codeobj_tab[18] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_seq2col, __pyx_mstate->__pyx_kp_b_iso88591_B_3fCq_MQcQR_AQ_4s_AQ_S_aq_S_aq, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[18])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 1, 10, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 268}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_nW, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_msg, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_nF, __pyx_mstate->__pyx_n_u_I, __pyx_mstate->__pyx_n_u_nL, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[19] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_seq2col, __pyx_mstate->__pyx_kp_b_iso88591_J_2V3a_VVWWYYZ_AQ_4r_q_AQ_RvQa, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[19])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 302}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_ids, __pyx_mstate->__pyx_n_u_seed, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_dest}; + __pyx_mstate_global->__pyx_codeobj_tab[20] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_hash, __pyx_mstate->__pyx_kp_b_iso88591_A_2_V2S_at4vUV_k_Q_E_as_1_as_4vQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[20])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 314}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[21] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_mean_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[21])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 314}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[22] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_mean_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[22])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 314}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_means}; + __pyx_mstate_global->__pyx_codeobj_tab[23] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_mean_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[23])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 335}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_means, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[24] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_mean_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wb_j_J_1_2S_R, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[24])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 335}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_means, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[25] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_mean_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wb_j_J_1_2S_R_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[25])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 335}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_means, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[26] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_mean_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wb_j_J_1_2S_R, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[26])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 361}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[27] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_sum_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_3, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[27])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 361}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[28] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_sum_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_4, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[28])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 361}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_sums}; + __pyx_mstate_global->__pyx_codeobj_tab[29] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_sum_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_2S_Rs_uF_7_T_q_3, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[29])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 382}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_sums, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[30] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_sum_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_V6_Q_JgRq_wb_j_J_1_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[30])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 382}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_sums, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[31] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_sum_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_V6_Q_JgRq_wb_j_J_1_2S_Rs_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[31])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 382}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_sums, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[32] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_sum_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_V6_Q_JgRq_wb_j_J_1_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[32])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 408}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[33] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_max_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_F_7_T_uF_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[33])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 408}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[34] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_max_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_F_7_T_uF_2S_Rs_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[34])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 408}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_maxes}; + __pyx_mstate_global->__pyx_codeobj_tab[35] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_reduce_max_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_QfAQ_QfAQ_F_7_T_uF_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[35])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 432}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_maxes, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[36] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_max_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wc_j_I_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[36])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 432}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_maxes, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[37] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_max_double2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wc_j_I_2S_Rs_2, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[37])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 9, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 432}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_maxes, __pyx_mstate->__pyx_n_u_which, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B, __pyx_mstate->__pyx_n_u_O, __pyx_mstate->__pyx_n_u_T, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_dX}; + __pyx_mstate_global->__pyx_codeobj_tab[38] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_reduce_max_float2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_WF_1_WF_1_Q_JgRq_wc_j_I_2S_Rs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[38])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 460}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices}; + __pyx_mstate_global->__pyx_codeobj_tab[39] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gather_add_float2d_t_int2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_q_Qc_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[39])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 460}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices}; + __pyx_mstate_global->__pyx_codeobj_tab[40] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gather_add_float2d_t_uint2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_q_Qc_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[40])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 460}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices}; + __pyx_mstate_global->__pyx_codeobj_tab[41] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gather_add_double2d_t_int2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_6_as_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[41])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 460}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices}; + __pyx_mstate_global->__pyx_codeobj_tab[42] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gather_add_double2d_t_uint2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_6_as_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[42])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 460}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices, __pyx_mstate->__pyx_n_u_cblas, __pyx_mstate->__pyx_n_u_rows, __pyx_mstate->__pyx_n_u_dims, __pyx_mstate->__pyx_n_u_output}; + __pyx_mstate_global->__pyx_codeobj_tab[43] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_gather_add_float2d_t_int2d_t, __pyx_mstate->__pyx_kp_b_iso88591_A_4vQ_wfAQ_uF_1_T_F_F_q_5_q_Qc_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[43])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 477}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_indices, __pyx_mstate->__pyx_n_u_values}; + __pyx_mstate_global->__pyx_codeobj_tab[44] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_scatter_add, __pyx_mstate->__pyx_kp_b_iso88591_A_5_s_G7_Q_F_A_E_q_G6_F_G6_A_E_s, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[44])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {10, 0, 0, 10, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 495}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_weights, __pyx_mstate->__pyx_n_u_gradient, __pyx_mstate->__pyx_n_u_mom1, __pyx_mstate->__pyx_n_u_mom2, __pyx_mstate->__pyx_n_u_beta1, __pyx_mstate->__pyx_n_u_beta2, __pyx_mstate->__pyx_n_u_eps, __pyx_mstate->__pyx_n_u_learn_rate, __pyx_mstate->__pyx_n_u_mod_rate}; + __pyx_mstate_global->__pyx_codeobj_tab[45] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_adam, __pyx_mstate->__pyx_kp_b_iso88591_A_q_q_q_axxwhd_Q_6_gWE_AXWA_HG1L, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[45])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 8, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 512}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_keys_2, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_output_2, __pyx_mstate->__pyx_n_u_output, __pyx_mstate->__pyx_n_u_i}; + __pyx_mstate_global->__pyx_codeobj_tab[46] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_ngrams, __pyx_mstate->__pyx_kp_b_iso88591_A_2Rq_4vRuF_1D_AS_F_2Rq_fBj_a_G1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[46])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 523}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_N, __pyx_mstate->__pyx_n_u_D, __pyx_mstate->__pyx_n_u_period, __pyx_mstate->__pyx_n_u_out, __pyx_mstate->__pyx_n_u_out_2}; + __pyx_mstate_global->__pyx_codeobj_tab[47] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_position_encode, __pyx_mstate->__pyx_kp_b_iso88591_a_4s_4vRs_fA_1_t6_S_t6_S_1HD_xs, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[47])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 535}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_ops, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_B}; + __pyx_mstate_global->__pyx_codeobj_tab[48] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_check_seq2col_lengths, __pyx_mstate->__pyx_kp_b_iso88591_xs_Zq_s_T_S_ay_4q_s_T_3c_1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[48])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 545}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_gradient, __pyx_mstate->__pyx_n_u_threshold, __pyx_mstate->__pyx_n_u_grad_norm}; + __pyx_mstate_global->__pyx_codeobj_tab[49] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_cpu_clip_gradient, __pyx_mstate->__pyx_kp_b_iso88591_5_4xvQa_z_A_81D_KxvQa, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[49])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 551}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_gradient, __pyx_mstate->__pyx_n_u_noise_level, __pyx_mstate->__pyx_n_u_timestep, __pyx_mstate->__pyx_n_u_variance}; + __pyx_mstate_global->__pyx_codeobj_tab[50] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_add_gradient_noise, __pyx_mstate->__pyx_kp_b_iso88591_Rr_3a_y_1_E_G7_6_4t5_1A_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[50])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 31, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 639}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_c_init, __pyx_mstate->__pyx_n_u_h_init, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_xp, __pyx_mstate->__pyx_n_u_depth, __pyx_mstate->__pyx_n_u_dirs, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_N, __pyx_mstate->__pyx_n_u_nI, __pyx_mstate->__pyx_n_u_nT, __pyx_mstate->__pyx_n_u_batch_size, __pyx_mstate->__pyx_n_u_G, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u_C, __pyx_mstate->__pyx_n_u_Yt2, __pyx_mstate->__pyx_n_u_Ct2, __pyx_mstate->__pyx_n_u_params_i, __pyx_mstate->__pyx_n_u_seq_i, __pyx_mstate->__pyx_n_u_orig_X, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_Yid, __pyx_mstate->__pyx_n_u_Cid, __pyx_mstate->__pyx_n_u_Gid, __pyx_mstate->__pyx_n_u_Wx, __pyx_mstate->__pyx_n_u_Wh, __pyx_mstate->__pyx_n_u_bias, __pyx_mstate->__pyx_n_u_d, __pyx_mstate->__pyx_n_u_layer_params, __pyx_mstate->__pyx_n_u_H}; + __pyx_mstate_global->__pyx_codeobj_tab[51] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_lstm_forward_training, __pyx_mstate->__pyx_kp_b_iso88591_F_6_q_vQa_q_aq_q_at3b_F_q_at5_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[51])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 45, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 768}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_dY, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_fwd_state, __pyx_mstate->__pyx_n_u_xp, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u_G, __pyx_mstate->__pyx_n_u_C, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_Yid, __pyx_mstate->__pyx_n_u_Cid, __pyx_mstate->__pyx_n_u_Gid, __pyx_mstate->__pyx_n_u_Wx, __pyx_mstate->__pyx_n_u_Wh, __pyx_mstate->__pyx_n_u_bias, __pyx_mstate->__pyx_n_u_dWx, __pyx_mstate->__pyx_n_u_dWh, __pyx_mstate->__pyx_n_u_d_bias, __pyx_mstate->__pyx_n_u_dYid, __pyx_mstate->__pyx_n_u_depth, __pyx_mstate->__pyx_n_u_dirs, __pyx_mstate->__pyx_n_u_N, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_nI, __pyx_mstate->__pyx_n_u_batch_size, __pyx_mstate->__pyx_n_u_nT, __pyx_mstate->__pyx_n_u_dC, __pyx_mstate->__pyx_n_u_dG, __pyx_mstate->__pyx_n_u_d_params, __pyx_mstate->__pyx_n_u_params_i, __pyx_mstate->__pyx_n_u_all_layer_params, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_n_inputs, __pyx_mstate->__pyx_n_u_d, __pyx_mstate->__pyx_n_u_layer_params, __pyx_mstate->__pyx_n_u_all_layer_grads, __pyx_mstate->__pyx_n_u_layer_grads, __pyx_mstate->__pyx_n_u_indices, __pyx_mstate->__pyx_n_u_seq_i, __pyx_mstate->__pyx_n_u_dX, __pyx_mstate->__pyx_n_u_Xs, __pyx_mstate->__pyx_n_u_dXs, __pyx_mstate->__pyx_n_u_grad_parts, __pyx_mstate->__pyx_n_u_dir_grads, __pyx_mstate->__pyx_n_u__10}; + __pyx_mstate_global->__pyx_codeobj_tab[52] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_backprop_lstm, __pyx_mstate->__pyx_kp_b_iso88591_s_T_QfAQ_AV1A_6_6_q_6_3e6_6_3b, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[52])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 5, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 870}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_dirs, __pyx_mstate->__pyx_n_u_X_2, __pyx_mstate->__pyx_n_u_Xs, __pyx_mstate->__pyx_n_u_d}; + __pyx_mstate_global->__pyx_codeobj_tab[53] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_split_directions, __pyx_mstate->__pyx_kp_b_iso88591_uCq_q_Qhb_q_S_Q_E_aq_gQe_Qb_A_q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[53])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 13, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 973}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_nI, __pyx_mstate->__pyx_n_u_params_i, __pyx_mstate->__pyx_n_u_Wx_size, __pyx_mstate->__pyx_n_u_bx_size, __pyx_mstate->__pyx_n_u_Wh_size, __pyx_mstate->__pyx_n_u_bh_size, __pyx_mstate->__pyx_n_u_Wx, __pyx_mstate->__pyx_n_u_bx, __pyx_mstate->__pyx_n_u_Wh, __pyx_mstate->__pyx_n_u_bh}; + __pyx_mstate_global->__pyx_codeobj_tab[54] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_split_weights, __pyx_mstate->__pyx_kp_b_iso88591_b_Rq_b_b_Rq_b_q_9Bhhb_D_q_9Bhhb, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[54])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 989}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_Wx, __pyx_mstate->__pyx_n_u_bx, __pyx_mstate->__pyx_n_u_Wh, __pyx_mstate->__pyx_n_u_bh, __pyx_mstate->__pyx_n_u_ascontig, __pyx_mstate->__pyx_n_u_bias}; + __pyx_mstate_global->__pyx_codeobj_tab[55] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_transpose_weights, __pyx_mstate->__pyx_kp_b_iso88591_T_t6_82T_BfBa_Rs_S_3b_b_82T_Jb, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[55])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 7, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 1006}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_params, __pyx_mstate->__pyx_n_u_Wx, __pyx_mstate->__pyx_n_u_Wh, __pyx_mstate->__pyx_n_u_bias, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_nI, __pyx_mstate->__pyx_n_u_zeros}; + __pyx_mstate_global->__pyx_codeobj_tab[56] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_untranspose_unsplit_weights, __pyx_mstate->__pyx_kp_b_iso88591_G1_6_6_82Qc_D_Cs_XRq_1_82Qc_D_C, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[56])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 1173}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_u, __pyx_mstate->__pyx_n_u_v, __pyx_mstate->__pyx_n_u_msg}; + __pyx_mstate_global->__pyx_codeobj_tab[57] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_backends_numpy_ops_pyx, __pyx_mstate->__pyx_n_u_check_compatible_shape, __pyx_mstate->__pyx_kp_b_iso88591_a_q_s_1_1_q_j, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[57])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* NumpyImportArray.init */ + /* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * numpy._import_array + */ + #ifdef NPY_FEATURE_VERSION + #ifndef NO_IMPORT_ARRAY + if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use 'numpy._import_array' to disable if you are certain you don't need it)."); + } + #endif + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* FusedFunction.init */ + if (likely(__pyx_FusedFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* UnicodeAsUCS4 (used by object_ord) */ +static void __Pyx_PyUnicode_AsPy_UCS4_error(Py_ssize_t length) { + if (likely(length >= 0)) { + PyErr_Format(PyExc_ValueError, + "only single character unicode strings can be converted to Py_UCS4, " + "got length %" CYTHON_FORMAT_SSIZE_T "d", length); + } +} +static CYTHON_INLINE Py_UCS4 __Pyx_PyUnicode_AsPy_UCS4(PyObject* x) { + Py_ssize_t length = __Pyx_PyUnicode_GET_LENGTH(x); + if (unlikely(length != 1)) { + __Pyx_PyUnicode_AsPy_UCS4_error(length); + return (Py_UCS4)-1; + } + return __Pyx_PyUnicode_READ_CHAR(x, 0); +} + +/* object_ord */ +static long __Pyx__PyObject_Ord(PyObject* c) { + Py_ssize_t size; + if (PyBytes_Check(c)) { + size = __Pyx_PyBytes_GET_SIZE(c); + if (likely(size == 1)) { +#if CYTHON_ASSUME_SAFE_MACROS + return (unsigned char) PyBytes_AS_STRING(c)[0]; +#else + char *data = PyBytes_AsString(c); + if (unlikely(!data)) return -1; + return (unsigned char) data[0]; +#endif + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(size < 0)) return -1; +#endif + } else if (PyByteArray_Check(c)) { + size = __Pyx_PyByteArray_GET_SIZE(c); + if (likely(size == 1)) { +#if CYTHON_ASSUME_SAFE_MACROS + return (unsigned char) PyByteArray_AS_STRING(c)[0]; +#else + char *data = PyByteArray_AsString(c); + if (unlikely(!data)) return -1; + return (unsigned char) data[0]; +#endif + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(size < 0)) return -1; +#endif + } else { + __Pyx_TypeName c_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(c)); + PyErr_Format(PyExc_TypeError, + "ord() expected string of length 1, but " __Pyx_FMT_TYPENAME " found", + c_type_name); + __Pyx_DECREF_TypeName(c_type_name); + return (long)(Py_UCS4)-1; + } + PyErr_Format(PyExc_TypeError, + "ord() expected a character, but string of length %zd found", size); + return (long)(Py_UCS4)-1; +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* memoryview_get_from_buffer */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#else +static Py_ssize_t __Pyx_PyMemoryView_Get_itemsize(PyObject *obj) { + Py_ssize_t result; + PyObject *attr = PyObject_GetAttr(obj, __pyx_mstate_global->__pyx_n_u_itemsize); + if (!attr) { + goto bad; + } + result = PyLong_AsSsize_t(attr); + Py_DECREF(attr); + return result; + bad: + Py_XDECREF(attr); + return -1; +} +#endif + +/* memoryview_get_from_buffer */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#else +static int __Pyx_PyMemoryView_Get_ndim(PyObject *obj) { + int result; + PyObject *attr = PyObject_GetAttr(obj, __pyx_mstate_global->__pyx_n_u_ndim); + if (!attr) { + goto bad; + } + result = PyLong_AsLong(attr); + Py_DECREF(attr); + return result; + bad: + Py_XDECREF(attr); + return -1; +} +#endif + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; itp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCall2Args) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCall2Args (used by CallUnboundCMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* UnpackUnboundCMethod (used by CallUnboundCMethod1) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod1 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + int flag = cfunc->flag; + if (flag == METH_O) { + return __Pyx_CallCFunction(cfunc, self, arg); + } else if (flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, &arg, 1); + } else if (flag == (METH_FASTCALL | METH_KEYWORDS)) { + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, &arg, 1, NULL); + } + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod1(&tmp_cfunc, self, arg); + } +#endif + PyObject* result = __Pyx__CallUnboundCMethod1(cfunc, self, arg); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg){ + PyObject *result = NULL; + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + } else +#endif + { + result = __Pyx_PyObject_Call2Args(cfunc->method, self, arg); + } + return result; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* dict_getitem_default */ +static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { + PyObject* value; +#if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 + value = PyDict_GetItemWithError(d, key); + if (unlikely(!value)) { + if (unlikely(PyErr_Occurred())) + return NULL; + value = default_value; + } + Py_INCREF(value); + if ((1)); +#else + if (PyBytes_CheckExact(key) || PyUnicode_CheckExact(key) || PyLong_CheckExact(key)) { + value = PyDict_GetItem(d, key); + if (unlikely(!value)) { + value = default_value; + } + Py_INCREF(value); + } +#endif + else { + if (default_value == Py_None) + value = __Pyx_CallUnboundCMethod1(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_get, d, key); + else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_get, d, key, default_value); + } + return value; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* IterFinish (used by dict_iter) */ +static CYTHON_INLINE int __Pyx_IterFinish(void) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return -1; + __Pyx_PyErr_Clear(); + return 0; + } + return 0; +} + +/* PyObjectCallNoArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectCallOneArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by dict_iter) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* RaiseNeedMoreValuesToUnpack (used by UnpackTuple2) */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* RaiseTooManyValuesToUnpack (used by UnpackItemEndCheck) */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* UnpackItemEndCheck (used by UnpackTuple2) */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } + return __Pyx_IterFinish(); +} + +/* RaiseNoneIterError (used by UnpackTupleError) */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); +} + +/* UnpackTupleError (used by UnpackTuple2) */ +static void __Pyx_UnpackTupleError(PyObject *t, Py_ssize_t index) { + if (t == Py_None) { + __Pyx_RaiseNoneNotIterableError(); + } else { + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(t); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(size < 0)) return; + #endif + if (size < index) { + __Pyx_RaiseNeedMoreValuesError(size); + } else { + __Pyx_RaiseTooManyValuesError(index); + } + } +} + +/* UnpackTuple2 (used by dict_iter) */ +static CYTHON_INLINE int __Pyx_unpack_tuple2( + PyObject* tuple, PyObject** value1, PyObject** value2, int is_tuple, int has_known_size, int decref_tuple) { + if (likely(is_tuple || PyTuple_Check(tuple))) { + Py_ssize_t size; + if (has_known_size) { + return __Pyx_unpack_tuple2_exact(tuple, value1, value2, decref_tuple); + } + size = __Pyx_PyTuple_GET_SIZE(tuple); + if (likely(size == 2)) { + return __Pyx_unpack_tuple2_exact(tuple, value1, value2, decref_tuple); + } + if (size >= 0) { + __Pyx_UnpackTupleError(tuple, 2); + } + return -1; + } else { + return __Pyx_unpack_tuple2_generic(tuple, value1, value2, has_known_size, decref_tuple); + } +} +static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( + PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, int decref_tuple) { + PyObject *value1 = NULL, *value2 = NULL; +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS + value1 = __Pyx_PySequence_ITEM(tuple, 0); if (unlikely(!value1)) goto bad; + value2 = __Pyx_PySequence_ITEM(tuple, 1); if (unlikely(!value2)) goto bad; +#else + value1 = PyTuple_GET_ITEM(tuple, 0); Py_INCREF(value1); + value2 = PyTuple_GET_ITEM(tuple, 1); Py_INCREF(value2); +#endif + if (decref_tuple) { + Py_DECREF(tuple); + } + *pvalue1 = value1; + *pvalue2 = value2; + return 0; +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +bad: + Py_XDECREF(value1); + Py_XDECREF(value2); + if (decref_tuple) { Py_XDECREF(tuple); } + return -1; +#endif +} +static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, + int has_known_size, int decref_tuple) { + Py_ssize_t index; + PyObject *value1 = NULL, *value2 = NULL, *iter = NULL; + iternextfunc iternext; + iter = PyObject_GetIter(tuple); + if (unlikely(!iter)) goto bad; + if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } + iternext = __Pyx_PyObject_GetIterNextFunc(iter); + value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } + value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } + if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; + Py_DECREF(iter); + *pvalue1 = value1; + *pvalue2 = value2; + return 0; +unpacking_failed: + if (!has_known_size && __Pyx_IterFinish() == 0) + __Pyx_RaiseNeedMoreValuesError(index); +bad: + Py_XDECREF(iter); + Py_XDECREF(value1); + Py_XDECREF(value2); + if (decref_tuple) { Py_XDECREF(tuple); } + return -1; +} + +/* dict_iter */ +#if CYTHON_COMPILING_IN_PYPY +#include +#endif +static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* iterable, int is_dict, PyObject* method_name, + Py_ssize_t* p_orig_length, int* p_source_is_dict) { + is_dict = is_dict || likely(PyDict_CheckExact(iterable)); + *p_source_is_dict = is_dict; + if (is_dict) { +#if !CYTHON_COMPILING_IN_PYPY + *p_orig_length = PyDict_Size(iterable); + Py_INCREF(iterable); + return iterable; +#else + static PyObject *py_items = NULL, *py_keys = NULL, *py_values = NULL; + PyObject **pp = NULL; + if (method_name) { + const char *name = PyUnicode_AsUTF8(method_name); + if (strcmp(name, "iteritems") == 0) pp = &py_items; + else if (strcmp(name, "iterkeys") == 0) pp = &py_keys; + else if (strcmp(name, "itervalues") == 0) pp = &py_values; + if (pp) { + if (!*pp) { + *pp = PyUnicode_FromString(name + 4); + if (!*pp) + return NULL; + } + method_name = *pp; + } + } +#endif + } + *p_orig_length = 0; + if (method_name) { + PyObject* iter; + iterable = __Pyx_PyObject_CallMethod0(iterable, method_name); + if (!iterable) + return NULL; +#if !CYTHON_COMPILING_IN_PYPY + if (PyTuple_CheckExact(iterable) || PyList_CheckExact(iterable)) + return iterable; +#endif + iter = PyObject_GetIter(iterable); + Py_DECREF(iterable); + return iter; + } + return PyObject_GetIter(iterable); +} +#if !CYTHON_AVOID_BORROWED_REFS +static CYTHON_INLINE int __Pyx_dict_iter_next_source_is_dict( + PyObject* iter_obj, CYTHON_NCP_UNUSED Py_ssize_t orig_length, CYTHON_NCP_UNUSED Py_ssize_t* ppos, + PyObject** pkey, PyObject** pvalue, PyObject** pitem) { + PyObject *key, *value; + if (unlikely(orig_length != PyDict_Size(iter_obj))) { + PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); + return -1; + } + if (unlikely(!PyDict_Next(iter_obj, ppos, &key, &value))) { + return 0; + } + if (pitem) { + PyObject* tuple = PyTuple_New(2); + if (unlikely(!tuple)) { + return -1; + } + Py_INCREF(key); + Py_INCREF(value); + #if CYTHON_ASSUME_SAFE_MACROS + PyTuple_SET_ITEM(tuple, 0, key); + PyTuple_SET_ITEM(tuple, 1, value); + #else + if (unlikely(PyTuple_SetItem(tuple, 0, key) < 0)) { + Py_DECREF(value); + Py_DECREF(tuple); + return -1; + } + if (unlikely(PyTuple_SetItem(tuple, 1, value) < 0)) { + Py_DECREF(tuple); + return -1; + } + #endif + *pitem = tuple; + } else { + if (pkey) { + Py_INCREF(key); + *pkey = key; + } + if (pvalue) { + Py_INCREF(value); + *pvalue = value; + } + } + return 1; +} +#endif +static CYTHON_INLINE int __Pyx_dict_iter_next( + PyObject* iter_obj, CYTHON_NCP_UNUSED Py_ssize_t orig_length, CYTHON_NCP_UNUSED Py_ssize_t* ppos, + PyObject** pkey, PyObject** pvalue, PyObject** pitem, int source_is_dict) { + PyObject* next_item; +#if !CYTHON_AVOID_BORROWED_REFS + if (source_is_dict) { + int result; +#if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(iter_obj); +#endif + result = __Pyx_dict_iter_next_source_is_dict(iter_obj, orig_length, ppos, pkey, pvalue, pitem); +#if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); +#endif + return result; + } else if (PyTuple_CheckExact(iter_obj)) { + Py_ssize_t pos = *ppos; + Py_ssize_t tuple_size = __Pyx_PyTuple_GET_SIZE(iter_obj); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(tuple_size < 0)) return -1; + #endif + if (unlikely(pos >= tuple_size)) return 0; + *ppos = pos + 1; + #if CYTHON_ASSUME_SAFE_MACROS + next_item = PyTuple_GET_ITEM(iter_obj, pos); + #else + next_item = PyTuple_GetItem(iter_obj, pos); + if (unlikely(!next_item)) return -1; + #endif + Py_INCREF(next_item); + } else if (PyList_CheckExact(iter_obj)) { + Py_ssize_t pos = *ppos; + Py_ssize_t list_size = __Pyx_PyList_GET_SIZE(iter_obj); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(list_size < 0)) return -1; + #endif + if (unlikely(pos >= list_size)) return 0; + *ppos = pos + 1; + next_item = __Pyx_PyList_GetItemRef(iter_obj, pos); + if (unlikely(!next_item)) return -1; + } else +#endif + { + next_item = PyIter_Next(iter_obj); + if (unlikely(!next_item)) { + return __Pyx_IterFinish(); + } + } + if (pitem) { + *pitem = next_item; + } else if (pkey && pvalue) { + if (__Pyx_unpack_tuple2(next_item, pkey, pvalue, source_is_dict, source_is_dict, 1)) + return -1; + } else if (pkey) { + *pkey = next_item; + } else { + *pvalue = next_item; + } + return 1; +} + +/* PyObjectFastCallMethod */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) { + PyObject *result; + PyObject *attr = PyObject_GetAttr(args[0], name); + if (unlikely(!attr)) + return NULL; + result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1); + Py_DECREF(attr); + return result; +} +#endif + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; ifunc)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* CIntToDigits (used by CIntToPyUnicode) */ +static const char DIGIT_PAIRS_10[2*10*10+1] = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; +static const char DIGIT_PAIRS_8[2*8*8+1] = { + "0001020304050607" + "1011121314151617" + "2021222324252627" + "3031323334353637" + "4041424344454647" + "5051525354555657" + "6061626364656667" + "7071727374757677" +}; +static const char DIGITS_HEX[2*16+1] = { + "0123456789abcdef" + "0123456789ABCDEF" +}; + +/* BuildPyUnicode (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char) { + PyObject *uval; + Py_ssize_t uoffset = ulength - clength; +#if CYTHON_USE_UNICODE_INTERNALS + Py_ssize_t i; + void *udata; + uval = PyUnicode_New(ulength, 127); + if (unlikely(!uval)) return NULL; + udata = PyUnicode_DATA(uval); + if (uoffset > 0) { + i = 0; + if (prepend_sign) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); + i++; + } + for (; i < uoffset; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); + } + } + for (i=0; i < clength; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); + } +#else + { + PyObject *sign = NULL, *padding = NULL; + uval = NULL; + if (uoffset > 0) { + prepend_sign = !!prepend_sign; + if (uoffset > prepend_sign) { + padding = PyUnicode_FromOrdinal(padding_char); + if (likely(padding) && uoffset > prepend_sign + 1) { + PyObject *tmp = PySequence_Repeat(padding, uoffset - prepend_sign); + Py_DECREF(padding); + padding = tmp; + } + if (unlikely(!padding)) goto done_or_error; + } + if (prepend_sign) { + sign = PyUnicode_FromOrdinal('-'); + if (unlikely(!sign)) goto done_or_error; + } + } + uval = PyUnicode_DecodeASCII(chars, clength, NULL); + if (likely(uval) && padding) { + PyObject *tmp = PyUnicode_Concat(padding, uval); + Py_DECREF(uval); + uval = tmp; + } + if (likely(uval) && sign) { + PyObject *tmp = PyUnicode_Concat(sign, uval); + Py_DECREF(uval); + uval = tmp; + } +done_or_error: + Py_XDECREF(padding); + Py_XDECREF(sign); + } +#endif + return uval; +} + +/* COrdinalToPyUnicode (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value) { + return value <= 1114111; +} +static PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t ulength, char padding_char) { + Py_ssize_t padding_length = ulength - 1; + if (likely((padding_length <= 250) && (value < 0xD800 || value > 0xDFFF))) { + char chars[256]; + if (value <= 255) { + memset(chars, padding_char, (size_t) padding_length); + chars[ulength-1] = (char) value; + return PyUnicode_DecodeLatin1(chars, ulength, NULL); + } + char *cpos = chars + sizeof(chars); + if (value < 0x800) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xc0 | (value & 0x1f)); + } else if (value < 0x10000) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xe0 | (value & 0x0f)); + } else { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xf0 | (value & 0x07)); + } + cpos -= padding_length; + memset(cpos, padding_char, (size_t) padding_length); + return PyUnicode_DecodeUTF8(cpos, chars + sizeof(chars) - cpos, NULL); + } + if (value <= 127 && CYTHON_USE_UNICODE_INTERNALS) { + const char chars[1] = {(char) value}; + return __Pyx_PyUnicode_BuildFromAscii(ulength, chars, 1, 0, padding_char); + } + { + PyObject *uchar, *padding_uchar, *padding, *result; + padding_uchar = PyUnicode_FromOrdinal(padding_char); + if (unlikely(!padding_uchar)) return NULL; + padding = PySequence_Repeat(padding_uchar, padding_length); + Py_DECREF(padding_uchar); + if (unlikely(!padding)) return NULL; + uchar = PyUnicode_FromOrdinal(value); + if (unlikely(!uchar)) { + Py_DECREF(padding); + return NULL; + } + result = PyUnicode_Concat(padding, uchar); + Py_DECREF(padding); + Py_DECREF(uchar); + return result; + } +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (int) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(int)*3+2]; + char *dpos, *end = digits + sizeof(int)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + int remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (int) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (int) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (int) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (Py_ssize_t) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(Py_ssize_t)*3+2]; + char *dpos, *end = digits + sizeof(Py_ssize_t)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + Py_ssize_t remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (Py_ssize_t) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (Py_ssize_t) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (Py_ssize_t) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* GetAttr */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) + return __Pyx_PyObject_GetAttrStr(o, n); +#endif + return PyObject_GetAttr(o, n); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* ObjectGetItem */ +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + PyObject *runerr = NULL; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1, 1); + } + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, __pyx_mstate_global->__pyx_n_u_class_getitem); + if (!meth) { + PyErr_Clear(); + } else { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + +/* RejectKeywords */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) { + PyObject *key = NULL; + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) { + key = __Pyx_PySequence_ITEM(kwds, 0); + } else { +#if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; +#else + Py_ssize_t pos = 0; +#endif +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return; +#endif + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); +#endif + } + if (likely(key)) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + Py_DECREF(key); + } +} + +/* DivInt[Py_ssize_t] */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t a, Py_ssize_t b, int b_is_constant) { + Py_ssize_t q = a / b; + Py_ssize_t r = a - q*b; + Py_ssize_t adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* GetAttr3 */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int res = PyObject_GetOptionalAttr(o, n, &r); + return (res != 0) ? r : __Pyx_NewRef(d); +#else + #if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } + #endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +#endif +} + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); + return 0; +} + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* SwapException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* PySequenceMultiply */ +static PyObject* __Pyx_PySequence_Multiply_Generic(PyObject *seq, Py_ssize_t mul) { + PyObject *result, *pymul = PyLong_FromSsize_t(mul); + if (unlikely(!pymul)) + return NULL; + result = PyNumber_Multiply(seq, pymul); + Py_DECREF(pymul); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul) { +#if CYTHON_USE_TYPE_SLOTS + PyTypeObject *type = Py_TYPE(seq); + if (likely(type->tp_as_sequence && type->tp_as_sequence->sq_repeat)) { + return type->tp_as_sequence->sq_repeat(seq, mul); + } else +#endif + { + return __Pyx_PySequence_Multiply_Generic(seq, mul); + } +} + +/* PyObjectFormatAndDecref */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f) { + if (unlikely(!s)) return NULL; + if (likely(PyUnicode_CheckExact(s))) return s; + return __Pyx_PyObject_FormatAndDecref(s, f); +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f) { + PyObject *result; + if (unlikely(!s)) return NULL; + result = PyObject_Format(s, f); + Py_DECREF(s); + return result; +} + +/* PyObjectFormat */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { + int ret; + _PyUnicodeWriter writer; + if (likely(PyFloat_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else if (likely(PyLong_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyLong_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else { + return PyObject_Format(obj, format_spec); + } + if (unlikely(ret == -1)) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} +#endif + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* DivInt[long] */ +static CYTHON_INLINE long __Pyx_div_long(long a, long b, int b_is_constant) { + long q = a / b; + long r = a - q*b; + long adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* SoftComplexToDouble */ +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_nogil) { + if (unlikely(__Pyx_CIMAG(value))) { + PyGILState_STATE gilstate; + if (have_nogil) + gilstate = PyGILState_Ensure(); + PyErr_SetString(PyExc_TypeError, + "Cannot convert 'complex' with non-zero imaginary component to 'double' " + "(this most likely comes from the '**' operator; " + "use 'cython.cpow(True)' to return 'nan' instead of a " + "complex number)."); + if (have_nogil) + PyGILState_Release(gilstate); + return -1.; + } + return __Pyx_CREAL(value); +} + +/* ErrOccurredWithGIL */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + PyGILState_STATE _save = PyGILState_Ensure(); + err = !!PyErr_Occurred(); + PyGILState_Release(_save); + return err; +} + +/* WriteUnraisableException */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); + else state = (PyGILState_STATE)0; + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(0); + } + ctx = PyUnicode_FromString(name); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } + if (nogil) + PyGILState_Release(state); +} + +/* PyObjectSetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_setattro)) + return tp->tp_setattro(obj, attr_name, value); + return PyObject_SetAttr(obj, attr_name, value); +} +#endif + +/* PyObjectVectorCallKwBuilder (used by PyObjectVectorCallMethodKwBuilder) */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* PyObjectVectorCallMethodKwBuilder */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyObject *result; + PyObject *obj = PyObject_GetAttr(args[0], name); + if (unlikely(!obj)) + return NULL; + result = __Pyx_Object_Vectorcall_CallFromBuilder(obj, args+1, nargsf-1, kwnames); + Py_DECREF(obj); + return result; +} +#endif + +/* DictGetItem */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { + PyObject *value; + if (unlikely(__Pyx_PyDict_GetItemRef(d, key, &value) == 0)) { // no value, no error + if (unlikely(PyTuple_Check(key))) { + PyObject* args = PyTuple_Pack(1, key); + if (likely(args)) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + } else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + return value; +} +#endif + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_long(long value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (long) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_long(long value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(long)*3+2]; + char *dpos, *end = digits + sizeof(long)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + long remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (long) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (long) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (long) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* BufferIndexError */ +static void __Pyx_RaiseBufferIndexError(int axis) { + PyErr_Format(PyExc_IndexError, + "Out of bounds on buffer access (axis %d)", axis); +} + +/* PyLongCompare */ +static CYTHON_INLINE int __Pyx_PyLong_BoolNeObjC(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); + if (op1 == op2) { + return 0; + } + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + int unequal; + unsigned long uintval; + Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + if (intval == 0) { + return (__Pyx_PyLong_IsZero(op1) != 1); + } else if (intval < 0) { + if (__Pyx_PyLong_IsNonNeg(op1)) + return 1; + intval = -intval; + } else { + if (__Pyx_PyLong_IsNeg(op1)) + return 1; + } + uintval = (unsigned long) intval; +#if PyLong_SHIFT * 4 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 4)) { + unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 3 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 3)) { + unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 2 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 2)) { + unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 1 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 1)) { + unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + return (unequal != 0); + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(op1); + return ((double)a != (double)b); + } + return __Pyx_PyObject_IsTrueAndDecref( + PyObject_RichCompare(op1, op2, Py_NE)); +} + +/* IsLittleEndian (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } S; + S.u32 = 0x01020304; + return S.u8[0] == 4; +} + +/* BufferFormatCheck (used by BufferGetAndValidate) */ +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type) { + stack[0].field = &ctx->root; + stack[0].parent_offset = 0; + ctx->root.type = type; + ctx->root.name = "buffer dtype"; + ctx->root.offset = 0; + ctx->head = stack; + ctx->head->field = &ctx->root; + ctx->fmt_offset = 0; + ctx->head->parent_offset = 0; + ctx->new_packmode = '@'; + ctx->enc_packmode = '@'; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->is_complex = 0; + ctx->is_valid_array = 0; + ctx->struct_alignment = 0; + while (type->typegroup == 'S') { + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = 0; + type = type->fields->type; + } +} +static int __Pyx_BufFmt_ParseNumber(const char** ts) { + int count; + const char* t = *ts; + if (*t < '0' || *t > '9') { + return -1; + } else { + count = *t++ - '0'; + while (*t >= '0' && *t <= '9') { + count *= 10; + count += *t++ - '0'; + } + } + *ts = t; + return count; +} +static int __Pyx_BufFmt_ExpectNumber(const char **ts) { + int number = __Pyx_BufFmt_ParseNumber(ts); + if (number == -1) + PyErr_Format(PyExc_ValueError,\ + "Does not understand character buffer dtype format string ('%c')", **ts); + return number; +} +static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) { + PyErr_Format(PyExc_ValueError, + "Unexpected format string character: '%c'", ch); +} +static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { + switch (ch) { + case '?': return "'bool'"; + case 'c': return "'char'"; + case 'b': return "'signed char'"; + case 'B': return "'unsigned char'"; + case 'h': return "'short'"; + case 'H': return "'unsigned short'"; + case 'i': return "'int'"; + case 'I': return "'unsigned int'"; + case 'l': return "'long'"; + case 'L': return "'unsigned long'"; + case 'q': return "'long long'"; + case 'Q': return "'unsigned long long'"; + case 'f': return (is_complex ? "'complex float'" : "'float'"); + case 'd': return (is_complex ? "'complex double'" : "'double'"); + case 'g': return (is_complex ? "'complex long double'" : "'long double'"); + case 'T': return "a struct"; + case 'O': return "Python object"; + case 'P': return "a pointer"; + case 's': case 'p': return "a string"; + case 0: return "end"; + default: return "unparsable format string"; + } +} +static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return 2; + case 'i': case 'I': case 'l': case 'L': return 4; + case 'q': case 'Q': return 8; + case 'f': return (is_complex ? 8 : 4); + case 'd': return (is_complex ? 16 : 8); + case 'g': { + PyErr_SetString(PyExc_ValueError, "Python does not define a standard format string size for long double ('g').."); + return 0; + } + case 'O': case 'P': return sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(short); + case 'i': case 'I': return sizeof(int); + case 'l': case 'L': return sizeof(long); + case 'q': case 'Q': return sizeof(PY_LONG_LONG); + case 'f': return sizeof(float) * (is_complex ? 2 : 1); + case 'd': return sizeof(double) * (is_complex ? 2 : 1); + case 'g': return sizeof(long double) * (is_complex ? 2 : 1); + case 'O': case 'P': return sizeof(void*); + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +typedef struct { char c; short x; } __Pyx_st_short; +typedef struct { char c; int x; } __Pyx_st_int; +typedef struct { char c; long x; } __Pyx_st_long; +typedef struct { char c; float x; } __Pyx_st_float; +typedef struct { char c; double x; } __Pyx_st_double; +typedef struct { char c; long double x; } __Pyx_st_longdouble; +typedef struct { char c; void *x; } __Pyx_st_void_p; +typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_st_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_st_float) - sizeof(float); + case 'd': return sizeof(__Pyx_st_double) - sizeof(double); + case 'g': return sizeof(__Pyx_st_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_st_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +/* These are for computing the padding at the end of the struct to align + on the first member of the struct. This will probably the same as above, + but we don't have any guarantees. + */ +typedef struct { short x; char c; } __Pyx_pad_short; +typedef struct { int x; char c; } __Pyx_pad_int; +typedef struct { long x; char c; } __Pyx_pad_long; +typedef struct { float x; char c; } __Pyx_pad_float; +typedef struct { double x; char c; } __Pyx_pad_double; +typedef struct { long double x; char c; } __Pyx_pad_longdouble; +typedef struct { void *x; char c; } __Pyx_pad_void_p; +typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_pad_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_pad_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_pad_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_pad_float) - sizeof(float); + case 'd': return sizeof(__Pyx_pad_double) - sizeof(double); + case 'g': return sizeof(__Pyx_pad_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_pad_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) { + switch (ch) { + case 'c': + return 'H'; + case 'b': case 'h': case 'i': + case 'l': case 'q': case 's': case 'p': + return 'I'; + case '?': case 'B': case 'H': case 'I': case 'L': case 'Q': + return 'U'; + case 'f': case 'd': case 'g': + return (is_complex ? 'C' : 'R'); + case 'O': + return 'O'; + case 'P': + return 'P'; + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) { + if (ctx->head == NULL || ctx->head->field == &ctx->root) { + const char* expected; + const char* quote; + if (ctx->head == NULL) { + expected = "end"; + quote = ""; + } else { + expected = ctx->head->field->type->name; + quote = "'"; + } + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected %s%s%s but got %s", + quote, expected, quote, + __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex)); + } else { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_StructField* parent = (ctx->head - 1)->field; + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected '%s' but got %s in '%s.%s'", + field->type->name, __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex), + parent->type->name, field->name); + } +} +static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { + char group; + size_t size, offset, arraysize = 1; + if (ctx->enc_type == 0) return 0; + if (ctx->head->field->type->arraysize[0]) { + int i, ndim = 0; + if (ctx->enc_type == 's' || ctx->enc_type == 'p') { + ctx->is_valid_array = ctx->head->field->type->ndim == 1; + ndim = 1; + if (ctx->enc_count != ctx->head->field->type->arraysize[0]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %zu", + ctx->head->field->type->arraysize[0], ctx->enc_count); + return -1; + } + } + if (!ctx->is_valid_array) { + PyErr_Format(PyExc_ValueError, "Expected %d dimensions, got %d", + ctx->head->field->type->ndim, ndim); + return -1; + } + for (i = 0; i < ctx->head->field->type->ndim; i++) { + arraysize *= ctx->head->field->type->arraysize[i]; + } + ctx->is_valid_array = 0; + ctx->enc_count = 1; + } + group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex); + do { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_TypeInfo* type = field->type; + if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') { + size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex); + } else { + size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); + } + if (ctx->enc_packmode == '@') { + size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); + size_t align_mod_offset; + if (align_at == 0) return -1; + align_mod_offset = ctx->fmt_offset % align_at; + if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset; + if (ctx->struct_alignment == 0) + ctx->struct_alignment = __Pyx_BufFmt_TypeCharToPadding(ctx->enc_type, + ctx->is_complex); + } + if (type->size != size || type->typegroup != group) { + if (type->typegroup == 'C' && type->fields != NULL) { + size_t parent_offset = ctx->head->parent_offset + field->offset; + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = parent_offset; + continue; + } + if ((type->typegroup == 'H' || group == 'H') && type->size == size) { + } else { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + } + offset = ctx->head->parent_offset + field->offset; + if (ctx->fmt_offset != offset) { + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch; next field is at offset %" CYTHON_FORMAT_SSIZE_T "d but %" CYTHON_FORMAT_SSIZE_T "d expected", + (Py_ssize_t)ctx->fmt_offset, (Py_ssize_t)offset); + return -1; + } + ctx->fmt_offset += size; + if (arraysize) + ctx->fmt_offset += (arraysize - 1) * size; + --ctx->enc_count; + while (1) { + if (field == &ctx->root) { + ctx->head = NULL; + if (ctx->enc_count != 0) { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + break; + } + ctx->head->field = ++field; + if (field->type == NULL) { + --ctx->head; + field = ctx->head->field; + continue; + } else if (field->type->typegroup == 'S') { + size_t parent_offset = ctx->head->parent_offset + field->offset; + if (field->type->fields->type == NULL) continue; + field = field->type->fields; + ++ctx->head; + ctx->head->field = field; + ctx->head->parent_offset = parent_offset; + break; + } else { + break; + } + } + } while (ctx->enc_count); + ctx->enc_type = 0; + ctx->is_complex = 0; + return 0; +} +static int +__pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp) +{ + const char *ts = *tsp; + int i = 0, number, ndim; + ++ts; + if (ctx->new_count != 1) { + PyErr_SetString(PyExc_ValueError, + "Cannot handle repeated arrays in format string"); + return -1; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return -1; + ndim = ctx->head->field->type->ndim; + while (*ts && *ts != ')') { + switch (*ts) { + case ' ': case '\f': case '\r': case '\n': case '\t': case '\v': continue; + default: break; + } + number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return -1; + if (i < ndim && (size_t) number != ctx->head->field->type->arraysize[i]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %d", + ctx->head->field->type->arraysize[i], number); + return -1; + } + if (*ts != ',' && *ts != ')') { + PyErr_Format(PyExc_ValueError, + "Expected a comma in format string, got '%c'", *ts); + return -1; + } + if (*ts == ',') ts++; + i++; + } + if (i != ndim) { + PyErr_Format(PyExc_ValueError, "Expected %d dimension(s), got %d", + ctx->head->field->type->ndim, i); + return -1; + } + if (!*ts) { + PyErr_SetString(PyExc_ValueError, + "Unexpected end of format string, expected ')'"); + return -1; + } + ctx->is_valid_array = 1; + ctx->new_count = 1; + *tsp = ++ts; + return 0; +} +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { + int got_Z = 0; + while (1) { + switch(*ts) { + case 0: + if (ctx->enc_type != 0 && ctx->head == NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + if (ctx->head != NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + return ts; + case ' ': + case '\r': + case '\n': + ++ts; + break; + case '<': + if (!__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '>': + case '!': + if (__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '=': + case '@': + case '^': + ctx->new_packmode = *ts++; + break; + case 'T': + { + const char* ts_after_sub; + size_t i, struct_count = ctx->new_count; + size_t struct_alignment = ctx->struct_alignment; + ctx->new_count = 1; + ++ts; + if (*ts != '{') { + PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'"); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + ctx->enc_count = 0; + ctx->struct_alignment = 0; + ++ts; + ts_after_sub = ts; + for (i = 0; i != struct_count; ++i) { + ts_after_sub = __Pyx_BufFmt_CheckString(ctx, ts); + if (!ts_after_sub) return NULL; + } + ts = ts_after_sub; + if (struct_alignment) ctx->struct_alignment = struct_alignment; + } + break; + case '}': + { + size_t alignment = ctx->struct_alignment; + ++ts; + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + if (alignment && ctx->fmt_offset % alignment) { + ctx->fmt_offset += alignment - (ctx->fmt_offset % alignment); + } + } + return ts; + case 'x': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->fmt_offset += ctx->new_count; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->enc_packmode = ctx->new_packmode; + ++ts; + break; + case 'Z': + got_Z = 1; + ++ts; + if (*ts != 'f' && *ts != 'd' && *ts != 'g') { + __Pyx_BufFmt_RaiseUnexpectedChar('Z'); + return NULL; + } + CYTHON_FALLTHROUGH; + case '?': case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': + case 'l': case 'L': case 'q': case 'Q': + case 'f': case 'd': case 'g': + case 'O': case 'p': + if ((ctx->enc_type == *ts) && (got_Z == ctx->is_complex) && + (ctx->enc_packmode == ctx->new_packmode) && (!ctx->is_valid_array)) { + ctx->enc_count += ctx->new_count; + ctx->new_count = 1; + got_Z = 0; + ++ts; + break; + } + CYTHON_FALLTHROUGH; + case 's': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_count = ctx->new_count; + ctx->enc_packmode = ctx->new_packmode; + ctx->enc_type = *ts; + ctx->is_complex = got_Z; + ++ts; + ctx->new_count = 1; + got_Z = 0; + break; + case ':': + ++ts; + while(*ts != ':') ++ts; + ++ts; + break; + case '(': + if (__pyx_buffmt_parse_array(ctx, &ts) < 0) return NULL; + break; + default: + { + int number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return NULL; + ctx->new_count = (size_t)number; + } + } + } +} + +/* BufferGetAndValidate */ + static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) { + if (unlikely(info->buf == NULL)) return; + if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL; + PyBuffer_Release(info); + } + static void __Pyx_ZeroBuffer(Py_buffer* buf) { + buf->buf = NULL; + buf->obj = NULL; + buf->strides = __Pyx_zeros; + buf->shape = __Pyx_zeros; + buf->suboffsets = __Pyx_minusones; + } + static int __Pyx__GetBufferAndValidate( + Py_buffer* buf, PyObject* obj, const __Pyx_TypeInfo* dtype, int flags, + int nd, int cast, __Pyx_BufFmt_StackElem* stack) + { + buf->buf = NULL; + if (unlikely(PyObject_GetBuffer(obj, buf, flags) == -1)) { + __Pyx_ZeroBuffer(buf); + return -1; + } + if (unlikely(buf->ndim != nd)) { + PyErr_Format(PyExc_ValueError, + "Buffer has wrong number of dimensions (expected %d, got %d)", + nd, buf->ndim); + goto fail; + } + if (!cast) { + __Pyx_BufFmt_Context ctx; + __Pyx_BufFmt_Init(&ctx, stack, dtype); + if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail; + } + if (unlikely((size_t)buf->itemsize != dtype->size)) { + PyErr_Format(PyExc_ValueError, + "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)", + buf->itemsize, (buf->itemsize > 1) ? "s" : "", + dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : ""); + goto fail; + } + if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones; + return 0; + fail:; + __Pyx_SafeReleaseBuffer(buf); + return -1; + } + +/* SliceObject */ + static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, + Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, + int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + __Pyx_TypeName obj_type_name; + #if CYTHON_USE_TYPE_SLOTS + PyMappingMethods* mp = Py_TYPE(obj)->tp_as_mapping; + if (likely(mp && mp->mp_ass_subscript)) + #endif + { + int result; + PyObject *py_slice, *py_start, *py_stop; + if (_py_slice) { + py_slice = *_py_slice; + } else { + PyObject* owned_start = NULL; + PyObject* owned_stop = NULL; + if (_py_start) { + py_start = *_py_start; + } else { + if (has_cstart) { + owned_start = py_start = PyLong_FromSsize_t(cstart); + if (unlikely(!py_start)) goto bad; + } else + py_start = Py_None; + } + if (_py_stop) { + py_stop = *_py_stop; + } else { + if (has_cstop) { + owned_stop = py_stop = PyLong_FromSsize_t(cstop); + if (unlikely(!py_stop)) { + Py_XDECREF(owned_start); + goto bad; + } + } else + py_stop = Py_None; + } + py_slice = PySlice_New(py_start, py_stop, Py_None); + Py_XDECREF(owned_start); + Py_XDECREF(owned_stop); + if (unlikely(!py_slice)) goto bad; + } + #if CYTHON_USE_TYPE_SLOTS + result = mp->mp_ass_subscript(obj, py_slice, value); + #else + result = value ? PyObject_SetItem(obj, py_slice, value) : PyObject_DelItem(obj, py_slice); + #endif + if (!_py_slice) { + Py_DECREF(py_slice); + } + return result; + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object does not support slice %.10s", + obj_type_name, value ? "assignment" : "deletion"); + __Pyx_DECREF_TypeName(obj_type_name); + bad: + return -1; + } + +/* SliceObject */ + static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice(PyObject* obj, + Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, + int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + __Pyx_TypeName obj_type_name; + #if CYTHON_USE_TYPE_SLOTS + PyMappingMethods* mp = Py_TYPE(obj)->tp_as_mapping; + if (likely(mp && mp->mp_subscript)) + #endif + { + PyObject* result; + PyObject *py_slice, *py_start, *py_stop; + if (_py_slice) { + py_slice = *_py_slice; + } else { + PyObject* owned_start = NULL; + PyObject* owned_stop = NULL; + if (_py_start) { + py_start = *_py_start; + } else { + if (has_cstart) { + owned_start = py_start = PyLong_FromSsize_t(cstart); + if (unlikely(!py_start)) goto bad; + } else + py_start = Py_None; + } + if (_py_stop) { + py_stop = *_py_stop; + } else { + if (has_cstop) { + owned_stop = py_stop = PyLong_FromSsize_t(cstop); + if (unlikely(!py_stop)) { + Py_XDECREF(owned_start); + goto bad; + } + } else + py_stop = Py_None; + } + py_slice = PySlice_New(py_start, py_stop, Py_None); + Py_XDECREF(owned_start); + Py_XDECREF(owned_stop); + if (unlikely(!py_slice)) goto bad; + } + #if CYTHON_USE_TYPE_SLOTS + result = mp->mp_subscript(obj, py_slice); + #else + result = PyObject_GetItem(obj, py_slice); + #endif + if (!_py_slice) { + Py_DECREF(py_slice); + } + return result; + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + bad: + return NULL; + } + +/* PyObjectCallMethod1 (used by append) */ + #if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) + static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; + } + #endif + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + #if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + #else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); + #endif + } + +/* append */ + static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x) { + if (likely(PyList_CheckExact(L))) { + if (unlikely(__Pyx_PyList_Append(L, x) < 0)) return -1; + } else { + PyObject* retval = __Pyx_PyObject_CallMethod1(L, __pyx_mstate_global->__pyx_n_u_append, x); + if (unlikely(!retval)) + return -1; + Py_DECREF(retval); + } + return 0; + } + +/* PyLongCompare */ + static CYTHON_INLINE int __Pyx_PyLong_BoolEqObjC(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); + if (op1 == op2) { + return 1; + } + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + int unequal; + unsigned long uintval; + Py_ssize_t size = __Pyx_PyLong_DigitCount(op1); + const digit* digits = __Pyx_PyLong_Digits(op1); + if (intval == 0) { + return (__Pyx_PyLong_IsZero(op1) == 1); + } else if (intval < 0) { + if (__Pyx_PyLong_IsNonNeg(op1)) + return 0; + intval = -intval; + } else { + if (__Pyx_PyLong_IsNeg(op1)) + return 0; + } + uintval = (unsigned long) intval; + #if PyLong_SHIFT * 4 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 4)) { + unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else + #endif + #if PyLong_SHIFT * 3 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 3)) { + unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else + #endif + #if PyLong_SHIFT * 2 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 2)) { + unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else + #endif + #if PyLong_SHIFT * 1 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 1)) { + unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else + #endif + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + return (unequal == 0); + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = __Pyx_PyFloat_AS_DOUBLE(op1); + return ((double)a == (double)b); + } + return __Pyx_PyObject_IsTrueAndDecref( + PyObject_RichCompare(op1, op2, Py_EQ)); + } + +/* AllocateExtensionType */ + static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } + } + +/* CallTypeTraverse */ + #if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) + #else + static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; + } + #endif + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ + #if CYTHON_COMPILING_IN_LIMITED_API + static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; + } + static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); + } + #endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ + static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; + #if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; + #else + tp_dict = tp->tp_dict; + #endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; + } + +/* FixUpExtensionType */ + static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { + #if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + #else + const PyType_Slot *slot = spec->slots; + int changed = 0; + #if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { + #if !CYTHON_COMPILING_IN_CPYTHON + const + #endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } + #if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } + #endif // CYTHON_METH_FASTCALL + #if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + #endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } + #endif // !CYTHON_COMPILING_IN_LIMITED_API + #if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + #endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); + #endif // PY_VERSION_HEX > 0x030900B1 + return 0; + } + +/* ValidateBasesTuple (used by PyType_Ready) */ + #if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS + static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; + #if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); + #else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; + #endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; + #elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); + #else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; + #endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); + #endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; + #if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; + #else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; + #endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } + #if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); + #endif + return -1; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); + #endif + } + return 0; + } + #endif + +/* PyType_Ready */ + CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; + } + static int __Pyx_PyType_Ready(PyTypeObject *t) { + #if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; + #if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; + #endif + return PyType_Ready(t); + #else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; + #if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + #endif + #else + (void)__Pyx_PyObject_CallMethod0; + #endif + r = PyType_Ready(t); + #if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } + #endif + return r; + #endif + } + +/* SetVTable */ + static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; + #if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) + #else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) + #endif + goto bad; + Py_DECREF(ob); + return 0; + bad: + Py_XDECREF(ob); + return -1; + } + +/* GetVTable (used by MergeVTables) */ + static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); + #else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); + #endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; + bad: + Py_XDECREF(ob); + return NULL; + } + +/* MergeVTables */ + static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; + #if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; + #else + size = PyTuple_GET_SIZE(bases); + #endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; + #if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; + #elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; + #else + basei = PyTuple_GET_ITEM(bases, i); + #endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); + #endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; + bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); + #if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; + #elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; + #else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); + #endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); + #endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); + #if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS + really_bad: // bad has failed! + #endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); + #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS + other_failure: + #endif + PyMem_Free(base_vtables); + return -1; + } + +/* DelItemOnTypeDict (used by SetupReduce) */ + static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) { + int result; + PyObject *tp_dict; + #if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; + #else + tp_dict = tp->tp_dict; + #endif + result = PyDict_DelItem(tp_dict, k); + if (likely(!result)) PyType_Modified(tp); + return result; + } + +/* SetupReduce */ + static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name_2); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; + } + static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_getstate = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + PyObject *getstate = NULL; + #if CYTHON_USE_PYTYPE_LOOKUP + getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + #else + getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + if (!getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } + #endif + if (getstate) { + #if CYTHON_USE_PYTYPE_LOOKUP + object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + #else + object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + if (!object_getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } + #endif + if (object_getstate != getstate) { + goto __PYX_GOOD; + } + } + #if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; + #else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; + #endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { + #if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; + #else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; + #endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); + if (likely(reduce_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + goto __PYX_BAD; + } + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); + if (likely(setstate_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; + __PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; + __PYX_GOOD: + #if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); + Py_XDECREF(object_getstate); + Py_XDECREF(getstate); + #endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; + } + +/* TypeImport */ + #ifndef __PYX_HAVE_RT_ImportType_3_2_1 + #define __PYX_HAVE_RT_ImportType_3_2_1 + static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size) + { + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; + #if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; + #endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } + #if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; + #else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + #endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_1 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_1 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; + bad: + Py_XDECREF(result); + return NULL; + } + #endif + +/* PxdImportShared (used by FunctionImport) */ + #ifndef __PYX_HAVE_RT_ImportFromPxd_3_2_1 + #define __PYX_HAVE_RT_ImportFromPxd_3_2_1 + static int __Pyx_ImportFromPxd_3_2_1(PyObject *module, const char *name, void **p, const char *sig, const char *what) { + PyObject *d = 0; + PyObject *cobj = 0; + d = PyObject_GetAttrString(module, "__pyx_capi__"); + if (!d) + goto bad; + #if (defined(Py_LIMITED_API) && Py_LIMITED_API >= 0x030d0000) || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030d0000) + PyDict_GetItemStringRef(d, name, &cobj); + #else + cobj = PyDict_GetItemString(d, name); + Py_XINCREF(cobj); + #endif + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C %.8s %.200s", + PyModule_GetName(module), what, name); + goto bad; + } + if (!PyCapsule_IsValid(cobj, sig)) { + PyErr_Format(PyExc_TypeError, + "C %.8s %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + what, PyModule_GetName(module), name, sig, PyCapsule_GetName(cobj)); + goto bad; + } + *p = PyCapsule_GetPointer(cobj, sig); + if (!(*p)) + goto bad; + Py_DECREF(d); + Py_DECREF(cobj); + return 0; + bad: + Py_XDECREF(d); + Py_XDECREF(cobj); + return -1; + } + #endif + +/* FunctionImport */ + #ifndef __PYX_HAVE_RT_ImportFunction_3_2_1 + #define __PYX_HAVE_RT_ImportFunction_3_2_1 + static int __Pyx_ImportFunction_3_2_1(PyObject *module, const char *funcname, void (**f)(void), const char *sig) { + union { + void (*fp)(void); + void *p; + } tmp; + int result = __Pyx_ImportFromPxd_3_2_1(module, funcname, &tmp.p, sig, "function"); + if (result == 0) { + *f = tmp.fp; + } + return result; + } + #endif + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u__4); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; + } + +/* Py3UpdateBases */ + static PyObject* + __Pyx_PEP560_update_bases(PyObject *bases) + { + Py_ssize_t i, j, size_bases; + PyObject *base = NULL, *meth, *new_base, *result, *new_bases = NULL; + #if CYTHON_ASSUME_SAFE_SIZE + size_bases = PyTuple_GET_SIZE(bases); + #else + size_bases = PyTuple_Size(bases); + if (size_bases < 0) return NULL; + #endif + for (i = 0; i < size_bases; i++) { + #if CYTHON_AVOID_BORROWED_REFS + Py_CLEAR(base); + #endif + #if CYTHON_ASSUME_SAFE_MACROS + base = PyTuple_GET_ITEM(bases, i); + #else + base = PyTuple_GetItem(bases, i); + if (!base) goto error; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(base); + #endif + if (PyType_Check(base)) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + meth = __Pyx_PyObject_GetAttrStrNoError(base, __pyx_mstate_global->__pyx_n_u_mro_entries); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + PyObject *base_from_list; + #if CYTHON_ASSUME_SAFE_MACROS + base_from_list = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base_from_list); + Py_INCREF(base_from_list); + #else + base_from_list = PyTuple_GetItem(bases, j); + if (!base_from_list) goto error; + Py_INCREF(base_from_list); + if (PyList_SetItem(new_bases, j, base_from_list) < 0) goto error; + #endif + } + } + #if CYTHON_ASSUME_SAFE_SIZE + j = PyList_GET_SIZE(new_bases); + #else + j = PyList_Size(new_bases); + if (j < 0) goto error; + #endif + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(base); + #endif + return result; + error: + Py_XDECREF(new_bases); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(base); + #endif + return NULL; + } + +/* CalculateMetaclass */ + static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { + Py_ssize_t i, nbases; + #if CYTHON_ASSUME_SAFE_SIZE + nbases = PyTuple_GET_SIZE(bases); + #else + nbases = PyTuple_Size(bases); + if (nbases < 0) return NULL; + #endif + for (i=0; i < nbases; i++) { + PyTypeObject *tmptype; + #if CYTHON_ASSUME_SAFE_MACROS + PyObject *tmp = PyTuple_GET_ITEM(bases, i); + #else + PyObject *tmp = PyTuple_GetItem(bases, i); + if (!tmp) return NULL; + #endif + tmptype = Py_TYPE(tmp); + if (!metaclass) { + metaclass = tmptype; + continue; + } + if (PyType_IsSubtype(metaclass, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, metaclass)) { + metaclass = tmptype; + continue; + } + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + if (!metaclass) { + metaclass = &PyType_Type; + } + Py_INCREF((PyObject*) metaclass); + return (PyObject*) metaclass; + } + +/* dict_setdefault (used by FetchCommonType) */ + static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + #elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); + #elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); + #else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); + #endif + return value; + } + +/* AddModuleRef (used by FetchSharedCythonModule) */ + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } + #elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) + #else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } + #endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ + static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); + } + +/* FetchCommonType (used by CommonTypesMetaclass) */ + #if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); + #if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); + #else + result->ob_type = metaclass; + #endif + Py_DECREF(old_tp); + } + return result; + } + #else + #define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) + #endif + static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; + #else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; + #endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; + } + static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } + done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; + bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; + } + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ + static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); + } + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; + } + static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; + } + #endif + static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} + }; + static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} + }; + static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots + }; + static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; + } + +/* PyMethodNew (used by CythonFunctionShared) */ + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; + } + #else + static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); + } + #endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ + #if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) + { + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); + cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; + } + static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) + { + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : + #if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); + #else + PyDict_GET_SIZE(kw); + #endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } + #if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } + #endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); + } + #endif + +/* CythonFunctionShared (used by CythonFunction) */ + #if CYTHON_COMPILING_IN_LIMITED_API + static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; + } + static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); + } + #else + static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; + } + #endif + static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { + #if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); + #else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); + #endif + } + static PyObject * + __Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) + { + if (unlikely(op->func_doc == NULL)) { + #if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; + #else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } + #endif + } + Py_INCREF(op->func_doc); + return op->func_doc; + } + static PyObject * + __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static int + __Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) + { + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + static PyObject * + __Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) + { + if (unlikely(op->func_name == NULL)) { + #if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); + #else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); + #endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; + } + static PyObject * + __Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) + { + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static int + __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) + { + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + static PyObject * + __Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) + { + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static int + __Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) + { + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + static PyObject * + __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) + { + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; + } + #endif + static PyObject * + __Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) + { + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; + } + static PyObject * + __Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) + { + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; + } + static PyObject * + __Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) + { + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; + } + static int + __Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; + } + static int + __Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + static PyObject * + __Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; + } + static PyObject * + __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static int + __Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + static PyObject * + __Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; + } + static PyObject * + __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static int + __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; + } + static PyObject * + __Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; + } + static PyObject * + __Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static PyObject * + __Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); + #if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); + #else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } + #endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } + ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); + } + static PyObject * + __Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); + #else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); + #endif + } + static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); + #else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); + #endif + } + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject * + __Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); + } + static int + __Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); + } + #endif + static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + #else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + #endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, + #if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, + #endif + {0, 0, 0, 0, 0} + }; + static PyMemberDef __pyx_CyFunction_members[] = { + #if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, + #endif + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, + #endif + #if CYTHON_METH_FASTCALL + #if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, + #else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, + #endif + #if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, + #else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, + #endif + #endif + {0, 0, 0, 0, 0} + }; + static PyObject * + __Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) + { + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; + } + static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} + }; + #if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) + #else + #define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) + #endif + static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + #if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; + #endif + if (unlikely(op == NULL)) + return NULL; + #if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; + #endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; + #if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; + #endif + Py_XINCREF(closure); + op->func_closure = closure; + #if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; + #endif + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; + #endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; + #if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; + #else + ((PyCMethodObject*)op)->mm_class = NULL; + #endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; + #if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } + #endif + return (PyObject *) op; + } + static int + __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) + { + Py_CLEAR(m->func_closure); + #if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); + #else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); + #endif + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); + #elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); + #else + PyObject_ClearManagedDict((PyObject*)m); + #endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); + #if !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); + #else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } + #endif + #endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; + } + static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) + { + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); + } + static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) + { + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); + } + static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) + { + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); + #if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); + #else + Py_VISIT(((PyCFunctionObject*)m)->m_module); + #endif + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); + #else + { + int e = + #if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict + #else + PyObject_VisitManagedDict + #endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } + #endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); + #if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); + #endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; + } + static PyObject* + __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) + { + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; + } + static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; + #else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; + #endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + #if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); + #else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; + #endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + #if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); + #else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; + #endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; + } + static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; + #if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; + #else + self = ((PyCFunctionObject*)func)->m_self; + #endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; + } + static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + #if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { + #if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); + #else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); + #endif + } + #endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; + #if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); + #else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; + #endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; + } + #if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) + { + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; + } + static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) + { + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + #if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; + #else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; + #endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + #if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; + #else + self = ((PyCFunctionObject*)cyfunc)->m_self; + #endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); + } + static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) + { + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + #if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; + #else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; + #endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + #if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; + #else + self = ((PyCFunctionObject*)cyfunc)->m_self; + #endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); + } + static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) + { + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + #if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; + #else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; + #endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + #if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; + #else + self = ((PyCFunctionObject*)cyfunc)->m_self; + #endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); + } + static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) + { + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + #if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; + #else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; + #endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + #if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; + #else + self = ((PyCFunctionObject*)cyfunc)->m_self; + #endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); + } + #endif + static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, + }; + static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + #ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | + #endif + #if CYTHON_METH_FASTCALL + #if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | + #elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | + #endif + #endif // CYTHON_METH_FASTCALL + #if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | + #endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots + }; + static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; + } + static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; + } + static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); + } + static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); + } + static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); + } + +/* CythonFunction */ + static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; + } + +/* FusedFunction */ + static PyObject * + __pyx_FusedFunction_New(PyMethodDef *ml, int flags, + PyObject *qualname, PyObject *closure, + PyObject *module, PyObject *globals, + PyObject *code) + { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_FusedFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + __pyx_FusedFunctionObject *fusedfunc = (__pyx_FusedFunctionObject *) op; + fusedfunc->__signatures__ = NULL; + fusedfunc->self = NULL; + #if CYTHON_COMPILING_IN_LIMITED_API + fusedfunc->ml = ml; + #endif + PyObject_GC_Track(op); + } + return op; + } + static void + __pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) + { + PyObject_GC_UnTrack(self); + Py_CLEAR(self->self); + Py_CLEAR(self->__signatures__); + __Pyx__CyFunction_dealloc((__pyx_CyFunctionObject *) self); + } + static int + __pyx_FusedFunction_traverse(__pyx_FusedFunctionObject *self, + visitproc visit, + void *arg) + { + Py_VISIT(self->self); + Py_VISIT(self->__signatures__); + return __Pyx_CyFunction_traverse((__pyx_CyFunctionObject *) self, visit, arg); + } + static int + __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self) + { + Py_CLEAR(self->self); + Py_CLEAR(self->__signatures__); + return __Pyx_CyFunction_clear((__pyx_CyFunctionObject *) self); + } + static __pyx_FusedFunctionObject * + __pyx_FusedFunction_descr_get_locked(__pyx_FusedFunctionObject *func, PyObject *obj) + { + PyObject *module; + __pyx_FusedFunctionObject *meth; + #if CYTHON_COMPILING_IN_LIMITED_API + module = __Pyx_CyFunction_get_module((__pyx_CyFunctionObject *) func, NULL); + if ((unlikely(!module))) return NULL; + #else + module = ((PyCFunctionObject *) func)->m_module; + #endif + meth = (__pyx_FusedFunctionObject *) __pyx_FusedFunction_New( + #if CYTHON_COMPILING_IN_LIMITED_API + func->ml, + #else + ((PyCFunctionObject *) func)->m_ml, + #endif + ((__pyx_CyFunctionObject *) func)->flags, + ((__pyx_CyFunctionObject *) func)->func_qualname, + ((__pyx_CyFunctionObject *) func)->func_closure, + module, + ((__pyx_CyFunctionObject *) func)->func_globals, + ((__pyx_CyFunctionObject *) func)->func_code); + #if CYTHON_COMPILING_IN_LIMITED_API + Py_DECREF(module); + #endif + if (unlikely(!meth)) + return NULL; + Py_XINCREF(func->func.defaults); + meth->func.defaults = func->func.defaults; + __Pyx_CyFunction_SetClassObj(meth, __Pyx_CyFunction_GetClassObj(func)); + Py_XINCREF(func->__signatures__); + meth->__signatures__ = func->__signatures__; + Py_XINCREF(func->func.defaults_tuple); + meth->func.defaults_tuple = func->func.defaults_tuple; + Py_XINCREF(obj); + meth->self = obj; + return meth; + } + static PyObject * + __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) + { + __pyx_FusedFunctionObject *func, *meth; + func = (__pyx_FusedFunctionObject *) self; + if (func->self || func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(self); + return self; + } + if (obj == Py_None) + obj = NULL; + if (func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD) + obj = type; + if (obj == NULL) { + Py_INCREF(self); + return self; + } + __Pyx_BEGIN_CRITICAL_SECTION(func); + meth = __pyx_FusedFunction_descr_get_locked(func, obj); + __Pyx_END_CRITICAL_SECTION() + return (PyObject *) meth; + } + static PyObject * + _obj_to_string(PyObject *obj) + { + if (PyUnicode_CheckExact(obj)) + return __Pyx_NewRef(obj); + else if (PyType_Check(obj)) + return PyObject_GetAttr(obj, __pyx_mstate_global->__pyx_n_u_name_2); + else + return PyObject_Str(obj); + } + static PyObject * + __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx) + { + PyObject *signature = NULL; + PyObject *unbound_result_func; + PyObject *result_func = NULL; + if (unlikely(self->__signatures__ == NULL)) { + PyErr_SetString(PyExc_TypeError, "Function is not fused"); + return NULL; + } + if (PyTuple_Check(idx)) { + Py_ssize_t n = __Pyx_PyTuple_GET_SIZE(idx); + PyObject *list; + int i; + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n < 0)) return NULL; + #endif + list = PyList_New(n); + if (unlikely(!list)) + return NULL; + for (i = 0; i < n; i++) { + PyObject *string; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *item = PyTuple_GET_ITEM(idx, i); + #else + PyObject *item = __Pyx_PySequence_ITEM(idx, i); if (unlikely(!item)) goto __pyx_err; + #endif + string = _obj_to_string(item); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(item); + #endif + if (unlikely(!string)) goto __pyx_err; + if (__Pyx_PyList_SET_ITEM(list, i, string) < (0)) goto __pyx_err; + } + signature = PyUnicode_Join(__pyx_mstate_global->__pyx_kp_u__2, list); + __pyx_err:; + Py_DECREF(list); + } else { + signature = _obj_to_string(idx); + } + if (unlikely(!signature)) + return NULL; + unbound_result_func = PyObject_GetItem(self->__signatures__, signature); + if (likely(unbound_result_func)) { + if (self->self) { + __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func; + __Pyx_CyFunction_SetClassObj(unbound, __Pyx_CyFunction_GetClassObj(self)); + result_func = __pyx_FusedFunction_descr_get(unbound_result_func, + self->self, self->self); + } else { + result_func = unbound_result_func; + Py_INCREF(result_func); + } + } + Py_DECREF(signature); + Py_XDECREF(unbound_result_func); + return result_func; + } + static PyObject * + __pyx_FusedFunction_callfunction(PyObject *func, PyObject *args, PyObject *kw) + { + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + int static_specialized = (cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD && + !((__pyx_FusedFunctionObject *) func)->__signatures__); + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !static_specialized) { + return __Pyx_CyFunction_CallAsMethod(func, args, kw); + } else { + return __Pyx_CyFunction_Call(func, args, kw); + } + } + static PyObject * + __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) + { + __pyx_FusedFunctionObject *binding_func = (__pyx_FusedFunctionObject *) func; + Py_ssize_t argc = __Pyx_PyTuple_GET_SIZE(args); + PyObject *new_args = NULL; + __pyx_FusedFunctionObject *new_func = NULL; + PyObject *result = NULL; + int is_staticmethod = binding_func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD; + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(argc < 0)) return NULL; + #endif + if (binding_func->self) { + PyObject *self; + Py_ssize_t i; + new_args = PyTuple_New(argc + 1); + if (unlikely(!new_args)) + return NULL; + self = binding_func->self; + Py_INCREF(self); + if (__Pyx_PyTuple_SET_ITEM(new_args, 0, self) < (0)) goto bad; + self = NULL; + for (i = 0; i < argc; i++) { + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *item = PyTuple_GET_ITEM(args, i); + Py_INCREF(item); + #else + PyObject *item = __Pyx_PySequence_ITEM(args, i); if (unlikely(!item)) goto bad; + #endif + if (__Pyx_PyTuple_SET_ITEM(new_args, i + 1, item) < (0)) goto bad; + } + args = new_args; + } + if (binding_func->__signatures__) { + PyObject *tup; + if (is_staticmethod && binding_func->func.flags & __Pyx_CYFUNCTION_CCLASS) { + tup = PyTuple_Pack(3, args, + kw == NULL ? Py_None : kw, + binding_func->func.defaults_tuple); + if (unlikely(!tup)) goto bad; + new_func = (__pyx_FusedFunctionObject *) __Pyx_CyFunction_CallMethod( + func, binding_func->__signatures__, tup, NULL); + } else { + tup = PyTuple_Pack(4, binding_func->__signatures__, args, + kw == NULL ? Py_None : kw, + binding_func->func.defaults_tuple); + if (unlikely(!tup)) goto bad; + new_func = (__pyx_FusedFunctionObject *) __pyx_FusedFunction_callfunction(func, tup, NULL); + } + Py_DECREF(tup); + if (unlikely(!new_func)) + goto bad; + __Pyx_CyFunction_SetClassObj(new_func, __Pyx_CyFunction_GetClassObj(binding_func)); + func = (PyObject *) new_func; + } + result = __pyx_FusedFunction_callfunction(func, args, kw); + bad: + Py_XDECREF(new_args); + Py_XDECREF((PyObject *) new_func); + return result; + } + static PyMemberDef __pyx_FusedFunction_members[] = { + {"__signatures__", + T_OBJECT, + offsetof(__pyx_FusedFunctionObject, __signatures__), + READONLY, + 0}, + {"__self__", T_OBJECT_EX, offsetof(__pyx_FusedFunctionObject, self), READONLY, 0}, + #if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, + #endif + {0, 0, 0, 0, 0}, + }; + static PyGetSetDef __pyx_FusedFunction_getsets[] = { + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + #if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, + #endif + {0, 0, 0, 0, 0} + }; + static PyType_Slot __pyx_FusedFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__pyx_FusedFunction_dealloc}, + {Py_tp_call, (void *)__pyx_FusedFunction_call}, + {Py_tp_traverse, (void *)__pyx_FusedFunction_traverse}, + {Py_tp_clear, (void *)__pyx_FusedFunction_clear}, + {Py_tp_members, (void *)__pyx_FusedFunction_members}, + {Py_tp_getset, (void *)__pyx_FusedFunction_getsets}, + {Py_tp_descr_get, (void *)__pyx_FusedFunction_descr_get}, + {Py_mp_subscript, (void *)__pyx_FusedFunction_getitem}, + {0, 0}, + }; + static PyType_Spec __pyx_FusedFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", + sizeof(__pyx_FusedFunctionObject), + 0, + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC , + __pyx_FusedFunctionType_slots + }; + static int __pyx_FusedFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, mstate->__pyx_CyFunctionType); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_FusedFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_FusedFunctionType_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_FusedFunctionType == NULL)) { + return -1; + } + return 0; + } + +/* PyObjectLookupSpecial (used by Py3ClassCreate) */ + #if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; + } + #endif + +/* Py3ClassCreate */ + static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, + PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { + PyObject *ns; + if (metaclass) { + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, __pyx_mstate_global->__pyx_n_u_prepare); + if (prep) { + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_DECREF(prep); + } else { + if (unlikely(PyErr_Occurred())) + return NULL; + ns = PyDict_New(); + } + } else { + ns = PyDict_New(); + } + if (unlikely(!ns)) + return NULL; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_module, modname) < 0)) goto bad; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_qualname, qualname) < 0)) goto bad; + if (unlikely(doc && PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_doc, doc) < 0)) goto bad; + return ns; + bad: + Py_DECREF(ns); + return NULL; + } + static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, + PyObject *dict, PyObject *mkw, + int calculate_metaclass, int allow_py2_metaclass) { + PyObject *result; + PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; + if (allow_py2_metaclass) { + owned_metaclass = PyObject_GetItem(dict, __pyx_mstate_global->__pyx_n_u_metaclass); + if (owned_metaclass) { + metaclass = owned_metaclass; + } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { + PyErr_Clear(); + } else { + return NULL; + } + } + if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { + metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); + Py_XDECREF(owned_metaclass); + if (unlikely(!metaclass)) + return NULL; + owned_metaclass = metaclass; + } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_XDECREF(owned_metaclass); + return result; + } + +/* CyFunctionClassCell */ + static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj) { + Py_ssize_t i, count = __Pyx_PyList_GET_SIZE(cyfunctions); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(count < 0)) return -1; + #endif + for (i = 0; i < count; i++) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && !CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + PyList_GET_ITEM(cyfunctions, i); + #else + __Pyx_PySequence_ITEM(cyfunctions, i); + if (unlikely(!m)) + return -1; + #endif + __Pyx_CyFunction_SetClassObj(m, classobj); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && !CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + Py_DECREF((PyObject*)m); + #endif + } + return 0; + } + +/* CLineInTraceback (used by AddTraceback) */ + #if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + #define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) + #elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); + #else + PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; + } + #endif + static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; + } + #endif + +/* CodeObjectCache (used by AddTraceback) */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } + } + static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; + } + static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. + #else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } + #endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + #endif + return result; + #endif + } + static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) + { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); + } + static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. + #else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } + #endif + __pyx__insert_code_object(code_cache, code_line, code_object); + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); + #endif + #endif + } + +/* AddTraceback */ + #include "compile.h" + #include "frameobject.h" + #include "traceback.h" + #if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" + #endif + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; + } + static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); + } + #else + static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; + bad: + Py_XDECREF(py_funcname); + return NULL; + } + static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); + bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); + } + #endif + +/* MemviewRefcount */ + #include + #ifndef _Py_NO_RETURN + #define _Py_NO_RETURN + #endif + _Py_NO_RETURN + static void __pyx_fatalerror(const char *fmt, ...) { + va_list vargs; + char msg[200]; + #if PY_VERSION_HEX >= 0x030A0000 || defined(HAVE_STDARG_PROTOTYPES) + va_start(vargs, fmt); + #else + va_start(vargs); + #endif + vsnprintf(msg, 200, fmt, vargs); + va_end(vargs); + Py_FatalError(msg); + } + static CYTHON_INLINE int + __pyx_add_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) + { + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)++; + PyThread_release_lock(lock); + return result; + } + static CYTHON_INLINE int + __pyx_sub_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) + { + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)--; + PyThread_release_lock(lock); + return result; + } + static CYTHON_INLINE void + __Pyx_INC_MEMVIEW(__Pyx_memviewslice *memslice, int have_gil, int lineno) + { + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + return; + } + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count+1, lineno); + } + } + } + static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *memslice, + int have_gil, int lineno) { + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + memslice->memview = NULL; + return; + } + old_acquisition_count = __pyx_sub_acquisition_count(memview); + memslice->data = NULL; + if (likely(old_acquisition_count > 1)) { + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + if (have_gil) { + Py_CLEAR(memslice->memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_CLEAR(memslice->memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count-1, lineno); + } + } + +/* MemviewSliceIsContig */ + static int + __pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim) + { + int i, index, step, start; + Py_ssize_t itemsize = mvs.memview->view.itemsize; + if (order == 'F') { + step = 1; + start = 0; + } else { + step = -1; + start = ndim - 1; + } + for (i = 0; i < ndim; i++) { + index = start + step * i; + if (mvs.suboffsets[index] >= 0 || mvs.strides[index] != itemsize) + return 0; + itemsize *= mvs.shape[index]; + } + return 1; + } + +/* OverlappingSlices */ + static void + __pyx_get_array_memory_extents(__Pyx_memviewslice *slice, + void **out_start, void **out_end, + int ndim, size_t itemsize) + { + char *start, *end; + int i; + start = end = slice->data; + for (i = 0; i < ndim; i++) { + Py_ssize_t stride = slice->strides[i]; + Py_ssize_t extent = slice->shape[i]; + if (extent == 0) { + *out_start = *out_end = start; + return; + } else { + if (stride > 0) + end += stride * (extent - 1); + else + start += stride * (extent - 1); + } + } + *out_start = start; + *out_end = end + itemsize; + } + static int + __pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize) + { + void *start1, *end1, *start2, *end2; + __pyx_get_array_memory_extents(slice1, &start1, &end1, ndim, itemsize); + __pyx_get_array_memory_extents(slice2, &start2, &end2, ndim, itemsize); + return (start1 < end2) && (start2 < end1); + } + +/* MemviewSliceInit */ + static int + __Pyx_init_memviewslice(struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference) + { + __Pyx_RefNannyDeclarations + int i, retval=-1; + Py_buffer *buf = &memview->view; + __Pyx_RefNannySetupContext("init_memviewslice", 0); + if (unlikely(memviewslice->memview || memviewslice->data)) { + PyErr_SetString(PyExc_ValueError, + "memviewslice is already initialized!"); + goto fail; + } + if (buf->strides) { + for (i = 0; i < ndim; i++) { + memviewslice->strides[i] = buf->strides[i]; + } + } else { + Py_ssize_t stride = buf->itemsize; + for (i = ndim - 1; i >= 0; i--) { + memviewslice->strides[i] = stride; + stride *= buf->shape[i]; + } + } + for (i = 0; i < ndim; i++) { + memviewslice->shape[i] = buf->shape[i]; + if (buf->suboffsets) { + memviewslice->suboffsets[i] = buf->suboffsets[i]; + } else { + memviewslice->suboffsets[i] = -1; + } + } + memviewslice->memview = memview; + memviewslice->data = (char *)buf->buf; + if (__pyx_add_acquisition_count(memview) == 0 && !memview_is_new_reference) { + Py_INCREF((PyObject*)memview); + } + retval = 0; + goto no_fail; + fail: + memviewslice->memview = 0; + memviewslice->data = 0; + retval = -1; + no_fail: + __Pyx_RefNannyFinishContext(); + return retval; + } + +/* TypeInfoCompare (used by MemviewSliceValidateAndInit) */ + static int + __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b) + { + int i; + if (!a || !b) + return 0; + if (a == b) + return 1; + if (a->size != b->size || a->typegroup != b->typegroup || + a->is_unsigned != b->is_unsigned || a->ndim != b->ndim) { + if (a->typegroup == 'H' || b->typegroup == 'H') { + return a->size == b->size; + } else { + return 0; + } + } + if (a->ndim) { + for (i = 0; i < a->ndim; i++) + if (a->arraysize[i] != b->arraysize[i]) + return 0; + } + if (a->typegroup == 'S') { + if (a->flags != b->flags) + return 0; + if (a->fields || b->fields) { + if (!(a->fields && b->fields)) + return 0; + for (i = 0; a->fields[i].type && b->fields[i].type; i++) { + const __Pyx_StructField *field_a = a->fields + i; + const __Pyx_StructField *field_b = b->fields + i; + if (field_a->offset != field_b->offset || + !__pyx_typeinfo_cmp(field_a->type, field_b->type)) + return 0; + } + return !a->fields[i].type && !b->fields[i].type; + } + } + return 1; + } + +/* MemviewSliceValidateAndInit (used by ObjectToMemviewSlice) */ + static int + __pyx_check_strides(Py_buffer *buf, int dim, int ndim, int spec) + { + if (buf->shape[dim] <= 1) + return 1; + if (buf->strides) { + if (spec & __Pyx_MEMVIEW_CONTIG) { + if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL)) { + if (unlikely(buf->strides[dim] != sizeof(void *))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly contiguous " + "in dimension %d.", dim); + goto fail; + } + } else if (unlikely(buf->strides[dim] != buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_FOLLOW) { + Py_ssize_t stride = buf->strides[dim]; + if (stride < 0) + stride = -stride; + if (unlikely(stride < buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + } else { + if (unlikely(spec & __Pyx_MEMVIEW_CONTIG && dim != ndim - 1)) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not contiguous in " + "dimension %d", dim); + goto fail; + } else if (unlikely(spec & (__Pyx_MEMVIEW_PTR))) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not indirect in " + "dimension %d", dim); + goto fail; + } else if (unlikely(buf->suboffsets)) { + PyErr_SetString(PyExc_ValueError, + "Buffer exposes suboffsets but no strides"); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) + { + CYTHON_UNUSED_VAR(ndim); + if (spec & __Pyx_MEMVIEW_DIRECT) { + if (unlikely(buf->suboffsets && buf->suboffsets[dim] >= 0)) { + PyErr_Format(PyExc_ValueError, + "Buffer not compatible with direct access " + "in dimension %d.", dim); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_PTR) { + if (unlikely(!buf->suboffsets || (buf->suboffsets[dim] < 0))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly accessible " + "in dimension %d.", dim); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_verify_contig(Py_buffer *buf, int ndim, int c_or_f_flag) + { + int i; + if (c_or_f_flag & __Pyx_IS_F_CONTIG) { + Py_ssize_t stride = 1; + for (i = 0; i < ndim; i++) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not fortran contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } else if (c_or_f_flag & __Pyx_IS_C_CONTIG) { + Py_ssize_t stride = 1; + for (i = ndim - 1; i >- 1; i--) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not C contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } + return 1; + fail: + return 0; + } + static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj) + { + struct __pyx_memoryview_obj *memview, *new_memview; + __Pyx_RefNannyDeclarations + Py_buffer *buf; + int i, spec = 0, retval = -1; + __Pyx_BufFmt_Context ctx; + int from_memoryview = __pyx_memoryview_check(original_obj); + __Pyx_RefNannySetupContext("ValidateAndInit_memviewslice", 0); + if (from_memoryview && __pyx_typeinfo_cmp(dtype, ((struct __pyx_memoryview_obj *) + original_obj)->typeinfo)) { + memview = (struct __pyx_memoryview_obj *) original_obj; + new_memview = NULL; + } else { + memview = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + original_obj, buf_flags, 0, dtype); + new_memview = memview; + if (unlikely(!memview)) + goto fail; + } + buf = &memview->view; + if (unlikely(buf->ndim != ndim)) { + PyErr_Format(PyExc_ValueError, + "Buffer has wrong number of dimensions (expected %d, got %d)", + ndim, buf->ndim); + goto fail; + } + if (new_memview) { + __Pyx_BufFmt_Init(&ctx, stack, dtype); + if (unlikely(!__Pyx_BufFmt_CheckString(&ctx, buf->format))) goto fail; + } + if (unlikely((unsigned) buf->itemsize != dtype->size)) { + PyErr_Format(PyExc_ValueError, + "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "u byte%s) " + "does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "u byte%s)", + buf->itemsize, + (buf->itemsize > 1) ? "s" : "", + dtype->name, + dtype->size, + (dtype->size > 1) ? "s" : ""); + goto fail; + } + if (buf->len > 0) { + for (i = 0; i < ndim; i++) { + spec = axes_specs[i]; + if (unlikely(!__pyx_check_strides(buf, i, ndim, spec))) + goto fail; + if (unlikely(!__pyx_check_suboffsets(buf, i, ndim, spec))) + goto fail; + } + if (unlikely(buf->strides && !__pyx_verify_contig(buf, ndim, c_or_f_flag))) + goto fail; + } + if (unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice, + new_memview != NULL) == -1)) { + goto fail; + } + retval = 0; + goto no_fail; + fail: + Py_XDECREF((PyObject*)new_memview); + retval = -1; + no_fail: + __Pyx_RefNannyFinishContext(); + return retval; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_float(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 3, + &__Pyx_TypeInfo_float, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_d_dc_double(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 3, + &__Pyx_TypeInfo_double, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 2, + &__Pyx_TypeInfo_float, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_double(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 2, + &__Pyx_TypeInfo_double, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_int(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 2, + &__Pyx_TypeInfo_int, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_unsigned_int(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 2, + &__Pyx_TypeInfo_unsigned_int, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) + #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) + #define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_int(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 1, + &__Pyx_TypeInfo_int, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* MemviewDtypeToObject */ + static CYTHON_INLINE PyObject *__pyx_memview_get_int(const char *itemp) { + return (PyObject *) __Pyx_PyLong_From_int(*(int const *) itemp); + } + static CYTHON_INLINE int __pyx_memview_set_int(char *itemp, PyObject *obj) { + int value = __Pyx_PyLong_As_int(obj); + if (unlikely((value == (int)-1) && PyErr_Occurred())) + return 0; + *(int *) itemp = value; + return 1; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_nn_uint64_t__const__(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 1, + &__Pyx_TypeInfo_nn_uint64_t__const__, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_nn___pyx_t_5thinc_8backends_9numpy_ops_weight_t(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 1, + &__Pyx_TypeInfo_nn___pyx_t_5thinc_8backends_9numpy_ops_weight_t, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_dc_float(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 1, + &__Pyx_TypeInfo_float, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* MemviewDtypeToObject */ + static CYTHON_INLINE PyObject *__pyx_memview_get_float(const char *itemp) { + return (PyObject *) PyFloat_FromDouble(*(float const *) itemp); + } + static CYTHON_INLINE int __pyx_memview_set_float(char *itemp, PyObject *obj) { + float value = __Pyx_PyFloat_AsFloat(obj); + if (unlikely((value == (float)-1) && PyErr_Occurred())) + return 0; + *(float *) itemp = value; + return 1; + } + +/* MemviewDtypeToObject */ + static CYTHON_INLINE PyObject *__pyx_memview_get_double(const char *itemp) { + return (PyObject *) PyFloat_FromDouble(*(double const *) itemp); + } + static CYTHON_INLINE int __pyx_memview_set_double(char *itemp, PyObject *obj) { + double value = __Pyx_PyFloat_AsDouble(obj); + if (unlikely((value == (double)-1) && PyErr_Occurred())) + return 0; + *(double *) itemp = value; + return 1; + } + +/* MemviewDtypeToObject */ + static CYTHON_INLINE PyObject *__pyx_memview_get_nn_uint64_t__const__(const char *itemp) { + return (PyObject *) __Pyx_PyLong_From_uint64_t(*(uint64_t const *) itemp); + } + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return ::std::complex< double >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return x + y*(__pyx_t_double_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + __pyx_t_double_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabs(b.real) >= fabs(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + double r = b.imag / b.real; + double s = (double)(1.0) / (b.real + b.imag * r); + return __pyx_t_double_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + double r = b.real / b.imag; + double s = (double)(1.0) / (b.imag + b.real * r); + return __pyx_t_double_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + double denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_double_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrt(z.real*z.real + z.imag*z.imag); + #else + return hypot(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + double r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + double denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_double(a, a); + case 3: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, a); + case 4: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = pow(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_double(a); + theta = atan2(a.imag, a.real); + } + lnr = log(r); + z_r = exp(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cos(z_theta); + z.imag = z_r * sin(z_theta); + return z; + } + #endif + #endif + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + return ::std::complex< float >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + return x + y*(__pyx_t_float_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + __pyx_t_float_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sum_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_diff_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prod_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + if (b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabsf(b.real) >= fabsf(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + float r = b.imag / b.real; + float s = (float)(1.0) / (b.real + b.imag * r); + return __pyx_t_float_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + float r = b.real / b.imag; + float s = (float)(1.0) / (b.imag + b.real * r); + return __pyx_t_float_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + if (b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + float denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_float_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_neg_float(__pyx_t_float_complex a) { + __pyx_t_float_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_float(__pyx_t_float_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conj_float(__pyx_t_float_complex a) { + __pyx_t_float_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE float __Pyx_c_abs_float(__pyx_t_float_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrtf(z.real*z.real + z.imag*z.imag); + #else + return hypotf(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_pow_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + float r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + float denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_float(a, a); + case 3: + z = __Pyx_c_prod_float(a, a); + return __Pyx_c_prod_float(z, a); + case 4: + z = __Pyx_c_prod_float(a, a); + return __Pyx_c_prod_float(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = powf(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2f(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_float(a); + theta = atan2f(a.imag, a.real); + } + lnr = logf(r); + z_r = expf(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cosf(z_theta); + z.imag = z_r * sinf(z_theta); + return z; + } + #endif + #endif + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + return ::std::complex< long double >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + return x + y*(__pyx_t_long_double_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + __pyx_t_long_double_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_sum_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_diff_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_prod_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + if (b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabsl(b.real) >= fabsl(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + long double r = b.imag / b.real; + long double s = (long double)(1.0) / (b.real + b.imag * r); + return __pyx_t_long_double_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + long double r = b.real / b.imag; + long double s = (long double)(1.0) / (b.imag + b.real * r); + return __pyx_t_long_double_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + if (b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + long double denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_long_double_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_neg_long__double(__pyx_t_long_double_complex a) { + __pyx_t_long_double_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_long__double(__pyx_t_long_double_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_conj_long__double(__pyx_t_long_double_complex a) { + __pyx_t_long_double_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE long double __Pyx_c_abs_long__double(__pyx_t_long_double_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrtl(z.real*z.real + z.imag*z.imag); + #else + return hypotl(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_pow_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + long double r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + long double denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_long__double(a, a); + case 3: + z = __Pyx_c_prod_long__double(a, a); + return __Pyx_c_prod_long__double(z, a); + case 4: + z = __Pyx_c_prod_long__double(a, a); + return __Pyx_c_prod_long__double(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = powl(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2l(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_long__double(a); + theta = atan2l(a.imag, a.real); + } + lnr = logl(r); + z_r = expl(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cosl(z_theta); + z.imag = z_r * sinl(z_theta); + return z; + } + #endif + #endif + +/* MemviewSliceCopy */ + static __Pyx_memviewslice + __pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object) + { + __Pyx_RefNannyDeclarations + int i; + __Pyx_memviewslice new_mvs = __Pyx_MEMSLICE_INIT; + struct __pyx_memoryview_obj *from_memview = from_mvs->memview; + Py_buffer *buf = &from_memview->view; + PyObject *shape_tuple = NULL; + PyObject *temp_int = NULL; + struct __pyx_array_obj *array_obj = NULL; + struct __pyx_memoryview_obj *memview_obj = NULL; + __Pyx_RefNannySetupContext("__pyx_memoryview_copy_new_contig", 0); + for (i = 0; i < ndim; i++) { + if (unlikely(from_mvs->suboffsets[i] >= 0)) { + PyErr_Format(PyExc_ValueError, "Cannot copy memoryview slice with " + "indirect dimensions (axis %d)", i); + goto fail; + } + } + shape_tuple = PyTuple_New(ndim); + if (unlikely(!shape_tuple)) { + goto fail; + } + __Pyx_GOTREF(shape_tuple); + for(i = 0; i < ndim; i++) { + temp_int = PyLong_FromSsize_t(from_mvs->shape[i]); + if(unlikely(!temp_int)) { + goto fail; + } else { + #if CYTHON_ASSUME_SAFE_MACROS + PyTuple_SET_ITEM(shape_tuple, i, temp_int); + #else + if (PyTuple_SetItem(shape_tuple, i, temp_int) < 0) { + goto fail; + } + #endif + temp_int = NULL; + } + } + array_obj = __pyx_array_new(shape_tuple, sizeof_dtype, buf->format, mode, NULL); + if (unlikely(!array_obj)) { + goto fail; + } + __Pyx_GOTREF(array_obj); + memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + (PyObject *) array_obj, contig_flag, + dtype_is_object, + from_mvs->memview->typeinfo); + if (unlikely(!memview_obj)) + goto fail; + if (unlikely(__Pyx_init_memviewslice(memview_obj, ndim, &new_mvs, 1) < 0)) + goto fail; + if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim, + dtype_is_object) < 0)) + goto fail; + goto no_fail; + fail: + __Pyx_XDECREF((PyObject *) new_mvs.memview); + new_mvs.memview = NULL; + new_mvs.data = NULL; + no_fail: + __Pyx_XDECREF(shape_tuple); + __Pyx_XDECREF(temp_int); + __Pyx_XDECREF((PyObject *) array_obj); + __Pyx_RefNannyFinishContext(); + return new_mvs; + } + +/* ImportNumPyArray */ + static PyObject* __Pyx__ImportNumPyArray(void) { + PyObject *numpy_module, *ndarray_object = NULL; + numpy_module = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_numpy, NULL, 0, NULL, 0); + if (likely(numpy_module)) { + ndarray_object = PyObject_GetAttrString(numpy_module, "ndarray"); + Py_DECREF(numpy_module); + } + if (unlikely(!ndarray_object)) { + PyErr_Clear(); + } + if (unlikely(!ndarray_object || !PyObject_TypeCheck(ndarray_object, &PyType_Type))) { + Py_XDECREF(ndarray_object); + Py_INCREF(Py_None); + ndarray_object = Py_None; + } + return ndarray_object; + } + static CYTHON_INLINE PyObject* __Pyx__ImportNumPyArrayTypeIfAvailable(void) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyMutex ndarray_mutex = {0}; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + PyObject *result = (PyObject*)__pyx_atomic_pointer_load_relaxed(&__pyx_mstate_global->__pyx_numpy_ndarray); + if (unlikely(!result)) { + PyMutex_Lock(&ndarray_mutex); + result = (PyObject*)__pyx_atomic_pointer_load_acquire(&__pyx_mstate_global->__pyx_numpy_ndarray); + if (!result) { + result = __Pyx__ImportNumPyArray(); + __pyx_atomic_pointer_exchange(&__pyx_mstate_global->__pyx_numpy_ndarray, result); + } + PyMutex_Unlock(&ndarray_mutex); + } + return result; + #else + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING // but not atomics + PyMutex_Lock(&ndarray_mutex); + #endif + if (unlikely(!__pyx_mstate_global->__pyx_numpy_ndarray)) + { + __pyx_mstate_global->__pyx_numpy_ndarray = __Pyx__ImportNumPyArray(); + } + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING // but not atomics + PyMutex_Unlock(&ndarray_mutex); + #endif + return __pyx_mstate_global->__pyx_numpy_ndarray; + #endif + } + static CYTHON_INLINE PyObject* __Pyx_ImportNumPyArrayTypeIfAvailable(void) { + PyObject *result = __Pyx__ImportNumPyArrayTypeIfAvailable(); + Py_INCREF(result); + return result; + } + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE uint32_t __Pyx_PyLong_As_uint32_t(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const uint32_t neg_one = (uint32_t) -1, const_zero = (uint32_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + uint32_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (uint32_t) -1; + val = __Pyx_PyLong_As_uint32_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(uint32_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(uint32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) >= 2 * PyLong_SHIFT)) { + return (uint32_t) (((((uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(uint32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) >= 3 * PyLong_SHIFT)) { + return (uint32_t) (((((((uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(uint32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) >= 4 * PyLong_SHIFT)) { + return (uint32_t) (((((((((uint32_t)digits[3]) << PyLong_SHIFT) | (uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (uint32_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(uint32_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(uint32_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(uint32_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(uint32_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(uint32_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(uint32_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 2 * PyLong_SHIFT)) { + return (uint32_t) (((uint32_t)-1)*(((((uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(uint32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 2 * PyLong_SHIFT)) { + return (uint32_t) ((((((uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(uint32_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 3 * PyLong_SHIFT)) { + return (uint32_t) (((uint32_t)-1)*(((((((uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(uint32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 3 * PyLong_SHIFT)) { + return (uint32_t) ((((((((uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(uint32_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 4 * PyLong_SHIFT)) { + return (uint32_t) (((uint32_t)-1)*(((((((((uint32_t)digits[3]) << PyLong_SHIFT) | (uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(uint32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint32_t) - 1 > 4 * PyLong_SHIFT)) { + return (uint32_t) ((((((((((uint32_t)digits[3]) << PyLong_SHIFT) | (uint32_t)digits[2]) << PyLong_SHIFT) | (uint32_t)digits[1]) << PyLong_SHIFT) | (uint32_t)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(uint32_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(uint32_t, long, PyLong_AsLong(x)) + } else if ((sizeof(uint32_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(uint32_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + uint32_t val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (uint32_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (uint32_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (uint32_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (uint32_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(uint32_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((uint32_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(uint32_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((uint32_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((uint32_t) 1) << (sizeof(uint32_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (uint32_t) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to uint32_t"); + return (uint32_t) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to uint32_t"); + return (uint32_t) -1; + } + +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + size_t val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_uint64_t(uint64_t value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint64_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(uint64_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(uint64_t), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(uint64_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (long) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_npy_intp(npy_intp value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const npy_intp neg_one = (npy_intp) -1, const_zero = (npy_intp) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(npy_intp) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(npy_intp) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(npy_intp) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(npy_intp) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(npy_intp) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(npy_intp), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(npy_intp)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE npy_intp __Pyx_PyLong_As_npy_intp(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const npy_intp neg_one = (npy_intp) -1, const_zero = (npy_intp) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + npy_intp val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (npy_intp) -1; + val = __Pyx_PyLong_As_npy_intp(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(npy_intp, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(npy_intp) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) >= 2 * PyLong_SHIFT)) { + return (npy_intp) (((((npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(npy_intp) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) >= 3 * PyLong_SHIFT)) { + return (npy_intp) (((((((npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(npy_intp) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) >= 4 * PyLong_SHIFT)) { + return (npy_intp) (((((((((npy_intp)digits[3]) << PyLong_SHIFT) | (npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (npy_intp) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(npy_intp) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(npy_intp, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(npy_intp) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(npy_intp, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(npy_intp, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(npy_intp) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 2 * PyLong_SHIFT)) { + return (npy_intp) (((npy_intp)-1)*(((((npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(npy_intp) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 2 * PyLong_SHIFT)) { + return (npy_intp) ((((((npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(npy_intp) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 3 * PyLong_SHIFT)) { + return (npy_intp) (((npy_intp)-1)*(((((((npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(npy_intp) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 3 * PyLong_SHIFT)) { + return (npy_intp) ((((((((npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(npy_intp) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 4 * PyLong_SHIFT)) { + return (npy_intp) (((npy_intp)-1)*(((((((((npy_intp)digits[3]) << PyLong_SHIFT) | (npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(npy_intp) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(npy_intp, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(npy_intp) - 1 > 4 * PyLong_SHIFT)) { + return (npy_intp) ((((((((((npy_intp)digits[3]) << PyLong_SHIFT) | (npy_intp)digits[2]) << PyLong_SHIFT) | (npy_intp)digits[1]) << PyLong_SHIFT) | (npy_intp)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(npy_intp) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(npy_intp, long, PyLong_AsLong(x)) + } else if ((sizeof(npy_intp) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(npy_intp, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + npy_intp val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (npy_intp) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (npy_intp) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (npy_intp) -1; + } else { + stepval = v; + } + v = NULL; + val = (npy_intp) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(npy_intp) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((npy_intp) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(npy_intp) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((npy_intp) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((npy_intp) 1) << (sizeof(npy_intp) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (npy_intp) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to npy_intp"); + return (npy_intp) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to npy_intp"); + return (npy_intp) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int32_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int32_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int32_t), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int32_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int32_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int32_t) -1; + val = __Pyx_PyLong_As_int32_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 2 * PyLong_SHIFT)) { + return (int32_t) (((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 3 * PyLong_SHIFT)) { + return (int32_t) (((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 4 * PyLong_SHIFT)) { + return (int32_t) (((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int32_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int32_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int32_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) ((((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) ((((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) ((((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int32_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, long, PyLong_AsLong(x)) + } else if ((sizeof(int32_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int32_t val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int32_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int32_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int32_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int32_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int32_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int32_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int32_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int32_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int32_t) 1) << (sizeof(int32_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int32_t) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int32_t"); + return (int32_t) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int32_t"); + return (int32_t) -1; + } + +/* PyUCS4InUnicode */ + static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character) { + Py_ssize_t idx = PyUnicode_FindChar(unicode, character, 0, PY_SSIZE_T_MAX, 1); + if (unlikely(idx == -2)) return -1; + return idx >= 0; + } + +/* UpdateUnpickledDict */ + static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + PyObject *state_dict = __Pyx_PySequence_ITEM(state, index); + if (unlikely(!state_dict)) { + return -1; + } + int non_empty = PyObject_IsTrue(state_dict); + if (non_empty == 0) { + Py_DECREF(state_dict); + return 0; + } else if (unlikely(non_empty == -1)) { + return -1; + } + PyObject *dict; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = PyObject_GetAttrString(obj, "__dict__"); + #else + dict = PyObject_GenericGetDict(obj, NULL); + #endif + if (unlikely(!dict)) { + Py_DECREF(state_dict); + return -1; + } + int result; + if (likely(PyDict_CheckExact(dict))) { + result = PyDict_Update(dict, state_dict); + } else { + PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict); + if (likely(obj_result)) { + Py_DECREF(obj_result); + result = 0; + } else { + result = -1; + } + } + Py_DECREF(state_dict); + Py_DECREF(dict); + return result; + } + static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(state_size == -1)) return -1; + #endif + if (state_size <= index) { + return 0; + } + return __Pyx__UpdateUnpickledDict(obj, state, index); + } + +/* CheckUnpickleChecksum */ + static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + PyObject *pickle_module = PyImport_ImportModule("pickle"); + if (unlikely(!pickle_module)) return; + PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (unlikely(!pickle_error)) return; + if (checksum2 == checksum1) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))", + checksum, checksum1, members); + } else if (checksum3 == checksum2) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, members); + } else { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, checksum3, members); + } + Py_DECREF(pickle_error); + } + static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + int found = 0; + found |= checksum1 == checksum; + found |= checksum2 == checksum; + found |= checksum3 == checksum; + if (likely(found)) + return 0; + __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members); + return -1; + } + +/* CIntFromPy */ + static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const char neg_one = (char) -1, const_zero = (char) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + char val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (char) -1; + val = __Pyx_PyLong_As_char(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 2 * PyLong_SHIFT)) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 3 * PyLong_SHIFT)) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 4 * PyLong_SHIFT)) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(char) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(char) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(char) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(char) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) + } else if ((sizeof(char) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + char val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (char) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (char) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (char) -1; + } else { + stepval = v; + } + v = NULL; + val = (char) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(char) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((char) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(char) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((char) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((char) 1) << (sizeof(char) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (char) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; + } + +/* FormatTypeName */ + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 + static __Pyx_TypeName + __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) + { + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__9); + } + goto done; + } + #endif + +/* GetRuntimeVersion */ + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + void __Pyx_init_runtime_version(void) { + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + } + #endif + static unsigned long __Pyx_get_runtime_version(void) { + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; + #else + return __Pyx_cached_runtime_version; + #endif + } + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } + } + +/* NewCodeObj */ + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } + #elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } + #elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #endif + static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map + ) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); + done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; + } + +/* DecompressString */ + static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; + import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); + bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; + } + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + + /* MultiPhaseInitModuleState */ + #if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE + #ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 + #else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 + #endif + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS + #error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" + #endif + #if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #define __Pyx_ModuleStateLookup_Lock() + #define __Pyx_ModuleStateLookup_Unlock() + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; + #define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(__cplusplus) && __cplusplus >= 201103L + #include + static std::mutex __Pyx_ModuleStateLookup_mutex; + #define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() + #define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() + #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) + #include + static mtx_t __Pyx_ModuleStateLookup_mutex; + static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; + static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); + } + #define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(HAVE_PTHREAD_H) + #include + static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; + #define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(_WIN32) + #include // synchapi.h on its own doesn't work + static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; + #define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #else + #error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." + #endif + typedef struct { + int64_t id; + PyObject *module; + } __Pyx_InterpreterIdAndModule; + typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; + } __Pyx_ModuleStateLookupData; + #define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; + #else + static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; + #endif + static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; + } + static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; + #else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + #endif + return result; + } + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); + } + #else + #define __Pyx_ModuleStateLookup_wait_until_no_readers() + #endif + static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; + } + static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; + } + static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); + #else + __Pyx_ModuleStateLookup_data = new_data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return result; + } + static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; + #endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); + #else + __Pyx_ModuleStateLookup_data = data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; + } + #endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pxd new file mode 100644 index 0000000000000000000000000000000000000000..8d9b2e2abec704c1c7f74670ec3e4a9a00494c6f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pxd @@ -0,0 +1,50 @@ +from .cblas cimport saxpy_ptr + +ctypedef double[:, ::1] double2d_t +ctypedef double[:, :, ::1] double3d_t +ctypedef float[:, ::1] float2d_t +ctypedef float[:, :, ::1] float3d_t +ctypedef int[:, ::1] int2d_t +ctypedef unsigned int[:, ::1] uint2d_t + +cdef fused ints2d_ft: + int2d_t + uint2d_t + +cdef fused reals2d_ft: + float2d_t + double2d_t + +cdef fused reals3d_ft: + float3d_t + double3d_t + + +cdef extern from "cpu_kernels.hh": + cdef cppclass axpy[T]: + ctypedef void (*ptr)(int N, T alpha, const T* X, int incX, T *Y, int incY); + + void cpu_maxout[A, L](A* best__bo, L* which__bo, const A* cands_bop, + L B, L O, L P) + void cpu_backprop_maxout[A, L](A* dX__bop, const A* dX__bo, const L* which__bo, + L B, L O, L P) except + + void cpu_reduce_max[A, L](A* maxes__bo, L* which_bo, const A* X__to, + const L* lengths__b, L B, L T, L O) except + + + void cpu_backprop_reduce_max[A, L](A* dX__to, const A* d_maxes__bo, const L* which__bo, + const L* lengths__b, L B, L T, L O) except + + void cpu_reduce_mean[A, L](A* means__bo, const A* X__to, const L* lengths__b, + L B, L T, L O) except + + void cpu_backprop_reduce_mean[A, L](A* dX__to, const A* d_means__bo, const L* lengths__b, + L B, L T, L O) + void cpu_mish[A, L](A* Y, L N, A threshold) + void cpu_backprop_mish[A, L](A* dX, const A* X, L N, A threshold) + void cpu_reduce_sum[A, L](A* sums__bo, const A* X__to, const L* lengths__b, + L B, L T, L O) except + + void cpu_backprop_reduce_sum[A, L](A* dX__to, const A* d_sums__bo, const L* lengths__b, + L B, L T, L O) + void cpu_relu[A, L](A* X, L N) + void backprop_seq2col[A, L](A* d_seqs, const A* d_cols, const L* lengths, L B, L I, L nW, L nL) + void seq2col[A, L](A* output, const A* X, const L* lengths, L nW, L B, L I, L nL) + void cpu_gather_add[F, I, L](axpy[F].ptr axpy, F* out_bo, const F* table_to, const I* indices_bk, + L T, L O, L B, L K) except + diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pyx new file mode 100644 index 0000000000000000000000000000000000000000..50cbb55832a92b259c8075f28e3ce0835c37a0b1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/numpy_ops.pyx @@ -0,0 +1,1183 @@ +# cython: cdivision=True +# cython: infer_types=True +from collections.abc import Sized +from typing import Optional + +import numpy + +cimport blis.cy +cimport cython +cimport numpy as np +from cymem.cymem cimport Pool +from libc.math cimport isnan +from libc.stdint cimport uint32_t, uint64_t +from libc.stdlib cimport calloc, free, malloc +from libc.string cimport memcpy, memset +from murmurhash.mrmr cimport hash64 +from preshed.maps cimport PreshMap + +from .. import registry +from ..types import ArrayXd, DeviceTypes, DTypes, Shape +from ..util import copy_array, get_array_module + +from .cblas cimport CBlas, daxpy, saxpy +from .linalg cimport Vec, VecVec + +from .ops import Ops + +try: + import blis.py + has_blis = True +except ImportError: + has_blis = False + + +ctypedef float weight_t + + +cdef extern from "math.h": + float logf(float x) noexcept nogil + float sqrtf(float x) noexcept nogil + float expf(float x) noexcept nogil + float tanhf(float x) noexcept nogil + float sinf(float x) noexcept nogil + float cosf(float x) noexcept nogil + + +@registry.ops("NumpyOps") +class NumpyOps(Ops): + name = "numpy" + xp = numpy + + def __init__( + self, + device_type: DeviceTypes = "cpu", + device_id: int = -1, + *, + use_blis: bool = True + ) -> None: + self.device_type = device_type + self.device_id = device_id + self.use_blis = use_blis + if self.use_blis and not has_blis: + raise ValueError("BLIS support requires blis: pip install blis") + + def asarray(self, data, dtype=None): + if isinstance(data, self.xp.ndarray): + array = data + elif hasattr(data, 'numpy'): + # Handles PyTorch Tensor + array = data.numpy() + elif hasattr(data, "get"): + array = data.get() + else: + array = self.xp.array(data, dtype=dtype) + + if dtype is not None: + array = array.astype(dtype=dtype, copy=False) + + return array + + + def alloc(self, shape: Shape, *, dtype: Optional[DTypes] = "float32", zeros: bool = True) -> ArrayXd: + if zeros: + return self.xp.zeros(shape, dtype=dtype) + else: + return self.xp.empty(shape, dtype=dtype) + + def cblas(self) -> CBlas: + return CBlas() + + def gemm(self, np.ndarray x, np.ndarray y, *, np.ndarray out=None, trans1=False, trans2=False): + if x.ndim != 2: + raise ValueError(f"Provided 'x' array should be 2-dimensional, but found {x.ndim} dimension(s).") + if y.ndim != 2: + raise ValueError(f"Provided 'y' array should be 2-dimensional, but found {y.ndim} dimension(s).") + if not self.use_blis: # delegate to base Ops + return super().gemm(x, y, out=out, trans1=trans1, trans2=trans2) + x = self.as_contig(x) + y = self.as_contig(y) + if out is not None: + out = self.as_contig(out) + return blis.py.gemm(x, y, out=out, trans1=trans1, trans2=trans2, beta=0.) + + def relu(self, np.ndarray X, inplace=False): + cdef np.ndarray Y + + if X.dtype == "float32": + Y = _inplace_or_copy(X, inplace) + cpu_relu(Y.data, Y.size) + return Y + elif X.dtype == "float64": + Y = _inplace_or_copy(X, inplace) + cpu_relu(Y.data, Y.size) + return Y + else: + return super().relu(X, inplace=inplace) + + def backprop_relu(self, np.ndarray dY, np.ndarray Y, inplace=False): + _check_compatible_shape(dY, Y) + + cdef size_t size = Y.size + cdef weight_t* dX_ptr + cdef const weight_t* Y_ptr = Y.data + cdef np.ndarray dX + if dY.dtype == "float32" and Y.dtype == "float32": + dX = _inplace_or_copy(dY, inplace) + dX_ptr = dX.data + for i in range(size): + if Y_ptr[i] <= 0: + dX_ptr[i] = 0. + return dX + else: + return super().backprop_relu(dY, Y, inplace) + + def lstm_forward_training( + self, + np.ndarray params, + np.ndarray H0, + np.ndarray C0, + np.ndarray X, + np.ndarray size_at_t + ): + assert H0.shape[0] == C0.shape[0] + assert H0.shape[1] == C0.shape[1] + Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) + return Y, fwd_state + + def lstm_forward_inference( + self, + np.ndarray params, + np.ndarray H0, + np.ndarray C0, + np.ndarray X, + np.ndarray size_at_t + ): + Y, _ = lstm_forward_training(params, H0, C0, X, size_at_t) + return Y + + def backprop_lstm( + self, np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state + ): + dX, d_params = backprop_lstm(dY, lengths, params, fwd_state) + return dX, d_params + + def maxout(self, reals3d_ft X): + cdef int B = X.shape[0] + cdef int O = X.shape[1] + cdef int P = X.shape[2] + + cdef np.ndarray best + cdef np.ndarray which = self.alloc(shape=(B, O), dtype='int32', zeros=False) + if reals3d_ft is float3d_t: + best = self.alloc(shape=(B, O), dtype="float32", zeros=False) + if len(X) > 0: + cpu_maxout(best.data, which.data, + &X[0, 0, 0], B, O, P) + else: + best = self.alloc(shape=(B, O), dtype="float64", zeros=False) + if len(X) > 0: + cpu_maxout(best.data, which.data, + &X[0, 0, 0], B, O, P) + return best, which + + def backprop_maxout(self, reals2d_ft dY, int[:, ::1] which, int P): + cdef int B = dY.shape[0] + cdef int O = dY.shape[1] + + cdef np.ndarray dX + if reals2d_ft == float2d_t: + dX = numpy.zeros((B, O, P), dtype='float32') + cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + else: + dX = numpy.zeros((B, O, P), dtype='float64') + cpu_backprop_maxout(dX.data, &dY[0, 0], &which[0, 0], B, O, P) + + return dX + + def mish(self, np.ndarray X, threshold=20.0, inplace: bool = False): + cdef np.ndarray Y + + if X.dtype == "float32": + Y = _inplace_or_copy(X, inplace) + cpu_mish(Y.data, Y.size, threshold) + return Y + elif X.dtype == "float64": + Y = _inplace_or_copy(X, inplace) + cpu_mish(Y.data, Y.size, threshold) + return Y + else: + return super().mish(X, inplace=inplace) + + def backprop_mish(self, np.ndarray dY, np.ndarray X, threshold=20.0, inplace=False): + _check_compatible_shape(dY, X) + + cdef np.ndarray dX + + if dY.dtype == "float32" and X.dtype == "float32": + dX = _inplace_or_copy(dY, inplace) + cpu_backprop_mish(dX.data, X.data, X.size, threshold) + return dX + elif dY.dtype == "float64" and X.dtype == "float64": + dX = _inplace_or_copy(dY, inplace) + cpu_backprop_mish(dX.data, X.data, X.size, threshold) + return dX + else: + return super().backprop_mish(dY, X, threshold, inplace) + + def seq2col(self, np.ndarray seq, int nW, *, int[::1] lengths=None): + """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + sequence. The new sequence is constructed by concatenating nW preceding + and succeeding vectors onto each column in the sequence, to extract a + window of features. + """ + + # Note: the type of seq should be changed to reals2d_ft once + # cython/cython#4697 is fixed. The following checks can then be + # removed, because they are guaranteed by the reals2d_ft + # type. + + if seq.ndim != 2: + msg = f"seq2col requires sequence array of dimensionality 2, was {seq.ndim}" + raise ValueError(msg) + if not seq.flags.c_contiguous: + msg = f"seq2col requires sequence array that is in C order and contiguous" + raise ValueError(msg) + + + cdef int B = seq.shape[0] + cdef int I = seq.shape[1] + + lengths = check_seq2col_lengths(self, lengths, B) + cdef int nL = lengths.shape[0] + + cdef np.ndarray cols + if seq.dtype == "float32": + cols = self.alloc((B, (2*nW + 1) * I), dtype="float32") + if seq.size != 0 and lengths.size != 0: + seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + return cols + elif seq.dtype == "float64": + cols = self.alloc((B, (2*nW + 1) * I), dtype="float64") + if seq.size != 0 and lengths.size != 0: + seq2col(cols.data, seq.data, &lengths[0], nW, B, I, nL) + return cols + else: + return super().seq2col(seq, nW, lengths=lengths) + + def backprop_seq2col(self, np.ndarray dY, int nW, *, int[::1] lengths=None): + # Note: the type of dY should be changed to reals2d_ft once + # cython/cython#4697 is fixed. The following checks can then be + # removed, because they are guaranteed by the reals2d_ft + # type. + + if dY.ndim != 2: + msg = f"backprop_seq2col requires gradient array of dimensionality 2, was {dY.ndim}" + raise ValueError(msg) + if not dY.flags.c_contiguous: + msg = f"backprop_seq2col requires gradient array that is in C order and contiguous" + raise ValueError(msg) + + cdef int B = dY.shape[0] + cdef int nF = nW*2+1 + cdef int I = dY.shape[1] / nF + + lengths = check_seq2col_lengths(self, lengths, B) + cdef int nL = lengths.shape[0] + + cdef np.ndarray dX + if dY.dtype == "float32": + dX = self.alloc((B, I), dtype='float32') + if dY.size != 0 and lengths.size != 0: + backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + return dX + elif dY.dtype == "float64": + dX = self.alloc((B, I), dtype='float64') + if dY.size != 0 and lengths.size != 0: + backprop_seq2col(dX.data, dY.data, &lengths[0], B, I, nW, nL) + return dX + else: + return super().backprop_seq2col(dY, nW, lengths=lengths) + + @cython.boundscheck(False) + @cython.wraparound(False) + def hash(self, const uint64_t[::1] ids, uint32_t seed): + """Hash a sequence of 64-bit keys into a table with 4 32-bit keys.""" + # Written to mirror the GPU implementation + cdef np.ndarray[uint32_t, ndim=2] keys = self.alloc((ids.shape[0], 4), dtype='uint32') + cdef int i + cdef uint32_t* dest = keys.data + for i in range(len(ids)): + MurmurHash3_x86_128_uint64(ids[i], seed, &dest[i*4]) + return keys + + def reduce_mean(self, reals2d_ft X, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = X.shape[1] + cdef int T = X.shape[0] + + if B == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(B, O), dtype="float32") + else: + return numpy.zeros(shape=(B, O), dtype="float64") + + cdef np.ndarray means + if reals2d_ft is float2d_t: + means = numpy.zeros(shape=(B, O), dtype="float32") + cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + else: + means = numpy.zeros(shape=(B, O), dtype="float64") + cpu_reduce_mean(means.data, &X[0, 0], &lengths[0], B, T, O) + + return means + + def backprop_reduce_mean(self, reals2d_ft d_means, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = d_means.shape[1] + cdef int T = 0 + + for length in lengths[:B]: + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + T += length + + if T == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(T, O), dtype="float32") + else: + return numpy.zeros(shape=(T, O), dtype="float64") + + cdef np.ndarray dX + if reals2d_ft is float2d_t: + dX = numpy.zeros((T, O), dtype="float32") + cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + else: + dX = numpy.zeros((T, O), dtype="float64") + cpu_backprop_reduce_mean(dX.data, &d_means[0,0], &lengths[0], B, T, O) + + return dX + + def reduce_sum(self, reals2d_ft X, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = X.shape[1] + cdef int T = X.shape[0] + + if B == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(B, O), dtype="float32") + else: + return numpy.zeros(shape=(B, O), dtype="float64") + + cdef np.ndarray sums + if reals2d_ft is float2d_t: + sums = numpy.zeros(shape=(B, O), dtype="float32") + cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + else: + sums = numpy.zeros(shape=(B, O), dtype="float64") + cpu_reduce_sum(sums.data, &X[0, 0], &lengths[0], B, T, O) + + return sums + + def backprop_reduce_sum(self, reals2d_ft d_sums, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = d_sums.shape[1] + cdef int T = 0 + + for length in lengths[:B]: + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + T += length + + if T == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(T, O), dtype="float32") + else: + return numpy.zeros(shape=(T, O), dtype="float64") + + cdef np.ndarray dX + if reals2d_ft is float2d_t: + dX = numpy.zeros((T, O), dtype="float32") + cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + else: + dX = numpy.zeros((T, O), dtype="float64") + cpu_backprop_reduce_sum(dX.data, &d_sums[0,0], &lengths[0], B, T, O) + + return dX + + def reduce_max(self, reals2d_ft X, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = X.shape[1] + cdef int T = X.shape[0] + + # Needs to be zero-initialized as we start by assuming that the first element is the max value. + cdef np.ndarray which = self.alloc(shape=(B, O), dtype="i", zeros=True) + + if B == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(B, O), dtype="float32"), which + else: + return numpy.zeros(shape=(B, O), dtype="float64"), which + + cdef np.ndarray maxes + if reals2d_ft is float2d_t: + maxes = self.alloc(shape=(B, O), dtype="float32", zeros=False) + cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + else: + maxes = self.alloc(shape=(B, O), dtype="float64", zeros=False) + cpu_reduce_max(maxes.data, which.data, &X[0, 0], &lengths[0], B, T, O) + + return maxes, which + + def backprop_reduce_max(self, reals2d_ft d_maxes, int[:, ::1] which, int[::1] lengths): + cdef int B = lengths.shape[0] + cdef int O = d_maxes.shape[1] + cdef int T = 0 + + for length in lengths[:B]: + if length <= 0: + raise ValueError(f"all sequence lengths must be > 0, got {length}") + T += length + + if T == 0 or O == 0: + if reals2d_ft is float2d_t: + return numpy.zeros(shape=(T, O), dtype="float32") + else: + return numpy.zeros(shape=(T, O), dtype="float64") + + cdef np.ndarray dX + if reals2d_ft is float2d_t: + dX = numpy.zeros((T, O), dtype="float32") + cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + &lengths[0], B, T, O) + else: + dX = numpy.zeros((T, O), dtype="float64") + cpu_backprop_reduce_max(dX.data, &d_maxes[0,0], &which[0, 0], + &lengths[0], B, T, O) + + return dX + + def gather_add(self, reals2d_ft table, ints2d_ft indices): + cdef CBlas cblas = self.cblas() + rows = indices.shape[0] + dims = table.shape[1] + + cdef np.ndarray output + if reals2d_ft is float2d_t: + output = self.xp.zeros((rows, dims), dtype="float32") + cpu_gather_add(saxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + table.shape[0], dims, rows, indices.shape[1]) + else: + output = self.xp.zeros((rows, dims), dtype="float64") + cpu_gather_add(daxpy(cblas), output.data, &table[0, 0], &indices[0, 0], + table.shape[0], dims, rows, indices.shape[1]) + + return output + + def scatter_add(self, np.ndarray table, np.ndarray indices, np.ndarray values): + if table.dtype == 'float32' \ + and indices.dtype == 'int32' \ + and values.dtype == 'float32' \ + and table.flags.c_contiguous \ + and indices.flags.c_contiguous \ + and values.flags.c_contiguous \ + and indices.ndim == 1 \ + and table.ndim == 2 \ + and values.ndim == 2 \ + and values.shape[0] == indices.shape[0] \ + and values.shape[1] == table.shape[1]: + cpu_scatter_add(table.data, + indices.data, values.data, + indices.shape[0], table.shape[1]) + else: + self.xp.add.at(table, indices, values) + + @cython.boundscheck(False) + @cython.wraparound(False) + def adam(self, np.ndarray[np.float32_t] weights, np.ndarray[np.float32_t] gradient, + np.ndarray[np.float32_t] mom1, np.ndarray[np.float32_t] mom2, + const float beta1, const float beta2, float eps, + float learn_rate, float mod_rate=1.): + _check_compatible_shape(weights, gradient) + _check_compatible_shape(weights, mom1) + _check_compatible_shape(weights, mom2) + + _adam_momentum(gradient.data, mom1.data, mom2.data, + weights.shape[0], beta1, beta2, eps, learn_rate) + VecVec.add_i(weights.data, + gradient.data, -learn_rate, weights.shape[0]) + memset(gradient.data, 0, gradient.size * sizeof(float)) + return weights, gradient, mom1, mom2 + + def ngrams(self, int n, const uint64_t[::1] keys): + if n < 1: + return self.alloc((0,), dtype="uint64") + keys_ = &keys[0] + length = max(0, keys.shape[0]-(n-1)) + cdef np.ndarray output_ = self.alloc((length,), dtype="uint64") + output = output_.data + for i in range(keys.shape[0]-(n-1)): + output[i] = hash64(&keys_[i], n*sizeof(keys_[0]), 0) + return output_ + + def position_encode(self, int N, int D, int period=10000, out=None): + cdef np.ndarray out_ + if out is None: + out_ = self.alloc((N, D), zeros=False) + else: + out_ = out + assert out_.shape[0] == N + assert out_.shape[1] == D + cpu_position_encode(out_.data, period, N, D) + return out_ + + +def check_seq2col_lengths(ops, lengths, B): + if lengths is None: + lengths = ops.asarray1i([B]) + else: + assert ops.xp.all(ops.xp.array(lengths) >= 0), "All sequence lengths must be >= 0" + assert ops.xp.sum(lengths) == B, "The lengths must sum up to the batch length" + + return lengths + + +def cpu_clip_gradient(weight_t[::1] gradient, weight_t threshold): + grad_norm = Vec.norm(&gradient[0], gradient.shape[0]) + if grad_norm >= threshold: + Vec.mul_i(&gradient[0], threshold / grad_norm, gradient.shape[0]) + + +def add_gradient_noise(float[::1] gradient, weight_t noise_level, + weight_t timestep): + cdef weight_t variance = noise_level / ((1 + timestep) ** 0.55) + if variance >= 0.000001: + gradient += numpy.asarray( + numpy.random.normal(scale=variance, loc=0., size=len(gradient)), + dtype='float32') + + +cdef void cpu_position_encode(float* output, float period, int N, int D) noexcept nogil: + cdef float pos, d + cdef int j + cdef float dimensions = D + for i in range(N): + pos = i + j = 0 + d = 0 + while (j+1) < D: + d = j + output[j] = sinf(pos / period ** (2 * d / dimensions)) + output[j+1] = cosf(pos / period ** (2 * d / dimensions)) + j += 2 + if j < D: + output[j] = sinf(pos / period ** (2 * d / dimensions)) + output += D + + +cdef void cpu_scatter_add(float* dest, + const int* indices, const float* src, + int nr_id, int nr_col) noexcept nogil: + cdef int i + for i in range(nr_id): + id_ = indices[i] + if id_ >= 0: + VecVec.add_i(&dest[id_*nr_col], + &src[i*nr_col], 1., nr_col) + + +@cython.cdivision(True) +cdef void _adam_momentum(weight_t* gradient, weight_t* mom1, weight_t* mom2, + int nr_weight, weight_t beta1, weight_t beta2, weight_t eps, + weight_t learn_rate) noexcept nogil: + # Calculate Adam on CPU, fused. + # Assumes the learning rate adjustment is calculated by the caller; + # a_t = learn_rate * sqrt(1-beta2**timestep) / (1-beta1**timestep) + cdef weight_t one_minus_beta1 = 1-beta1 + cdef weight_t one_minus_beta2 = 1-beta2 + cdef weight_t m1, m2, g + cdef int i + # Blockwise implementation is a bit faster. Adam is slooow :( + cdef weight_t[64] buff + cdef int steps = nr_weight // 64 + if steps * 64 < nr_weight: + steps += 1 + idx = 0 + for i in range(steps): + step_size = min(64, nr_weight-idx) + Vec.mul_i(mom1, beta1, step_size) + VecVec.add_i(mom1, gradient, one_minus_beta1, step_size) + Vec.mul_i(mom2, beta2, step_size) + for j in range(step_size): + mom2[j] += one_minus_beta2 * gradient[j] ** 2 + for j in range(step_size): + buff[j] = sqrtf(mom2[j]) + for j in range(step_size): + buff[j] += eps + for j in range(step_size): + buff[j] = mom1[j] / buff[j] + for j in range(step_size): + gradient[j] = buff[j] + mom1 += step_size + mom2 += step_size + gradient += step_size + idx += step_size + + +@cython.cdivision(True) +cdef void cpu_update_averages(weight_t* ema, + const weight_t* weights, int nr_weight, weight_t t, weight_t max_decay) noexcept nogil: + cdef weight_t decay = (1.0 + t) / (10.0 + t) + if decay > max_decay: + decay = max_decay + cdef weight_t one_minus_decay = 1-decay + cdef int i + for i in range(nr_weight): # num_threads=4, schedule='static'): + ema[i] -= one_minus_decay * (ema[i] - weights[i]) + + +def lstm_forward_training( + np.ndarray params, np.ndarray c_init, np.ndarray h_init, + np.ndarray X, np.ndarray lengths +): + xp = numpy + depth = c_init.shape[0] + dirs = c_init.shape[1] + nO = c_init.shape[2] + N = X.shape[0] + nI = X.shape[1] + nT = lengths.shape[0] + cdef int batch_size = lengths[0] + # Preallocate these so we can pass them through for loop. + cdef np.ndarray G = xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f") + cdef np.ndarray Y = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + cdef np.ndarray C = xp.zeros((depth, dirs, X.shape[0], nO), dtype="f") + cdef np.ndarray Yt2 = numpy.zeros((batch_size, nO), dtype="f") + cdef np.ndarray Ct2 = numpy.zeros((batch_size, nO), dtype="f") + + cdef int params_i = 0 + cdef int seq_i = 0 + orig_X = X + cdef int i + cdef np.ndarray Yid + cdef np.ndarray Cid + cdef np.ndarray Gid + cdef np.ndarray Wx + cdef np.ndarray Wh + cdef np.ndarray bias + for i in range(depth): + nI = X.shape[1] + for d in range(dirs): + # The inits are shaped (depth, dirs, nO). We add the internal dimension + # to make them set correctly. + Yt2[:] = h_init[i, d].reshape((1, nO)) + Ct2[:] = c_init[i, d].reshape((1, nO)) + layer_params, params_i = _split_weights(params, i, nO, nI, params_i) + Wx, Wh, bias = _transpose_weights(layer_params) + Yid = Y[i, d] + Cid = C[i, d] + Gid = G[i, d] + _lstm_forward_training( + d, N, nO, nI, nT, + Gid, + Yid.data, + Cid.data, + X.data, + Wx.data, + Wh.data, + bias, + lengths.data, + Yt2.data, + Ct2.data + ) + H = Y[i].transpose((1, 0, 2)).reshape((N, -1)) + if dirs == 2: + H = xp.ascontiguousarray(H) + X = H + return H, (Y, G, C, orig_X) + + +cdef int _lstm_forward_training( + int d, int N, int nO, int nI, int nT, + np.ndarray G, + float* Y, + float* C, + float* X, + float* Wx, + float* Wh, + np.ndarray bias, + int* lengths, + float* Yt2, + float* Ct2, +) except -1: + cdef double one = 1.0 + blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, + N, nO*4, nI, + one, + X, nI, 1, + Wx, nI, 1, + one, + G.data, nO*4, 1 + ) + cdef int t, batch_size + cdef int seq_i = 0 if d == 0 else N + cdef int i, j + cdef np.ndarray Gt3_ + for t in range(nT): + if d == 0: + batch_size = lengths[t] + else: + batch_size = lengths[nT-(t+1)] + seq_i -= batch_size + # Prepare the inputs + Yt3 = &Y[seq_i*nO] + Ct3 = &C[seq_i*nO] + Gt3_ = G[seq_i : seq_i+batch_size] + Gt3 = Gt3_.data + # Now do the actual calculation + blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.TRANSPOSE, + batch_size, nO*4, nO, + one, + Yt2, nO, 1, + Wh, nO, 1, + one, + Gt3, nO*4, 1 + ) + # This is super weird: if we remove this add, it gets slower? I guess + # it does cache prefetching or something? + # It's annoying though --- it means I can't really refactor further, + # because speed goes down if I remove this. + Gt3_ += bias + #for i in range(batch_size): + # for j in range(nO*4): + # Gt3[i*nO*4+j] += bias[j] + cpu_lstm_activate_fwd(Gt3, + batch_size, nO) + cpu_lstm_gates_fwd(Yt3, Ct3, + Gt3, Ct2, batch_size, nO) + if d == 0: + seq_i += batch_size + # We need to keep a full-sized array here, padded with the sequence-start + # values. This isn't necessary for the l2r part, but for the r2l part + # it's necessary, as we otherwise would have the previous step smaller + # than the current. + memcpy(Yt2, Yt3, sizeof(Yt3[0]) * batch_size * nO) + memcpy(Ct2, Ct3, sizeof(Ct3[0]) * batch_size * nO) + + +def backprop_lstm(np.ndarray dY, np.ndarray lengths, np.ndarray params, fwd_state): + xp = numpy + cdef np.ndarray Y + cdef np.ndarray G + cdef np.ndarray C + cdef np.ndarray X + cdef np.ndarray Yid + cdef np.ndarray Cid + cdef np.ndarray Gid + cdef np.ndarray Wx, Wh, bias + cdef np.ndarray dWx, dWh, d_bias + cdef np.ndarray dYid + Y, G, C, X = fwd_state + cdef int depth = C.shape[0] + cdef int dirs = C.shape[1] + cdef int N = C.shape[2] + cdef int nO = C.shape[3] + cdef int nI = X.shape[1] + cdef int batch_size = lengths[0] + cdef int nT = lengths.shape[0] + # We don't need to store all the cells for all the layers. + cdef np.ndarray dC = xp.zeros((N, nO), dtype=C.dtype) + cdef np.ndarray dG = xp.zeros((N, nO*4), dtype=C.dtype) + cdef np.ndarray d_params = xp.zeros((params.shape[0],), dtype=params.dtype) + # Collect the params and slices. It makes it a bit easier to get the indexing + # right, when we're iterating backwards. + params_i = 0 + all_layer_params = [] + for i in range(depth): + all_layer_params.append([]) + n_inputs = nI if i == 0 else (nO * dirs) + for d in range(dirs): + layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) + layer_params = _transpose_weights(layer_params) + all_layer_params[-1].append((layer_params, params_i)) + params_i = 0 + all_layer_grads = [] + for i in range(depth): + all_layer_grads.append([]) + n_inputs = nI if i == 0 else (nO * dirs) + for d in range(dirs): + layer_grads, params_i = _split_weights(params, i, nO, n_inputs, params_i) + layer_grads = _transpose_weights(layer_grads) + all_layer_grads[-1].append((layer_grads, params_i)) + # Similarly, we want to compute the indices first + indices = [] + seq_i = 0 + for batch_size in lengths: + indices.append((seq_i, batch_size)) + seq_i += batch_size + + cdef np.ndarray dX + Xs = [X] + [Y[i].transpose(1, 0, 2).reshape((N, -1)) for i in range(depth-1)] + dXs = [xp.zeros((X.shape[0], X.shape[1]), dtype=X.dtype) for X in Xs] + # Okay, now do the actual looping + for i in reversed(range(depth)): + dY = dY.reshape((N, dirs, nO)).transpose((1, 0, 2)) + dX = dXs[i] + X = Xs[i] + if dirs >= 2: + dY = numpy.ascontiguousarray(dY) + for d in range(dirs): + Wx, Wh, bias = all_layer_params[i][d][0] + dWx, dWh, d_bias = all_layer_grads[i][d][0] + assert Wx.shape[1] == dWx.shape[1] == X.shape[1] == dX.shape[1], (Wx.shape[1], dWx.shape[1], X.shape[1], dX.shape[1]) + dYid = dY[d] + dC.fill(0.) + dG.fill(0.) + Cid = C[i, d] + Gid = G[i, d] + Yid = Y[i, d] + assert (Cid.shape[0], Cid.shape[1]) == (N, nO) + assert (Yid.shape[0], Yid.shape[1]) == (N, nO) + assert (Gid.shape[0], Gid.shape[1]) == (N, nO*4) + assert (dYid.shape[0], dYid.shape[1]) == (N, nO) + assert (dC.shape[0], dC.shape[1]) == (N, nO) + assert (dG.shape[0], dG.shape[1]) == (N, nO*4) + _lstm_backward_training(d, N, nO, dX.shape[1], nT, + dX.data, + dYid.data, + dC.data, + dG.data, + dWx.data, + dWh.data, + d_bias.data, + Cid.data, + Gid.data, + Yid.data, + X.data, + Wx.data, + Wh.data, + list(indices) + ) + dY = dX + assert dX.shape[1] == X.shape[1] + grad_parts = [] + for layer_grads in all_layer_grads: + for dir_grads, _ in layer_grads: + grad_parts.append(_untranspose_unsplit_weights(dir_grads)) + return dX, numpy.concatenate(grad_parts) + + +def _split_directions(X, dirs): + if dirs == 1: + return [X] + else: + X_ = X.reshape((X.shape[0], -1, dirs)) + Xs = [] + for d in range(dirs): + Xs.append(numpy.ascontiguousarray(X_[:, d])) + return Xs + + +cdef int _lstm_backward_training( + int d, int N, int nO, int nI, int nT, + float* dX, + float* dY, + float* dC, + float* dG, + float* dWx, + float* dWh, + float* d_bias, + const float* C, + const float* G, + const float* Y, + const float* X, + const float* Wx, + const float* Wh, + indices, +) except -1: + cdef int seq_t2 + cdef int seq_t3 + cdef double one = 1.0 + if d == 0: + seq_t3, size_t3 = indices[-1] + indices = indices[:-1] + indices.reverse() + else: + seq_t3, size_t3 = indices[0] + indices = indices[1:] + cdef int batch_size + for seq_t2, size_t2 in indices: + dGt3 = &dG[seq_t3*nO*4] + dXt3 = &dX[seq_t3*nI] + dYt3 = &dY[seq_t3*nO] + dCt3 = &dC[seq_t3*nO] + dYt2 = &dY[seq_t2*nO] + dCt2 = &dC[seq_t2*nO] + Ct3 = &C[seq_t3*nO] + Gt3 = &G[seq_t3*nO*4] + Ct2 = &C[seq_t2*nO] + + batch_size = min(size_t2, size_t3) + cpu_lstm_gates_bwd(dGt3, dCt2, + dYt3, dCt3, Gt3, Ct3, Ct2, batch_size * nO + ) + # Backprop hidden-to-hidden w.r.t. hidden. + # dYt2 += dGt3 @ Wh + blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.NO_TRANSPOSE, + batch_size, nO, nO*4, + one, + dGt3, nO*4, 1, + Wh, nO, 1, + one, + dYt2, nO, 1 + ) + seq_t3 = seq_t2 + size_t3 = size_t2 + + # Backprop input-to-hidden w.r.t. weights. + # dWx += dG @ X + blis.cy.gemm(blis.cy.TRANSPOSE, blis.cy.NO_TRANSPOSE, + nO*4, nI, N, + one, + dG, nO*4, 1, + X, nI, 1, + one, + dWx, nI, 1 + ) + # Backprop hidden-to-hidden w.r.t weights. + # dWh += dG @ Y + blis.cy.gemm(blis.cy.TRANSPOSE, blis.cy.NO_TRANSPOSE, + nO*4, nO, N, + one, + dG, nO*4, 1, + Y, nO, 1, + one, + dWh, nO, 1 + ) + # Backprop bias + for i in range(N): + for j in range(nO*4): + d_bias[j] += dG[i*nO*4+j] + + # Backprop input-to-hidden w.r.t. input + blis.cy.gemm(blis.cy.NO_TRANSPOSE, blis.cy.NO_TRANSPOSE, + N, nI, nO*4, + one, + dG, nO*4, 1, + Wx, nI, 1, + one, + dX, nI, 1 + ) + + +def _split_weights(np.ndarray params, int i, int nO, int nI, int params_i): + Wx_size = 4 * nO * nI + bx_size = 4 * nO + Wh_size = 4 * nO * nO + bh_size = 4 * nO + Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) + params_i += Wx_size + bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) + params_i += bx_size + Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) + params_i += Wh_size + bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) + params_i += bh_size + return ((Wx, bx), (Wh, bh)), params_i + + +def _transpose_weights(params): + # Transpose the parameters so that the gates are the last dimension. This + # makes it easier to fuse. + (Wx, bx), (Wh, bh) = params + Wx = Wx.reshape((4, -1, Wx.shape[-1])) + Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + Wh = Wh.reshape((4, -1, Wh.shape[-1])) + Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) + bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + ascontig = numpy.ascontiguousarray + Wx = ascontig(Wx) + Wh = ascontig(Wh) + bias = ascontig(bx) + bh + return Wx, Wh, bias + + +def _untranspose_unsplit_weights(params): + Wx, Wh, bias = params + nO = Wh.shape[1] + nI = Wx.shape[1] + Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) + zeros = numpy.zeros(bias.shape, dtype="f") + return numpy.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) + + +cdef inline float sigmoid(float X) noexcept nogil: + return 1./(1. + expf(-X)) + + +cdef inline float dsigmoid(float y) noexcept nogil: + return y*(1-y) + + +cdef inline float dtanh(float y) noexcept nogil: + return 1-y**2 + + +cdef void cpu_lstm_activate_fwd(float* gates, int B, int N) noexcept nogil: + """Apply sigmoid activation in-place to columns 0, 1, 2 and tanh to column 3. + The data is assumed to have the gates in the last dimension. + """ + # This just does the following, but unrolled slightly to give + # a better chance at simd. + # + # gates[g+i+0] = sigmoid(gates[g+i+0]) + # gates[g+i+1] = sigmoid(gates[g+i+1]) + # gates[g+i+2] = sigmoid(gates[g+i+2]) + # gates[g+i+3] = tanh(gates[g+i+3]) + # + # I would've hoped the compiler would find this itself? It seems to make + # it like, 10% faster. It feels like a dumb thing to do but it's not much + # code. The problem with this sort of thing is it needs to be rebenchmarked + # later...It's fine to revert this at a later date to the simpler loop. + # Shrug. The weird thing is, why should the batch entries be a good loop + # stride here? Surely something to do with cache lines would make more sense? + cdef int i, b, g + g = 0 + for b in range(B): + g = b * N * 4 + end = g + N*4 + while g < end: + gates[g+0] = expf(-gates[g+0]) + gates[g+1] = expf(-gates[g+1]) + gates[g+2] = expf(-gates[g+2]) + g += 4 + g = b * N * 4 + while g < end: + gates[g+0] += 1 + gates[g+1] += 1 + gates[g+2] += 1 + g += 4 + g = b * N * 4 + while g < end: + gates[g+0] = 1.0 / gates[g+0] + gates[g+1] = 1.0 / gates[g+1] + gates[g+2] = 1.0 / gates[g+2] + g += 4 + g = b * N * 4 + while g < end: + gates[g+3] = tanhf(gates[g+3]) + g += 4 + + +cdef void cpu_lstm_gates_fwd(float* hiddens, float* cells, + const float* gates, const float* prevcells, int B, int N) noexcept nogil: + cdef float hf, hi, ho, hc, ct2, ct3 + cdef int i, b, g, c, h + g = 0 + c = 0 + h = 0 + while g < B*N*4: + hf = gates[g+0] + hi = gates[g+1] + ho = gates[g+2] + hc = gates[g+3] + ct2 = prevcells[c] + ct3 = hf * ct2 + hi * hc + hiddens[h] = tanhf(ct3) * ho + cells[c] = ct3 + g += 4 + c += 1 + h += 1 + + +cdef void cpu_lstm_gates_bwd( + float* dGt3, + float* dCt2, + const float* dYt3, + const float* dCt3, + const float* Gt3, + const float* Ct3, + const float* Ct2, + int N +) noexcept nogil: + cdef int i + cdef float ct2, ct3, hf, hi, ho, hc, tanh_ct3 + cdef float d_ho, d_tanh_ct3, dct3, d_hi, d_hc, d_hf + for i in range(N): + ct2 = Ct2[i] + ct3 = Ct3[i] + dct3 = dCt3[i] + dyt3 = dYt3[i] + hf = Gt3[i*4+0] + hi = Gt3[i*4+1] + ho = Gt3[i*4+2] + hc = Gt3[i*4+3] + + tanh_ct3 = tanhf(ct3) + # 3b: Yt3 = tanhCt3 * ho + d_ho = dyt3 * tanh_ct3 + d_tanh_ct3 = dyt3 * ho + # 3a: tanhCt3 = tanh(Ct3) + dct3 += d_tanh_ct3 * dtanh(tanh_ct3) + # 2b: Ct3 += hi * hc + d_hi = dct3 * hc + d_hc = dct3 * hi + # 2a: Ct3 = hf * Ct2 + d_hf = dct3 * ct2 + dCt2[i] = dct3 * hf + dGt3[i*4+0] = d_hf * dsigmoid(hf) # 1a + dGt3[i*4+1] = d_hi * dsigmoid(hi) # 1b + dGt3[i*4+2] = d_ho * dsigmoid(ho) # 1c + dGt3[i*4+3] = d_hc * dtanh(hc) # 1d + + +cdef void MurmurHash3_x86_128_uint64( + const uint64_t val, + const uint32_t seed, + uint32_t *out +) noexcept nogil: + cdef uint64_t h1, h2 + + h1 = val + h1 *= 0x87c37b91114253d5ull + h1 = (h1 << 31) | (h1 >> 33) + h1 *= 0x4cf5ad432745937full + h1 ^= seed + h1 ^= 8 + h2 = seed + h2 ^= 8 + h1 += h2 + h2 += h1 + h1 ^= h1 >> 33 + h1 *= 0xff51afd7ed558ccdull + h1 ^= h1 >> 33 + h1 *= 0xc4ceb9fe1a85ec53ull + h1 ^= h1 >> 33 + h2 ^= h2 >> 33 + h2 *= 0xff51afd7ed558ccdull + h2 ^= h2 >> 33 + h2 *= 0xc4ceb9fe1a85ec53ull + h2 ^= h2 >> 33 + h1 += h2 + h2 += h1 + + out[0] = h1 & 0xffffffffu + out[1] = h1 >> 32 + out[2] = h2 & 0xffffffffu + out[3] = h2 >> 32 + + +def _check_compatible_shape(u: np.ndarray, v: np.ndarray): + if u.shape != v.shape: + msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + raise ValueError(msg) + + +cdef inline np.ndarray _inplace_or_copy(np.ndarray X, inplace): + if inplace: + return X + else: + return numpy.array(X) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/ops.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f51fa380f8e7215ccdd89de6ff57f186725102fc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/backends/ops.py @@ -0,0 +1,1708 @@ +import itertools +import math +from typing import ( + Any, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + cast, + overload, +) + +import numpy + +from ..types import ( + Array1d, + Array2d, + Array3d, + Array4d, + ArrayXd, + Batchable, + DeviceTypes, + DTypes, + DTypesFloat, + DTypesInt, + Floats1d, + Floats2d, + Floats3d, + Floats4d, + FloatsXd, + FloatsXdT, + Generator, + Ints1d, + Ints2d, + Ints3d, + Ints4d, + IntsXd, + List2d, + ListXd, + Padded, + Shape, + SizedGenerator, + Xp, + _Floats, +) +from ..util import get_array_module, is_xp_array, to_numpy +from .cblas import CBlas + +ArrayT = TypeVar("ArrayT", bound=ArrayXd) +FloatsT = TypeVar("FloatsT", bound=_Floats) +SQRT2PI = math.sqrt(2.0 / math.pi) +INV_SQRT2 = 1.0 / math.sqrt(2.0) +INV_SQRT_2PI = 1.0 / math.sqrt(2.0 * math.pi) + + +class Ops: + name: str = "base" + xp: Xp = numpy + + def __init__( + self, device_type: DeviceTypes = "cpu", device_id: int = -1, **kwargs + ) -> None: + self.device_type = device_type + self.device_id = device_id + + def cblas(self) -> CBlas: + """Return C BLAS function table.""" + err = f"{type(self).__name__} does not provide C BLAS functions" + raise NotImplementedError(err) + + def to_numpy(self, data, *, byte_order=None): # pragma: no cover + if isinstance(data, numpy.ndarray): + if byte_order: + dtype = data.dtype.newbyteorder(byte_order) + data = numpy.asarray(data, dtype=dtype) + return data + else: + raise ValueError("Cannot convert non-numpy from base Ops class") + + def minibatch( + self, + size: Union[int, Generator], + sequence: Batchable, + *, + shuffle: bool = False, + buffer: int = 1, + ) -> SizedGenerator: + """Iterate slices from a sequence, optionally shuffled. Slices + may be either views or copies of the underlying data. + + The `size` argument may be either an integer, or a sequence of integers. + If a sequence, a new size is drawn before every output. + + If shuffle is True, shuffled batches are produced by first generating + an index array, shuffling it, and then using it to slice into the + sequence. + + An internal queue of `buffer` items is accumulated before being each + output. Buffering is useful for some devices, to allow the + network to run asynchronously without blocking on every batch. + """ + if not hasattr(sequence, "__len__"): + err = f"Can't minibatch data. Expected sequence, got {type(sequence)}" + raise ValueError(err) + sizes = self._get_batch_sizes( + len(sequence), itertools.repeat(size) if isinstance(size, int) else size + ) + indices = numpy.arange(len(sequence)) + + # This is a bit convoluted, but it's a time where convenience makes + # trickery worthwhile: instead of being an actual generator, we + # return our SizedGenerator object, which provides a __len__. + def _iter_items(): + if shuffle: + numpy.random.shuffle(indices) + queue = [] + i = 0 + for size in sizes: + size = int(size) + queue.append(self._get_batch(sequence, indices[i : i + size])) + if len(queue) >= buffer: + yield from queue + queue = [] + i += size + yield from queue + + return SizedGenerator(_iter_items, len(sizes)) + + def multibatch( + self, + size: Union[int, Generator], + sequence: Batchable, + *others: Batchable, + shuffle: bool = False, + buffer: int = 1, + ) -> SizedGenerator: + """Minibatch one or more sequences of data, and yield + lists with one batch per sequence. See ops.minibatch. + """ + # You'd think we could just do this by calling into minibatch and zip... + # But the shuffling makes it really hard. + sequences = (sequence,) + tuple(others) + if not all(hasattr(seq, "__len__") for seq in sequences): + values = ", ".join([f"{type(seq)}" for seq in sequences]) + err = f"Can't multibatch data. Expected sequences, got {values}" + raise ValueError(err) + sizes = self._get_batch_sizes( + len(sequence), itertools.repeat(size) if isinstance(size, int) else size + ) + indices = numpy.arange(len(sequence)) + + def _iter_items(): + if shuffle: + numpy.random.shuffle(indices) + queue = [] + i = 0 + for size in sizes: + size = int(size) + idx_batch = indices[i : i + size] + queue.append([]) + for sequence in sequences: + queue[-1].append(self._get_batch(sequence, idx_batch)) + if len(queue) >= buffer: + yield from queue + queue = [] + i += size + yield from queue + + return SizedGenerator(_iter_items, len(sizes)) + + def _get_batch(self, sequence, indices): + if isinstance(sequence, list): + subseq = [sequence[i] for i in indices] + elif isinstance(sequence, tuple): + subseq = tuple(sequence[i] for i in indices) + else: + subseq = sequence[indices] + if is_xp_array(subseq): + subseq = self.as_contig(self.xp.asarray(subseq)) + return subseq + + def _get_batch_sizes(self, length: int, sizes: Iterator[int]): + output = [] + i = 0 + while i < length: + output.append(next(sizes)) + i += output[-1] + return output + + def seq2col( + self, seq: Floats2d, nW: int, *, lengths: Optional[Ints1d] = None + ) -> Floats2d: + """Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1)) + sequence. The new sequence is constructed by concatenating nW preceding + and succeeding vectors onto each column in the sequence, to extract a + window of features. + """ + # This is a test implementation that only supports nW=1 and lengths=None + assert nW == 1 + assert lengths == None + B = seq.shape[0] + I = seq.shape[1] + cols = self.alloc3f(B, (nW * 2 + 1), I) + # Copy left contexts. The last words aren't the left-context for anything. + cols[nW:, :nW] = self.reshape3f(seq[:-nW], -1, nW, I) + cols[:, nW] = seq + cols[:-nW, nW + 1 :] = self.reshape3f(seq[nW:], -1, nW, I) + return self.reshape2f(cols, B, I * (2 * nW + 1)) + + def backprop_seq2col( + self, dY: Floats2d, nW: int, *, lengths: Optional[Ints1d] = None + ) -> Floats2d: + """The reverse/backward operation of the `seq2col` function: calculate + the gradient of the original `(M, N)` sequence, as a function of the + gradient of the output `(M, N*(nW*2+1))` sequence. + """ + # This is a test implementation that only supports nW=1 and lengths=None + assert nW == 1 + assert lengths == None + nF = nW * 2 + 1 + B = dY.shape[0] + I = dY.shape[1] // nF + # Having trouble getting the kernel to work... + dX = self.alloc2f(B, I) + dY3d = self.reshape3f(dY, B, nF, I) + dX[:-nW] += self.reshape2f(dY3d[nW:, :nW], -1, I) + dX += dY3d[:, nW] + dX[nW:] += self.reshape2f(dY3d[:-nW, nW + 1 :], -1, I) + return dX + + def gemm( + self, + x: Floats2d, + y: Floats2d, + out: Optional[Floats2d] = None, + trans1: bool = False, + trans2: bool = False, + ) -> Floats2d: + """Perform General Matrix Multiplication (GeMM) and optionally store + the result in the specified output variable. + """ + if trans1: + x = x.T + if trans2: + y = y.T + if out is None: + return self.xp.dot(x, y) + else: + self.xp.dot(x, y, out=out) + return out + + def tile(self, X: Floats2d, reps: int) -> Floats2d: + return self.xp.tile(X, reps) + + def affine(self, X: Floats2d, W: Floats2d, b: Floats1d) -> Floats2d: + """Apply a weights layer and a bias to some inputs, i.e. + Y = X @ W.T + b + """ + Y = self.gemm(X, W, trans2=True) + Y += b + return Y + + @overload + def flatten( + self, + X: List[Floats2d], + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> Floats2d: + ... + + @overload + def flatten( + self, + X: List[Ints1d], + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> Ints1d: + ... + + @overload + def flatten( + self, + X: List2d, + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> Array2d: + ... + + # further specific typed signatures can be added as necessary + + @overload + def flatten( + self, + X: ListXd, + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> ArrayXd: + ... + + @overload + def flatten( + self, + X: Sequence[ArrayXd], + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> ArrayXd: + ... + + def flatten( + self, + X: Sequence[ArrayXd], + dtype: Optional[DTypes] = None, + pad: int = 0, + ndim_if_empty: int = 2, + ) -> ArrayXd: + """Flatten a list of arrays into one large array.""" + if X is None or len(X) == 0: + return self.alloc((0,) * ndim_if_empty, dtype=dtype or "f") + xp = get_array_module(X[0]) + shape_if_empty = X[0].shape + X = [x for x in X if x.size != 0] + if len(X) == 0: + return self.alloc(shape_if_empty, dtype=dtype or "f") + if int(pad) >= 1: + padded = [] + for x in X: + padded.append(xp.zeros((pad,) + x.shape[1:], dtype=x.dtype)) + padded.append(x) + padded.append(xp.zeros((pad,) + x.shape[1:], dtype=x.dtype)) + X = padded + result = xp.concatenate(X) + if dtype is not None: + result = xp.asarray(result, dtype=dtype) + return result + + @overload + def unflatten(self, X: Floats2d, lengths: Ints1d, pad: int = 0) -> List[Floats2d]: + ... + + @overload + def unflatten(self, X: Ints1d, lengths: Ints1d, pad: int = 0) -> List[Ints1d]: + ... + + @overload + def unflatten(self, X: Array2d, lengths: Ints1d, pad: int = 0) -> List2d: + ... + + # further specific typed signatures can be added as necessary + + @overload + def unflatten(self, X: ArrayXd, lengths: Ints1d, pad: int = 0) -> ListXd: + ... + + def unflatten(self, X: ArrayXd, lengths: Ints1d, pad: int = 0) -> ListXd: + """The reverse/backward operation of the `flatten` function: unflatten + a large array into a list of arrays according to the given lengths. + """ + # cupy.split requires lengths to be in CPU memory. + lengths = to_numpy(lengths) + + if pad > 0: + lengths = numpy.where(lengths > 0, lengths + pad, 0) # type: ignore + unflat = self.xp.split(X, numpy.cumsum(lengths))[:-1] # type: ignore + if pad > 0: + unflat = [a[pad:] for a in unflat] + + assert len(unflat) == len(lengths) + + return unflat + + @overload + def pad(self, seqs: List[Ints2d], round_to=1) -> Ints3d: + ... + + @overload # noqa: F811 + def pad(self, seqs: List[Floats2d], round_to=1) -> Floats3d: + ... + + def pad( # noqa: F811 + self, seqs: Union[List[Ints2d], List[Floats2d]], round_to=1 + ) -> Array3d: + """Perform padding on a list of arrays so that they each have the same + length, by taking the maximum dimension across each axis. This only + works on non-empty sequences with the same `ndim` and `dtype`. + """ + if round_to < 1: + raise ValueError( + f"Rounding for padding must at least be 1, was: {round_to}" + ) + + # TODO: This should be generalized to handle different ranks + if not seqs: + raise ValueError("Cannot pad empty sequence") + if len(set(seq.ndim for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences with different ndims") + if len(set(seq.dtype for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences with different dtypes") + if len(set(seq.shape[1:] for seq in seqs)) != 1: + raise ValueError("Cannot pad sequences that differ on other dimensions") + # Find the maximum dimension along each axis. That's what we'll pad to. + max_seq_len = max(len(seq) for seq in seqs) + # Round the length to nearest bucket -- helps on GPU, to make similar + # array sizes. + max_seq_len += -max_seq_len % round_to + final_shape = (len(seqs), max_seq_len) + seqs[0].shape[1:] + output: Array3d = cast(Array3d, self.alloc(final_shape, dtype=seqs[0].dtype)) + for i, arr in enumerate(seqs): + # It's difficult to convince this that the dtypes will match. + output[i, : arr.shape[0]] = arr # type: ignore[assignment, call-overload] + return output + + def unpad(self, padded: Array3d, lengths: List[int]) -> List2d: + """The reverse/backward operation of the `pad` function: transform an + array back into a list of arrays, each with their original length. + """ + output = [] + for i, length in enumerate(lengths): + output.append(padded[i, :length]) + return cast(List2d, output) + + def list2padded(self, seqs: List2d) -> Padded: + """Pack a sequence of 2d arrays into a Padded datatype.""" + if not seqs: + return Padded( + self.alloc3f(0, 0, 0), self.alloc1i(0), self.alloc1i(0), self.alloc1i(0) + ) + elif len(seqs) == 1: + data = self.reshape3(seqs[0], seqs[0].shape[0], 1, seqs[0].shape[1]) + size_at_t = self.asarray1i([1] * data.shape[0]) + lengths = self.asarray1i([data.shape[0]]) + indices = self.asarray1i([0]) + return Padded(data, size_at_t, lengths, indices) + lengths_indices = [(len(seq), i) for i, seq in enumerate(seqs)] + lengths_indices.sort(reverse=True) + indices_ = [i for length, i in lengths_indices] + lengths_ = [length for length, i in lengths_indices] + nS = max([seq.shape[0] for seq in seqs]) + nB = len(seqs) + nO = seqs[0].shape[1] + # Reorder the sequences, by length. This looks the same in either + # direction: you're swapping elements between their original and sorted + # position. + seqs = cast(List2d, [seqs[i] for i in indices_]) + arr: Array3d = self.pad(seqs) + assert arr.shape == (nB, nS, nO), (nB, nS, nO) + arr = self.as_contig(arr.transpose((1, 0, 2))) + assert arr.shape == (nS, nB, nO) + # Build a lookup table so we can find how big the batch is at point t. + batch_size_at_t_ = [0 for _ in range(nS)] + current_size = len(lengths_) + for t in range(nS): + while current_size and t >= lengths_[current_size - 1]: + current_size -= 1 + batch_size_at_t_[t] = current_size + assert sum(lengths_) == sum(batch_size_at_t_) + return Padded( + arr, + self.asarray1i(batch_size_at_t_), + self.asarray1i(lengths_), + self.asarray1i(indices_), + ) + + def padded2list(self, padded: Padded) -> List2d: + """Unpack a Padded datatype to a list of 2-dimensional arrays.""" + data = padded.data + indices = to_numpy(padded.indices) + lengths = to_numpy(padded.lengths) + unpadded: List[Optional[Array2d]] = [None] * len(lengths) + # Transpose from (length, batch, data) to (batch, length, data) + data = self.as_contig(data.transpose((1, 0, 2))) + for i in range(data.shape[0]): + unpadded[indices[i]] = data[i, : int(lengths[i])] + return cast(List2d, unpadded) + + def get_dropout_mask(self, shape: Shape, drop: Optional[float]) -> FloatsXd: + """Create a random mask for applying dropout, with a certain percent of + the mask (defined by `drop`) will contain zeros. The neurons at those + positions will be deactivated during training, resulting in a more + robust network and less overfitting. + """ + if drop is None or drop <= 0: + return self.xp.ones(shape, dtype="f") + elif drop >= 1.0: + return self.alloc_f(shape) + coinflips = self.xp.random.uniform(0.0, 1.0, shape) + mask = (coinflips >= drop) / (1.0 - drop) + return cast(FloatsXd, self.asarray(mask, dtype="float32")) + + def alloc1f( + self, + d0: int, + *, + dtype: Optional[DTypesFloat] = "float32", + zeros: bool = True, + ) -> Floats1d: + return cast(Floats1d, self.alloc((d0,), dtype=dtype, zeros=zeros)) + + def alloc2f( + self, + d0: int, + d1: int, + *, + dtype: Optional[DTypesFloat] = "float32", + zeros: bool = True, + ) -> Floats2d: + return cast(Floats2d, self.alloc((d0, d1), dtype=dtype, zeros=zeros)) + + def alloc3f( + self, + d0: int, + d1: int, + d2: int, + *, + dtype: Optional[DTypesFloat] = "float32", + zeros: bool = True, + ) -> Floats3d: + return cast(Floats3d, self.alloc((d0, d1, d2), dtype=dtype, zeros=zeros)) + + def alloc4f( + self, + d0: int, + d1: int, + d2: int, + d3: int, + *, + dtype: Optional[DTypesFloat] = "float32", + zeros: bool = True, + ) -> Floats4d: + return cast(Floats4d, self.alloc((d0, d1, d2, d3), dtype=dtype, zeros=zeros)) + + def alloc_f( + self, + shape: Shape, + *, + dtype: Optional[DTypesFloat] = "float32", + zeros: bool = True, + ) -> FloatsXd: + return cast(FloatsXd, self.alloc(shape, dtype=dtype, zeros=zeros)) + + def alloc1i( + self, + d0: int, + *, + dtype: Optional[DTypesInt] = "int32", + zeros: bool = True, + ) -> Ints1d: + return cast(Ints1d, self.alloc((d0,), dtype=dtype, zeros=zeros)) + + def alloc2i( + self, + d0: int, + d1: int, + *, + dtype: Optional[DTypesInt] = "int32", + zeros: bool = True, + ) -> Ints2d: + return cast(Ints2d, self.alloc((d0, d1), dtype=dtype, zeros=zeros)) + + def alloc3i( + self, + d0: int, + d1: int, + d2: int, + *, + dtype: Optional[DTypesInt] = "int32", + zeros: bool = True, + ) -> Ints3d: + return cast(Ints3d, self.alloc((d0, d1, d2), dtype=dtype, zeros=zeros)) + + def alloc4i( + self, + d0: int, + d1: int, + d2: int, + d3: int, + *, + dtype: Optional[DTypesInt] = "int32", + zeros: bool = True, + ) -> Ints4d: + return cast(Ints4d, self.alloc((d0, d1, d2, d3), dtype=dtype, zeros=zeros)) + + def alloc_i( + self, + shape: Shape, + *, + dtype: Optional[DTypesInt] = "int32", + zeros: bool = True, + ) -> IntsXd: + return cast(IntsXd, self.alloc(shape, dtype=dtype, zeros=zeros)) + + def alloc( + self, + shape: Shape, + *, + dtype: Optional[DTypes] = "float32", + zeros: bool = True, + ) -> Any: + """Allocate an array of a certain shape.""" + if isinstance(shape, int): + shape = (shape,) + + if zeros: + return self.xp.zeros(shape, dtype=dtype) + else: + return self.xp.empty(shape, dtype=dtype) + + def reshape1(self, array: ArrayXd, d0: int) -> Array1d: + return cast(Array1d, self.reshape(array, (d0,))) + + def reshape2(self, array: ArrayXd, d0: int, d1: int) -> Array2d: + return cast(Array2d, self.reshape(array, (d0, d1))) + + def reshape3(self, array: ArrayXd, d0: int, d1: int, d2: int) -> Array3d: + return cast(Array3d, self.reshape(array, (d0, d1, d2))) + + def reshape4(self, array: ArrayXd, d0: int, d1: int, d2: int, d3: int) -> Array4d: + return cast(Array4d, self.reshape(array, (d0, d1, d2, d3))) + + def reshape1f(self, array: FloatsXd, d0: int) -> Floats1d: + return cast(Floats1d, self.reshape(array, (d0,))) + + def reshape2f(self, array: FloatsXd, d0: int, d1: int) -> Floats2d: + return cast(Floats2d, self.reshape(array, (d0, d1))) + + def reshape3f(self, array: FloatsXd, d0: int, d1: int, d2: int) -> Floats3d: + return cast(Floats3d, self.reshape(array, (d0, d1, d2))) + + def reshape4f( + self, array: FloatsXd, d0: int, d1: int, d2: int, d3: int + ) -> Floats4d: + return cast(Floats4d, self.reshape(array, (d0, d1, d2, d3))) + + def reshape_f(self, array: FloatsXd, shape: Shape) -> FloatsXd: + return self.reshape(array, shape) + + def reshape1i(self, array: IntsXd, d0: int) -> Ints1d: + return cast(Ints1d, self.reshape(array, (d0,))) + + def reshape2i(self, array: IntsXd, d0: int, d1: int) -> Ints2d: + return cast(Ints2d, self.reshape(array, (d0, d1))) + + def reshape3i(self, array: IntsXd, d0: int, d1: int, d2: int) -> Ints3d: + return cast(Ints3d, self.reshape(array, (d0, d1, d2))) + + def reshape4i(self, array: IntsXd, d0: int, d1: int, d2: int, d3: int) -> Ints4d: + return cast(Ints4d, self.reshape(array, (d0, d1, d2, d3))) + + def reshape_i(self, array: IntsXd, shape: Shape) -> IntsXd: + return self.reshape(array, shape) + + def reshape(self, array: ArrayT, shape: Shape) -> ArrayT: + """Reshape an array.""" + if isinstance(shape, int): + shape = (shape,) + return cast(ArrayT, array.reshape(shape)) + + def asarray4f( + self, + data: Union[Floats4d, Sequence[Sequence[Sequence[Sequence[float]]]]], + *, + dtype: Optional[DTypes] = "float32", + ) -> Floats4d: + return cast(Floats4d, self.asarray(data, dtype=dtype)) + + def asarray3f( + self, + data: Union[Floats3d, Sequence[Sequence[Sequence[float]]]], + *, + dtype: Optional[DTypes] = "float32", + ) -> Floats3d: + return cast(Floats3d, self.asarray(data, dtype=dtype)) + + def asarray2f( + self, + data: Union[Floats2d, Sequence[Sequence[float]]], + *, + dtype: Optional[DTypes] = "float32", + ) -> Floats2d: + return cast(Floats2d, self.asarray(data, dtype=dtype)) + + def asarray1f( + self, + data: Union[Floats1d, Sequence[float]], + *, + dtype: Optional[DTypes] = "float32", + ) -> Floats1d: + return cast(Floats1d, self.asarray(data, dtype=dtype)) + + def asarray_f( + self, + data: Union[FloatsXd, Sequence[Any]], + *, + dtype: Optional[DTypes] = "float32", + ) -> FloatsXd: + return cast(FloatsXd, self.asarray(data, dtype=dtype)) + + def asarray1i( + self, data: Union[Ints1d, Sequence[int]], *, dtype: Optional[DTypes] = "int32" + ) -> Ints1d: + return cast(Ints1d, self.asarray(data, dtype=dtype)) + + def asarray2i( + self, + data: Union[Ints2d, Sequence[Sequence[int]]], + *, + dtype: Optional[DTypes] = "int32", + ) -> Ints2d: + return cast(Ints2d, self.asarray(data, dtype=dtype)) + + def asarray3i( + self, + data: Union[Ints3d, Sequence[Sequence[Sequence[int]]]], + *, + dtype: Optional[DTypes] = "int32", + ) -> Ints3d: + return cast(Ints3d, self.asarray(data, dtype=dtype)) + + def asarray4i( + self, + data: Union[Ints4d, Sequence[Sequence[Sequence[Sequence[int]]]]], + *, + dtype: Optional[DTypes] = "int32", + ) -> Ints4d: + return cast(Ints4d, self.asarray(data, dtype=dtype)) + + def asarray_i( + self, data: Union[IntsXd, Sequence[Any]], *, dtype: Optional[DTypes] = "int32" + ) -> IntsXd: + return cast(IntsXd, self.asarray(data, dtype=dtype)) + + def asarray( + self, + data: Union[ArrayXd, Sequence[ArrayXd], Sequence[Any]], + *, + dtype: Optional[DTypes] = None, + ) -> ArrayXd: + """Ensure a given array is of the correct type.""" + if isinstance(data, self.xp.ndarray): + if dtype is None: + return data + elif data.dtype == dtype: + return data + else: + return self.xp.asarray(data, dtype=dtype) + elif hasattr(data, "numpy"): + # Handles PyTorch Tensor + return data.numpy() # type: ignore[union-attr] + elif dtype is not None: + return self.xp.array(data, dtype=dtype) + else: + return self.xp.array(data) + + def as_contig(self, data: ArrayT, dtype: Optional[DTypes] = None) -> ArrayT: + """Allow the backend to make a contiguous copy of an array. + Implementations of `Ops` do not have to make a copy or make it + contiguous if that would not improve efficiency for the execution engine. + """ + if data.flags["C_CONTIGUOUS"] and dtype in (None, data.dtype): + return data + kwargs = {"dtype": dtype} if dtype is not None else {} + return self.xp.ascontiguousarray(data, **kwargs) + + def sigmoid(self, X: FloatsXdT, *, inplace: bool = False) -> FloatsXdT: + if inplace: + # To prevent overflows and help with regularization/numerical stability + X = self.xp.clip(X, -20.0, 20.0, out=X) + self.xp.exp(-X, out=X) + X += 1.0 + X **= -1.0 + return X + else: + X = self.xp.clip(X, -20.0, 20.0) + return 1.0 / (1.0 + self.xp.exp(-X)) + + def backprop_sigmoid( + self, dY: FloatsXdT, Y: FloatsXdT, *, inplace: bool = False + ) -> FloatsXdT: + if inplace: + self.dsigmoid(Y, inplace=True) + Y *= dY + return Y + else: + return dY * self.dsigmoid(Y, inplace=inplace) + + def dsigmoid(self, Y: FloatsXdT, *, inplace: bool = False) -> FloatsXdT: + if inplace: + Y *= 1 - Y + return Y + else: + return Y * (1.0 - Y) + + def dtanh(self, Y: FloatsT, *, inplace: bool = False) -> FloatsT: + if inplace: + Y **= 2 + Y *= -1.0 + Y += 1.0 + return Y + else: + return 1 - Y**2 + + def softmax( + self, + x: FloatsT, + *, + inplace: bool = False, + axis: int = -1, + temperature: float = 1.0, + ) -> FloatsT: + if temperature != 1.0: + x = x / temperature + maxes = self.xp.max(x, axis=axis, keepdims=True) + shifted = x - maxes + new_x = self.xp.exp(shifted) + new_x /= new_x.sum(axis=axis, keepdims=True) + return new_x + + def softmax_sequences( + self, Xs: Floats2d, lengths: Ints1d, *, inplace: bool = False, axis: int = -1 + ) -> Floats2d: + if Xs.ndim >= 3: + err = f"Softmax currently only supports 2d. Got: {Xs.ndim}" + raise NotImplementedError(err) + # This loses almost no fidelity, and helps the numerical stability. + Xs = self.xp.clip(Xs, -20.0, 20.0) + new_x = self.xp.exp(Xs) + summed = self.backprop_reduce_sum(self.reduce_sum(new_x, lengths), lengths) + new_x /= summed + return new_x + + def backprop_softmax( + self, Y: FloatsT, dY: FloatsT, *, axis: int = -1, temperature: float = 1.0 + ) -> FloatsT: + if temperature != 1.0: + dY = dY / temperature + + dX = Y * dY + dX -= Y * dX.sum(axis=axis, keepdims=True) + return dX + + def backprop_softmax_sequences( + self, dY: Floats2d, Y: Floats2d, lengths: Ints1d + ) -> Floats2d: + dX = Y * dY + sum_dX = self.backprop_reduce_sum(self.reduce_sum(dX, lengths), lengths) + dX -= Y * sum_dX + return dX + + def lstm_forward_training( + self, + params: Floats1d, + H0: Floats3d, + C0: Floats3d, + X: Floats2d, + size_at_t: Ints1d, + ) -> Tuple[Floats2d, Tuple]: + assert H0.shape == C0.shape + assert H0.shape[1] == C0.shape[1] + Y, fwd_state = lstm_forward_training(params, H0, C0, X, size_at_t) + return Y, fwd_state + + def lstm_forward_inference( + self, + params: Floats1d, + H0: Floats3d, + C0: Floats3d, + X: Floats2d, + size_at_t: Ints1d, + ) -> Floats2d: + Y, _ = lstm_forward_training(params, H0, C0, X, size_at_t) + return Y + + def backprop_lstm( + self, dY: Floats2d, lengths: Ints1d, params: Floats1d, fwd_state: Tuple + ) -> Tuple[Floats2d, Floats1d]: + dX, d_params = backprop_lstm(dY, lengths, params, fwd_state) + return dX, d_params + + def maxout(self, X: Floats3d) -> Tuple[Floats2d, Ints2d]: + which = X.argmax(axis=-1) + return X.max(axis=-1), which + + def backprop_maxout(self, dY: Floats2d, which: Ints2d, P: int) -> Floats3d: + dX = self.alloc3f(dY.shape[0], dY.shape[1], P, dtype=dY.dtype) + for b in range(dY.shape[0]): + for o in range(dY.shape[1]): + dX[b, o, which[b, o]] = dY[b, o] + return dX + + def relu(self, X: Floats2d, inplace: bool = False) -> Floats2d: + if not inplace: + return X * (X > 0) + else: + X *= X > 0 + return X + + def backprop_relu( + self, dY: Floats2d, Y: Floats2d, inplace: bool = False + ) -> Floats2d: + if not inplace: + return dY * (Y > 0) + dY *= Y > 0 + return dY + + def clipped_linear( + self, + X: FloatsXdT, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, + inplace: bool = False, + ) -> FloatsXdT: + if inplace: + X *= slope + X += offset + return self.xp.clip(X, min_val, max_val, out=X) + out = X * slope + offset + return self.xp.clip(out, min_val, max_val) + + def backprop_clipped_linear( + self, + dY: FloatsXdT, + X: FloatsXdT, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, + inplace: bool = False, + ) -> FloatsXdT: + low = (min_val - offset) / slope + high = (max_val - offset) / slope + slope = self.xp.float64(slope).astype(X.dtype) + zero = self.xp.float64(0.0).astype(X.dtype) + dX = self.xp.where((low < X) & (X < high), slope, zero) + if inplace: + dY *= dX + return dY + return dY * dX + + def relu_k(self, X: FloatsXdT, n: float = 6.0, inplace: bool = False) -> FloatsXdT: + return self.clipped_linear(X, max_val=n, inplace=inplace) + + def backprop_relu_k( + self, dY: FloatsXdT, X: FloatsXdT, n: float = 6.0, inplace: bool = False + ) -> FloatsXdT: + return self.backprop_clipped_linear(dY, X, max_val=n, inplace=inplace) + + def hard_sigmoid(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + return self.clipped_linear(X, slope=0.2, offset=0.5, inplace=inplace) + + def backprop_hard_sigmoid( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + return self.backprop_clipped_linear(dY, X, slope=0.2, offset=0.5) + + def hard_tanh(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + return self.clipped_linear(X, min_val=-1.0, max_val=1.0, inplace=inplace) + + def backprop_hard_tanh( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + return self.backprop_clipped_linear(dY, X, min_val=-1.0, max_val=1.0) + + def swish(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + if inplace: + X *= self.sigmoid(X) + return X + out = X * self.sigmoid(X) + return out + + def backprop_swish( + self, dY: FloatsXdT, X: FloatsXdT, Y: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + Y = Y + self.sigmoid(X) * (1 - Y) + if inplace: + dY *= Y + return dY + out = dY * Y + return out + + # Following https://www.scitepress.org/Papers/2019/74696/74696.pdf + def hard_swish(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + if inplace: + X *= self.hard_sigmoid(X) + return X + out = X * self.hard_sigmoid(X) + return out + + def backprop_hard_swish( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + dX = X * 0.4 + 0.5 + dX[X > 2.5] = 1.0 + dX[X < -2.5] = 0 + if inplace: + dY *= dX + return dY + return dY * dX + + # From https://arxiv.org/pdf/1905.02244v5.pdf + def hard_swish_mobilenet(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + if inplace: + X *= self.relu_k(X + 3) / 6 + return X + return X * (self.relu_k(X + 3) / 6) + + def backprop_hard_swish_mobilenet( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + dX = (1 / 6) * (X * 2.0 + 3.0) + dX[X > 3.0] = 1.0 + dX[X < -3.0] = 0 + if inplace: + dY *= dX + return dY + return dX * dY + + def dish(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + tmp = self.xp.square(X) + tmp += 1.0 + self.xp.sqrt(tmp, out=tmp) + tmp = X / tmp + tmp += 1 + tmp *= 0.5 + if inplace: + X *= tmp + return X + else: + return X * tmp + + def backprop_dish( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + x_sq = self.xp.square(X) + x_sq_plus_one = x_sq + 1.0 + deriv = X / self.xp.sqrt(x_sq_plus_one) + second = 0.5 * X * x_sq + second /= x_sq_plus_one**1.5 + deriv -= second + deriv += 0.5 + if inplace: + dY *= deriv + return dY + else: + return dY * deriv + + # Code snippet taken from: + # https://www.johndcook.com/blog/2009/01/19/stand-alone-error-function-erf/ + def erf(self, X: FloatsXdT) -> FloatsXdT: + # save the sign of x + sign = self.xp.sign(X) + X = self.xp.abs(X) + + a1 = 0.254829592 + a2 = -0.284496736 + a3 = 1.421413741 + a4 = -1.453152027 + a5 = 1.061405429 + p = 0.3275911 + + t = 1.0 / (1.0 + p * X) + y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * self.xp.exp( + -X * X + ) + out = sign * y + out = out.astype(X.dtype) + return out + + def sechsq(self, X: FloatsXdT) -> FloatsXdT: + # Avoid overflow in cosh. Clipping at |20| has an error of 1.7e-17. + X = self.xp.clip(X, -20.0, 20.0) + return (1 / self.xp.cosh(X)) ** 2 + + def gelu_approx(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + tmp = 1.0 + self.xp.tanh(SQRT2PI * (X + 0.044715 * self.xp.power(X, 3))) + tmp *= 0.5 + tmp = tmp.astype(X.dtype) + if inplace: + X *= tmp + return X + else: + Y = self.xp.array(X) + Y *= tmp + return Y + + def backprop_gelu_approx( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + dX = cast(FloatsXdT, self.alloc_f(X.shape)) + Xp3 = self.xp.power(X, 3) + tmp = 0.5 * self.xp.tanh(0.0356774 * Xp3 + 0.797885 * X) + tmp += (0.0535161 * Xp3 + 0.398942 * X) * self.sechsq( + 0.0356774 * Xp3 + 0.797885 * X + ) + tmp += 0.5 + dX += tmp + if inplace: + dY *= dX + return dY + return dY * dX + + def gelu(self, X: FloatsXdT, inplace: bool = False) -> FloatsXdT: + # GELU(x) = x · Φ(x) + cdf = gaussian_cdf(self, X) + if inplace: + X *= cdf + return X + return X * cdf + + def backprop_gelu( + self, dY: FloatsXdT, X: FloatsXdT, inplace: bool = False + ) -> FloatsXdT: + # GELU'(x) = Φ(x) + x · PDF(x) + dX = gaussian_cdf(self, X) + X * gaussian_pdf(self, X) + if inplace: + dY *= dX + return dY + return dY * dX + + def mish( + self, X: FloatsXdT, threshold: float = 20.0, inplace: bool = False + ) -> FloatsXdT: + tmp = X * self.xp.tanh(self.xp.log(1.0 + self.xp.exp(X))) + Y = self.xp.where(X >= threshold, X, tmp) + if inplace: + X[:] = Y + return X + else: + return Y + + def backprop_mish( + self, + dY: FloatsXdT, + X: Floats2d, + threshold: float = 20.0, + inplace: bool = False, + ) -> FloatsXdT: + if dY.shape != X.shape: + msg = f"arrays have incompatible shapes: {dY.shape} and {X.shape}" + raise ValueError(msg) + + xp = get_array_module(X) + indices = X < threshold + Xsub = X[indices] + dYsub = dY[indices] + omega = 4.0 * (Xsub + 1.0) + omega += 4.0 * xp.exp(2.0 * Xsub) + omega += xp.exp(3.0 * Xsub) + omega += xp.exp(Xsub) * ((4.0 * Xsub) + 6.0) + delta = xp.exp(Xsub) + 1.0 + delta *= delta + delta += 1.0 + dXsub = dYsub * ((xp.exp(Xsub) * omega) / (delta**2)) + # Gradient when above threshold will ignore softplus. + if inplace: + out = dY + else: + out = xp.copy(dY) + out[indices] = dXsub + return out + + def update_averages( + self, ema: FloatsT, weights: FloatsT, t: int, max_decay: float = 0.9999 + ) -> None: + # Internals for optimizer + decay = (1.0 + t) / (10.0 + t) + if decay > max_decay: + decay = max_decay + ema -= (1 - decay) * (ema - weights) + + def adam( + self, + weights: Floats1d, + gradient: Floats1d, + mom1: Floats1d, + mom2: Floats1d, + beta1: float, + beta2: float, + eps: float, + learn_rate: float, + mod_rate: float = 1.0, + ) -> Tuple[Floats1d, Floats1d, Floats1d, Floats1d]: + _check_compatible_shape(weights, gradient) + _check_compatible_shape(weights, mom1) + _check_compatible_shape(weights, mom2) + + # Internals for optimizer + mom1 *= beta1 + mom2 *= beta2 + mom1 += gradient * (1.0 - beta1) + mom2 += gradient * gradient * (1.0 - beta2) + # Here we assume learn rate is calculated by the caller. + # cdef weight_t a_t = learn_rate * sqrt(1-beta2**hp.t) / (1-beta1**hp.t); + weights -= learn_rate * (mom1 / (mod_rate * self.xp.sqrt(mom2) + eps)) + return weights, gradient, mom1, mom2 + + def clip_gradient(self, gradient: FloatsT, threshold: float) -> FloatsT: + # Internals for optimizer + xp = get_array_module(gradient) + grad_norm = xp.linalg.norm(gradient) + if grad_norm >= threshold: + gradient *= threshold / grad_norm + return gradient + + def logloss(self, y_true: FloatsT, y_pred: FloatsT) -> float: + # Currently not used + log_yp = self.xp.log(y_pred + 1e-8) + loss = (y_true * log_yp) + (1 - y_true) * self.xp.log((1 - y_pred) + 1e-8) + return -loss + + def reduce_sum(self, X: Floats2d, lengths: Ints1d) -> Floats2d: + Y = self.alloc2f(lengths.shape[0], X.shape[1], zeros=False) + start = 0 + for i, length in enumerate(lengths): + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + elif start + length > X.shape[0]: + raise IndexError("lengths must sum up to the number of rows") + elif length: + Y[i] = X[start : start + length].sum(axis=0) + start += length + else: + Y[i] = 0.0 + return Y + + def reduce_first(self, X: Floats2d, lengths: Ints1d) -> Tuple[Floats2d, Ints1d]: + if lengths.size == 0: + return self.alloc2f(0, X.shape[1]), lengths + if not self.xp.all(lengths > 0): + raise ValueError(f"all sequence lengths must be > 0") + starts_ends = self.alloc1i(lengths.shape[0] + 1, zeros=False) + starts_ends[0] = 0 + starts_ends[1:] = lengths.cumsum() + if starts_ends[-1] != X.shape[0]: + raise IndexError("lengths must sum up to the number of rows") + + return X[starts_ends[:-1]], starts_ends + + def reduce_last(self, X: Floats2d, lengths: Ints1d) -> Tuple[Floats2d, Ints1d]: + if lengths.size == 0: + return self.alloc2f(0, X.shape[1]), lengths + if not self.xp.all(lengths > 0): + raise ValueError(f"all sequence lengths must be > 0") + lasts = lengths.cumsum() - 1 + if lasts[-1] + 1 != X.shape[0]: + raise IndexError("lengths must sum up to the number of rows") + return X[lasts], lasts + + def reduce_mean(self, X: Floats2d, lengths: Ints1d) -> Floats2d: + Y = self.alloc2f(lengths.shape[0], X.shape[1], zeros=False) + start = 0 + for i, length in enumerate(lengths): + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + elif start + length > X.shape[0]: + raise IndexError("lengths must sum up to the number of rows") + elif length: + Y[i] = X[start : start + length].mean(axis=0) + else: + Y[i] = 0.0 + start += length + return Y + + def reduce_max(self, X: Floats2d, lengths: Ints1d) -> Tuple[Floats2d, Ints2d]: + Y = self.alloc2f(lengths.shape[0], X.shape[1], dtype=X.dtype, zeros=False) + which = self.alloc2i(lengths.shape[0], X.shape[1], zeros=False) + start = 0 + for i, length in enumerate(lengths): + if length <= 0: + raise ValueError(f"all sequence lengths must be > 0, got {length}") + elif start + length > X.shape[0]: + raise IndexError("lengths must sum up to the number of rows") + elif length: + which[i] = X[start : start + length].argmax(axis=0) + Y[i] = X[start : start + length].max(axis=0) + start += length + return Y, which + + def backprop_reduce_first( + self, d_firsts: Floats2d, starts_ends: Ints1d + ) -> Floats2d: + if starts_ends.size == 0: + return self.alloc2f(0, d_firsts.shape[1], dtype=d_firsts.dtype, zeros=True) + elif starts_ends.size == 1: + raise ValueError(f"starts_ends must not have size 1") + dX = self.alloc2f( + int(starts_ends[-1]), d_firsts.shape[1], dtype=d_firsts.dtype, zeros=True + ) + dX[starts_ends[:-1]] = d_firsts + return dX + + def backprop_reduce_last(self, d_lasts: Floats2d, lasts: Ints1d) -> Floats2d: + if lasts.size == 0: + return self.alloc2f(0, d_lasts.shape[1], dtype=d_lasts.dtype, zeros=True) + dX = self.alloc2f( + int(lasts[-1]) + 1, d_lasts.shape[1], dtype=d_lasts.dtype, zeros=True + ) + dX[lasts] = d_lasts + return dX + + def backprop_reduce_sum(self, d_sums: Floats2d, lengths: Ints1d) -> Floats2d: + dX = self.alloc2f( + lengths.sum(), d_sums.shape[1], dtype=d_sums.dtype, zeros=False + ) + start = 0 + for i, length in enumerate(lengths): + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + dX[start : start + length] = d_sums[i] + start += length + return dX + + def backprop_reduce_mean(self, d_means: Floats2d, lengths: Ints1d) -> Floats2d: + dX = self.alloc2f( + lengths.sum(), d_means.shape[1], dtype=d_means.dtype, zeros=False + ) + start = 0 + for i, length in enumerate(lengths): + if length < 0: + raise ValueError(f"all sequence lengths must be >= 0, got {length}") + dX[start : start + length] = d_means[i] / length + start += length + return dX + + def backprop_reduce_max( + self, d_maxes: Floats2d, which: Ints2d, lengths: Ints1d + ) -> Floats2d: + dX = self.alloc2f(lengths.sum(), d_maxes.shape[1], dtype=d_maxes.dtype) + start = 0 + for i, length in enumerate(lengths): + if length <= 0: + raise ValueError(f"all sequence lengths must be > 0, got {length}") + + self.xp.put_along_axis( + dX[start : start + length], which[i].reshape((1, -1)), d_maxes[i], 0 + ) + start += length + return dX + + def hash(self, ids: Ints1d, seed: int) -> Ints2d: + """Hash a sequence of 64-bit keys into a table with 4 32-bit keys, using + murmurhash3. + """ + from .numpy_ops import NumpyOps + + numpy_ops = NumpyOps() + return self.asarray2i( + numpy_ops.hash(numpy_ops.asarray(ids, dtype="uint64"), seed) + ) + + def ngrams(self, n: int, keys: Ints1d) -> Ints1d: + from .numpy_ops import NumpyOps + + numpy_ops = NumpyOps() + return self.asarray1i( + numpy_ops.ngrams(n, numpy_ops.asarray(keys, dtype="uint64")) + ) + + def position_encode( + self, N: int, D: int, period: int = 10000, out: Optional[Floats2d] = None + ) -> Floats2d: + # Currently internals only + from .numpy_ops import NumpyOps + + numpy_ops = NumpyOps() + return self.asarray2f(numpy_ops.position_encode(N, D, period, out)) + + def gather_add(self, table: Floats2d, indices: Ints2d) -> Floats2d: + return table[indices].sum(axis=1) # type: ignore[return-value] + + def scatter_add( + self, table: FloatsXd, indices: IntsXd, values: FloatsXd + ) -> FloatsXd: + return self.xp.add.at(table, indices, values) + + def insert_into(self, shape, Xs): + """Maybe don't need this? Just a quicky to get Jax working.""" + output = self.alloc(shape, dtype=Xs[0].dtype) + for i, x in enumerate(Xs): + output[i, : x.shape[0]] = x + return output + + +""" +LSTM Notation (kind of involved, but made it a lot easier to write) + +X: Inputs +Y: Outputs (aka hiddens) +C: Cells +G: Gates (Output of non-linearity, i.e. lstm_gates(X @ W.T) +A: Activations (X @ W.T, before non-linearity) + +Imagine we have the input: +batch = [ + ["apple", "banana", "cantaloupe", "date", "elderberry"], + ["aardvark", "bat", "capybara", "dingo", "elephant"] +] + +The input variable X will have one vector per word, so X[0, 1] will be banana's +vector, X[0, 1, 0] will be a float, the first element of that vector. + +We're computing an output variable Y of shape (nL, nB, nO), so that Y[0, 1] is +the output variable of banana. + +A problem with variables for RNNs is keeping the timesteps straight. It's hard +to distinguish the current, previous, and next timesteps. To solve this problem, +we follow the convention that **we are at timestep 3**. + +Additionally, the variables for Y and C are offset by one, as the 0th elements +have the initial hiddens and initial cells. So: + + t=3 + Xt3: The input vectors for 'dingo' and 'date', i.e. X[t] + Yt3: The output vectors for 'dingo' and 'date', i.e. Y[t+1] (Y is offset.) + Ct2: The cells calculated at 'c...', that are the input for 'd...' + Ct3: The cells calculated at 'd...', that are the input for 'e...' + At3: The activations at 'd...' + Gt3: The gates at 'd...' +""" + + +def lstm_forward_training( + params: Floats1d, c_init: Floats3d, h_init: Floats3d, X: Floats2d, lengths: Ints1d +) -> Tuple[Floats2d, Tuple]: + xp = get_array_module(params) + depth, dirs, nO = c_init.shape + N, nI = X.shape + batch_size = lengths[0] + # Preallocate these so we can pass them through for loop. + G = cast(Floats4d, xp.zeros((depth, dirs, X.shape[0], nO * 4), dtype="f")) + Y = cast(Floats4d, xp.zeros((depth, dirs, X.shape[0], nO), dtype="f")) + C = cast(Floats4d, xp.zeros((depth, dirs, X.shape[0], nO), dtype="f")) + Yt2 = cast(Floats2d, xp.zeros((batch_size, nO), dtype="f")) + Ct2 = cast(Floats2d, xp.zeros((batch_size, nO), dtype="f")) + # Compute the start and end indices first. + indices = [] + start = 0 + for batch_size in lengths: + indices.append((start, start + batch_size)) + start += batch_size + params_i = 0 + orig_X = X + for i in range(depth): + nI = X.shape[1] + for d in range(dirs): + # The inits are shaped (depth, dirs, nO). We add the internal dimension + # to make them set correctly. + Yt2 = h_init[i, d].reshape((1, nO)) # type: ignore[assignment] + Ct2 = c_init[i, d].reshape((1, nO)) # type: ignore[assignment] + layer_params, params_i = _split_weights(params, i, nO, nI, params_i) + Wx, Wh, bias = _transpose_weights(layer_params) + G[i, d] += xp.dot(X, Wx.T) + G[i, d] += bias + for start, end in indices if d == 0 else reversed(indices): + # When we iterate left-to-right, t2 might be longer than t3. + Yt2 = Yt2[: end - start] + Ct2 = Ct2[: end - start] + # But in right-to-left, it's the opposite: t3 can be longer. + Gt3 = G[i, d, start:end] + Gt3 = Gt3[: Yt2.shape[0]] + Gt3 += xp.dot(Yt2, Wh.T) + Gt3_ = cast(Floats3d, Gt3.reshape((-1, nO, 4))) + hf = sigmoid(Gt3_[:, :, 0]) + hi = sigmoid(Gt3_[:, :, 1]) + ho = sigmoid(Gt3_[:, :, 2]) + hc = xp.tanh(Gt3_[:, :, 3]) + Ct3 = hf * Ct2 + Ct3 += hi * hc + # Store results + Gt3 = ( + xp.hstack((hf, hi, ho, hc)) + .reshape((-1, 4, nO)) + .transpose((0, 2, 1)) + .reshape((-1, nO * 4)) + ) + # Fix the endpoint to account for shorter slices when iterating + # reversed. Not 100% sure this is right. If there's a bug, look + # here? + end = min(end, start + ho.shape[0]) + Y[i, d, start:end] = xp.tanh(Ct3) * ho + G[i, d, start:end] = Gt3 + C[i, d, start:end] = Ct3 + # Set the t2 variables to the current t3 variables. + Ct2 = Ct3 + Yt2 = Y[i, d, start:end] + H = cast(Floats2d, Y[i].transpose((1, 0, 2)).reshape((N, -1))) + if dirs == 2: + H = xp.ascontiguousarray(H) + X = H + return H, (Y, G, C, orig_X) + + +def backprop_lstm(dY: Floats2d, lengths: Ints1d, params: Floats1d, fwd_state: Tuple): + xp = get_array_module(params) + + Y: Floats4d + G: Floats4d + C: Floats4d + X: Floats2d + Wx: Floats2d + Wh: Floats2d + bias: Floats1d + dWx: Floats2d + dWh: Floats2d + d_bias: Floats1d + Y, G, C, X = fwd_state + depth, dirs, N, nO = C.shape + nI = X.shape[1] + batch_size = lengths[0] + # We don't need to store all the cells for all the layers. + dC = cast(Floats2d, xp.zeros((N, nO), dtype=C.dtype)) + dG = cast(Floats2d, xp.zeros((N, nO * 4), dtype=C.dtype)) + d_params = cast(Floats1d, xp.zeros((params.shape[0],), dtype=params.dtype)) + # Collect the params and slices. It makes it a bit easier to get the indexing + # right, when we're iterating backwards. + params_i = 0 + all_layer_params: List[List[Tuple[Tuple[Floats2d, Floats2d, Floats1d], int]]] = [] + for i in range(depth): + all_layer_params.append([]) + n_inputs = nI if i == 0 else (nO * dirs) + for d in range(dirs): + layer_params, params_i = _split_weights(params, i, nO, n_inputs, params_i) + layer_params = _transpose_weights(layer_params) + all_layer_params[-1].append((layer_params, params_i)) + params_i = 0 + all_layer_grads: List[List[Tuple[Tuple[Floats2d, Floats2d, Floats1d], int]]] = [] + for i in range(depth): + all_layer_grads.append([]) + n_inputs = nI if i == 0 else (nO * dirs) + for d in range(dirs): + layer_grads, params_i = _split_weights(d_params, i, nO, n_inputs, params_i) + layer_grads = _transpose_weights(layer_grads) + all_layer_grads[-1].append((layer_grads, params_i)) + # Similarly, we want to compute the indices first + indices = [] + start = 0 + for batch_size in lengths: + indices.append((start, start + batch_size)) + start += batch_size + + Xs = [X] + [ + cast(Floats2d, Y[i].transpose((1, 0, 2)).reshape((N, -1))) + for i in range(depth - 1) + ] + dXs = [xp.zeros((X.shape[0], X.shape[1]), dtype=X.dtype) for X in Xs] + # Okay, now do the actual looping + for i in reversed(range(depth)): + dY3d = cast(Floats3d, dY.reshape((N, dirs, nO)).transpose((1, 0, 2))) + dX = dXs[i] + X = Xs[i] + if dirs >= 2: + dY3d = xp.ascontiguousarray(dY3d) + for d in range(dirs): + Wx, Wh, bias = all_layer_params[i][d][0] + dWx, dWh, d_bias = all_layer_grads[i][d][0] + if d == 0: + start_t3, end_t3 = indices[-1] + layer_indices = indices[:-1] + layer_indices.reverse() + else: + start_t3, end_t3 = indices[0] + layer_indices = indices[1:] + for start_t2, end_t2 in layer_indices: + size = min(end_t2 - start_t2, end_t3 - start_t3) + dGt3, dCt2 = backprop_lstm_gates( + dY3d[d, start_t3 : start_t3 + size], + dC[start_t3 : start_t3 + size], + G[i, d, start_t3 : start_t3 + size], + C[i, d, start_t3 : start_t3 + size], + C[i, d, start_t2 : start_t2 + size], + ) + # Backprop hidden-to-hidden w.r.t. hidden. + dY3d[d, start_t2 : start_t2 + size] += dGt3 @ Wh + # Update iteration variables + dC[start_t2 : start_t2 + size] = dCt2 + start_t3 = start_t2 + end_t3 = end_t2 + # Backprop input-to-hidden w.r.t. weights. + dWx += dG.T @ X + # Backprop hidden-to-hidden w.r.t. weights. + dWh += dG.T @ Y[i, d] + # Backprop bias + d_bias += dG.sum(axis=0) + # Backprop input-to-hidden w.r.t. input + dX += dG @ Wx + dY = dX + assert dX.shape[1] == X.shape[1] + grad_parts = [] + for layer_grads in all_layer_grads: + for dir_grads, _ in layer_grads: + grad_parts.append(_untranspose_unsplit_weights(dir_grads)) + return dX, xp.concatenate(grad_parts) + + +def _split_weights(params: Floats1d, i: int, nO: int, nI: int, params_i: int): + Wx_size = 4 * nO * nI + bx_size = 4 * nO + Wh_size = 4 * nO * nO + bh_size = 4 * nO + Wx = params[params_i : params_i + Wx_size].reshape((4 * nO, nI)) + params_i += Wx_size + bx = params[params_i : params_i + bx_size].reshape((4 * nO,)) + params_i += bx_size + Wh = params[params_i : params_i + Wh_size].reshape((4 * nO, nO)) + params_i += Wh_size + bh = params[params_i : params_i + bh_size].reshape((4 * nO,)) + params_i += bh_size + return ((Wx, bx), (Wh, bh)), params_i + + +def _transpose_weights(params): + # Transpose the parameters so that the gates are the last dimension. This + # makes it easier to fuse. + (Wx, bx), (Wh, bh) = params + xp = get_array_module(Wx) + Wx = Wx.reshape((4, -1, Wx.shape[-1])) + Wx = Wx.transpose((1, 0, 2)).reshape((-1, Wx.shape[-1])) + bx = bx.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + Wh = Wh.reshape((4, -1, Wh.shape[-1])) + Wh = Wh.transpose((1, 0, 2)).reshape((-1, Wh.shape[-1])) + bh = bh.reshape((4, -1)).transpose((1, 0)).reshape((-1,)) + ascontig = xp.ascontiguousarray + Wx = ascontig(Wx) + Wh = ascontig(Wh) + bias = ascontig(bx) + bh + return Wx, Wh, bias + + +def _untranspose_unsplit_weights(params): + Wx, Wh, bias = params + xp = get_array_module(Wx) + nO = Wh.shape[1] + nI = Wx.shape[1] + Wx = Wx.reshape((-1, 4, nI)).transpose((1, 0, 2)).reshape((-1, nI)) + Wh = Wh.reshape((-1, 4, nO)).transpose((1, 0, 2)).reshape((-1, nO)) + bias = bias.reshape((-1, 4)).transpose((1, 0)).reshape((-1,)) + zeros = xp.zeros(bias.shape, dtype="f") + return xp.concatenate((Wx.ravel(), bias, Wh.ravel(), zeros)) + + +def backprop_lstm_gates( + dYt3: Floats2d, dCt3: Floats2d, Gt3: Floats2d, Ct3: Floats2d, Ct2: Floats2d +) -> Tuple[Floats2d, Floats2d]: + # See above for notation. Step numbering refers to forward_lstm_gates + xp = get_array_module(dYt3) + hf, hi, ho, hc = xp.split(Gt3, 4, axis=-1) + assert hf.shape[0] == hi.shape[0] == ho.shape[0] == hc.shape[0] + assert hf.shape[0] == dYt3.shape[0] == dCt3.shape[0] == Ct3.shape[0] == Ct2.shape[0] + tanhCt3 = xp.tanh(Ct3) + # 3b: Yt3 = tanhCt3 * ho + d_ho = dYt3 * tanhCt3 + d_tanhCt3 = dYt3 * ho + # 3a: tanhCt3 = tanh(Ct3) + dCt3 += d_tanhCt3 * dtanh(tanhCt3) + # 2b: Ct3 += hi * hc + d_hi = dCt3 * hc + d_hc = dCt3 * hi + # 2a: Ct3 = hf * Ct2 + d_hf = dCt3 * Ct2 + dCt2 = dCt3 * hf + d_At3_hc = d_hc * dtanh(hc) # 1d + d_At3_ho = d_ho * dsigmoid(ho) # 1c + d_At3_hi = d_hi * dsigmoid(hi) # 1b + d_At3_hf = d_hf * dsigmoid(hf) # 1a + dAt3 = xp.concatenate((d_At3_hf, d_At3_hi, d_At3_ho, d_At3_hc), axis=-1) + return dAt3, dCt2 + + +def sigmoid(X, out=None): + xp = get_array_module(X) + + # To prevent overflows and help with regularization/numerical stability + X = xp.clip(X, -20.0, 20.0) + return 1.0 / (1.0 + xp.exp(-X)) + + +def dsigmoid(Y: ArrayT) -> ArrayT: + return Y * (1.0 - Y) + + +def dtanh(Y: ArrayT) -> ArrayT: + return 1 - Y**2 + + +def gaussian_cdf(ops: Ops, X: FloatsXdT) -> FloatsXdT: + """Gaussian CDF for distribution with mean 0 and stdev 1.""" + return 0.5 * (1.0 + ops.erf(INV_SQRT2 * X)) + + +def gaussian_pdf(ops: Ops, X: FloatsXdT) -> FloatsXdT: + """Gaussian PDF for distribution with mean 0 and stdev 1.""" + return INV_SQRT_2PI * ops.xp.exp(-0.5 * X * X) + + +def _check_compatible_shape(u: FloatsXd, v: FloatsXd): + if u.shape != v.shape: + msg = f"arrays have incompatible shapes: {u.shape} and {v.shape}" + raise ValueError(msg) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__init__.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__init__.pxd new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..356209aa19cdcdcc625fcff9ca07e7de6b7b07fa Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1ef718e82cfb01ad7e116cd7677de348c479e4b7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.cpp @@ -0,0 +1,19181 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "include_dirs": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.extra.search", + "sources": [ + "thinc/extra/search.pyx" + ] + }, + "module_name": "thinc.extra.search" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__extra__search +#define __PYX_HAVE_API__thinc__extra__search +/* Early includes */ +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include + + #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include + + namespace cython_std { + template typename std::remove_reference::type&& move(T& t) noexcept { return std::move(t); } + template typename std::remove_reference::type&& move(T&& t) noexcept { return std::move(t); } + } + + #endif + +#include +#include +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/extra/search.pyx", + "thinc/extra/search.pxd", + "", + "cymem/cymem.pxd", + "preshed/maps.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* #### Code section: numeric_typedefs ### */ + +/* "preshed/maps.pxd":5 + * + * + * ctypedef uint64_t key_t # <<<<<<<<<<<<<< + * + * +*/ +typedef uint64_t __pyx_t_7preshed_4maps_key_t; + +/* "thinc/extra/search.pxd":7 + * from libcpp.vector cimport vector + * + * ctypedef uint64_t hash_t # <<<<<<<<<<<<<< + * ctypedef uint64_t class_t + * ctypedef float weight_t +*/ +typedef uint64_t __pyx_t_5thinc_5extra_6search_hash_t; + +/* "thinc/extra/search.pxd":8 + * + * ctypedef uint64_t hash_t + * ctypedef uint64_t class_t # <<<<<<<<<<<<<< + * ctypedef float weight_t + * +*/ +typedef uint64_t __pyx_t_5thinc_5extra_6search_class_t; + +/* "thinc/extra/search.pxd":9 + * ctypedef uint64_t hash_t + * ctypedef uint64_t class_t + * ctypedef float weight_t # <<<<<<<<<<<<<< + * + * +*/ +typedef float __pyx_t_5thinc_5extra_6search_weight_t; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc; +struct __pyx_obj_5cymem_5cymem_PyFree; +struct __pyx_obj_5cymem_5cymem_Pool; +struct __pyx_obj_5cymem_5cymem_Address; +struct __pyx_obj_7preshed_4maps_PreshMap; +struct __pyx_obj_7preshed_4maps_PreshMapArray; +struct __pyx_obj_5thinc_5extra_6search_Beam; +struct __pyx_obj_5thinc_5extra_6search_MaxViolation; +struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr; + +/* "cymem/cymem.pxd":1 + * ctypedef void* (*malloc_t)(size_t n) # <<<<<<<<<<<<<< + * ctypedef void (*free_t)(void *p) + * +*/ +typedef void *(*__pyx_t_5cymem_5cymem_malloc_t)(size_t); + +/* "cymem/cymem.pxd":2 + * ctypedef void* (*malloc_t)(size_t n) + * ctypedef void (*free_t)(void *p) # <<<<<<<<<<<<<< + * + * cdef class PyMalloc: +*/ +typedef void (*__pyx_t_5cymem_5cymem_free_t)(void *); +struct __pyx_t_7preshed_4maps_Cell; +struct __pyx_t_7preshed_4maps_Result; +struct __pyx_t_7preshed_4maps_MapStruct; + +/* "preshed/maps.pxd":8 + * + * + * cdef struct Cell: # <<<<<<<<<<<<<< + * key_t key + * void* value +*/ +struct __pyx_t_7preshed_4maps_Cell { + __pyx_t_7preshed_4maps_key_t key; + void *value; +}; + +/* "preshed/maps.pxd":13 + * + * + * cdef struct Result: # <<<<<<<<<<<<<< + * int found + * void* value +*/ +struct __pyx_t_7preshed_4maps_Result { + int found; + void *value; +}; + +/* "preshed/maps.pxd":18 + * + * + * cdef struct MapStruct: # <<<<<<<<<<<<<< + * Cell* cells + * void* value_for_empty_key +*/ +struct __pyx_t_7preshed_4maps_MapStruct { + struct __pyx_t_7preshed_4maps_Cell *cells; + void *value_for_empty_key; + void *value_for_del_key; + __pyx_t_7preshed_4maps_key_t length; + __pyx_t_7preshed_4maps_key_t filled; + int is_empty_key_set; + int is_del_key_set; +}; +struct __pyx_t_5thinc_5extra_6search__State; + +/* "thinc/extra/search.pxd":12 + * + * + * ctypedef pair[weight_t, size_t] Entry # <<<<<<<<<<<<<< + * ctypedef priority_queue[Entry] Queue + * +*/ +typedef std::pair<__pyx_t_5thinc_5extra_6search_weight_t,size_t> __pyx_t_5thinc_5extra_6search_Entry; + +/* "thinc/extra/search.pxd":13 + * + * ctypedef pair[weight_t, size_t] Entry + * ctypedef priority_queue[Entry] Queue # <<<<<<<<<<<<<< + * + * +*/ +typedef std::priority_queue<__pyx_t_5thinc_5extra_6search_Entry> __pyx_t_5thinc_5extra_6search_Queue; + +/* "thinc/extra/search.pxd":16 + * + * + * ctypedef int (*trans_func_t)(void* dest, void* src, class_t clas, void* x) except -1 # <<<<<<<<<<<<<< + * + * ctypedef void* (*init_func_t)(Pool mem, int n, void* extra_args) except NULL +*/ +typedef int (*__pyx_t_5thinc_5extra_6search_trans_func_t)(void *, void *, __pyx_t_5thinc_5extra_6search_class_t, void *); + +/* "thinc/extra/search.pxd":18 + * ctypedef int (*trans_func_t)(void* dest, void* src, class_t clas, void* x) except -1 + * + * ctypedef void* (*init_func_t)(Pool mem, int n, void* extra_args) except NULL # <<<<<<<<<<<<<< + * + * ctypedef int (*del_func_t)(Pool mem, void* state, void* extra_args) except -1 +*/ +typedef void *(*__pyx_t_5thinc_5extra_6search_init_func_t)(struct __pyx_obj_5cymem_5cymem_Pool *, int, void *); + +/* "thinc/extra/search.pxd":20 + * ctypedef void* (*init_func_t)(Pool mem, int n, void* extra_args) except NULL + * + * ctypedef int (*del_func_t)(Pool mem, void* state, void* extra_args) except -1 # <<<<<<<<<<<<<< + * + * ctypedef int (*finish_func_t)(void* state, void* extra_args) except -1 +*/ +typedef int (*__pyx_t_5thinc_5extra_6search_del_func_t)(struct __pyx_obj_5cymem_5cymem_Pool *, void *, void *); + +/* "thinc/extra/search.pxd":22 + * ctypedef int (*del_func_t)(Pool mem, void* state, void* extra_args) except -1 + * + * ctypedef int (*finish_func_t)(void* state, void* extra_args) except -1 # <<<<<<<<<<<<<< + * + * ctypedef hash_t (*hash_func_t)(void* state, void* x) except 0 +*/ +typedef int (*__pyx_t_5thinc_5extra_6search_finish_func_t)(void *, void *); + +/* "thinc/extra/search.pxd":24 + * ctypedef int (*finish_func_t)(void* state, void* extra_args) except -1 + * + * ctypedef hash_t (*hash_func_t)(void* state, void* x) except 0 # <<<<<<<<<<<<<< + * + * +*/ +typedef __pyx_t_5thinc_5extra_6search_hash_t (*__pyx_t_5thinc_5extra_6search_hash_func_t)(void *, void *); + +/* "thinc/extra/search.pxd":27 + * + * + * cdef struct _State: # <<<<<<<<<<<<<< + * void* content + * class_t* hist +*/ +struct __pyx_t_5thinc_5extra_6search__State { + void *content; + __pyx_t_5thinc_5extra_6search_class_t *hist; + __pyx_t_5thinc_5extra_6search_weight_t score; + __pyx_t_5thinc_5extra_6search_weight_t loss; + int i; + int t; + int is_done; +}; + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtab; + __pyx_t_5cymem_5cymem_malloc_t malloc; +}; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ +struct __pyx_obj_5cymem_5cymem_PyFree { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtab; + __pyx_t_5cymem_5cymem_free_t free; +}; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ +struct __pyx_obj_5cymem_5cymem_Pool { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtab; + size_t size; + PyObject *addresses; + PyObject *refs; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "cymem/cymem.pxd":28 + * + * + * cdef class Address: # <<<<<<<<<<<<<< + * cdef void* ptr + * cdef readonly PyMalloc pymalloc +*/ +struct __pyx_obj_5cymem_5cymem_Address { + PyObject_HEAD + void *ptr; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ +struct __pyx_obj_7preshed_4maps_PreshMap { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtab; + struct __pyx_t_7preshed_4maps_MapStruct *c_map; + struct __pyx_obj_5cymem_5cymem_Pool *mem; +}; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ +struct __pyx_obj_7preshed_4maps_PreshMapArray { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtab; + struct __pyx_obj_5cymem_5cymem_Pool *mem; + struct __pyx_t_7preshed_4maps_MapStruct *maps; + size_t length; +}; + + +/* "thinc/extra/search.pxd":37 + * + * + * cdef class Beam: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef class_t nr_class +*/ +struct __pyx_obj_5thinc_5extra_6search_Beam { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_5extra_6search_Beam *__pyx_vtab; + struct __pyx_obj_5cymem_5cymem_Pool *mem; + __pyx_t_5thinc_5extra_6search_class_t nr_class; + __pyx_t_5thinc_5extra_6search_class_t width; + __pyx_t_5thinc_5extra_6search_class_t size; + __pyx_t_5thinc_5extra_6search_weight_t min_density; + int t; + int is_done; + PyObject *histories; + PyObject *_parent_histories; + __pyx_t_5thinc_5extra_6search_weight_t **scores; + int **is_valid; + __pyx_t_5thinc_5extra_6search_weight_t **costs; + struct __pyx_t_5thinc_5extra_6search__State *_parents; + struct __pyx_t_5thinc_5extra_6search__State *_states; + __pyx_t_5thinc_5extra_6search_del_func_t del_func; +}; + + +/* "thinc/extra/search.pxd":75 + * + * + * cdef class MaxViolation: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef weight_t cost +*/ +struct __pyx_obj_5thinc_5extra_6search_MaxViolation { + PyObject_HEAD + struct __pyx_vtabstruct_5thinc_5extra_6search_MaxViolation *__pyx_vtab; + struct __pyx_obj_5cymem_5cymem_Pool *mem; + __pyx_t_5thinc_5extra_6search_weight_t cost; + __pyx_t_5thinc_5extra_6search_weight_t delta; + __pyx_t_5thinc_5extra_6search_weight_t p_score; + __pyx_t_5thinc_5extra_6search_weight_t g_score; + double Z; + double gZ; + __pyx_t_5thinc_5extra_6search_class_t n; + PyObject *p_hist; + PyObject *g_hist; + PyObject *p_probs; + PyObject *g_probs; +}; + + +/* "thinc/extra/search.pyx":302 + * max_ = max(nums) + * nums = [(exp(n-max_) if n is not None else None) for n in nums] + * Z = sum(n for n in nums if n is not None) # <<<<<<<<<<<<<< + * return [(n/Z if n is not None else None) for n in nums] +*/ +struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr { + PyObject_HEAD + PyObject *__pyx_genexpr_arg_0; + PyObject *__pyx_v_n; + PyObject *__pyx_t_0; + Py_ssize_t __pyx_t_1; + PyObject *(*__pyx_t_2)(PyObject *); +}; + + + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyMalloc *, __pyx_t_5cymem_5cymem_malloc_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtabptr_5cymem_5cymem_PyMalloc; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyFree { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyFree *, __pyx_t_5cymem_5cymem_free_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtabptr_5cymem_5cymem_PyFree; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_Pool { + void *(*alloc)(struct __pyx_obj_5cymem_5cymem_Pool *, size_t, size_t); + void (*free)(struct __pyx_obj_5cymem_5cymem_Pool *, void *); + void *(*realloc)(struct __pyx_obj_5cymem_5cymem_Pool *, void *, size_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtabptr_5cymem_5cymem_Pool; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMap { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtabptr_7preshed_4maps_PreshMap; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtabptr_7preshed_4maps_PreshMapArray; + + +/* "thinc/extra/search.pyx":12 + * + * + * cdef class Beam: # <<<<<<<<<<<<<< + * def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): + * assert nr_class != 0 +*/ + +struct __pyx_vtabstruct_5thinc_5extra_6search_Beam { + int (*_fill)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_Queue *, __pyx_t_5thinc_5extra_6search_weight_t **, int **); + void *(*at)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int); + int (*initialize)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_init_func_t, __pyx_t_5thinc_5extra_6search_del_func_t, int, void *); + int (*advance)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_trans_func_t, __pyx_t_5thinc_5extra_6search_hash_func_t, void *); + int (*check_done)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_finish_func_t, void *); + void (*set_cell)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int, int, __pyx_t_5thinc_5extra_6search_weight_t, int, __pyx_t_5thinc_5extra_6search_weight_t); + int (*set_row)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int, __pyx_t_5thinc_5extra_6search_weight_t const *, int const *, __pyx_t_5thinc_5extra_6search_weight_t const *); + int (*set_table)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_weight_t **, int **, __pyx_t_5thinc_5extra_6search_weight_t **); +}; +static struct __pyx_vtabstruct_5thinc_5extra_6search_Beam *__pyx_vtabptr_5thinc_5extra_6search_Beam; +static CYTHON_INLINE void *__pyx_f_5thinc_5extra_6search_4Beam_at(struct __pyx_obj_5thinc_5extra_6search_Beam *, int); +static CYTHON_INLINE void __pyx_f_5thinc_5extra_6search_4Beam_set_cell(struct __pyx_obj_5thinc_5extra_6search_Beam *, int, int, __pyx_t_5thinc_5extra_6search_weight_t, int, __pyx_t_5thinc_5extra_6search_weight_t); + + +/* "thinc/extra/search.pyx":218 + * + * + * cdef class MaxViolation: # <<<<<<<<<<<<<< + * def __init__(self): + * self.p_score = 0.0 +*/ + +struct __pyx_vtabstruct_5thinc_5extra_6search_MaxViolation { + int (*check)(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *, struct __pyx_obj_5thinc_5extra_6search_Beam *, struct __pyx_obj_5thinc_5extra_6search_Beam *, int __pyx_skip_dispatch); + int (*check_crf)(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *, struct __pyx_obj_5thinc_5extra_6search_Beam *, struct __pyx_obj_5thinc_5extra_6search_Beam *, int __pyx_skip_dispatch); +}; +static struct __pyx_vtabstruct_5thinc_5extra_6search_MaxViolation *__pyx_vtabptr_5thinc_5extra_6search_MaxViolation; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallCFunction.proto (used by CallUnboundCMethod0) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallOneArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ListAppend.proto (used by append) */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_PyList_Append(L,x) PyList_Append(L,x) +#endif + +/* PyObjectCall2Args.proto (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod1) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod1.proto (used by append) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* append.proto */ +static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x); + +/* RejectKeywords.export */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds); + +/* PyTypeError_Check.proto */ +#define __Pyx_PyExc_TypeError_Check(obj) __Pyx_TypeCheck(obj, PyExc_TypeError) + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* PyObjectCallNoArg.proto (used by pyfrozenset_new) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* pyfrozenset_new.proto (used by PySetContains) */ +static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it); + +/* PySetContains.proto */ +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq); + +/* SliceObject.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice( + PyObject* obj, Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** py_start, PyObject** py_stop, PyObject** py_slice, + int has_cstart, int has_cstop, int wraparound); + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* PyObjectFastCallMethod.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL) +#else +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf); +#endif + +/* ExtTypeTest.proto */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* CheckTypeForFreelists.proto */ +#if CYTHON_USE_FREELISTS +#if CYTHON_USE_TYPE_SPECS +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t) == (expected_tp))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS Py_TPFLAGS_IS_ABSTRACT +#else +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t)->tp_basicsize == (expected_size))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE) +#endif +#define __PYX_CHECK_TYPE_FOR_FREELISTS(t, expected_tp, expected_size)\ + (__PYX_CHECK_FINAL_TYPE_FOR_FREELISTS((t), (expected_tp), (expected_size)) &\ + (int) (!__Pyx_PyType_HasFeature((t), __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS))) +#endif + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* DelItemOnTypeDict.proto (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k); +#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k) + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_1 +#define __PYX_HAVE_RT_ImportType_proto_3_2_1 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_2_1 { + __Pyx_ImportType_CheckSize_Error_3_2_1 = 0, + __Pyx_ImportType_CheckSize_Warn_3_2_1 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_2_1 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size); +#endif + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CheckUnpickleChecksum.proto */ +static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members); + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* PyObjectVectorCallKwBuilder.proto (used by CIntToPy) */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_uint64_t(uint64_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE uint64_t __Pyx_PyLong_As_uint64_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* UpdateUnpickledDict.proto */ +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* SwapException.proto (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* IterNextPlain.proto (used by CoroutineBase) */ +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator); +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void); +#endif + +/* ReturnWithStopIteration.proto (used by CoroutineBase) */ +static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext); + +/* CoroutineBase.proto (used by Generator) */ +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); +#if CYTHON_USE_EXC_INFO_STACK +#define __Pyx_ExcInfoStruct _PyErr_StackItem +#else +typedef struct { + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; +} __Pyx_ExcInfoStruct; +#endif +typedef struct __pyx_CoroutineObject { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + __Pyx_ExcInfoStruct gi_exc_state; +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *gi_weakreflist; +#endif + PyObject *classobj; + PyObject *yieldfrom; + __Pyx_pyiter_sendfunc yieldfrom_am_send; + PyObject *gi_name; + PyObject *gi_qualname; + PyObject *gi_modulename; + PyObject *gi_code; + PyObject *gi_frame; +#if CYTHON_USE_SYS_MONITORING && (CYTHON_PROFILE || CYTHON_TRACE) + PyMonitoringState __pyx_pymonitoring_state[__Pyx_MonitoringEventTypes_CyGen_count]; + uint64_t __pyx_pymonitoring_version; +#endif + int resume_label; + char is_running; +} __pyx_CoroutineObject; +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); +static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *self); +static int __Pyx_Coroutine_clear(PyObject *self); +static __Pyx_PySendResult __Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval); +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); +static __Pyx_PySendResult __Pyx_Coroutine_Close(PyObject *self, PyObject **retval); +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#if CYTHON_USE_EXC_INFO_STACK +#define __Pyx_Coroutine_SwapException(self) +#define __Pyx_Coroutine_ResetAndClearException(self) __Pyx_Coroutine_ExceptionClear(&(self)->gi_exc_state) +#else +#define __Pyx_Coroutine_SwapException(self) {\ + __Pyx_ExceptionSwap(&(self)->gi_exc_state.exc_type, &(self)->gi_exc_state.exc_value, &(self)->gi_exc_state.exc_traceback);\ + __Pyx_Coroutine_ResetFrameBackpointer(&(self)->gi_exc_state);\ + } +#define __Pyx_Coroutine_ResetAndClearException(self) {\ + __Pyx_ExceptionReset((self)->gi_exc_state.exc_type, (self)->gi_exc_state.exc_value, (self)->gi_exc_state.exc_traceback);\ + (self)->gi_exc_state.exc_type = (self)->gi_exc_state.exc_value = (self)->gi_exc_state.exc_traceback = NULL;\ + } +#endif +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__pyx_tstate, pvalue) +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue)\ + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue) +#endif +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state); +static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen); +static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen); +static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen); +static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure); +#if __PYX_HAS_PY_AM_SEND == 2 +static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send); +#endif +static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg); + +/* Generator.proto */ +#define __Pyx_Generator_USED +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_GeneratorType) +#define __Pyx_Generator_New(body, code, closure, name, qualname, module_name)\ + __Pyx__Coroutine_New(__pyx_mstate_global->__pyx_GeneratorType, body, code, closure, name, qualname, module_name) +static PyObject *__Pyx_Generator_Next(PyObject *self); +static int __pyx_Generator_init(PyObject *module); +static CYTHON_INLINE PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static CYTHON_INLINE void *__pyx_f_5thinc_5extra_6search_4Beam_at(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i); /* proto*/ +static CYTHON_INLINE void __pyx_f_5thinc_5extra_6search_4Beam_set_cell(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i, int __pyx_v_j, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_score, int __pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_cost); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam_set_row(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i, __pyx_t_5thinc_5extra_6search_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t const *__pyx_v_costs); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam_set_table(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_scores, int **__pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_costs); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam_initialize(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_init_func_t __pyx_v_init_func, __pyx_t_5thinc_5extra_6search_del_func_t __pyx_v_del_func, int __pyx_v_n, void *__pyx_v_extra_args); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam_advance(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_trans_func_t __pyx_v_transition_func, __pyx_t_5thinc_5extra_6search_hash_func_t __pyx_v_hash_func, void *__pyx_v_extra_args); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam_check_done(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_finish_func_t __pyx_v_finish_func, void *__pyx_v_extra_args); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_4Beam__fill(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_Queue *__pyx_v_q, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_scores, int **__pyx_v_is_valid); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_12MaxViolation_check(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold, int __pyx_skip_dispatch); /* proto*/ +static int __pyx_f_5thinc_5extra_6search_12MaxViolation_check_crf(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold, int __pyx_skip_dispatch); /* proto*/ + +/* Module declarations from "cymem.cymem" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "libcpp.utility" */ + +/* Module declarations from "libcpp.pair" */ + +/* Module declarations from "libcpp.queue" */ + +/* Module declarations from "libcpp.vector" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "libc.math" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "preshed.maps" */ + +/* Module declarations from "thinc.extra.search" */ +static PyObject *__pyx_f_5thinc_5extra_6search___pyx_unpickle_MaxViolation__set_state(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.extra.search" +extern int __pyx_module_is_main_thinc__extra__search; +int __pyx_module_is_main_thinc__extra__search = 0; + +/* Implementation of "thinc.extra.search" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_zip; +static PyObject *__pyx_builtin_max; +static PyObject *__pyx_builtin_sum; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static int __pyx_pf_5thinc_5extra_6search_4Beam___init__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_class_t __pyx_v_nr_class, __pyx_t_5thinc_5extra_6search_class_t __pyx_v_width, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_min_density); /* proto */ +static Py_ssize_t __pyx_pf_5thinc_5extra_6search_4Beam_2__len__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_5score___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_9min_score___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_4loss___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_5probs___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_6scores___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_9histories___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static void __pyx_pf_5thinc_5extra_6search_4Beam_4__dealloc__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_11min_density___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static int __pyx_pf_5thinc_5extra_6search_4Beam_11min_density_2__set__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_7is_done___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_6__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_8__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_pf_5thinc_5extra_6search_12MaxViolation___init__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_2check(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_4check_crf(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_score___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_score___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_1Z___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_2gZ___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6p_hist___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6g_hist___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_probs___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_probs___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_8_softmax_genexpr(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search__softmax(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nums); /* proto */ +static PyObject *__pyx_pf_5thinc_5extra_6search_2__pyx_unpickle_MaxViolation(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_tp_new_5thinc_5extra_6search_Beam(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_5extra_6search_MaxViolation(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5thinc_5extra_6search___pyx_scope_struct__genexpr(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyMalloc; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyFree; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Pool; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Address; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMap; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMapArray; + PyObject *__pyx_type_5thinc_5extra_6search_Beam; + PyObject *__pyx_type_5thinc_5extra_6search_MaxViolation; + PyObject *__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr; + PyTypeObject *__pyx_ptype_5thinc_5extra_6search_Beam; + PyTypeObject *__pyx_ptype_5thinc_5extra_6search_MaxViolation; + PyTypeObject *__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[9]; + PyObject *__pyx_string_tab[89]; + PyObject *__pyx_number_tab[2]; +/* #### Code section: module_state_contents ### */ + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *__pyx_freelist_5thinc_5extra_6search___pyx_scope_struct__genexpr[8]; +int __pyx_freecount_5thinc_5extra_6search___pyx_scope_struct__genexpr; +#endif +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* IterNextPlain.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +PyObject *__Pyx_GetBuiltinNext_LimitedAPI_cache; +#endif + +/* Generator.module_state_decls */ +PyTypeObject *__pyx_GeneratorType; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[1] +#define __pyx_kp_u_add_note __pyx_string_tab[2] +#define __pyx_kp_u_disable __pyx_string_tab[3] +#define __pyx_kp_u_enable __pyx_string_tab[4] +#define __pyx_kp_u_gc __pyx_string_tab[5] +#define __pyx_kp_u_isenabled __pyx_string_tab[6] +#define __pyx_kp_u_self__parents_self__states_self __pyx_string_tab[7] +#define __pyx_kp_u_self_g_hist_is_not_None_or_self __pyx_string_tab[8] +#define __pyx_kp_u_stringsource __pyx_string_tab[9] +#define __pyx_kp_u_thinc_extra_math __pyx_string_tab[10] +#define __pyx_kp_u_thinc_extra_search_pyx __pyx_string_tab[11] +#define __pyx_n_u_Beam __pyx_string_tab[12] +#define __pyx_n_u_Beam___reduce_cython __pyx_string_tab[13] +#define __pyx_n_u_Beam___setstate_cython __pyx_string_tab[14] +#define __pyx_n_u_MaxViolation __pyx_string_tab[15] +#define __pyx_n_u_MaxViolation___reduce_cython __pyx_string_tab[16] +#define __pyx_n_u_MaxViolation___setstate_cython __pyx_string_tab[17] +#define __pyx_n_u_MaxViolation_check __pyx_string_tab[18] +#define __pyx_n_u_MaxViolation_check_crf __pyx_string_tab[19] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[20] +#define __pyx_n_u_Z __pyx_string_tab[21] +#define __pyx_n_u_append __pyx_string_tab[22] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[23] +#define __pyx_n_u_check __pyx_string_tab[24] +#define __pyx_n_u_check_crf __pyx_string_tab[25] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[26] +#define __pyx_n_u_close __pyx_string_tab[27] +#define __pyx_n_u_dict __pyx_string_tab[28] +#define __pyx_n_u_dict_2 __pyx_string_tab[29] +#define __pyx_n_u_func __pyx_string_tab[30] +#define __pyx_n_u_genexpr __pyx_string_tab[31] +#define __pyx_n_u_getstate __pyx_string_tab[32] +#define __pyx_n_u_gold __pyx_string_tab[33] +#define __pyx_n_u_is_coroutine __pyx_string_tab[34] +#define __pyx_n_u_items __pyx_string_tab[35] +#define __pyx_n_u_loss __pyx_string_tab[36] +#define __pyx_n_u_main __pyx_string_tab[37] +#define __pyx_n_u_math __pyx_string_tab[38] +#define __pyx_n_u_max __pyx_string_tab[39] +#define __pyx_n_u_max_2 __pyx_string_tab[40] +#define __pyx_n_u_min_density __pyx_string_tab[41] +#define __pyx_n_u_module __pyx_string_tab[42] +#define __pyx_n_u_n __pyx_string_tab[43] +#define __pyx_n_u_name __pyx_string_tab[44] +#define __pyx_n_u_new __pyx_string_tab[45] +#define __pyx_n_u_next __pyx_string_tab[46] +#define __pyx_n_u_nr_class __pyx_string_tab[47] +#define __pyx_n_u_nums __pyx_string_tab[48] +#define __pyx_n_u_pop __pyx_string_tab[49] +#define __pyx_n_u_pred __pyx_string_tab[50] +#define __pyx_n_u_pyx_checksum __pyx_string_tab[51] +#define __pyx_n_u_pyx_result __pyx_string_tab[52] +#define __pyx_n_u_pyx_state __pyx_string_tab[53] +#define __pyx_n_u_pyx_type __pyx_string_tab[54] +#define __pyx_n_u_pyx_unpickle_MaxViolation __pyx_string_tab[55] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[56] +#define __pyx_n_u_qualname __pyx_string_tab[57] +#define __pyx_n_u_reduce __pyx_string_tab[58] +#define __pyx_n_u_reduce_cython __pyx_string_tab[59] +#define __pyx_n_u_reduce_ex __pyx_string_tab[60] +#define __pyx_n_u_score __pyx_string_tab[61] +#define __pyx_n_u_self __pyx_string_tab[62] +#define __pyx_n_u_send __pyx_string_tab[63] +#define __pyx_n_u_set_name __pyx_string_tab[64] +#define __pyx_n_u_setdefault __pyx_string_tab[65] +#define __pyx_n_u_setstate __pyx_string_tab[66] +#define __pyx_n_u_setstate_cython __pyx_string_tab[67] +#define __pyx_n_u_softmax __pyx_string_tab[68] +#define __pyx_n_u_softmax_locals_genexpr __pyx_string_tab[69] +#define __pyx_n_u_state __pyx_string_tab[70] +#define __pyx_n_u_sum __pyx_string_tab[71] +#define __pyx_n_u_test __pyx_string_tab[72] +#define __pyx_n_u_thinc_extra_search __pyx_string_tab[73] +#define __pyx_n_u_throw __pyx_string_tab[74] +#define __pyx_n_u_update __pyx_string_tab[75] +#define __pyx_n_u_use_setstate __pyx_string_tab[76] +#define __pyx_n_u_value __pyx_string_tab[77] +#define __pyx_n_u_values __pyx_string_tab[78] +#define __pyx_n_u_width __pyx_string_tab[79] +#define __pyx_n_u_zip __pyx_string_tab[80] +#define __pyx_kp_b_iso88591_6 __pyx_string_tab[81] +#define __pyx_kp_b_iso88591_8_haq_haq_7_AQ_1F_RuD_c_3b_a_at __pyx_string_tab[82] +#define __pyx_kp_b_iso88591_A __pyx_string_tab[83] +#define __pyx_kp_b_iso88591_A_D_r_Q_S_q_Jat4uE_a_4vRr_d_2S __pyx_string_tab[84] +#define __pyx_kp_b_iso88591_Q __pyx_string_tab[85] +#define __pyx_kp_b_iso88591_T_T_HD_T_jPTT_bbhhllpptt_B_B_L __pyx_string_tab[86] +#define __pyx_kp_b_iso88591_q_0_kQR_xq_7_a_nA_1 __pyx_string_tab[87] +#define __pyx_kp_b_iso88591_t1_q_3aq_2S_9BgZvT_a_1A_2QauBgZ __pyx_string_tab[88] +#define __pyx_int_0 __pyx_number_tab[0] +#define __pyx_int_237080843 __pyx_number_tab[1] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_5extra_6search_Beam); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_5extra_6search_Beam); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_5extra_6search_MaxViolation); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_5extra_6search_MaxViolation); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr); + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<9; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<89; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* Generator.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_GeneratorType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_5extra_6search_Beam); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_5extra_6search_Beam); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_5extra_6search_MaxViolation); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_5extra_6search_MaxViolation); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr); + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<9; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<89; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* Generator.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_GeneratorType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "thinc/extra/search.pxd":56 + * cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1 + * + * cdef inline void* at(self, int i) nogil: # <<<<<<<<<<<<<< + * return self._states[i].content + * +*/ + +static CYTHON_INLINE void *__pyx_f_5thinc_5extra_6search_4Beam_at(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i) { + void *__pyx_r; + + /* "thinc/extra/search.pxd":57 + * + * cdef inline void* at(self, int i) nogil: + * return self._states[i].content # <<<<<<<<<<<<<< + * + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1 +*/ + __pyx_r = (__pyx_v_self->_states[__pyx_v_i]).content; + goto __pyx_L0; + + /* "thinc/extra/search.pxd":56 + * cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1 + * + * cdef inline void* at(self, int i) nogil: # <<<<<<<<<<<<<< + * return self._states[i].content + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/extra/search.pxd":65 + * + * + * cdef inline void set_cell(self, int i, int j, weight_t score, int is_valid, weight_t cost) nogil: # <<<<<<<<<<<<<< + * self.scores[i][j] = score + * self.is_valid[i][j] = is_valid +*/ + +static CYTHON_INLINE void __pyx_f_5thinc_5extra_6search_4Beam_set_cell(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i, int __pyx_v_j, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_score, int __pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_cost) { + + /* "thinc/extra/search.pxd":66 + * + * cdef inline void set_cell(self, int i, int j, weight_t score, int is_valid, weight_t cost) nogil: + * self.scores[i][j] = score # <<<<<<<<<<<<<< + * self.is_valid[i][j] = is_valid + * self.costs[i][j] = cost +*/ + ((__pyx_v_self->scores[__pyx_v_i])[__pyx_v_j]) = __pyx_v_score; + + /* "thinc/extra/search.pxd":67 + * cdef inline void set_cell(self, int i, int j, weight_t score, int is_valid, weight_t cost) nogil: + * self.scores[i][j] = score + * self.is_valid[i][j] = is_valid # <<<<<<<<<<<<<< + * self.costs[i][j] = cost + * +*/ + ((__pyx_v_self->is_valid[__pyx_v_i])[__pyx_v_j]) = __pyx_v_is_valid; + + /* "thinc/extra/search.pxd":68 + * self.scores[i][j] = score + * self.is_valid[i][j] = is_valid + * self.costs[i][j] = cost # <<<<<<<<<<<<<< + * + * cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, +*/ + ((__pyx_v_self->costs[__pyx_v_i])[__pyx_v_j]) = __pyx_v_cost; + + /* "thinc/extra/search.pxd":65 + * + * + * cdef inline void set_cell(self, int i, int j, weight_t score, int is_valid, weight_t cost) nogil: # <<<<<<<<<<<<<< + * self.scores[i][j] = score + * self.is_valid[i][j] = is_valid +*/ + + /* function exit code */ +} + +/* "thinc/extra/search.pyx":13 + * + * cdef class Beam: + * def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): # <<<<<<<<<<<<<< + * assert nr_class != 0 + * assert width != 0 +*/ + +/* Python wrapper */ +static int __pyx_pw_5thinc_5extra_6search_4Beam_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5thinc_5extra_6search_4Beam_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_nr_class; + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_width; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_min_density; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_nr_class,&__pyx_mstate_global->__pyx_n_u_width,&__pyx_mstate_global->__pyx_n_u_min_density,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 13, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 13, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 13, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 13, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 13, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, i); __PYX_ERR(0, 13, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 13, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 13, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 13, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_nr_class = __Pyx_PyLong_As_uint64_t(values[0]); if (unlikely((__pyx_v_nr_class == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L3_error) + __pyx_v_width = __Pyx_PyLong_As_uint64_t(values[1]); if (unlikely((__pyx_v_width == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L3_error) + if (values[2]) { + __pyx_v_min_density = __Pyx_PyFloat_AsFloat(values[2]); if (unlikely((__pyx_v_min_density == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 13, __pyx_L3_error) + } else { + __pyx_v_min_density = ((__pyx_t_5thinc_5extra_6search_weight_t)0.0); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, __pyx_nargs); __PYX_ERR(0, 13, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.Beam.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam___init__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self), __pyx_v_nr_class, __pyx_v_width, __pyx_v_min_density); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5thinc_5extra_6search_4Beam___init__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_class_t __pyx_v_nr_class, __pyx_t_5thinc_5extra_6search_class_t __pyx_v_width, __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_min_density) { + int __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + void *__pyx_t_5; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_6; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_7; + int __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "thinc/extra/search.pyx":14 + * cdef class Beam: + * def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): + * assert nr_class != 0 # <<<<<<<<<<<<<< + * assert width != 0 + * self.nr_class = nr_class +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_nr_class != 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 14, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 14, __pyx_L1_error) + #endif + + /* "thinc/extra/search.pyx":15 + * def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): + * assert nr_class != 0 + * assert width != 0 # <<<<<<<<<<<<<< + * self.nr_class = nr_class + * self.width = width +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_width != 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 15, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 15, __pyx_L1_error) + #endif + + /* "thinc/extra/search.pyx":16 + * assert nr_class != 0 + * assert width != 0 + * self.nr_class = nr_class # <<<<<<<<<<<<<< + * self.width = width + * self.min_density = min_density +*/ + __pyx_v_self->nr_class = __pyx_v_nr_class; + + /* "thinc/extra/search.pyx":17 + * assert width != 0 + * self.nr_class = nr_class + * self.width = width # <<<<<<<<<<<<<< + * self.min_density = min_density + * self.size = 1 +*/ + __pyx_v_self->width = __pyx_v_width; + + /* "thinc/extra/search.pyx":18 + * self.nr_class = nr_class + * self.width = width + * self.min_density = min_density # <<<<<<<<<<<<<< + * self.size = 1 + * self.t = 0 +*/ + __pyx_v_self->min_density = __pyx_v_min_density; + + /* "thinc/extra/search.pyx":19 + * self.width = width + * self.min_density = min_density + * self.size = 1 # <<<<<<<<<<<<<< + * self.t = 0 + * self.mem = Pool() +*/ + __pyx_v_self->size = 1; + + /* "thinc/extra/search.pyx":20 + * self.min_density = min_density + * self.size = 1 + * self.t = 0 # <<<<<<<<<<<<<< + * self.mem = Pool() + * self._parents = <_State*>self.mem.alloc(self.width, sizeof(_State)) +*/ + __pyx_v_self->t = 0; + + /* "thinc/extra/search.pyx":21 + * self.size = 1 + * self.t = 0 + * self.mem = Pool() # <<<<<<<<<<<<<< + * self._parents = <_State*>self.mem.alloc(self.width, sizeof(_State)) + * self._states = <_State*>self.mem.alloc(self.width, sizeof(_State)) +*/ + __pyx_t_3 = NULL; + __pyx_t_4 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_ptype_5cymem_5cymem_Pool, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 21, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_2); + } + __Pyx_GIVEREF((PyObject *)__pyx_t_2); + __Pyx_GOTREF((PyObject *)__pyx_v_self->mem); + __Pyx_DECREF((PyObject *)__pyx_v_self->mem); + __pyx_v_self->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":22 + * self.t = 0 + * self.mem = Pool() + * self._parents = <_State*>self.mem.alloc(self.width, sizeof(_State)) # <<<<<<<<<<<<<< + * self._states = <_State*>self.mem.alloc(self.width, sizeof(_State)) + * cdef int i +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->width, (sizeof(struct __pyx_t_5thinc_5extra_6search__State))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 22, __pyx_L1_error) + __pyx_v_self->_parents = ((struct __pyx_t_5thinc_5extra_6search__State *)__pyx_t_5); + + /* "thinc/extra/search.pyx":23 + * self.mem = Pool() + * self._parents = <_State*>self.mem.alloc(self.width, sizeof(_State)) + * self._states = <_State*>self.mem.alloc(self.width, sizeof(_State)) # <<<<<<<<<<<<<< + * cdef int i + * self.histories = [[] for i in range(self.width)] +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->width, (sizeof(struct __pyx_t_5thinc_5extra_6search__State))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_v_self->_states = ((struct __pyx_t_5thinc_5extra_6search__State *)__pyx_t_5); + + /* "thinc/extra/search.pyx":25 + * self._states = <_State*>self.mem.alloc(self.width, sizeof(_State)) + * cdef int i + * self.histories = [[] for i in range(self.width)] # <<<<<<<<<<<<<< + * self._parent_histories = [[] for i in range(self.width)] + * +*/ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = __pyx_v_self->width; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_3))) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_self->histories); + __Pyx_DECREF(__pyx_v_self->histories); + __pyx_v_self->histories = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":26 + * cdef int i + * self.histories = [[] for i in range(self.width)] + * self._parent_histories = [[] for i in range(self.width)] # <<<<<<<<<<<<<< + * + * self.scores = self.mem.alloc(self.width, sizeof(weight_t*)) +*/ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = __pyx_v_self->width; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_3))) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_self->_parent_histories); + __Pyx_DECREF(__pyx_v_self->_parent_histories); + __pyx_v_self->_parent_histories = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":28 + * self._parent_histories = [[] for i in range(self.width)] + * + * self.scores = self.mem.alloc(self.width, sizeof(weight_t*)) # <<<<<<<<<<<<<< + * self.is_valid = self.mem.alloc(self.width, sizeof(weight_t*)) + * self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->width, (sizeof(__pyx_t_5thinc_5extra_6search_weight_t *))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 28, __pyx_L1_error) + __pyx_v_self->scores = ((__pyx_t_5thinc_5extra_6search_weight_t **)__pyx_t_5); + + /* "thinc/extra/search.pyx":29 + * + * self.scores = self.mem.alloc(self.width, sizeof(weight_t*)) + * self.is_valid = self.mem.alloc(self.width, sizeof(weight_t*)) # <<<<<<<<<<<<<< + * self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) + * for i in range(self.width): +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->width, (sizeof(__pyx_t_5thinc_5extra_6search_weight_t *))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 29, __pyx_L1_error) + __pyx_v_self->is_valid = ((int **)__pyx_t_5); + + /* "thinc/extra/search.pyx":30 + * self.scores = self.mem.alloc(self.width, sizeof(weight_t*)) + * self.is_valid = self.mem.alloc(self.width, sizeof(weight_t*)) + * self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) # <<<<<<<<<<<<<< + * for i in range(self.width): + * self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->width, (sizeof(__pyx_t_5thinc_5extra_6search_weight_t *))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 30, __pyx_L1_error) + __pyx_v_self->costs = ((__pyx_t_5thinc_5extra_6search_weight_t **)__pyx_t_5); + + /* "thinc/extra/search.pyx":31 + * self.is_valid = self.mem.alloc(self.width, sizeof(weight_t*)) + * self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) + * for i in range(self.width): # <<<<<<<<<<<<<< + * self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * self.is_valid[i] = self.mem.alloc(self.nr_class, sizeof(int)) +*/ + __pyx_t_6 = __pyx_v_self->width; + __pyx_t_7 = __pyx_t_6; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_7; __pyx_t_8+=1) { + __pyx_v_i = __pyx_t_8; + + /* "thinc/extra/search.pyx":32 + * self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) + * for i in range(self.width): + * self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) # <<<<<<<<<<<<<< + * self.is_valid[i] = self.mem.alloc(self.nr_class, sizeof(int)) + * self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->nr_class, (sizeof(__pyx_t_5thinc_5extra_6search_weight_t))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 32, __pyx_L1_error) + (__pyx_v_self->scores[__pyx_v_i]) = ((__pyx_t_5thinc_5extra_6search_weight_t *)__pyx_t_5); + + /* "thinc/extra/search.pyx":33 + * for i in range(self.width): + * self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * self.is_valid[i] = self.mem.alloc(self.nr_class, sizeof(int)) # <<<<<<<<<<<<<< + * self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->nr_class, (sizeof(int))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 33, __pyx_L1_error) + (__pyx_v_self->is_valid[__pyx_v_i]) = ((int *)__pyx_t_5); + + /* "thinc/extra/search.pyx":34 + * self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * self.is_valid[i] = self.mem.alloc(self.nr_class, sizeof(int)) + * self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __pyx_t_5 = ((struct __pyx_vtabstruct_5cymem_5cymem_Pool *)__pyx_v_self->mem->__pyx_vtab)->alloc(__pyx_v_self->mem, __pyx_v_self->nr_class, (sizeof(__pyx_t_5thinc_5extra_6search_weight_t))); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 34, __pyx_L1_error) + (__pyx_v_self->costs[__pyx_v_i]) = ((__pyx_t_5thinc_5extra_6search_weight_t *)__pyx_t_5); + } + + /* "thinc/extra/search.pyx":13 + * + * cdef class Beam: + * def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): # <<<<<<<<<<<<<< + * assert nr_class != 0 + * assert width != 0 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("thinc.extra.search.Beam.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":36 + * self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self.size + * +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_pw_5thinc_5extra_6search_4Beam_3__len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_pw_5thinc_5extra_6search_4Beam_3__len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_2__len__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_pf_5thinc_5extra_6search_4Beam_2__len__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + Py_ssize_t __pyx_r; + + /* "thinc/extra/search.pyx":37 + * + * def __len__(self): + * return self.size # <<<<<<<<<<<<<< + * + * property score: +*/ + __pyx_r = __pyx_v_self->size; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":36 + * self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self.size + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/extra/search.pyx":40 + * + * property score: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[0].score + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_5score_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_5score_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_5score___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_5score___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":41 + * property score: + * def __get__(self): + * return self._states[0].score # <<<<<<<<<<<<<< + * + * property min_score: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_self->_states[0]).score); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":40 + * + * property score: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[0].score + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.Beam.score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":44 + * + * property min_score: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[self.size-1].score + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9min_score_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9min_score_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_9min_score___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_9min_score___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":45 + * property min_score: + * def __get__(self): + * return self._states[self.size-1].score # <<<<<<<<<<<<<< + * + * property loss: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_self->_states[(__pyx_v_self->size - 1)]).score); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":44 + * + * property min_score: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[self.size-1].score + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.Beam.min_score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":48 + * + * property loss: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[0].loss + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_4loss_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_4loss_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_4loss___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_4loss___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":49 + * property loss: + * def __get__(self): + * return self._states[0].loss # <<<<<<<<<<<<<< + * + * property probs: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_self->_states[0]).loss); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":48 + * + * property loss: + * def __get__(self): # <<<<<<<<<<<<<< + * return self._states[0].loss + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.Beam.loss.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":52 + * + * property probs: + * def __get__(self): # <<<<<<<<<<<<<< + * return _softmax([self._states[i].score for i in range(self.size)]) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_5probs_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_5probs_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_5probs___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_5probs___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_5; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_6; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":53 + * property probs: + * def __get__(self): + * return _softmax([self._states[i].score for i in range(self.size)]) # <<<<<<<<<<<<<< + * + * property scores: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_softmax); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_self->size; + __pyx_t_6 = __pyx_t_5; + for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) { + __pyx_v_i = __pyx_t_7; + __pyx_t_8 = PyFloat_FromDouble((__pyx_v_self->_states[__pyx_v_i]).score); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_4, (PyObject*)__pyx_t_8))) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":52 + * + * property probs: + * def __get__(self): # <<<<<<<<<<<<<< + * return _softmax([self._states[i].score for i in range(self.size)]) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.extra.search.Beam.probs.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":56 + * + * property scores: + * def __get__(self): # <<<<<<<<<<<<<< + * return [self._states[i].score for i in range(self.size)] + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_6scores_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_6scores_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_6scores___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_6scores___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_3; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":57 + * property scores: + * def __get__(self): + * return [self._states[i].score for i in range(self.size)] # <<<<<<<<<<<<<< + * + * property histories: +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_v_self->size; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + __pyx_t_5 = PyFloat_FromDouble((__pyx_v_self->_states[__pyx_v_i]).score); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":56 + * + * property scores: + * def __get__(self): # <<<<<<<<<<<<<< + * return [self._states[i].score for i in range(self.size)] + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.extra.search.Beam.scores.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":60 + * + * property histories: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.histories + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9histories_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9histories_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_9histories___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_9histories___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + + /* "thinc/extra/search.pyx":61 + * property histories: + * def __get__(self): + * return self.histories # <<<<<<<<<<<<<< + * + * cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->histories); + __pyx_r = __pyx_v_self->histories; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":60 + * + * property histories: + * def __get__(self): # <<<<<<<<<<<<<< + * return self.histories + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":63 + * return self.histories + * + * cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, # <<<<<<<<<<<<<< + * const weight_t* costs) except -1: + * cdef int j +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam_set_row(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, int __pyx_v_i, __pyx_t_5thinc_5extra_6search_weight_t const *__pyx_v_scores, int const *__pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t const *__pyx_v_costs) { + int __pyx_v_j; + int __pyx_r; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/extra/search.pyx":66 + * const weight_t* costs) except -1: + * cdef int j + * for j in range(self.nr_class): # <<<<<<<<<<<<<< + * self.scores[i][j] = scores[j] + * self.is_valid[i][j] = is_valid[j] +*/ + __pyx_t_1 = __pyx_v_self->nr_class; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_j = __pyx_t_3; + + /* "thinc/extra/search.pyx":67 + * cdef int j + * for j in range(self.nr_class): + * self.scores[i][j] = scores[j] # <<<<<<<<<<<<<< + * self.is_valid[i][j] = is_valid[j] + * self.costs[i][j] = costs[j] +*/ + ((__pyx_v_self->scores[__pyx_v_i])[__pyx_v_j]) = (__pyx_v_scores[__pyx_v_j]); + + /* "thinc/extra/search.pyx":68 + * for j in range(self.nr_class): + * self.scores[i][j] = scores[j] + * self.is_valid[i][j] = is_valid[j] # <<<<<<<<<<<<<< + * self.costs[i][j] = costs[j] + * +*/ + ((__pyx_v_self->is_valid[__pyx_v_i])[__pyx_v_j]) = (__pyx_v_is_valid[__pyx_v_j]); + + /* "thinc/extra/search.pyx":69 + * self.scores[i][j] = scores[j] + * self.is_valid[i][j] = is_valid[j] + * self.costs[i][j] = costs[j] # <<<<<<<<<<<<<< + * + * cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1: +*/ + ((__pyx_v_self->costs[__pyx_v_i])[__pyx_v_j]) = (__pyx_v_costs[__pyx_v_j]); + } + + /* "thinc/extra/search.pyx":63 + * return self.histories + * + * cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, # <<<<<<<<<<<<<< + * const weight_t* costs) except -1: + * cdef int j +*/ + + /* function exit code */ + __pyx_r = 0; + return __pyx_r; +} + +/* "thinc/extra/search.pyx":71 + * self.costs[i][j] = costs[j] + * + * cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1: # <<<<<<<<<<<<<< + * cdef int i, j + * for i in range(self.width): +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam_set_table(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_scores, int **__pyx_v_is_valid, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_costs) { + int __pyx_v_i; + int __pyx_r; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + int __pyx_t_3; + + /* "thinc/extra/search.pyx":73 + * cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1: + * cdef int i, j + * for i in range(self.width): # <<<<<<<<<<<<<< + * memcpy(self.scores[i], scores[i], sizeof(weight_t) * self.nr_class) + * memcpy(self.is_valid[i], is_valid[i], sizeof(bint) * self.nr_class) +*/ + __pyx_t_1 = __pyx_v_self->width; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/extra/search.pyx":74 + * cdef int i, j + * for i in range(self.width): + * memcpy(self.scores[i], scores[i], sizeof(weight_t) * self.nr_class) # <<<<<<<<<<<<<< + * memcpy(self.is_valid[i], is_valid[i], sizeof(bint) * self.nr_class) + * memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) +*/ + (void)(memcpy((__pyx_v_self->scores[__pyx_v_i]), (__pyx_v_scores[__pyx_v_i]), ((sizeof(__pyx_t_5thinc_5extra_6search_weight_t)) * __pyx_v_self->nr_class))); + + /* "thinc/extra/search.pyx":75 + * for i in range(self.width): + * memcpy(self.scores[i], scores[i], sizeof(weight_t) * self.nr_class) + * memcpy(self.is_valid[i], is_valid[i], sizeof(bint) * self.nr_class) # <<<<<<<<<<<<<< + * memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) + * +*/ + (void)(memcpy((__pyx_v_self->is_valid[__pyx_v_i]), (__pyx_v_is_valid[__pyx_v_i]), ((sizeof(int)) * __pyx_v_self->nr_class))); + + /* "thinc/extra/search.pyx":76 + * memcpy(self.scores[i], scores[i], sizeof(weight_t) * self.nr_class) + * memcpy(self.is_valid[i], is_valid[i], sizeof(bint) * self.nr_class) + * memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) # <<<<<<<<<<<<<< + * + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: +*/ + (void)(memcpy((__pyx_v_self->costs[__pyx_v_i]), (__pyx_v_costs[__pyx_v_i]), ((sizeof(int)) * __pyx_v_self->nr_class))); + } + + /* "thinc/extra/search.pyx":71 + * self.costs[i][j] = costs[j] + * + * cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1: # <<<<<<<<<<<<<< + * cdef int i, j + * for i in range(self.width): +*/ + + /* function exit code */ + __pyx_r = 0; + return __pyx_r; +} + +/* "thinc/extra/search.pyx":78 + * memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) + * + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: # <<<<<<<<<<<<<< + * for i in range(self.width): + * self._states[i].content = init_func(self.mem, n, extra_args) +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam_initialize(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_init_func_t __pyx_v_init_func, __pyx_t_5thinc_5extra_6search_del_func_t __pyx_v_del_func, int __pyx_v_n, void *__pyx_v_extra_args) { + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + void *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("initialize", 0); + + /* "thinc/extra/search.pyx":79 + * + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: + * for i in range(self.width): # <<<<<<<<<<<<<< + * self._states[i].content = init_func(self.mem, n, extra_args) + * self._parents[i].content = init_func(self.mem, n, extra_args) +*/ + __pyx_t_1 = __pyx_v_self->width; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/extra/search.pyx":80 + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: + * for i in range(self.width): + * self._states[i].content = init_func(self.mem, n, extra_args) # <<<<<<<<<<<<<< + * self._parents[i].content = init_func(self.mem, n, extra_args) + * self.del_func = del_func +*/ + __pyx_t_4 = ((PyObject *)__pyx_v_self->mem); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_init_func(((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_4), __pyx_v_n, __pyx_v_extra_args); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 80, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + (__pyx_v_self->_states[__pyx_v_i]).content = __pyx_t_5; + + /* "thinc/extra/search.pyx":81 + * for i in range(self.width): + * self._states[i].content = init_func(self.mem, n, extra_args) + * self._parents[i].content = init_func(self.mem, n, extra_args) # <<<<<<<<<<<<<< + * self.del_func = del_func + * +*/ + __pyx_t_4 = ((PyObject *)__pyx_v_self->mem); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_init_func(((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_4), __pyx_v_n, __pyx_v_extra_args); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(0, 81, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + (__pyx_v_self->_parents[__pyx_v_i]).content = __pyx_t_5; + } + + /* "thinc/extra/search.pyx":82 + * self._states[i].content = init_func(self.mem, n, extra_args) + * self._parents[i].content = init_func(self.mem, n, extra_args) + * self.del_func = del_func # <<<<<<<<<<<<<< + * + * def __dealloc__(self): +*/ + __pyx_v_self->del_func = __pyx_v_del_func; + + /* "thinc/extra/search.pyx":78 + * memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) + * + * cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: # <<<<<<<<<<<<<< + * for i in range(self.width): + * self._states[i].content = init_func(self.mem, n, extra_args) +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("thinc.extra.search.Beam.initialize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":84 + * self.del_func = del_func + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * for i in range(self.width): + * self.del_func(self.mem, self._states[i].content, NULL) +*/ + +/* Python wrapper */ +static void __pyx_pw_5thinc_5extra_6search_4Beam_5__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5thinc_5extra_6search_4Beam_5__dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_pf_5thinc_5extra_6search_4Beam_4__dealloc__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_pf_5thinc_5extra_6search_4Beam_4__dealloc__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + __Pyx_RefNannyDeclarations + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__dealloc__", 0); + + /* "thinc/extra/search.pyx":85 + * + * def __dealloc__(self): + * for i in range(self.width): # <<<<<<<<<<<<<< + * self.del_func(self.mem, self._states[i].content, NULL) + * self.del_func(self.mem, self._parents[i].content, NULL) +*/ + __pyx_t_1 = __pyx_v_self->width; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/extra/search.pyx":86 + * def __dealloc__(self): + * for i in range(self.width): + * self.del_func(self.mem, self._states[i].content, NULL) # <<<<<<<<<<<<<< + * self.del_func(self.mem, self._parents[i].content, NULL) + * +*/ + __pyx_t_4 = ((PyObject *)__pyx_v_self->mem); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_self->del_func(((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_4), (__pyx_v_self->_states[__pyx_v_i]).content, NULL); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/extra/search.pyx":87 + * for i in range(self.width): + * self.del_func(self.mem, self._states[i].content, NULL) + * self.del_func(self.mem, self._parents[i].content, NULL) # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + __pyx_t_4 = ((PyObject *)__pyx_v_self->mem); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __pyx_v_self->del_func(((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_4), (__pyx_v_self->_parents[__pyx_v_i]).content, NULL); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + + /* "thinc/extra/search.pyx":84 + * self.del_func = del_func + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * for i in range(self.width): + * self.del_func(self.mem, self._states[i].content, NULL) +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_WriteUnraisable("thinc.extra.search.Beam.__dealloc__", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0); + __pyx_L0:; + __Pyx_RefNannyFinishContext(); +} + +/* "thinc/extra/search.pyx":89 + * self.del_func(self.mem, self._parents[i].content, NULL) + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef int advance(self, trans_func_t transition_func, hash_func_t hash_func, + * void* extra_args) except -1: +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam_advance(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_trans_func_t __pyx_v_transition_func, __pyx_t_5thinc_5extra_6search_hash_func_t __pyx_v_hash_func, void *__pyx_v_extra_args) { + __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_scores; + int **__pyx_v_is_valid; + __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_costs; + __pyx_t_5thinc_5extra_6search_Queue *__pyx_v_q; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_score; + int __pyx_v_p_i; + int __pyx_v_i; + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_clas; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_v_parent; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_v_state; + __pyx_t_5thinc_5extra_6search_hash_t __pyx_v_key; + struct __pyx_obj_7preshed_4maps_PreshMap *__pyx_v_seen_states = 0; + uint64_t __pyx_v_is_seen; + uint64_t __pyx_v_one; + __pyx_t_5thinc_5extra_6search_Entry __pyx_v_data; + int __pyx_r; + __Pyx_RefNannyDeclarations + __pyx_t_5thinc_5extra_6search_weight_t **__pyx_t_1; + int **__pyx_t_2; + __pyx_t_5thinc_5extra_6search_Queue *__pyx_t_3; + int __pyx_t_4; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_t_5; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + size_t __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_15; + struct __pyx_t_5thinc_5extra_6search__State __pyx_t_16; + struct __pyx_t_5thinc_5extra_6search__State __pyx_t_17; + __pyx_t_5thinc_5extra_6search_hash_t __pyx_t_18; + __pyx_t_5thinc_5extra_6search_hash_t __pyx_t_19; + void *__pyx_t_20; + int __pyx_t_21; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_22; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_23; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("advance", 0); + + /* "thinc/extra/search.pyx":92 + * cdef int advance(self, trans_func_t transition_func, hash_func_t hash_func, + * void* extra_args) except -1: + * cdef weight_t** scores = self.scores # <<<<<<<<<<<<<< + * cdef int** is_valid = self.is_valid + * cdef weight_t** costs = self.costs +*/ + __pyx_t_1 = __pyx_v_self->scores; + __pyx_v_scores = __pyx_t_1; + + /* "thinc/extra/search.pyx":93 + * void* extra_args) except -1: + * cdef weight_t** scores = self.scores + * cdef int** is_valid = self.is_valid # <<<<<<<<<<<<<< + * cdef weight_t** costs = self.costs + * +*/ + __pyx_t_2 = __pyx_v_self->is_valid; + __pyx_v_is_valid = __pyx_t_2; + + /* "thinc/extra/search.pyx":94 + * cdef weight_t** scores = self.scores + * cdef int** is_valid = self.is_valid + * cdef weight_t** costs = self.costs # <<<<<<<<<<<<<< + * + * cdef Queue* q = new Queue() +*/ + __pyx_t_1 = __pyx_v_self->costs; + __pyx_v_costs = __pyx_t_1; + + /* "thinc/extra/search.pyx":96 + * cdef weight_t** costs = self.costs + * + * cdef Queue* q = new Queue() # <<<<<<<<<<<<<< + * self._fill(q, scores, is_valid) + * # For a beam of width k, we only ever need 2k state objects. How? +*/ + try { + __pyx_t_3 = new __pyx_t_5thinc_5extra_6search_Queue(); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 96, __pyx_L1_error) + } + __pyx_v_q = __pyx_t_3; + + /* "thinc/extra/search.pyx":97 + * + * cdef Queue* q = new Queue() + * self._fill(q, scores, is_valid) # <<<<<<<<<<<<<< + * # For a beam of width k, we only ever need 2k state objects. How? + * # Each transition takes a parent and a class and produces a new state. +*/ + __pyx_t_4 = ((struct __pyx_vtabstruct_5thinc_5extra_6search_Beam *)__pyx_v_self->__pyx_vtab)->_fill(__pyx_v_self, __pyx_v_q, __pyx_v_scores, __pyx_v_is_valid); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(0, 97, __pyx_L1_error) + + /* "thinc/extra/search.pyx":103 + * # each step, we take a parent, and apply one or more extensions to + * # it. + * self._parents, self._states = self._states, self._parents # <<<<<<<<<<<<<< + * self._parent_histories, self.histories = self.histories, self._parent_histories + * cdef weight_t score +*/ + __pyx_t_5 = __pyx_v_self->_states; + __pyx_t_6 = __pyx_v_self->_parents; + __pyx_v_self->_parents = __pyx_t_5; + __pyx_v_self->_states = __pyx_t_6; + + /* "thinc/extra/search.pyx":104 + * # it. + * self._parents, self._states = self._states, self._parents + * self._parent_histories, self.histories = self.histories, self._parent_histories # <<<<<<<<<<<<<< + * cdef weight_t score + * cdef int p_i +*/ + __pyx_t_7 = __pyx_v_self->histories; + __pyx_t_8 = __pyx_v_self->_parent_histories; + __pyx_v_self->_parent_histories = ((PyObject*)__pyx_t_7); + __pyx_t_7 = 0; + __pyx_v_self->histories = ((PyObject*)__pyx_t_8); + __pyx_t_8 = 0; + + /* "thinc/extra/search.pyx":107 + * cdef weight_t score + * cdef int p_i + * cdef int i = 0 # <<<<<<<<<<<<<< + * cdef class_t clas + * cdef _State* parent +*/ + __pyx_v_i = 0; + + /* "thinc/extra/search.pyx":112 + * cdef _State* state + * cdef hash_t key + * cdef PreshMap seen_states = PreshMap(self.width) # <<<<<<<<<<<<<< + * cdef uint64_t is_seen + * cdef uint64_t one = 1 +*/ + __pyx_t_10 = NULL; + __pyx_t_11 = __Pyx_PyLong_From_uint64_t(__pyx_v_self->width); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_10, __pyx_t_11}; + __pyx_t_9 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_ptype_7preshed_4maps_PreshMap, __pyx_callargs+__pyx_t_12, (2-__pyx_t_12) | (__pyx_t_12*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_9); + } + __pyx_v_seen_states = ((struct __pyx_obj_7preshed_4maps_PreshMap *)__pyx_t_9); + __pyx_t_9 = 0; + + /* "thinc/extra/search.pyx":114 + * cdef PreshMap seen_states = PreshMap(self.width) + * cdef uint64_t is_seen + * cdef uint64_t one = 1 # <<<<<<<<<<<<<< + * while i < self.width and not q.empty(): + * data = q.top() +*/ + __pyx_v_one = 1; + + /* "thinc/extra/search.pyx":115 + * cdef uint64_t is_seen + * cdef uint64_t one = 1 + * while i < self.width and not q.empty(): # <<<<<<<<<<<<<< + * data = q.top() + * p_i = data.second / self.nr_class +*/ + while (1) { + __pyx_t_14 = (__pyx_v_i < __pyx_v_self->width); + if (__pyx_t_14) { + } else { + __pyx_t_13 = __pyx_t_14; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_14 = (!__pyx_v_q->empty()); + __pyx_t_13 = __pyx_t_14; + __pyx_L5_bool_binop_done:; + if (!__pyx_t_13) break; + + /* "thinc/extra/search.pyx":116 + * cdef uint64_t one = 1 + * while i < self.width and not q.empty(): + * data = q.top() # <<<<<<<<<<<<<< + * p_i = data.second / self.nr_class + * clas = data.second % self.nr_class +*/ + __pyx_v_data = __pyx_v_q->top(); + + /* "thinc/extra/search.pyx":117 + * while i < self.width and not q.empty(): + * data = q.top() + * p_i = data.second / self.nr_class # <<<<<<<<<<<<<< + * clas = data.second % self.nr_class + * score = data.first +*/ + __pyx_v_p_i = (__pyx_v_data.second / __pyx_v_self->nr_class); + + /* "thinc/extra/search.pyx":118 + * data = q.top() + * p_i = data.second / self.nr_class + * clas = data.second % self.nr_class # <<<<<<<<<<<<<< + * score = data.first + * q.pop() +*/ + __pyx_v_clas = (__pyx_v_data.second % __pyx_v_self->nr_class); + + /* "thinc/extra/search.pyx":119 + * p_i = data.second / self.nr_class + * clas = data.second % self.nr_class + * score = data.first # <<<<<<<<<<<<<< + * q.pop() + * parent = &self._parents[p_i] +*/ + __pyx_t_15 = __pyx_v_data.first; + __pyx_v_score = __pyx_t_15; + + /* "thinc/extra/search.pyx":120 + * clas = data.second % self.nr_class + * score = data.first + * q.pop() # <<<<<<<<<<<<<< + * parent = &self._parents[p_i] + * # Indicates terminal state reached; i.e. state is done +*/ + __pyx_v_q->pop(); + + /* "thinc/extra/search.pyx":121 + * score = data.first + * q.pop() + * parent = &self._parents[p_i] # <<<<<<<<<<<<<< + * # Indicates terminal state reached; i.e. state is done + * if parent.is_done: +*/ + __pyx_v_parent = (&(__pyx_v_self->_parents[__pyx_v_p_i])); + + /* "thinc/extra/search.pyx":123 + * parent = &self._parents[p_i] + * # Indicates terminal state reached; i.e. state is done + * if parent.is_done: # <<<<<<<<<<<<<< + * # Now parent will not be changed, so we don't have to copy. + * # Once finished, should also be unbranching. +*/ + if (__pyx_v_parent->is_done) { + + /* "thinc/extra/search.pyx":126 + * # Now parent will not be changed, so we don't have to copy. + * # Once finished, should also be unbranching. + * self._states[i], parent[0] = parent[0], self._states[i] # <<<<<<<<<<<<<< + * parent.i = self._states[i].i + * parent.t = self._states[i].t +*/ + __pyx_t_16 = (__pyx_v_parent[0]); + __pyx_t_17 = (__pyx_v_self->_states[__pyx_v_i]); + (__pyx_v_self->_states[__pyx_v_i]) = __pyx_t_16; + (__pyx_v_parent[0]) = __pyx_t_17; + + /* "thinc/extra/search.pyx":127 + * # Once finished, should also be unbranching. + * self._states[i], parent[0] = parent[0], self._states[i] + * parent.i = self._states[i].i # <<<<<<<<<<<<<< + * parent.t = self._states[i].t + * parent.is_done = self._states[i].t +*/ + __pyx_t_4 = (__pyx_v_self->_states[__pyx_v_i]).i; + __pyx_v_parent->i = __pyx_t_4; + + /* "thinc/extra/search.pyx":128 + * self._states[i], parent[0] = parent[0], self._states[i] + * parent.i = self._states[i].i + * parent.t = self._states[i].t # <<<<<<<<<<<<<< + * parent.is_done = self._states[i].t + * self._states[i].score = score +*/ + __pyx_t_4 = (__pyx_v_self->_states[__pyx_v_i]).t; + __pyx_v_parent->t = __pyx_t_4; + + /* "thinc/extra/search.pyx":129 + * parent.i = self._states[i].i + * parent.t = self._states[i].t + * parent.is_done = self._states[i].t # <<<<<<<<<<<<<< + * self._states[i].score = score + * self.histories[i] = list(self._parent_histories[p_i]) +*/ + __pyx_t_4 = (__pyx_v_self->_states[__pyx_v_i]).t; + __pyx_v_parent->is_done = __pyx_t_4; + + /* "thinc/extra/search.pyx":130 + * parent.t = self._states[i].t + * parent.is_done = self._states[i].t + * self._states[i].score = score # <<<<<<<<<<<<<< + * self.histories[i] = list(self._parent_histories[p_i]) + * i += 1 +*/ + (__pyx_v_self->_states[__pyx_v_i]).score = __pyx_v_score; + + /* "thinc/extra/search.pyx":131 + * parent.is_done = self._states[i].t + * self._states[i].score = score + * self.histories[i] = list(self._parent_histories[p_i]) # <<<<<<<<<<<<<< + * i += 1 + * else: +*/ + if (unlikely(__pyx_v_self->_parent_histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 131, __pyx_L1_error) + } + __pyx_t_9 = __Pyx_GetItemInt_List(__pyx_v_self->_parent_histories, __pyx_v_p_i, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 131, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_11 = __Pyx_PySequence_ListKeepNew(__pyx_t_9); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 131, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(__pyx_v_self->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 131, __pyx_L1_error) + } + if (unlikely((__Pyx_SetItemInt(__pyx_v_self->histories, __pyx_v_i, __pyx_t_11, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference) < 0))) __PYX_ERR(0, 131, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/extra/search.pyx":132 + * self._states[i].score = score + * self.histories[i] = list(self._parent_histories[p_i]) + * i += 1 # <<<<<<<<<<<<<< + * else: + * state = &self._states[i] +*/ + __pyx_v_i = (__pyx_v_i + 1); + + /* "thinc/extra/search.pyx":123 + * parent = &self._parents[p_i] + * # Indicates terminal state reached; i.e. state is done + * if parent.is_done: # <<<<<<<<<<<<<< + * # Now parent will not be changed, so we don't have to copy. + * # Once finished, should also be unbranching. +*/ + goto __pyx_L7; + } + + /* "thinc/extra/search.pyx":134 + * i += 1 + * else: + * state = &self._states[i] # <<<<<<<<<<<<<< + * # The supplied transition function should adjust the destination + * # state to be the result of applying the class to the source state +*/ + /*else*/ { + __pyx_v_state = (&(__pyx_v_self->_states[__pyx_v_i])); + + /* "thinc/extra/search.pyx":137 + * # The supplied transition function should adjust the destination + * # state to be the result of applying the class to the source state + * transition_func(state.content, parent.content, clas, extra_args) # <<<<<<<<<<<<<< + * key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 + * is_seen = seen_states.get(key) +*/ + __pyx_t_4 = __pyx_v_transition_func(__pyx_v_state->content, __pyx_v_parent->content, __pyx_v_clas, __pyx_v_extra_args); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(0, 137, __pyx_L1_error) + + /* "thinc/extra/search.pyx":138 + * # state to be the result of applying the class to the source state + * transition_func(state.content, parent.content, clas, extra_args) + * key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 # <<<<<<<<<<<<<< + * is_seen = seen_states.get(key) + * if key == 0 or key == 1 or not is_seen: +*/ + __pyx_t_13 = (__pyx_v_hash_func != NULL); + if (__pyx_t_13) { + __pyx_t_19 = __pyx_v_hash_func(__pyx_v_state->content, __pyx_v_extra_args); if (unlikely(__pyx_t_19 == ((__pyx_t_5thinc_5extra_6search_hash_t)0))) __PYX_ERR(0, 138, __pyx_L1_error) + __pyx_t_18 = __pyx_t_19; + } else { + __pyx_t_18 = 0; + } + __pyx_v_key = __pyx_t_18; + + /* "thinc/extra/search.pyx":139 + * transition_func(state.content, parent.content, clas, extra_args) + * key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 + * is_seen = seen_states.get(key) # <<<<<<<<<<<<<< + * if key == 0 or key == 1 or not is_seen: + * if key != 0 and key != 1: +*/ + __pyx_t_20 = ((struct __pyx_vtabstruct_7preshed_4maps_PreshMap *)__pyx_v_seen_states->__pyx_vtab)->get(__pyx_v_seen_states, __pyx_v_key); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 139, __pyx_L1_error) + __pyx_v_is_seen = ((uint64_t)__pyx_t_20); + + /* "thinc/extra/search.pyx":140 + * key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 + * is_seen = seen_states.get(key) + * if key == 0 or key == 1 or not is_seen: # <<<<<<<<<<<<<< + * if key != 0 and key != 1: + * seen_states.set(key, one) +*/ + __pyx_t_14 = (__pyx_v_key == 0); + if (!__pyx_t_14) { + } else { + __pyx_t_13 = __pyx_t_14; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_14 = (__pyx_v_key == 1); + if (!__pyx_t_14) { + } else { + __pyx_t_13 = __pyx_t_14; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_14 = (!(__pyx_v_is_seen != 0)); + __pyx_t_13 = __pyx_t_14; + __pyx_L9_bool_binop_done:; + if (__pyx_t_13) { + + /* "thinc/extra/search.pyx":141 + * is_seen = seen_states.get(key) + * if key == 0 or key == 1 or not is_seen: + * if key != 0 and key != 1: # <<<<<<<<<<<<<< + * seen_states.set(key, one) + * state.score = score +*/ + switch (__pyx_v_key) { + case 0: + case 1: + __pyx_t_13 = 0; + break; + default: + __pyx_t_13 = 1; + break; + } + if (__pyx_t_13) { + + /* "thinc/extra/search.pyx":142 + * if key == 0 or key == 1 or not is_seen: + * if key != 0 and key != 1: + * seen_states.set(key, one) # <<<<<<<<<<<<<< + * state.score = score + * state.loss = parent.loss + costs[p_i][clas] +*/ + ((struct __pyx_vtabstruct_7preshed_4maps_PreshMap *)__pyx_v_seen_states->__pyx_vtab)->set(__pyx_v_seen_states, __pyx_v_key, ((void *)__pyx_v_one)); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 142, __pyx_L1_error) + + /* "thinc/extra/search.pyx":141 + * is_seen = seen_states.get(key) + * if key == 0 or key == 1 or not is_seen: + * if key != 0 and key != 1: # <<<<<<<<<<<<<< + * seen_states.set(key, one) + * state.score = score +*/ + } + + /* "thinc/extra/search.pyx":143 + * if key != 0 and key != 1: + * seen_states.set(key, one) + * state.score = score # <<<<<<<<<<<<<< + * state.loss = parent.loss + costs[p_i][clas] + * self.histories[i] = list(self._parent_histories[p_i]) +*/ + __pyx_v_state->score = __pyx_v_score; + + /* "thinc/extra/search.pyx":144 + * seen_states.set(key, one) + * state.score = score + * state.loss = parent.loss + costs[p_i][clas] # <<<<<<<<<<<<<< + * self.histories[i] = list(self._parent_histories[p_i]) + * self.histories[i].append(clas) +*/ + __pyx_v_state->loss = (__pyx_v_parent->loss + ((__pyx_v_costs[__pyx_v_p_i])[__pyx_v_clas])); + + /* "thinc/extra/search.pyx":145 + * state.score = score + * state.loss = parent.loss + costs[p_i][clas] + * self.histories[i] = list(self._parent_histories[p_i]) # <<<<<<<<<<<<<< + * self.histories[i].append(clas) + * i += 1 +*/ + if (unlikely(__pyx_v_self->_parent_histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 145, __pyx_L1_error) + } + __pyx_t_11 = __Pyx_GetItemInt_List(__pyx_v_self->_parent_histories, __pyx_v_p_i, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_9 = __Pyx_PySequence_ListKeepNew(__pyx_t_11); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(__pyx_v_self->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 145, __pyx_L1_error) + } + if (unlikely((__Pyx_SetItemInt(__pyx_v_self->histories, __pyx_v_i, __pyx_t_9, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference) < 0))) __PYX_ERR(0, 145, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "thinc/extra/search.pyx":146 + * state.loss = parent.loss + costs[p_i][clas] + * self.histories[i] = list(self._parent_histories[p_i]) + * self.histories[i].append(clas) # <<<<<<<<<<<<<< + * i += 1 + * del q +*/ + if (unlikely(__pyx_v_self->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 146, __pyx_L1_error) + } + __pyx_t_9 = __Pyx_GetItemInt_List(__pyx_v_self->histories, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_11 = __Pyx_PyLong_From_uint64_t(__pyx_v_clas); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_21 = __Pyx_PyObject_Append(__pyx_t_9, __pyx_t_11); if (unlikely(__pyx_t_21 == ((int)-1))) __PYX_ERR(0, 146, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/extra/search.pyx":147 + * self.histories[i] = list(self._parent_histories[p_i]) + * self.histories[i].append(clas) + * i += 1 # <<<<<<<<<<<<<< + * del q + * self.size = i +*/ + __pyx_v_i = (__pyx_v_i + 1); + + /* "thinc/extra/search.pyx":140 + * key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 + * is_seen = seen_states.get(key) + * if key == 0 or key == 1 or not is_seen: # <<<<<<<<<<<<<< + * if key != 0 and key != 1: + * seen_states.set(key, one) +*/ + } + } + __pyx_L7:; + } + + /* "thinc/extra/search.pyx":148 + * self.histories[i].append(clas) + * i += 1 + * del q # <<<<<<<<<<<<<< + * self.size = i + * assert self.size >= 1 +*/ + delete __pyx_v_q; + + /* "thinc/extra/search.pyx":149 + * i += 1 + * del q + * self.size = i # <<<<<<<<<<<<<< + * assert self.size >= 1 + * for i in range(self.width): +*/ + __pyx_v_self->size = __pyx_v_i; + + /* "thinc/extra/search.pyx":150 + * del q + * self.size = i + * assert self.size >= 1 # <<<<<<<<<<<<<< + * for i in range(self.width): + * memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_13 = (__pyx_v_self->size >= 1); + if (unlikely(!__pyx_t_13)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 150, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 150, __pyx_L1_error) + #endif + + /* "thinc/extra/search.pyx":151 + * self.size = i + * assert self.size >= 1 + * for i in range(self.width): # <<<<<<<<<<<<<< + * memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) +*/ + __pyx_t_22 = __pyx_v_self->width; + __pyx_t_23 = __pyx_t_22; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_23; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":152 + * assert self.size >= 1 + * for i in range(self.width): + * memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) # <<<<<<<<<<<<<< + * memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.is_valid[i], 0, sizeof(int) * self.nr_class) +*/ + (void)(memset((__pyx_v_self->scores[__pyx_v_i]), 0, ((sizeof(__pyx_t_5thinc_5extra_6search_weight_t)) * __pyx_v_self->nr_class))); + + /* "thinc/extra/search.pyx":153 + * for i in range(self.width): + * memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) # <<<<<<<<<<<<<< + * memset(self.is_valid[i], 0, sizeof(int) * self.nr_class) + * self.t += 1 +*/ + (void)(memset((__pyx_v_self->costs[__pyx_v_i]), 0, ((sizeof(__pyx_t_5thinc_5extra_6search_weight_t)) * __pyx_v_self->nr_class))); + + /* "thinc/extra/search.pyx":154 + * memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.is_valid[i], 0, sizeof(int) * self.nr_class) # <<<<<<<<<<<<<< + * self.t += 1 + * +*/ + (void)(memset((__pyx_v_self->is_valid[__pyx_v_i]), 0, ((sizeof(int)) * __pyx_v_self->nr_class))); + } + + /* "thinc/extra/search.pyx":155 + * memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) + * memset(self.is_valid[i], 0, sizeof(int) * self.nr_class) + * self.t += 1 # <<<<<<<<<<<<<< + * + * cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1: +*/ + __pyx_v_self->t = (__pyx_v_self->t + 1); + + /* "thinc/extra/search.pyx":89 + * self.del_func(self.mem, self._parents[i].content, NULL) + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef int advance(self, trans_func_t transition_func, hash_func_t hash_func, + * void* extra_args) except -1: +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("thinc.extra.search.Beam.advance", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_seen_states); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":157 + * self.t += 1 + * + * cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1: # <<<<<<<<<<<<<< + * cdef int i + * for i in range(self.size): +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam_check_done(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_finish_func_t __pyx_v_finish_func, void *__pyx_v_extra_args) { + int __pyx_v_i; + int __pyx_r; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "thinc/extra/search.pyx":159 + * cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1: + * cdef int i + * for i in range(self.size): # <<<<<<<<<<<<<< + * if not self._states[i].is_done: + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) +*/ + __pyx_t_1 = __pyx_v_self->size; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/extra/search.pyx":160 + * cdef int i + * for i in range(self.size): + * if not self._states[i].is_done: # <<<<<<<<<<<<<< + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) + * for i in range(self.size): +*/ + __pyx_t_4 = (!(__pyx_v_self->_states[__pyx_v_i]).is_done); + if (__pyx_t_4) { + + /* "thinc/extra/search.pyx":161 + * for i in range(self.size): + * if not self._states[i].is_done: + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) # <<<<<<<<<<<<<< + * for i in range(self.size): + * if not self._states[i].is_done: +*/ + __pyx_t_5 = __pyx_v_finish_func((__pyx_v_self->_states[__pyx_v_i]).content, __pyx_v_extra_args); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(0, 161, __pyx_L1_error) + (__pyx_v_self->_states[__pyx_v_i]).is_done = __pyx_t_5; + + /* "thinc/extra/search.pyx":160 + * cdef int i + * for i in range(self.size): + * if not self._states[i].is_done: # <<<<<<<<<<<<<< + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) + * for i in range(self.size): +*/ + } + } + + /* "thinc/extra/search.pyx":162 + * if not self._states[i].is_done: + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) + * for i in range(self.size): # <<<<<<<<<<<<<< + * if not self._states[i].is_done: + * self.is_done = False +*/ + __pyx_t_1 = __pyx_v_self->size; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "thinc/extra/search.pyx":163 + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) + * for i in range(self.size): + * if not self._states[i].is_done: # <<<<<<<<<<<<<< + * self.is_done = False + * break +*/ + __pyx_t_4 = (!(__pyx_v_self->_states[__pyx_v_i]).is_done); + if (__pyx_t_4) { + + /* "thinc/extra/search.pyx":164 + * for i in range(self.size): + * if not self._states[i].is_done: + * self.is_done = False # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_v_self->is_done = 0; + + /* "thinc/extra/search.pyx":165 + * if not self._states[i].is_done: + * self.is_done = False + * break # <<<<<<<<<<<<<< + * else: + * self.is_done = True +*/ + goto __pyx_L7_break; + + /* "thinc/extra/search.pyx":163 + * self._states[i].is_done = finish_func(self._states[i].content, extra_args) + * for i in range(self.size): + * if not self._states[i].is_done: # <<<<<<<<<<<<<< + * self.is_done = False + * break +*/ + } + } + /*else*/ { + + /* "thinc/extra/search.pyx":167 + * break + * else: + * self.is_done = True # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + __pyx_v_self->is_done = 1; + } + __pyx_L7_break:; + + /* "thinc/extra/search.pyx":157 + * self.t += 1 + * + * cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1: # <<<<<<<<<<<<<< + * cdef int i + * for i in range(self.size): +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.extra.search.Beam.check_done", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/extra/search.pyx":169 + * self.is_done = True + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1: + * """Populate the queue from a k * n matrix of scores, where k is the +*/ + +static int __pyx_f_5thinc_5extra_6search_4Beam__fill(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, __pyx_t_5thinc_5extra_6search_Queue *__pyx_v_q, __pyx_t_5thinc_5extra_6search_weight_t **__pyx_v_scores, int **__pyx_v_is_valid) { + __pyx_t_5thinc_5extra_6search_Entry __pyx_v_entry; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_v_s; + int __pyx_v_i; + int __pyx_v_j; + int __pyx_v_move_id; + std::vector<__pyx_t_5thinc_5extra_6search_Entry> __pyx_v_entries; + double __pyx_v_max_; + double __pyx_v_Z; + double __pyx_v_cutoff; + double __pyx_v_prob; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_2; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_6; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_7; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_8; + int __pyx_t_9; + std::vector<__pyx_t_5thinc_5extra_6search_Entry> ::size_type __pyx_t_10; + std::vector<__pyx_t_5thinc_5extra_6search_Entry> ::size_type __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_fill", 0); + + /* "thinc/extra/search.pyx":178 + * cdef _State* s + * cdef int i, j, move_id + * assert self.size >= 1 # <<<<<<<<<<<<<< + * cdef vector[Entry] entries + * for i in range(self.size): +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_self->size >= 1); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(0, 178, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 178, __pyx_L1_error) + #endif + + /* "thinc/extra/search.pyx":180 + * assert self.size >= 1 + * cdef vector[Entry] entries + * for i in range(self.size): # <<<<<<<<<<<<<< + * s = &self._states[i] + * move_id = i * self.nr_class +*/ + __pyx_t_2 = __pyx_v_self->size; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":181 + * cdef vector[Entry] entries + * for i in range(self.size): + * s = &self._states[i] # <<<<<<<<<<<<<< + * move_id = i * self.nr_class + * if s.is_done: +*/ + __pyx_v_s = (&(__pyx_v_self->_states[__pyx_v_i])); + + /* "thinc/extra/search.pyx":182 + * for i in range(self.size): + * s = &self._states[i] + * move_id = i * self.nr_class # <<<<<<<<<<<<<< + * if s.is_done: + * # Update score by path average, following TACL '13 paper. +*/ + __pyx_v_move_id = (__pyx_v_i * __pyx_v_self->nr_class); + + /* "thinc/extra/search.pyx":183 + * s = &self._states[i] + * move_id = i * self.nr_class + * if s.is_done: # <<<<<<<<<<<<<< + * # Update score by path average, following TACL '13 paper. + * if self.histories[i]: +*/ + if (__pyx_v_s->is_done) { + + /* "thinc/extra/search.pyx":185 + * if s.is_done: + * # Update score by path average, following TACL '13 paper. + * if self.histories[i]: # <<<<<<<<<<<<<< + * entry.first = s.score + (s.score / self.t) + * else: +*/ + if (unlikely(__pyx_v_self->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 185, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_GetItemInt_List(__pyx_v_self->histories, __pyx_v_i, int, 1, __Pyx_PyLong_From_int, 1, 1, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":186 + * # Update score by path average, following TACL '13 paper. + * if self.histories[i]: + * entry.first = s.score + (s.score / self.t) # <<<<<<<<<<<<<< + * else: + * entry.first = s.score +*/ + __pyx_v_entry.first = (__pyx_v_s->score + (__pyx_v_s->score / __pyx_v_self->t)); + + /* "thinc/extra/search.pyx":185 + * if s.is_done: + * # Update score by path average, following TACL '13 paper. + * if self.histories[i]: # <<<<<<<<<<<<<< + * entry.first = s.score + (s.score / self.t) + * else: +*/ + goto __pyx_L6; + } + + /* "thinc/extra/search.pyx":188 + * entry.first = s.score + (s.score / self.t) + * else: + * entry.first = s.score # <<<<<<<<<<<<<< + * entry.second = move_id + * entries.push_back(entry) +*/ + /*else*/ { + __pyx_t_6 = __pyx_v_s->score; + __pyx_v_entry.first = __pyx_t_6; + } + __pyx_L6:; + + /* "thinc/extra/search.pyx":189 + * else: + * entry.first = s.score + * entry.second = move_id # <<<<<<<<<<<<<< + * entries.push_back(entry) + * else: +*/ + __pyx_v_entry.second = __pyx_v_move_id; + + /* "thinc/extra/search.pyx":190 + * entry.first = s.score + * entry.second = move_id + * entries.push_back(entry) # <<<<<<<<<<<<<< + * else: + * for j in range(self.nr_class): +*/ + try { + __pyx_v_entries.push_back(__pyx_v_entry); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 190, __pyx_L1_error) + } + + /* "thinc/extra/search.pyx":183 + * s = &self._states[i] + * move_id = i * self.nr_class + * if s.is_done: # <<<<<<<<<<<<<< + * # Update score by path average, following TACL '13 paper. + * if self.histories[i]: +*/ + goto __pyx_L5; + } + + /* "thinc/extra/search.pyx":192 + * entries.push_back(entry) + * else: + * for j in range(self.nr_class): # <<<<<<<<<<<<<< + * if is_valid[i][j]: + * entry.first = s.score + scores[i][j] +*/ + /*else*/ { + __pyx_t_7 = __pyx_v_self->nr_class; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_j = __pyx_t_9; + + /* "thinc/extra/search.pyx":193 + * else: + * for j in range(self.nr_class): + * if is_valid[i][j]: # <<<<<<<<<<<<<< + * entry.first = s.score + scores[i][j] + * entry.second = move_id + j +*/ + __pyx_t_1 = (((__pyx_v_is_valid[__pyx_v_i])[__pyx_v_j]) != 0); + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":194 + * for j in range(self.nr_class): + * if is_valid[i][j]: + * entry.first = s.score + scores[i][j] # <<<<<<<<<<<<<< + * entry.second = move_id + j + * entries.push_back(entry) +*/ + __pyx_v_entry.first = (__pyx_v_s->score + ((__pyx_v_scores[__pyx_v_i])[__pyx_v_j])); + + /* "thinc/extra/search.pyx":195 + * if is_valid[i][j]: + * entry.first = s.score + scores[i][j] + * entry.second = move_id + j # <<<<<<<<<<<<<< + * entries.push_back(entry) + * cdef double max_, Z, cutoff +*/ + __pyx_v_entry.second = (__pyx_v_move_id + __pyx_v_j); + + /* "thinc/extra/search.pyx":196 + * entry.first = s.score + scores[i][j] + * entry.second = move_id + j + * entries.push_back(entry) # <<<<<<<<<<<<<< + * cdef double max_, Z, cutoff + * if self.min_density == 0.0: +*/ + try { + __pyx_v_entries.push_back(__pyx_v_entry); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 196, __pyx_L1_error) + } + + /* "thinc/extra/search.pyx":193 + * else: + * for j in range(self.nr_class): + * if is_valid[i][j]: # <<<<<<<<<<<<<< + * entry.first = s.score + scores[i][j] + * entry.second = move_id + j +*/ + } + } + } + __pyx_L5:; + } + + /* "thinc/extra/search.pyx":198 + * entries.push_back(entry) + * cdef double max_, Z, cutoff + * if self.min_density == 0.0: # <<<<<<<<<<<<<< + * for i in range(entries.size()): + * q.push(entries[i]) +*/ + __pyx_t_1 = (__pyx_v_self->min_density == 0.0); + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":199 + * cdef double max_, Z, cutoff + * if self.min_density == 0.0: + * for i in range(entries.size()): # <<<<<<<<<<<<<< + * q.push(entries[i]) + * elif not entries.empty(): +*/ + __pyx_t_10 = __pyx_v_entries.size(); + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_11; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":200 + * if self.min_density == 0.0: + * for i in range(entries.size()): + * q.push(entries[i]) # <<<<<<<<<<<<<< + * elif not entries.empty(): + * max_ = entries[0].first +*/ + __pyx_v_q->push((__pyx_v_entries[__pyx_v_i])); + } + + /* "thinc/extra/search.pyx":198 + * entries.push_back(entry) + * cdef double max_, Z, cutoff + * if self.min_density == 0.0: # <<<<<<<<<<<<<< + * for i in range(entries.size()): + * q.push(entries[i]) +*/ + goto __pyx_L10; + } + + /* "thinc/extra/search.pyx":201 + * for i in range(entries.size()): + * q.push(entries[i]) + * elif not entries.empty(): # <<<<<<<<<<<<<< + * max_ = entries[0].first + * Z = 0. +*/ + __pyx_t_1 = (!__pyx_v_entries.empty()); + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":202 + * q.push(entries[i]) + * elif not entries.empty(): + * max_ = entries[0].first # <<<<<<<<<<<<<< + * Z = 0. + * cutoff = 0. +*/ + __pyx_t_6 = (__pyx_v_entries[0]).first; + __pyx_v_max_ = __pyx_t_6; + + /* "thinc/extra/search.pyx":203 + * elif not entries.empty(): + * max_ = entries[0].first + * Z = 0. # <<<<<<<<<<<<<< + * cutoff = 0. + * # Softmax into probabilities, so we can prune +*/ + __pyx_v_Z = 0.; + + /* "thinc/extra/search.pyx":204 + * max_ = entries[0].first + * Z = 0. + * cutoff = 0. # <<<<<<<<<<<<<< + * # Softmax into probabilities, so we can prune + * for i in range(entries.size()): +*/ + __pyx_v_cutoff = 0.; + + /* "thinc/extra/search.pyx":206 + * cutoff = 0. + * # Softmax into probabilities, so we can prune + * for i in range(entries.size()): # <<<<<<<<<<<<<< + * if entries[i].first > max_: + * max_ = entries[i].first +*/ + __pyx_t_10 = __pyx_v_entries.size(); + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_11; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":207 + * # Softmax into probabilities, so we can prune + * for i in range(entries.size()): + * if entries[i].first > max_: # <<<<<<<<<<<<<< + * max_ = entries[i].first + * for i in range(entries.size()): +*/ + __pyx_t_1 = ((__pyx_v_entries[__pyx_v_i]).first > __pyx_v_max_); + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":208 + * for i in range(entries.size()): + * if entries[i].first > max_: + * max_ = entries[i].first # <<<<<<<<<<<<<< + * for i in range(entries.size()): + * Z += exp(entries[i].first-max_) +*/ + __pyx_t_6 = (__pyx_v_entries[__pyx_v_i]).first; + __pyx_v_max_ = __pyx_t_6; + + /* "thinc/extra/search.pyx":207 + * # Softmax into probabilities, so we can prune + * for i in range(entries.size()): + * if entries[i].first > max_: # <<<<<<<<<<<<<< + * max_ = entries[i].first + * for i in range(entries.size()): +*/ + } + } + + /* "thinc/extra/search.pyx":209 + * if entries[i].first > max_: + * max_ = entries[i].first + * for i in range(entries.size()): # <<<<<<<<<<<<<< + * Z += exp(entries[i].first-max_) + * cutoff = (1. / Z) * self.min_density +*/ + __pyx_t_10 = __pyx_v_entries.size(); + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_11; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":210 + * max_ = entries[i].first + * for i in range(entries.size()): + * Z += exp(entries[i].first-max_) # <<<<<<<<<<<<<< + * cutoff = (1. / Z) * self.min_density + * for i in range(entries.size()): +*/ + __pyx_v_Z = (__pyx_v_Z + exp(((__pyx_v_entries[__pyx_v_i]).first - __pyx_v_max_))); + } + + /* "thinc/extra/search.pyx":211 + * for i in range(entries.size()): + * Z += exp(entries[i].first-max_) + * cutoff = (1. / Z) * self.min_density # <<<<<<<<<<<<<< + * for i in range(entries.size()): + * prob = exp(entries[i].first-max_) / Z +*/ + __pyx_v_cutoff = ((1. / __pyx_v_Z) * __pyx_v_self->min_density); + + /* "thinc/extra/search.pyx":212 + * Z += exp(entries[i].first-max_) + * cutoff = (1. / Z) * self.min_density + * for i in range(entries.size()): # <<<<<<<<<<<<<< + * prob = exp(entries[i].first-max_) / Z + * if prob >= cutoff: +*/ + __pyx_t_10 = __pyx_v_entries.size(); + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_11; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "thinc/extra/search.pyx":213 + * cutoff = (1. / Z) * self.min_density + * for i in range(entries.size()): + * prob = exp(entries[i].first-max_) / Z # <<<<<<<<<<<<<< + * if prob >= cutoff: + * q.push(entries[i]) +*/ + __pyx_v_prob = (exp(((__pyx_v_entries[__pyx_v_i]).first - __pyx_v_max_)) / __pyx_v_Z); + + /* "thinc/extra/search.pyx":214 + * for i in range(entries.size()): + * prob = exp(entries[i].first-max_) / Z + * if prob >= cutoff: # <<<<<<<<<<<<<< + * q.push(entries[i]) + * +*/ + __pyx_t_1 = (__pyx_v_prob >= __pyx_v_cutoff); + if (__pyx_t_1) { + + /* "thinc/extra/search.pyx":215 + * prob = exp(entries[i].first-max_) / Z + * if prob >= cutoff: + * q.push(entries[i]) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_q->push((__pyx_v_entries[__pyx_v_i])); + + /* "thinc/extra/search.pyx":214 + * for i in range(entries.size()): + * prob = exp(entries[i].first-max_) / Z + * if prob >= cutoff: # <<<<<<<<<<<<<< + * q.push(entries[i]) + * +*/ + } + } + + /* "thinc/extra/search.pyx":201 + * for i in range(entries.size()): + * q.push(entries[i]) + * elif not entries.empty(): # <<<<<<<<<<<<<< + * max_ = entries[0].first + * Z = 0. +*/ + } + __pyx_L10:; + + /* "thinc/extra/search.pyx":169 + * self.is_done = True + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1: + * """Populate the queue from a k * n matrix of scores, where k is the +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.extra.search.Beam._fill", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":42 + * cdef class_t width + * cdef class_t size + * cdef public weight_t min_density # <<<<<<<<<<<<<< + * cdef int t + * cdef readonly bint is_done +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_11min_density_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_11min_density_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_11min_density___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_11min_density___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->min_density); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.Beam.min_density.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static int __pyx_pw_5thinc_5extra_6search_4Beam_11min_density_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_pw_5thinc_5extra_6search_4Beam_11min_density_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__set__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_11min_density_2__set__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5thinc_5extra_6search_4Beam_11min_density_2__set__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, PyObject *__pyx_v_value) { + int __pyx_r; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __pyx_t_1 = __Pyx_PyFloat_AsFloat(__pyx_v_value); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) __PYX_ERR(1, 42, __pyx_L1_error) + __pyx_v_self->min_density = __pyx_t_1; + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.extra.search.Beam.min_density.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/extra/search.pxd":44 + * cdef public weight_t min_density + * cdef int t + * cdef readonly bint is_done # <<<<<<<<<<<<<< + * cdef list histories + * cdef list _parent_histories +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_7is_done_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_7is_done_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_7is_done___get__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_7is_done___get__(struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->is_done); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.Beam.is_done.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_7__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_4Beam_6__reduce_cython__, "Beam.__reduce_cython__(self)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_4Beam_7__reduce_cython__ = {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_4Beam_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_4Beam_6__reduce_cython__}; +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_7__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_6__reduce_cython__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_6__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_self__parents_self__states_self, 0, 0); + __PYX_ERR(2, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.extra.search.Beam.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_4Beam_8__setstate_cython__, "Beam.__setstate_cython__(self, __pyx_state)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_4Beam_9__setstate_cython__ = {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_4Beam_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_4Beam_8__setstate_cython__}; +static PyObject *__pyx_pw_5thinc_5extra_6search_4Beam_9__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(2, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(2, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(2, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(2, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.Beam.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_5extra_6search_4Beam_8__setstate_cython__(((struct __pyx_obj_5thinc_5extra_6search_Beam *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_4Beam_8__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_self__parents_self__states_self, 0, 0); + __PYX_ERR(2, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.extra.search.Beam.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":219 + * + * cdef class MaxViolation: + * def __init__(self): # <<<<<<<<<<<<<< + * self.p_score = 0.0 + * self.g_score = 0.0 +*/ + +/* Python wrapper */ +static int __pyx_pw_5thinc_5extra_6search_12MaxViolation_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5thinc_5extra_6search_12MaxViolation_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 0, 0, __pyx_nargs); return -1; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return -1; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__init__", __pyx_kwds); return -1;} + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation___init__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5thinc_5extra_6search_12MaxViolation___init__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "thinc/extra/search.pyx":220 + * cdef class MaxViolation: + * def __init__(self): + * self.p_score = 0.0 # <<<<<<<<<<<<<< + * self.g_score = 0.0 + * self.Z = 0.0 +*/ + __pyx_v_self->p_score = 0.0; + + /* "thinc/extra/search.pyx":221 + * def __init__(self): + * self.p_score = 0.0 + * self.g_score = 0.0 # <<<<<<<<<<<<<< + * self.Z = 0.0 + * self.gZ = 0.0 +*/ + __pyx_v_self->g_score = 0.0; + + /* "thinc/extra/search.pyx":222 + * self.p_score = 0.0 + * self.g_score = 0.0 + * self.Z = 0.0 # <<<<<<<<<<<<<< + * self.gZ = 0.0 + * self.delta = -1 +*/ + __pyx_v_self->Z = 0.0; + + /* "thinc/extra/search.pyx":223 + * self.g_score = 0.0 + * self.Z = 0.0 + * self.gZ = 0.0 # <<<<<<<<<<<<<< + * self.delta = -1 + * self.cost = 0 +*/ + __pyx_v_self->gZ = 0.0; + + /* "thinc/extra/search.pyx":224 + * self.Z = 0.0 + * self.gZ = 0.0 + * self.delta = -1 # <<<<<<<<<<<<<< + * self.cost = 0 + * self.p_hist = [] +*/ + __pyx_v_self->delta = -1.0; + + /* "thinc/extra/search.pyx":225 + * self.gZ = 0.0 + * self.delta = -1 + * self.cost = 0 # <<<<<<<<<<<<<< + * self.p_hist = [] + * self.g_hist = [] +*/ + __pyx_v_self->cost = 0.0; + + /* "thinc/extra/search.pyx":226 + * self.delta = -1 + * self.cost = 0 + * self.p_hist = [] # <<<<<<<<<<<<<< + * self.g_hist = [] + * self.p_probs = [] +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 226, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->p_hist); + __Pyx_DECREF(__pyx_v_self->p_hist); + __pyx_v_self->p_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":227 + * self.cost = 0 + * self.p_hist = [] + * self.g_hist = [] # <<<<<<<<<<<<<< + * self.p_probs = [] + * self.g_probs = [] +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->g_hist); + __Pyx_DECREF(__pyx_v_self->g_hist); + __pyx_v_self->g_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":228 + * self.p_hist = [] + * self.g_hist = [] + * self.p_probs = [] # <<<<<<<<<<<<<< + * self.g_probs = [] + * +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->p_probs); + __Pyx_DECREF(__pyx_v_self->p_probs); + __pyx_v_self->p_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":229 + * self.g_hist = [] + * self.p_probs = [] + * self.g_probs = [] # <<<<<<<<<<<<<< + * + * cpdef int check(self, Beam pred, Beam gold) except -1: +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 229, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->g_probs); + __Pyx_DECREF(__pyx_v_self->g_probs); + __pyx_v_self->g_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":219 + * + * cdef class MaxViolation: + * def __init__(self): # <<<<<<<<<<<<<< + * self.p_score = 0.0 + * self.g_score = 0.0 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":231 + * self.g_probs = [] + * + * cpdef int check(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * cdef _State* p = &pred._states[0] + * cdef _State* g = &gold._states[0] +*/ + +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_3check(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static int __pyx_f_5thinc_5extra_6search_12MaxViolation_check(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold, int __pyx_skip_dispatch) { + struct __pyx_t_5thinc_5extra_6search__State *__pyx_v_p; + struct __pyx_t_5thinc_5extra_6search__State *__pyx_v_g; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_v_d; + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + double __pyx_v_prob; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_9; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_10; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_11; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_12; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("check", 0); + /* Check if called by wrapper */ + if (unlikely(__pyx_skip_dispatch)) ; + /* Check if overridden in Python */ + else if ( + #if !CYTHON_USE_TYPE_SLOTS + unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation && + __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC)) + #else + unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE))) + #endif + ) { + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; + if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) { + PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); + #endif + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_check); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_5thinc_5extra_6search_12MaxViolation_3check)) { + __pyx_t_3 = NULL; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = __pyx_t_1; + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, ((PyObject *)__pyx_v_pred), ((PyObject *)__pyx_v_gold)}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_6 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_6 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L0; + } + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); + __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self)); + if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) { + __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; + } + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + } + #endif + } + + /* "thinc/extra/search.pyx":232 + * + * cpdef int check(self, Beam pred, Beam gold) except -1: + * cdef _State* p = &pred._states[0] # <<<<<<<<<<<<<< + * cdef _State* g = &gold._states[0] + * cdef weight_t d = p.score - g.score +*/ + __pyx_v_p = (&(__pyx_v_pred->_states[0])); + + /* "thinc/extra/search.pyx":233 + * cpdef int check(self, Beam pred, Beam gold) except -1: + * cdef _State* p = &pred._states[0] + * cdef _State* g = &gold._states[0] # <<<<<<<<<<<<<< + * cdef weight_t d = p.score - g.score + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): +*/ + __pyx_v_g = (&(__pyx_v_gold->_states[0])); + + /* "thinc/extra/search.pyx":234 + * cdef _State* p = &pred._states[0] + * cdef _State* g = &gold._states[0] + * cdef weight_t d = p.score - g.score # <<<<<<<<<<<<<< + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): + * self.cost = p.loss +*/ + __pyx_v_d = (__pyx_v_p->score - __pyx_v_g->score); + + /* "thinc/extra/search.pyx":235 + * cdef _State* g = &gold._states[0] + * cdef weight_t d = p.score - g.score + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): # <<<<<<<<<<<<<< + * self.cost = p.loss + * self.delta = d +*/ + __pyx_t_8 = (__pyx_v_p->loss >= 1.0); + if (__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_8 = (__pyx_v_self->cost == 0.0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_8 = (__pyx_v_d > __pyx_v_self->delta); + __pyx_t_7 = __pyx_t_8; + __pyx_L4_bool_binop_done:; + if (__pyx_t_7) { + + /* "thinc/extra/search.pyx":236 + * cdef weight_t d = p.score - g.score + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): + * self.cost = p.loss # <<<<<<<<<<<<<< + * self.delta = d + * self.p_hist = list(pred.histories[0]) +*/ + __pyx_t_9 = __pyx_v_p->loss; + __pyx_v_self->cost = __pyx_t_9; + + /* "thinc/extra/search.pyx":237 + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): + * self.cost = p.loss + * self.delta = d # <<<<<<<<<<<<<< + * self.p_hist = list(pred.histories[0]) + * self.g_hist = list(gold.histories[0]) +*/ + __pyx_v_self->delta = __pyx_v_d; + + /* "thinc/extra/search.pyx":238 + * self.cost = p.loss + * self.delta = d + * self.p_hist = list(pred.histories[0]) # <<<<<<<<<<<<<< + * self.g_hist = list(gold.histories[0]) + * self.p_score = p.score +*/ + if (unlikely(__pyx_v_pred->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 238, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_GetItemInt_List(__pyx_v_pred->histories, 0, long, 1, __Pyx_PyLong_From_long, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 238, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PySequence_ListKeepNew(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 238, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_self->p_hist); + __Pyx_DECREF(__pyx_v_self->p_hist); + __pyx_v_self->p_hist = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":239 + * self.delta = d + * self.p_hist = list(pred.histories[0]) + * self.g_hist = list(gold.histories[0]) # <<<<<<<<<<<<<< + * self.p_score = p.score + * self.g_score = g.score +*/ + if (unlikely(__pyx_v_gold->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 239, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_gold->histories, 0, long, 1, __Pyx_PyLong_From_long, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 239, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PySequence_ListKeepNew(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 239, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->g_hist); + __Pyx_DECREF(__pyx_v_self->g_hist); + __pyx_v_self->g_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":240 + * self.p_hist = list(pred.histories[0]) + * self.g_hist = list(gold.histories[0]) + * self.p_score = p.score # <<<<<<<<<<<<<< + * self.g_score = g.score + * self.Z = 1e-10 +*/ + __pyx_t_9 = __pyx_v_p->score; + __pyx_v_self->p_score = __pyx_t_9; + + /* "thinc/extra/search.pyx":241 + * self.g_hist = list(gold.histories[0]) + * self.p_score = p.score + * self.g_score = g.score # <<<<<<<<<<<<<< + * self.Z = 1e-10 + * self.gZ = 1e-10 +*/ + __pyx_t_9 = __pyx_v_g->score; + __pyx_v_self->g_score = __pyx_t_9; + + /* "thinc/extra/search.pyx":242 + * self.p_score = p.score + * self.g_score = g.score + * self.Z = 1e-10 # <<<<<<<<<<<<<< + * self.gZ = 1e-10 + * for i in range(pred.size): +*/ + __pyx_v_self->Z = 1e-10; + + /* "thinc/extra/search.pyx":243 + * self.g_score = g.score + * self.Z = 1e-10 + * self.gZ = 1e-10 # <<<<<<<<<<<<<< + * for i in range(pred.size): + * if pred._states[i].loss > 0: +*/ + __pyx_v_self->gZ = 1e-10; + + /* "thinc/extra/search.pyx":244 + * self.Z = 1e-10 + * self.gZ = 1e-10 + * for i in range(pred.size): # <<<<<<<<<<<<<< + * if pred._states[i].loss > 0: + * self.Z += exp(pred._states[i].score) +*/ + __pyx_t_10 = __pyx_v_pred->size; + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_11; __pyx_t_12+=1) { + __pyx_v_i = __pyx_t_12; + + /* "thinc/extra/search.pyx":245 + * self.gZ = 1e-10 + * for i in range(pred.size): + * if pred._states[i].loss > 0: # <<<<<<<<<<<<<< + * self.Z += exp(pred._states[i].score) + * for i in range(gold.size): +*/ + __pyx_t_7 = ((__pyx_v_pred->_states[__pyx_v_i]).loss > 0.0); + if (__pyx_t_7) { + + /* "thinc/extra/search.pyx":246 + * for i in range(pred.size): + * if pred._states[i].loss > 0: + * self.Z += exp(pred._states[i].score) # <<<<<<<<<<<<<< + * for i in range(gold.size): + * if gold._states[i].loss == 0: +*/ + __pyx_v_self->Z = (__pyx_v_self->Z + exp((__pyx_v_pred->_states[__pyx_v_i]).score)); + + /* "thinc/extra/search.pyx":245 + * self.gZ = 1e-10 + * for i in range(pred.size): + * if pred._states[i].loss > 0: # <<<<<<<<<<<<<< + * self.Z += exp(pred._states[i].score) + * for i in range(gold.size): +*/ + } + } + + /* "thinc/extra/search.pyx":247 + * if pred._states[i].loss > 0: + * self.Z += exp(pred._states[i].score) + * for i in range(gold.size): # <<<<<<<<<<<<<< + * if gold._states[i].loss == 0: + * prob = exp(gold._states[i].score) +*/ + __pyx_t_10 = __pyx_v_gold->size; + __pyx_t_11 = __pyx_t_10; + for (__pyx_t_12 = 0; __pyx_t_12 < __pyx_t_11; __pyx_t_12+=1) { + __pyx_v_i = __pyx_t_12; + + /* "thinc/extra/search.pyx":248 + * self.Z += exp(pred._states[i].score) + * for i in range(gold.size): + * if gold._states[i].loss == 0: # <<<<<<<<<<<<<< + * prob = exp(gold._states[i].score) + * self.Z += prob +*/ + __pyx_t_7 = ((__pyx_v_gold->_states[__pyx_v_i]).loss == 0.0); + if (__pyx_t_7) { + + /* "thinc/extra/search.pyx":249 + * for i in range(gold.size): + * if gold._states[i].loss == 0: + * prob = exp(gold._states[i].score) # <<<<<<<<<<<<<< + * self.Z += prob + * self.gZ += prob +*/ + __pyx_v_prob = exp((__pyx_v_gold->_states[__pyx_v_i]).score); + + /* "thinc/extra/search.pyx":250 + * if gold._states[i].loss == 0: + * prob = exp(gold._states[i].score) + * self.Z += prob # <<<<<<<<<<<<<< + * self.gZ += prob + * +*/ + __pyx_v_self->Z = (__pyx_v_self->Z + __pyx_v_prob); + + /* "thinc/extra/search.pyx":251 + * prob = exp(gold._states[i].score) + * self.Z += prob + * self.gZ += prob # <<<<<<<<<<<<<< + * + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: +*/ + __pyx_v_self->gZ = (__pyx_v_self->gZ + __pyx_v_prob); + + /* "thinc/extra/search.pyx":248 + * self.Z += exp(pred._states[i].score) + * for i in range(gold.size): + * if gold._states[i].loss == 0: # <<<<<<<<<<<<<< + * prob = exp(gold._states[i].score) + * self.Z += prob +*/ + } + } + + /* "thinc/extra/search.pyx":235 + * cdef _State* g = &gold._states[0] + * cdef weight_t d = p.score - g.score + * if p.loss >= 1 and (self.cost == 0 or d > self.delta): # <<<<<<<<<<<<<< + * self.cost = p.loss + * self.delta = d +*/ + } + + /* "thinc/extra/search.pyx":231 + * self.g_probs = [] + * + * cpdef int check(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * cdef _State* p = &pred._states[0] + * cdef _State* g = &gold._states[0] +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_3check(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_12MaxViolation_2check, "MaxViolation.check(self, Beam pred, Beam gold) -> int"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_12MaxViolation_3check = {"check", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_3check, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_2check}; +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_3check(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred = 0; + struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("check (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pred,&__pyx_mstate_global->__pyx_n_u_gold,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 231, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 231, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 231, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "check", 0) < (0)) __PYX_ERR(0, 231, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("check", 1, 2, 2, i); __PYX_ERR(0, 231, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 231, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 231, __pyx_L3_error) + } + __pyx_v_pred = ((struct __pyx_obj_5thinc_5extra_6search_Beam *)values[0]); + __pyx_v_gold = ((struct __pyx_obj_5thinc_5extra_6search_Beam *)values[1]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("check", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 231, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_pred), __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_Beam, 1, "pred", 0))) __PYX_ERR(0, 231, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_gold), __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_Beam, 1, "gold", 0))) __PYX_ERR(0, 231, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_2check(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self), __pyx_v_pred, __pyx_v_gold); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_2check(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("check", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_5thinc_5extra_6search_12MaxViolation_check(__pyx_v_self, __pyx_v_pred, __pyx_v_gold, 1); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 231, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":253 + * self.gZ += prob + * + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) +*/ + +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_5check_crf(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static int __pyx_f_5thinc_5extra_6search_12MaxViolation_check_crf(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold, int __pyx_skip_dispatch) { + PyObject *__pyx_v_d = NULL; + PyObject *__pyx_v_seen_golds = NULL; + PyObject *__pyx_v_p_hist = NULL; + PyObject *__pyx_v_p_scores = NULL; + PyObject *__pyx_v_g_hist = NULL; + PyObject *__pyx_v_g_scores = NULL; + __pyx_t_5thinc_5extra_6search_class_t __pyx_v_i; + PyObject *__pyx_v_all_probs = NULL; + PyObject *__pyx_v_p_probs = NULL; + PyObject *__pyx_v_g_probs_all = NULL; + PyObject *__pyx_v_g_probs = NULL; + PyObject *__pyx_v_x = NULL; + PyObject *__pyx_v_y = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_t_6; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_7; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_8; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_9; + int __pyx_t_10; + int __pyx_t_11; + int __pyx_t_12; + Py_ssize_t __pyx_t_13; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_14; + PyObject *(*__pyx_t_15)(PyObject *); + PyObject *__pyx_t_16 = NULL; + PyObject *__pyx_t_17 = NULL; + PyObject *(*__pyx_t_18)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("check_crf", 0); + /* Check if called by wrapper */ + if (unlikely(__pyx_skip_dispatch)) ; + /* Check if overridden in Python */ + else if ( + #if !CYTHON_USE_TYPE_SLOTS + unlikely(Py_TYPE(((PyObject *)__pyx_v_self)) != __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation && + __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), Py_TPFLAGS_HAVE_GC)) + #else + unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0 || __Pyx_PyType_HasFeature(Py_TYPE(((PyObject *)__pyx_v_self)), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE))) + #endif + ) { + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + static PY_UINT64_T __pyx_tp_dict_version = __PYX_DICT_VERSION_INIT, __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; + if (unlikely(!__Pyx_object_dict_version_matches(((PyObject *)__pyx_v_self), __pyx_tp_dict_version, __pyx_obj_dict_version))) { + PY_UINT64_T __pyx_typedict_guard = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); + #endif + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_check_crf); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!__Pyx_IsSameCFunction(__pyx_t_1, (void(*)(void)) __pyx_pw_5thinc_5extra_6search_12MaxViolation_5check_crf)) { + __pyx_t_3 = NULL; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = __pyx_t_1; + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, ((PyObject *)__pyx_v_pred), ((PyObject *)__pyx_v_gold)}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_6 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_6 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L0; + } + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + __pyx_tp_dict_version = __Pyx_get_tp_dict_version(((PyObject *)__pyx_v_self)); + __pyx_obj_dict_version = __Pyx_get_object_dict_version(((PyObject *)__pyx_v_self)); + if (unlikely(__pyx_typedict_guard != __pyx_tp_dict_version)) { + __pyx_tp_dict_version = __pyx_obj_dict_version = __PYX_DICT_VERSION_INIT; + } + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS + } + #endif + } + + /* "thinc/extra/search.pyx":254 + * + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: + * d = pred.score - gold.score # <<<<<<<<<<<<<< + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_pred), __pyx_mstate_global->__pyx_n_u_score); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_gold), __pyx_mstate_global->__pyx_n_u_score); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = PyNumber_Subtract(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_d = __pyx_t_4; + __pyx_t_4 = 0; + + /* "thinc/extra/search.pyx":255 + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) # <<<<<<<<<<<<<< + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): + * p_hist = [] +*/ + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = __pyx_v_gold->size; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_i = __pyx_t_9; + if (unlikely(__pyx_v_gold->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 255, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_GetItemInt_List(__pyx_v_gold->histories, __pyx_v_i, __pyx_t_5thinc_5extra_6search_class_t, 0, __Pyx_PyLong_From_uint64_t, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PySequence_Tuple(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(__Pyx_ListComp_Append(__pyx_t_4, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_t_1 = PySet_New(__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_seen_golds = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":256 + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): # <<<<<<<<<<<<<< + * p_hist = [] + * p_scores = [] +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_pred), __pyx_mstate_global->__pyx_n_u_loss); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_mstate_global->__pyx_int_0, Py_GT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely((__pyx_t_11 < 0))) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_11) { + } else { + __pyx_t_10 = __pyx_t_11; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_11 = (__pyx_v_self->cost == 0.0); + if (!__pyx_t_11) { + } else { + __pyx_t_10 = __pyx_t_11; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_4 = PyFloat_FromDouble(__pyx_v_self->delta); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyObject_RichCompare(__pyx_v_d, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_11 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_11 < 0))) __PYX_ERR(0, 256, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_10 = __pyx_t_11; + __pyx_L6_bool_binop_done:; + if (__pyx_t_10) { + + /* "thinc/extra/search.pyx":257 + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): + * p_hist = [] # <<<<<<<<<<<<<< + * p_scores = [] + * g_hist = [] +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_p_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":258 + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): + * p_hist = [] + * p_scores = [] # <<<<<<<<<<<<<< + * g_hist = [] + * g_scores = [] +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 258, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_p_scores = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":259 + * p_hist = [] + * p_scores = [] + * g_hist = [] # <<<<<<<<<<<<<< + * g_scores = [] + * for i in range(pred.size): +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 259, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_g_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":260 + * p_scores = [] + * g_hist = [] + * g_scores = [] # <<<<<<<<<<<<<< + * for i in range(pred.size): + * if pred._states[i].loss > 0: +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 260, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_g_scores = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":261 + * g_hist = [] + * g_scores = [] + * for i in range(pred.size): # <<<<<<<<<<<<<< + * if pred._states[i].loss > 0: + * p_scores.append(pred._states[i].score) +*/ + __pyx_t_7 = __pyx_v_pred->size; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_i = __pyx_t_9; + + /* "thinc/extra/search.pyx":262 + * g_scores = [] + * for i in range(pred.size): + * if pred._states[i].loss > 0: # <<<<<<<<<<<<<< + * p_scores.append(pred._states[i].score) + * p_hist.append(list(pred.histories[i])) +*/ + __pyx_t_10 = ((__pyx_v_pred->_states[__pyx_v_i]).loss > 0.0); + if (__pyx_t_10) { + + /* "thinc/extra/search.pyx":263 + * for i in range(pred.size): + * if pred._states[i].loss > 0: + * p_scores.append(pred._states[i].score) # <<<<<<<<<<<<<< + * p_hist.append(list(pred.histories[i])) + * # This can happen from non-monotonic actions +*/ + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_pred->_states[__pyx_v_i]).score); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_p_scores, __pyx_t_1); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":264 + * if pred._states[i].loss > 0: + * p_scores.append(pred._states[i].score) + * p_hist.append(list(pred.histories[i])) # <<<<<<<<<<<<<< + * # This can happen from non-monotonic actions + * # If we find a better gold analysis this way, be sure to keep it. +*/ + if (unlikely(__pyx_v_pred->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 264, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_GetItemInt_List(__pyx_v_pred->histories, __pyx_v_i, __pyx_t_5thinc_5extra_6search_class_t, 0, __Pyx_PyLong_From_uint64_t, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PySequence_ListKeepNew(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_p_hist, __pyx_t_4); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 264, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/extra/search.pyx":262 + * g_scores = [] + * for i in range(pred.size): + * if pred._states[i].loss > 0: # <<<<<<<<<<<<<< + * p_scores.append(pred._states[i].score) + * p_hist.append(list(pred.histories[i])) +*/ + goto __pyx_L11; + } + + /* "thinc/extra/search.pyx":267 + * # This can happen from non-monotonic actions + * # If we find a better gold analysis this way, be sure to keep it. + * elif pred._states[i].loss <= 0 \ # <<<<<<<<<<<<<< + * and tuple(pred.histories[i]) not in seen_golds: + * g_scores.append(pred._states[i].score) +*/ + __pyx_t_11 = ((__pyx_v_pred->_states[__pyx_v_i]).loss <= 0.0); + if (__pyx_t_11) { + } else { + __pyx_t_10 = __pyx_t_11; + goto __pyx_L12_bool_binop_done; + } + + /* "thinc/extra/search.pyx":268 + * # If we find a better gold analysis this way, be sure to keep it. + * elif pred._states[i].loss <= 0 \ + * and tuple(pred.histories[i]) not in seen_golds: # <<<<<<<<<<<<<< + * g_scores.append(pred._states[i].score) + * g_hist.append(list(pred.histories[i])) +*/ + if (unlikely(__pyx_v_pred->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 268, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_List(__pyx_v_pred->histories, __pyx_v_i, __pyx_t_5thinc_5extra_6search_class_t, 0, __Pyx_PyLong_From_uint64_t, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PySequence_Tuple(__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_11 = (__Pyx_PySet_ContainsTF(__pyx_t_1, __pyx_v_seen_golds, Py_NE)); if (unlikely((__pyx_t_11 < 0))) __PYX_ERR(0, 268, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_10 = __pyx_t_11; + __pyx_L12_bool_binop_done:; + + /* "thinc/extra/search.pyx":267 + * # This can happen from non-monotonic actions + * # If we find a better gold analysis this way, be sure to keep it. + * elif pred._states[i].loss <= 0 \ # <<<<<<<<<<<<<< + * and tuple(pred.histories[i]) not in seen_golds: + * g_scores.append(pred._states[i].score) +*/ + if (__pyx_t_10) { + + /* "thinc/extra/search.pyx":269 + * elif pred._states[i].loss <= 0 \ + * and tuple(pred.histories[i]) not in seen_golds: + * g_scores.append(pred._states[i].score) # <<<<<<<<<<<<<< + * g_hist.append(list(pred.histories[i])) + * for i in range(gold.size): +*/ + __pyx_t_1 = PyFloat_FromDouble((__pyx_v_pred->_states[__pyx_v_i]).score); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_g_scores, __pyx_t_1); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":270 + * and tuple(pred.histories[i]) not in seen_golds: + * g_scores.append(pred._states[i].score) + * g_hist.append(list(pred.histories[i])) # <<<<<<<<<<<<<< + * for i in range(gold.size): + * if gold._states[i].loss == 0: +*/ + if (unlikely(__pyx_v_pred->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 270, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_GetItemInt_List(__pyx_v_pred->histories, __pyx_v_i, __pyx_t_5thinc_5extra_6search_class_t, 0, __Pyx_PyLong_From_uint64_t, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 270, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PySequence_ListKeepNew(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 270, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_g_hist, __pyx_t_4); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 270, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/extra/search.pyx":267 + * # This can happen from non-monotonic actions + * # If we find a better gold analysis this way, be sure to keep it. + * elif pred._states[i].loss <= 0 \ # <<<<<<<<<<<<<< + * and tuple(pred.histories[i]) not in seen_golds: + * g_scores.append(pred._states[i].score) +*/ + } + __pyx_L11:; + } + + /* "thinc/extra/search.pyx":271 + * g_scores.append(pred._states[i].score) + * g_hist.append(list(pred.histories[i])) + * for i in range(gold.size): # <<<<<<<<<<<<<< + * if gold._states[i].loss == 0: + * g_scores.append(gold._states[i].score) +*/ + __pyx_t_7 = __pyx_v_gold->size; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_i = __pyx_t_9; + + /* "thinc/extra/search.pyx":272 + * g_hist.append(list(pred.histories[i])) + * for i in range(gold.size): + * if gold._states[i].loss == 0: # <<<<<<<<<<<<<< + * g_scores.append(gold._states[i].score) + * g_hist.append(list(gold.histories[i])) +*/ + __pyx_t_10 = ((__pyx_v_gold->_states[__pyx_v_i]).loss == 0.0); + if (__pyx_t_10) { + + /* "thinc/extra/search.pyx":273 + * for i in range(gold.size): + * if gold._states[i].loss == 0: + * g_scores.append(gold._states[i].score) # <<<<<<<<<<<<<< + * g_hist.append(list(gold.histories[i])) + * +*/ + __pyx_t_4 = PyFloat_FromDouble((__pyx_v_gold->_states[__pyx_v_i]).score); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_g_scores, __pyx_t_4); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/extra/search.pyx":274 + * if gold._states[i].loss == 0: + * g_scores.append(gold._states[i].score) + * g_hist.append(list(gold.histories[i])) # <<<<<<<<<<<<<< + * + * all_probs = _softmax(p_scores + g_scores) +*/ + if (unlikely(__pyx_v_gold->histories == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 274, __pyx_L1_error) + } + __pyx_t_4 = __Pyx_GetItemInt_List(__pyx_v_gold->histories, __pyx_v_i, __pyx_t_5thinc_5extra_6search_class_t, 0, __Pyx_PyLong_From_uint64_t, 1, 0, 1, 1, __Pyx_ReferenceSharing_SharedReference); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PySequence_ListKeepNew(__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_12 = __Pyx_PyList_Append(__pyx_v_g_hist, __pyx_t_1); if (unlikely(__pyx_t_12 == ((int)-1))) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":272 + * g_hist.append(list(pred.histories[i])) + * for i in range(gold.size): + * if gold._states[i].loss == 0: # <<<<<<<<<<<<<< + * g_scores.append(gold._states[i].score) + * g_hist.append(list(gold.histories[i])) +*/ + } + } + + /* "thinc/extra/search.pyx":276 + * g_hist.append(list(gold.histories[i])) + * + * all_probs = _softmax(p_scores + g_scores) # <<<<<<<<<<<<<< + * p_probs = all_probs[:len(p_scores)] + * g_probs_all = all_probs[len(p_scores):] +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_softmax); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Add(__pyx_v_p_scores, __pyx_v_g_scores); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_all_probs = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":277 + * + * all_probs = _softmax(p_scores + g_scores) + * p_probs = all_probs[:len(p_scores)] # <<<<<<<<<<<<<< + * g_probs_all = all_probs[len(p_scores):] + * g_probs = _softmax(g_scores) +*/ + __pyx_t_13 = __Pyx_PyList_GET_SIZE(__pyx_v_p_scores); if (unlikely(__pyx_t_13 == ((Py_ssize_t)-1))) __PYX_ERR(0, 277, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetSlice(__pyx_v_all_probs, 0, __pyx_t_13, NULL, NULL, NULL, 0, 1, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_p_probs = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":278 + * all_probs = _softmax(p_scores + g_scores) + * p_probs = all_probs[:len(p_scores)] + * g_probs_all = all_probs[len(p_scores):] # <<<<<<<<<<<<<< + * g_probs = _softmax(g_scores) + * +*/ + __pyx_t_13 = __Pyx_PyList_GET_SIZE(__pyx_v_p_scores); if (unlikely(__pyx_t_13 == ((Py_ssize_t)-1))) __PYX_ERR(0, 278, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetSlice(__pyx_v_all_probs, __pyx_t_13, 0, NULL, NULL, NULL, 1, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_g_probs_all = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":279 + * p_probs = all_probs[:len(p_scores)] + * g_probs_all = all_probs[len(p_scores):] + * g_probs = _softmax(g_scores) # <<<<<<<<<<<<<< + * + * self.cost = pred.loss +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_softmax); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 279, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_g_scores}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 279, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_g_probs = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":281 + * g_probs = _softmax(g_scores) + * + * self.cost = pred.loss # <<<<<<<<<<<<<< + * self.delta = d + * self.p_hist = p_hist +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_pred), __pyx_mstate_global->__pyx_n_u_loss); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 281, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_14 = __Pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_14 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 281, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_self->cost = __pyx_t_14; + + /* "thinc/extra/search.pyx":282 + * + * self.cost = pred.loss + * self.delta = d # <<<<<<<<<<<<<< + * self.p_hist = p_hist + * self.g_hist = g_hist +*/ + __pyx_t_14 = __Pyx_PyFloat_AsFloat(__pyx_v_d); if (unlikely((__pyx_t_14 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 282, __pyx_L1_error) + __pyx_v_self->delta = __pyx_t_14; + + /* "thinc/extra/search.pyx":283 + * self.cost = pred.loss + * self.delta = d + * self.p_hist = p_hist # <<<<<<<<<<<<<< + * self.g_hist = g_hist + * # TODO: These variables are misnamed! These are the gradients of the loss. +*/ + __Pyx_INCREF(__pyx_v_p_hist); + __Pyx_GIVEREF(__pyx_v_p_hist); + __Pyx_GOTREF(__pyx_v_self->p_hist); + __Pyx_DECREF(__pyx_v_self->p_hist); + __pyx_v_self->p_hist = __pyx_v_p_hist; + + /* "thinc/extra/search.pyx":284 + * self.delta = d + * self.p_hist = p_hist + * self.g_hist = g_hist # <<<<<<<<<<<<<< + * # TODO: These variables are misnamed! These are the gradients of the loss. + * self.p_probs = p_probs +*/ + __Pyx_INCREF(__pyx_v_g_hist); + __Pyx_GIVEREF(__pyx_v_g_hist); + __Pyx_GOTREF(__pyx_v_self->g_hist); + __Pyx_DECREF(__pyx_v_self->g_hist); + __pyx_v_self->g_hist = __pyx_v_g_hist; + + /* "thinc/extra/search.pyx":286 + * self.g_hist = g_hist + * # TODO: These variables are misnamed! These are the gradients of the loss. + * self.p_probs = p_probs # <<<<<<<<<<<<<< + * # Intuition here: + * # The gradient of the loss is: +*/ + __pyx_t_1 = __pyx_v_p_probs; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_1))) __PYX_ERR(0, 286, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->p_probs); + __Pyx_DECREF(__pyx_v_self->p_probs); + __pyx_v_self->p_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":294 + * # to create a distribution over the gold, proportional to the scores + * # awarded. + * self.g_probs = [x-y for x, y in zip(g_probs_all, g_probs)] # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = NULL; + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_v_g_probs_all, __pyx_v_g_probs}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_zip, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + if (likely(PyList_CheckExact(__pyx_t_3)) || PyTuple_CheckExact(__pyx_t_3)) { + __pyx_t_2 = __pyx_t_3; __Pyx_INCREF(__pyx_t_2); + __pyx_t_13 = 0; + __pyx_t_15 = NULL; + } else { + __pyx_t_13 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_15 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 294, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + for (;;) { + if (likely(!__pyx_t_15)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 294, __pyx_L1_error) + #endif + if (__pyx_t_13 >= __pyx_temp) break; + } + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_13, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_13; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 294, __pyx_L1_error) + #endif + if (__pyx_t_13 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_13)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_13); + #endif + ++__pyx_t_13; + } + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 294, __pyx_L1_error) + } else { + __pyx_t_3 = __pyx_t_15(__pyx_t_2); + if (unlikely(!__pyx_t_3)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 294, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_3); + if ((likely(PyTuple_CheckExact(__pyx_t_3))) || (PyList_CheckExact(__pyx_t_3))) { + PyObject* sequence = __pyx_t_3; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 294, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_16 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_16); + } else { + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_16 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_16); + } + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_16 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_16); + #endif + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_17 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_17); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_18 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_17); + index = 0; __pyx_t_4 = __pyx_t_18(__pyx_t_17); if (unlikely(!__pyx_t_4)) goto __pyx_L19_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_16 = __pyx_t_18(__pyx_t_17); if (unlikely(!__pyx_t_16)) goto __pyx_L19_unpacking_failed; + __Pyx_GOTREF(__pyx_t_16); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_18(__pyx_t_17), 2) < (0)) __PYX_ERR(0, 294, __pyx_L1_error) + __pyx_t_18 = NULL; + __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; + goto __pyx_L20_unpacking_done; + __pyx_L19_unpacking_failed:; + __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; + __pyx_t_18 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 294, __pyx_L1_error) + __pyx_L20_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_x, __pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_y, __pyx_t_16); + __pyx_t_16 = 0; + __pyx_t_3 = PyNumber_Subtract(__pyx_v_x, __pyx_v_y); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_3))) __PYX_ERR(0, 294, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v_self->g_probs); + __Pyx_DECREF(__pyx_v_self->g_probs); + __pyx_v_self->g_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/extra/search.pyx":256 + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) + * if pred.loss > 0 and (self.cost == 0 or d > self.delta): # <<<<<<<<<<<<<< + * p_hist = [] + * p_scores = [] +*/ + } + + /* "thinc/extra/search.pyx":253 + * self.gZ += prob + * + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_16); + __Pyx_XDECREF(__pyx_t_17); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check_crf", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_d); + __Pyx_XDECREF(__pyx_v_seen_golds); + __Pyx_XDECREF(__pyx_v_p_hist); + __Pyx_XDECREF(__pyx_v_p_scores); + __Pyx_XDECREF(__pyx_v_g_hist); + __Pyx_XDECREF(__pyx_v_g_scores); + __Pyx_XDECREF(__pyx_v_all_probs); + __Pyx_XDECREF(__pyx_v_p_probs); + __Pyx_XDECREF(__pyx_v_g_probs_all); + __Pyx_XDECREF(__pyx_v_g_probs); + __Pyx_XDECREF(__pyx_v_x); + __Pyx_XDECREF(__pyx_v_y); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_5check_crf(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_12MaxViolation_4check_crf, "MaxViolation.check_crf(self, Beam pred, Beam gold) -> int"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_12MaxViolation_5check_crf = {"check_crf", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_5check_crf, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_4check_crf}; +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_5check_crf(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred = 0; + struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("check_crf (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pred,&__pyx_mstate_global->__pyx_n_u_gold,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 253, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 253, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 253, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "check_crf", 0) < (0)) __PYX_ERR(0, 253, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("check_crf", 1, 2, 2, i); __PYX_ERR(0, 253, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 253, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 253, __pyx_L3_error) + } + __pyx_v_pred = ((struct __pyx_obj_5thinc_5extra_6search_Beam *)values[0]); + __pyx_v_gold = ((struct __pyx_obj_5thinc_5extra_6search_Beam *)values[1]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("check_crf", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 253, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check_crf", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_pred), __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_Beam, 1, "pred", 0))) __PYX_ERR(0, 253, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_gold), __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_Beam, 1, "gold", 0))) __PYX_ERR(0, 253, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_4check_crf(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self), __pyx_v_pred, __pyx_v_gold); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_4check_crf(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_pred, struct __pyx_obj_5thinc_5extra_6search_Beam *__pyx_v_gold) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("check_crf", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_5thinc_5extra_6search_12MaxViolation_check_crf(__pyx_v_self, __pyx_v_pred, __pyx_v_gold, 1); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 253, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyLong_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.check_crf", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":79 + * cdef weight_t cost + * cdef weight_t delta + * cdef readonly weight_t p_score # <<<<<<<<<<<<<< + * cdef readonly weight_t g_score + * cdef readonly double Z +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_score_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_score_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_score___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_score___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->p_score); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.p_score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":80 + * cdef weight_t delta + * cdef readonly weight_t p_score + * cdef readonly weight_t g_score # <<<<<<<<<<<<<< + * cdef readonly double Z + * cdef readonly double gZ +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_score_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_score_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_score___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_score___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->g_score); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 80, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.g_score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":81 + * cdef readonly weight_t p_score + * cdef readonly weight_t g_score + * cdef readonly double Z # <<<<<<<<<<<<<< + * cdef readonly double gZ + * cdef class_t n +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_1Z_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_1Z_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_1Z___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_1Z___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->Z); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 81, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.Z.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":82 + * cdef readonly weight_t g_score + * cdef readonly double Z + * cdef readonly double gZ # <<<<<<<<<<<<<< + * cdef class_t n + * cdef readonly list p_hist +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_2gZ_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_2gZ_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_2gZ___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_2gZ___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->gZ); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.gZ.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":84 + * cdef readonly double gZ + * cdef class_t n + * cdef readonly list p_hist # <<<<<<<<<<<<<< + * cdef readonly list g_hist + * cdef readonly list p_probs +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_6p_hist_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_6p_hist_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_6p_hist___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6p_hist___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->p_hist); + __pyx_r = __pyx_v_self->p_hist; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":85 + * cdef class_t n + * cdef readonly list p_hist + * cdef readonly list g_hist # <<<<<<<<<<<<<< + * cdef readonly list p_probs + * cdef readonly list g_probs +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_6g_hist_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_6g_hist_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_6g_hist___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6g_hist___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->g_hist); + __pyx_r = __pyx_v_self->g_hist; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":86 + * cdef readonly list p_hist + * cdef readonly list g_hist + * cdef readonly list p_probs # <<<<<<<<<<<<<< + * cdef readonly list g_probs + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_probs_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_probs_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_probs___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7p_probs___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->p_probs); + __pyx_r = __pyx_v_self->p_probs; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pxd":87 + * cdef readonly list g_hist + * cdef readonly list p_probs + * cdef readonly list g_probs # <<<<<<<<<<<<<< + * + * cpdef int check(self, Beam pred, Beam gold) except -1 +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_probs_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_probs_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_probs___get__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_7g_probs___get__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->g_probs); + __pyx_r = __pyx_v_self->g_probs; + goto __pyx_L0; + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__, "MaxViolation.__reduce_cython__(self)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__ = {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__}; +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self) { + PyObject *__pyx_v_state = 0; + PyObject *__pyx_v__dict = 0; + int __pyx_v_use_setstate; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":5 + * cdef object _dict + * cdef bint use_setstate + * state = (self.Z, self.cost, self.delta, self.gZ, self.g_hist, self.g_probs, self.g_score, self.mem, self.n, self.p_hist, self.p_probs, self.p_score) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: +*/ + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->Z); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_self->cost); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->delta); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyFloat_FromDouble(__pyx_v_self->gZ); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyFloat_FromDouble(__pyx_v_self->g_score); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyLong_From_uint64_t(__pyx_v_self->n); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyFloat_FromDouble(__pyx_v_self->p_score); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(12); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_1) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_2) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_3) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 3, __pyx_t_4) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_self->g_hist); + __Pyx_GIVEREF(__pyx_v_self->g_hist); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 4, __pyx_v_self->g_hist) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_self->g_probs); + __Pyx_GIVEREF(__pyx_v_self->g_probs); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 5, __pyx_v_self->g_probs) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 6, __pyx_t_5) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_INCREF((PyObject *)__pyx_v_self->mem); + __Pyx_GIVEREF((PyObject *)__pyx_v_self->mem); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 7, ((PyObject *)__pyx_v_self->mem)) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 8, __pyx_t_6) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_self->p_hist); + __Pyx_GIVEREF(__pyx_v_self->p_hist); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 9, __pyx_v_self->p_hist) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_self->p_probs); + __Pyx_GIVEREF(__pyx_v_self->p_probs); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 10, __pyx_v_self->p_probs) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 11, __pyx_t_7) != (0)) __PYX_ERR(2, 5, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_2 = 0; + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_6 = 0; + __pyx_t_7 = 0; + __pyx_v_state = ((PyObject*)__pyx_t_8); + __pyx_t_8 = 0; + + /* "(tree fragment)":6 + * cdef bint use_setstate + * state = (self.Z, self.cost, self.delta, self.gZ, self.g_hist, self.g_probs, self.g_score, self.mem, self.n, self.p_hist, self.p_probs, self.p_score) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None and _dict: + * state += (_dict,) +*/ + __pyx_t_8 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_v__dict = __pyx_t_8; + __pyx_t_8 = 0; + + /* "(tree fragment)":7 + * state = (self.Z, self.cost, self.delta, self.gZ, self.g_hist, self.g_probs, self.g_score, self.mem, self.n, self.p_hist, self.p_probs, self.p_score) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + __pyx_t_10 = (__pyx_v__dict != Py_None); + if (__pyx_t_10) { + } else { + __pyx_t_9 = __pyx_t_10; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_10 < 0))) __PYX_ERR(2, 7, __pyx_L1_error) + __pyx_t_9 = __pyx_t_10; + __pyx_L4_bool_binop_done:; + if (__pyx_t_9) { + + /* "(tree fragment)":8 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: +*/ + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v__dict) != (0)) __PYX_ERR(2, 8, __pyx_L1_error); + __pyx_t_7 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_7)); + __pyx_t_7 = 0; + + /* "(tree fragment)":9 + * if _dict is not None and _dict: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) +*/ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":7 + * state = (self.Z, self.cost, self.delta, self.gZ, self.g_hist, self.g_probs, self.g_score, self.mem, self.n, self.p_hist, self.p_probs, self.p_score) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + goto __pyx_L3; + } + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state +*/ + /*else*/ { + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_mstate_global->__pyx_tuple[0]); if (unlikely((__pyx_t_9 == (int)-1) && PyErr_Occurred())) __PYX_ERR(2, 11, __pyx_L1_error) + __pyx_v_use_setstate = __pyx_t_9; + } + __pyx_L3:; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state + * else: +*/ + if (__pyx_v_use_setstate) { + + /* "(tree fragment)":13 + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) + * if use_setstate: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_MaxViolation); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_237080843); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_237080843); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_mstate_global->__pyx_int_237080843) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, Py_None) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __pyx_t_6 = PyTuple_New(3); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_7) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_8) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 2, __pyx_v_state) != (0)) __PYX_ERR(2, 13, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_8 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state + * else: +*/ + } + + /* "(tree fragment)":15 + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state + * else: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_MaxViolation__set_state(self, __pyx_state) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_MaxViolation); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(2, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_237080843); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_237080843); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_mstate_global->__pyx_int_237080843) != (0)) __PYX_ERR(2, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_v_state) != (0)) __PYX_ERR(2, 15, __pyx_L1_error); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_6) != (0)) __PYX_ERR(2, 15, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_8) != (0)) __PYX_ERR(2, 15, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_8 = 0; + __pyx_r = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":16 + * else: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_MaxViolation__set_state(self, __pyx_state) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__, "MaxViolation.__setstate_cython__(self, __pyx_state)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__ = {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__}; +static PyObject *__pyx_pw_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(2, 16, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 16, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(2, 16, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(2, 16, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 16, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(2, 16, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":17 + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_MaxViolation__set_state(self, __pyx_state) # <<<<<<<<<<<<<< +*/ + __pyx_t_1 = __pyx_f_5thinc_5extra_6search___pyx_unpickle_MaxViolation__set_state(__pyx_v_self, __pyx_v___pyx_state); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_MaxViolation__set_state(self, __pyx_state) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.MaxViolation.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":297 + * + * + * def _softmax(nums): # <<<<<<<<<<<<<< + * if not nums: + * return [] +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_1_softmax(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search__softmax, "_softmax(nums)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_1_softmax = {"_softmax", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_1_softmax, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search__softmax}; +static PyObject *__pyx_pw_5thinc_5extra_6search_1_softmax(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_nums = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_softmax (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_nums,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 297, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 297, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_softmax", 0) < (0)) __PYX_ERR(0, 297, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_softmax", 1, 1, 1, i); __PYX_ERR(0, 297, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 297, __pyx_L3_error) + } + __pyx_v_nums = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_softmax", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 297, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search._softmax", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_5extra_6search__softmax(__pyx_self, __pyx_v_nums); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +static PyObject *__pyx_gb_5thinc_5extra_6search_8_softmax_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value); /* proto */ + +/* "thinc/extra/search.pyx":302 + * max_ = max(nums) + * nums = [(exp(n-max_) if n is not None else None) for n in nums] + * Z = sum(n for n in nums if n is not None) # <<<<<<<<<<<<<< + * return [(n/Z if n is not None else None) for n in nums] +*/ + +static PyObject *__pyx_pf_5thinc_5extra_6search_8_softmax_genexpr(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_genexpr_arg_0) { + struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("genexpr", 0); + __pyx_cur_scope = (struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)__pyx_tp_new_5thinc_5extra_6search___pyx_scope_struct__genexpr(__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 302, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_genexpr_arg_0 = __pyx_genexpr_arg_0; + __Pyx_INCREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_genexpr_arg_0); + { + __pyx_CoroutineObject *gen = __Pyx_Generator_New((__pyx_coroutine_body_t) __pyx_gb_5thinc_5extra_6search_8_softmax_2generator, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0]), (PyObject *) __pyx_cur_scope, __pyx_mstate_global->__pyx_n_u_genexpr, __pyx_mstate_global->__pyx_n_u_softmax_locals_genexpr, __pyx_mstate_global->__pyx_n_u_thinc_extra_search); if (unlikely(!gen)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.extra.search._softmax.genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5thinc_5extra_6search_8_softmax_2generator(__pyx_CoroutineObject *__pyx_generator, CYTHON_UNUSED PyThreadState *__pyx_tstate, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *__pyx_cur_scope = ((struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + Py_ssize_t __pyx_t_2; + PyObject *(*__pyx_t_3)(PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("genexpr", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L7_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(__pyx_sent_value != Py_None)) { + if (unlikely(__pyx_sent_value)) PyErr_SetString(PyExc_TypeError, "can't send non-None value to a just-started generator"); + __PYX_ERR(0, 302, __pyx_L1_error) + } + if (unlikely(!__pyx_cur_scope->__pyx_genexpr_arg_0)) { __Pyx_RaiseUnboundLocalError(".0"); __PYX_ERR(0, 302, __pyx_L1_error) } + if (likely(PyList_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) || PyTuple_CheckExact(__pyx_cur_scope->__pyx_genexpr_arg_0)) { + __pyx_t_1 = __pyx_cur_scope->__pyx_genexpr_arg_0; __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = 0; + __pyx_t_3 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_cur_scope->__pyx_genexpr_arg_0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 302, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_3)) { + if (likely(PyList_CheckExact(__pyx_t_1))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 302, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_1, __pyx_t_2, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 302, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2)); + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 302, __pyx_L1_error) + } else { + __pyx_t_4 = __pyx_t_3(__pyx_t_1); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 302, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_n); + __Pyx_XDECREF_SET(__pyx_cur_scope->__pyx_v_n, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_5 = (__pyx_cur_scope->__pyx_v_n != Py_None); + if (__pyx_t_5) { + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_n); + __pyx_r = __pyx_cur_scope->__pyx_v_n; + __Pyx_XGIVEREF(__pyx_t_1); + __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_2 = __pyx_t_3; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L7_resume_from_yield:; + __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; + __pyx_cur_scope->__pyx_t_0 = 0; + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_2; + if (unlikely(!__pyx_sent_value)) __PYX_ERR(0, 302, __pyx_L1_error) + } + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + CYTHON_MAYBE_UNUSED_VAR(__pyx_cur_scope); + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + if (__Pyx_PyErr_Occurred()) { + __Pyx_AddTraceback("genexpr", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + #if !CYTHON_USE_EXC_INFO_STACK + __Pyx_Coroutine_ResetAndClearException(__pyx_generator); + #endif + __pyx_generator->resume_label = -1; + __Pyx_Coroutine_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/extra/search.pyx":297 + * + * + * def _softmax(nums): # <<<<<<<<<<<<<< + * if not nums: + * return [] +*/ + +static PyObject *__pyx_pf_5thinc_5extra_6search__softmax(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nums) { + PyObject *__pyx_v_max_ = NULL; + PyObject *__pyx_v_Z = NULL; + PyObject *__pyx_v_n = NULL; + PyObject *__pyx_gb_5thinc_5extra_6search_8_softmax_2generator = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + double __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_softmax", 0); + __Pyx_INCREF(__pyx_v_nums); + + /* "thinc/extra/search.pyx":298 + * + * def _softmax(nums): + * if not nums: # <<<<<<<<<<<<<< + * return [] + * max_ = max(nums) +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_nums); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 298, __pyx_L1_error) + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "thinc/extra/search.pyx":299 + * def _softmax(nums): + * if not nums: + * return [] # <<<<<<<<<<<<<< + * max_ = max(nums) + * nums = [(exp(n-max_) if n is not None else None) for n in nums] +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 299, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":298 + * + * def _softmax(nums): + * if not nums: # <<<<<<<<<<<<<< + * return [] + * max_ = max(nums) +*/ + } + + /* "thinc/extra/search.pyx":300 + * if not nums: + * return [] + * max_ = max(nums) # <<<<<<<<<<<<<< + * nums = [(exp(n-max_) if n is not None else None) for n in nums] + * Z = sum(n for n in nums if n is not None) +*/ + __pyx_t_4 = NULL; + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_v_nums}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_max, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 300, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_v_max_ = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/extra/search.pyx":301 + * return [] + * max_ = max(nums) + * nums = [(exp(n-max_) if n is not None else None) for n in nums] # <<<<<<<<<<<<<< + * Z = sum(n for n in nums if n is not None) + * return [(n/Z if n is not None else None) for n in nums] +*/ + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (likely(PyList_CheckExact(__pyx_v_nums)) || PyTuple_CheckExact(__pyx_v_nums)) { + __pyx_t_4 = __pyx_v_nums; __Pyx_INCREF(__pyx_t_4); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_v_nums); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_4); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 301, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_4))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 301, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_4, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 301, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_6)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_4, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 301, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_4); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 301, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_n, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_t_2 = (__pyx_v_n != Py_None); + if (__pyx_t_2) { + __pyx_t_9 = PyNumber_Subtract(__pyx_v_n, __pyx_v_max_); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = __Pyx_PyFloat_AsDouble(__pyx_t_9); if (unlikely((__pyx_t_10 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_9 = PyFloat_FromDouble(exp(__pyx_t_10)); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_8 = __pyx_t_9; + __pyx_t_9 = 0; + } else { + __Pyx_INCREF(Py_None); + __pyx_t_8 = Py_None; + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_8))) __PYX_ERR(0, 301, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF_SET(__pyx_v_nums, __pyx_t_3); + __pyx_t_3 = 0; + + /* "thinc/extra/search.pyx":302 + * max_ = max(nums) + * nums = [(exp(n-max_) if n is not None else None) for n in nums] + * Z = sum(n for n in nums if n is not None) # <<<<<<<<<<<<<< + * return [(n/Z if n is not None else None) for n in nums] +*/ + __pyx_t_4 = NULL; + __pyx_t_8 = __pyx_pf_5thinc_5extra_6search_8_softmax_genexpr(NULL, __pyx_v_nums); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_t_8}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin_sum, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 302, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_v_Z = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/extra/search.pyx":303 + * nums = [(exp(n-max_) if n is not None else None) for n in nums] + * Z = sum(n for n in nums if n is not None) + * return [(n/Z if n is not None else None) for n in nums] # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 303, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (likely(PyList_CheckExact(__pyx_v_nums)) || PyTuple_CheckExact(__pyx_v_nums)) { + __pyx_t_8 = __pyx_v_nums; __Pyx_INCREF(__pyx_t_8); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_nums); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 303, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_8); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 303, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_8))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_8); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 303, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(__pyx_t_8, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_8); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 303, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_4 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6)); + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(__pyx_t_8, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 303, __pyx_L1_error) + } else { + __pyx_t_4 = __pyx_t_7(__pyx_t_8); + if (unlikely(!__pyx_t_4)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 303, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_4); + __Pyx_XDECREF_SET(__pyx_v_n, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_2 = (__pyx_v_n != Py_None); + if (__pyx_t_2) { + __pyx_t_9 = __Pyx_PyNumber_Divide(__pyx_v_n, __pyx_v_Z); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 303, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_4 = __pyx_t_9; + __pyx_t_9 = 0; + } else { + __Pyx_INCREF(Py_None); + __pyx_t_4 = Py_None; + } + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_4))) __PYX_ERR(0, 303, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/extra/search.pyx":297 + * + * + * def _softmax(nums): # <<<<<<<<<<<<<< + * if not nums: + * return [] +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.extra.search._softmax", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_max_); + __Pyx_XDECREF(__pyx_v_Z); + __Pyx_XDECREF(__pyx_v_n); + __Pyx_XDECREF(__pyx_gb_5thinc_5extra_6search_8_softmax_2generator); + __Pyx_XDECREF(__pyx_v_nums); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_MaxViolation(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_5extra_6search_3__pyx_unpickle_MaxViolation(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_5extra_6search_2__pyx_unpickle_MaxViolation, "__pyx_unpickle_MaxViolation(__pyx_type, long __pyx_checksum, tuple __pyx_state)"); +static PyMethodDef __pyx_mdef_5thinc_5extra_6search_3__pyx_unpickle_MaxViolation = {"__pyx_unpickle_MaxViolation", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_3__pyx_unpickle_MaxViolation, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_2__pyx_unpickle_MaxViolation}; +static PyObject *__pyx_pw_5thinc_5extra_6search_3__pyx_unpickle_MaxViolation(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_MaxViolation (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(2, 4, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(2, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(2, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_MaxViolation", 0) < (0)) __PYX_ERR(2, 4, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_MaxViolation", 1, 3, 3, i); __PYX_ERR(2, 4, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(2, 4, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(2, 4, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(2, 4, __pyx_L3_error) + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(2, 4, __pyx_L3_error) + __pyx_v___pyx_state = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_MaxViolation", 1, 3, 3, __pyx_nargs); __PYX_ERR(2, 4, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.extra.search.__pyx_unpickle_MaxViolation", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(2, 4, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_5extra_6search_2__pyx_unpickle_MaxViolation(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_5extra_6search_2__pyx_unpickle_MaxViolation(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_MaxViolation", 0); + + /* "(tree fragment)":6 + * def __pyx_unpickle_MaxViolation(__pyx_type, long __pyx_checksum, tuple __pyx_state): + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') # <<<<<<<<<<<<<< + * __pyx_result = MaxViolation.__new__(__pyx_type) + * if __pyx_state is not None: +*/ + __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, ((char const *)"Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score")); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 6, __pyx_L1_error) + + /* "(tree fragment)":7 + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') + * __pyx_result = MaxViolation.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) +*/ + __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v___pyx_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') + * __pyx_result = MaxViolation.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None)); + if (__pyx_t_5) { + + /* "(tree fragment)":9 + * __pyx_result = MaxViolation.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): +*/ + __pyx_t_2 = __pyx_f_5thinc_5extra_6search___pyx_unpickle_MaxViolation__set_state(((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') + * __pyx_result = MaxViolation.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + } + + /* "(tree fragment)":10 + * if __pyx_state is not None: + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): + * __pyx_result.Z = __pyx_state[0]; __pyx_result.cost = __pyx_state[1]; __pyx_result.delta = __pyx_state[2]; __pyx_result.gZ = __pyx_state[3]; __pyx_result.g_hist = __pyx_state[4]; __pyx_result.g_probs = __pyx_state[5]; __pyx_result.g_score = __pyx_state[6]; __pyx_result.mem = __pyx_state[7]; __pyx_result.n = __pyx_state[8]; __pyx_result.p_hist = __pyx_state[9]; __pyx_result.p_probs = __pyx_state[10]; __pyx_result.p_score = __pyx_state[11] +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_MaxViolation(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("thinc.extra.search.__pyx_unpickle_MaxViolation", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":11 + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.Z = __pyx_state[0]; __pyx_result.cost = __pyx_state[1]; __pyx_result.delta = __pyx_state[2]; __pyx_result.gZ = __pyx_state[3]; __pyx_result.g_hist = __pyx_state[4]; __pyx_result.g_probs = __pyx_state[5]; __pyx_result.g_score = __pyx_state[6]; __pyx_result.mem = __pyx_state[7]; __pyx_result.n = __pyx_state[8]; __pyx_result.p_hist = __pyx_state[9]; __pyx_result.p_probs = __pyx_state[10]; __pyx_result.p_score = __pyx_state[11] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 12) +*/ + +static PyObject *__pyx_f_5thinc_5extra_6search___pyx_unpickle_MaxViolation__set_state(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + double __pyx_t_2; + __pyx_t_5thinc_5extra_6search_weight_t __pyx_t_3; + __pyx_t_5thinc_5extra_6search_class_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_MaxViolation__set_state", 0); + + /* "(tree fragment)":12 + * return __pyx_result + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): + * __pyx_result.Z = __pyx_state[0]; __pyx_result.cost = __pyx_state[1]; __pyx_result.delta = __pyx_state[2]; __pyx_result.gZ = __pyx_state[3]; __pyx_result.g_hist = __pyx_state[4]; __pyx_result.g_probs = __pyx_state[5]; __pyx_result.g_score = __pyx_state[6]; __pyx_result.mem = __pyx_state[7]; __pyx_result.n = __pyx_state[8]; __pyx_result.p_hist = __pyx_state[9]; __pyx_result.p_probs = __pyx_state[10]; __pyx_result.p_score = __pyx_state[11] # <<<<<<<<<<<<<< + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 12) +*/ + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyFloat_AsDouble(__pyx_t_1); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->Z = __pyx_t_2; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 1, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->cost = __pyx_t_3; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 2, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->delta = __pyx_t_3; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 3, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyFloat_AsDouble(__pyx_t_1); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->gZ = __pyx_t_2; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 4, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_1))) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->g_hist); + __Pyx_DECREF(__pyx_v___pyx_result->g_hist); + __pyx_v___pyx_result->g_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 5, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_1))) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->g_probs); + __Pyx_DECREF(__pyx_v___pyx_result->g_probs); + __pyx_v___pyx_result->g_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 6, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->g_score = __pyx_t_3; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 7, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5cymem_5cymem_Pool))))) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF((PyObject *)__pyx_v___pyx_result->mem); + __Pyx_DECREF((PyObject *)__pyx_v___pyx_result->mem); + __pyx_v___pyx_result->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 8, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyLong_As_uint64_t(__pyx_t_1); if (unlikely((__pyx_t_4 == ((uint64_t)-1)) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->n = __pyx_t_4; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 9, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_1))) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->p_hist); + __Pyx_DECREF(__pyx_v___pyx_result->p_hist); + __pyx_v___pyx_result->p_hist = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 10, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("list", __pyx_t_1))) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->p_probs); + __Pyx_DECREF(__pyx_v___pyx_result->p_probs); + __pyx_v___pyx_result->p_probs = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v___pyx_state, 11, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyFloat_AsFloat(__pyx_t_1); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) __PYX_ERR(2, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v___pyx_result->p_score = __pyx_t_3; + + /* "(tree fragment)":13 + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): + * __pyx_result.Z = __pyx_state[0]; __pyx_result.cost = __pyx_state[1]; __pyx_result.delta = __pyx_state[2]; __pyx_result.gZ = __pyx_state[3]; __pyx_result.g_hist = __pyx_state[4]; __pyx_result.g_probs = __pyx_state[5]; __pyx_result.g_score = __pyx_state[6]; __pyx_result.mem = __pyx_state[7]; __pyx_result.n = __pyx_state[8]; __pyx_result.p_hist = __pyx_state[9]; __pyx_result.p_probs = __pyx_state[10]; __pyx_result.p_score = __pyx_state[11] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 12) # <<<<<<<<<<<<<< +*/ + __pyx_t_5 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 12); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(2, 13, __pyx_L1_error) + + /* "(tree fragment)":11 + * __pyx_unpickle_MaxViolation__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_MaxViolation__set_state(MaxViolation __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.Z = __pyx_state[0]; __pyx_result.cost = __pyx_state[1]; __pyx_result.delta = __pyx_state[2]; __pyx_result.gZ = __pyx_state[3]; __pyx_result.g_hist = __pyx_state[4]; __pyx_result.g_probs = __pyx_state[5]; __pyx_result.g_score = __pyx_state[6]; __pyx_result.mem = __pyx_state[7]; __pyx_result.n = __pyx_state[8]; __pyx_result.p_hist = __pyx_state[9]; __pyx_result.p_probs = __pyx_state[10]; __pyx_result.p_score = __pyx_state[11] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 12) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("thinc.extra.search.__pyx_unpickle_MaxViolation__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ +static struct __pyx_vtabstruct_5thinc_5extra_6search_Beam __pyx_vtable_5thinc_5extra_6search_Beam; + +static PyObject *__pyx_tp_new_5thinc_5extra_6search_Beam(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_5extra_6search_Beam *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_5extra_6search_Beam *)o); + p->__pyx_vtab = __pyx_vtabptr_5thinc_5extra_6search_Beam; + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + p->histories = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_parent_histories = ((PyObject*)Py_None); Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_5thinc_5extra_6search_Beam(PyObject *o) { + struct __pyx_obj_5thinc_5extra_6search_Beam *p = (struct __pyx_obj_5thinc_5extra_6search_Beam *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_5extra_6search_Beam) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_pw_5thinc_5extra_6search_4Beam_5__dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->mem); + Py_CLEAR(p->histories); + Py_CLEAR(p->_parent_histories); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_5thinc_5extra_6search_Beam(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_5extra_6search_Beam *p = (struct __pyx_obj_5thinc_5extra_6search_Beam *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->mem) { + e = (*v)(((PyObject *)p->mem), a); if (e) return e; + } + if (p->histories) { + e = (*v)(p->histories, a); if (e) return e; + } + if (p->_parent_histories) { + e = (*v)(p->_parent_histories, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_5extra_6search_Beam(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5thinc_5extra_6search_Beam *p = (struct __pyx_obj_5thinc_5extra_6search_Beam *)o; + tmp = ((PyObject*)p->mem); + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->histories); + p->histories = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_parent_histories); + p->_parent_histories = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_score(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_5score_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_min_score(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_9min_score_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_loss(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_4loss_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_probs(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_5probs_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_scores(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_6scores_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_histories(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_9histories_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_min_density(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_11min_density_1__get__(o); +} + +static int __pyx_setprop_5thinc_5extra_6search_4Beam_min_density(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { + if (v) { + return __pyx_pw_5thinc_5extra_6search_4Beam_11min_density_3__set__(o, v); + } + else { + PyErr_SetString(PyExc_NotImplementedError, "__del__"); + return -1; + } +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_4Beam_is_done(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_4Beam_7is_done_1__get__(o); +} + +static PyMethodDef __pyx_methods_5thinc_5extra_6search_Beam[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_4Beam_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_4Beam_6__reduce_cython__}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_4Beam_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_4Beam_8__setstate_cython__}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5thinc_5extra_6search_Beam[] = { + {"score", __pyx_getprop_5thinc_5extra_6search_4Beam_score, 0, 0, 0}, + {"min_score", __pyx_getprop_5thinc_5extra_6search_4Beam_min_score, 0, 0, 0}, + {"loss", __pyx_getprop_5thinc_5extra_6search_4Beam_loss, 0, 0, 0}, + {"probs", __pyx_getprop_5thinc_5extra_6search_4Beam_probs, 0, 0, 0}, + {"scores", __pyx_getprop_5thinc_5extra_6search_4Beam_scores, 0, 0, 0}, + {"histories", __pyx_getprop_5thinc_5extra_6search_4Beam_histories, 0, 0, 0}, + {"min_density", __pyx_getprop_5thinc_5extra_6search_4Beam_min_density, __pyx_setprop_5thinc_5extra_6search_4Beam_min_density, PyDoc_STR("min_density: 'weight_t'"), 0}, + {"is_done", __pyx_getprop_5thinc_5extra_6search_4Beam_is_done, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_5extra_6search_Beam_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_5extra_6search_Beam}, + {Py_sq_length, (void *)__pyx_pw_5thinc_5extra_6search_4Beam_3__len__}, + {Py_mp_length, (void *)__pyx_pw_5thinc_5extra_6search_4Beam_3__len__}, + {Py_tp_doc, (void *)PyDoc_STR("Beam(class_t nr_class, class_t width, weight_t min_density=0.0)")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_5extra_6search_Beam}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_5extra_6search_Beam}, + {Py_tp_methods, (void *)__pyx_methods_5thinc_5extra_6search_Beam}, + {Py_tp_getset, (void *)__pyx_getsets_5thinc_5extra_6search_Beam}, + {Py_tp_init, (void *)__pyx_pw_5thinc_5extra_6search_4Beam_1__init__}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_5extra_6search_Beam}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_5extra_6search_Beam_spec = { + "thinc.extra.search.Beam", + sizeof(struct __pyx_obj_5thinc_5extra_6search_Beam), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_5extra_6search_Beam_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_Beam = { + __pyx_pw_5thinc_5extra_6search_4Beam_3__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Beam = { + __pyx_pw_5thinc_5extra_6search_4Beam_3__len__, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyTypeObject __pyx_type_5thinc_5extra_6search_Beam = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.extra.search.""Beam", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_5extra_6search_Beam), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_5extra_6search_Beam, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_Beam, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Beam, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + PyDoc_STR("Beam(class_t nr_class, class_t width, weight_t min_density=0.0)"), /*tp_doc*/ + __pyx_tp_traverse_5thinc_5extra_6search_Beam, /*tp_traverse*/ + __pyx_tp_clear_5thinc_5extra_6search_Beam, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5thinc_5extra_6search_Beam, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5thinc_5extra_6search_Beam, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_pw_5thinc_5extra_6search_4Beam_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_5extra_6search_Beam, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_5thinc_5extra_6search_MaxViolation __pyx_vtable_5thinc_5extra_6search_MaxViolation; + +static PyObject *__pyx_tp_new_5thinc_5extra_6search_MaxViolation(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5thinc_5extra_6search_MaxViolation *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)o); + p->__pyx_vtab = __pyx_vtabptr_5thinc_5extra_6search_MaxViolation; + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + p->p_hist = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->g_hist = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->p_probs = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->g_probs = ((PyObject*)Py_None); Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_5thinc_5extra_6search_MaxViolation(PyObject *o) { + struct __pyx_obj_5thinc_5extra_6search_MaxViolation *p = (struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_5extra_6search_MaxViolation) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->mem); + Py_CLEAR(p->p_hist); + Py_CLEAR(p->g_hist); + Py_CLEAR(p->p_probs); + Py_CLEAR(p->g_probs); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_5thinc_5extra_6search_MaxViolation(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_5extra_6search_MaxViolation *p = (struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->mem) { + e = (*v)(((PyObject *)p->mem), a); if (e) return e; + } + if (p->p_hist) { + e = (*v)(p->p_hist, a); if (e) return e; + } + if (p->g_hist) { + e = (*v)(p->g_hist, a); if (e) return e; + } + if (p->p_probs) { + e = (*v)(p->p_probs, a); if (e) return e; + } + if (p->g_probs) { + e = (*v)(p->g_probs, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_5extra_6search_MaxViolation(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5thinc_5extra_6search_MaxViolation *p = (struct __pyx_obj_5thinc_5extra_6search_MaxViolation *)o; + tmp = ((PyObject*)p->mem); + p->mem = ((struct __pyx_obj_5cymem_5cymem_Pool *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->p_hist); + p->p_hist = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->g_hist); + p->g_hist = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->p_probs); + p->p_probs = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->g_probs); + p->g_probs = ((PyObject*)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_score(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_score_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_score(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_score_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_Z(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_1Z_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_gZ(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_2gZ_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_hist(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_6p_hist_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_hist(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_6g_hist_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_probs(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_7p_probs_1__get__(o); +} + +static PyObject *__pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_probs(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5thinc_5extra_6search_12MaxViolation_7g_probs_1__get__(o); +} + +static PyMethodDef __pyx_methods_5thinc_5extra_6search_MaxViolation[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_6__reduce_cython__}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_5extra_6search_12MaxViolation_8__setstate_cython__}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5thinc_5extra_6search_MaxViolation[] = { + {"p_score", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_score, 0, 0, 0}, + {"g_score", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_score, 0, 0, 0}, + {"Z", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_Z, 0, 0, 0}, + {"gZ", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_gZ, 0, 0, 0}, + {"p_hist", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_hist, 0, 0, 0}, + {"g_hist", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_hist, 0, 0, 0}, + {"p_probs", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_p_probs, 0, 0, 0}, + {"g_probs", __pyx_getprop_5thinc_5extra_6search_12MaxViolation_g_probs, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_5extra_6search_MaxViolation_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_5extra_6search_MaxViolation}, + {Py_tp_doc, (void *)PyDoc_STR("MaxViolation()")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_5extra_6search_MaxViolation}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_5extra_6search_MaxViolation}, + {Py_tp_methods, (void *)__pyx_methods_5thinc_5extra_6search_MaxViolation}, + {Py_tp_getset, (void *)__pyx_getsets_5thinc_5extra_6search_MaxViolation}, + {Py_tp_init, (void *)__pyx_pw_5thinc_5extra_6search_12MaxViolation_1__init__}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_5extra_6search_MaxViolation}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_5extra_6search_MaxViolation_spec = { + "thinc.extra.search.MaxViolation", + sizeof(struct __pyx_obj_5thinc_5extra_6search_MaxViolation), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_5extra_6search_MaxViolation_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_5extra_6search_MaxViolation = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.extra.search.""MaxViolation", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_5extra_6search_MaxViolation), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_5extra_6search_MaxViolation, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + PyDoc_STR("MaxViolation()"), /*tp_doc*/ + __pyx_tp_traverse_5thinc_5extra_6search_MaxViolation, /*tp_traverse*/ + __pyx_tp_clear_5thinc_5extra_6search_MaxViolation, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5thinc_5extra_6search_MaxViolation, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5thinc_5extra_6search_MaxViolation, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_pw_5thinc_5extra_6search_12MaxViolation_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_5extra_6search_MaxViolation, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_5thinc_5extra_6search___pyx_scope_struct__genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_5extra_6search___pyx_scope_struct__genexpr > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr, sizeof(struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_5thinc_5extra_6search___pyx_scope_struct__genexpr[--__pyx_mstate_global->__pyx_freecount_5thinc_5extra_6search___pyx_scope_struct__genexpr]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_5thinc_5extra_6search___pyx_scope_struct__genexpr(PyObject *o) { + struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *p = (struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_5extra_6search___pyx_scope_struct__genexpr) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_genexpr_arg_0); + Py_CLEAR(p->__pyx_v_n); + Py_CLEAR(p->__pyx_t_0); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_5extra_6search___pyx_scope_struct__genexpr < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr, sizeof(struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr)))) + { + __pyx_mstate_global->__pyx_freelist_5thinc_5extra_6search___pyx_scope_struct__genexpr[__pyx_mstate_global->__pyx_freecount_5thinc_5extra_6search___pyx_scope_struct__genexpr++] = ((struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_5thinc_5extra_6search___pyx_scope_struct__genexpr(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *p = (struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_genexpr_arg_0) { + e = (*v)(p->__pyx_genexpr_arg_0, a); if (e) return e; + } + if (p->__pyx_v_n) { + e = (*v)(p->__pyx_v_n, a); if (e) return e; + } + if (p->__pyx_t_0) { + e = (*v)(p->__pyx_t_0, a); if (e) return e; + } + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_5extra_6search___pyx_scope_struct__genexpr}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_5extra_6search___pyx_scope_struct__genexpr}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_5extra_6search___pyx_scope_struct__genexpr}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr_spec = { + "thinc.extra.search.__pyx_scope_struct__genexpr", + sizeof(struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.extra.search.""__pyx_scope_struct__genexpr", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_5extra_6search___pyx_scope_struct__genexpr), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_5extra_6search___pyx_scope_struct__genexpr, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5thinc_5extra_6search___pyx_scope_struct__genexpr, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_5extra_6search___pyx_scope_struct__genexpr, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __pyx_vtabptr_5thinc_5extra_6search_Beam = &__pyx_vtable_5thinc_5extra_6search_Beam; + __pyx_vtable_5thinc_5extra_6search_Beam._fill = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_Queue *, __pyx_t_5thinc_5extra_6search_weight_t **, int **))__pyx_f_5thinc_5extra_6search_4Beam__fill; + __pyx_vtable_5thinc_5extra_6search_Beam.at = (void *(*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int))__pyx_f_5thinc_5extra_6search_4Beam_at; + __pyx_vtable_5thinc_5extra_6search_Beam.initialize = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_init_func_t, __pyx_t_5thinc_5extra_6search_del_func_t, int, void *))__pyx_f_5thinc_5extra_6search_4Beam_initialize; + __pyx_vtable_5thinc_5extra_6search_Beam.advance = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_trans_func_t, __pyx_t_5thinc_5extra_6search_hash_func_t, void *))__pyx_f_5thinc_5extra_6search_4Beam_advance; + __pyx_vtable_5thinc_5extra_6search_Beam.check_done = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_finish_func_t, void *))__pyx_f_5thinc_5extra_6search_4Beam_check_done; + __pyx_vtable_5thinc_5extra_6search_Beam.set_cell = (void (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int, int, __pyx_t_5thinc_5extra_6search_weight_t, int, __pyx_t_5thinc_5extra_6search_weight_t))__pyx_f_5thinc_5extra_6search_4Beam_set_cell; + __pyx_vtable_5thinc_5extra_6search_Beam.set_row = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, int, __pyx_t_5thinc_5extra_6search_weight_t const *, int const *, __pyx_t_5thinc_5extra_6search_weight_t const *))__pyx_f_5thinc_5extra_6search_4Beam_set_row; + __pyx_vtable_5thinc_5extra_6search_Beam.set_table = (int (*)(struct __pyx_obj_5thinc_5extra_6search_Beam *, __pyx_t_5thinc_5extra_6search_weight_t **, int **, __pyx_t_5thinc_5extra_6search_weight_t **))__pyx_f_5thinc_5extra_6search_4Beam_set_table; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_5extra_6search_Beam_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam)) __PYX_ERR(0, 12, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_5extra_6search_Beam_spec, __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam = &__pyx_type_5thinc_5extra_6search_Beam; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam, __pyx_vtabptr_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_Beam, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_Beam) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __pyx_vtabptr_5thinc_5extra_6search_MaxViolation = &__pyx_vtable_5thinc_5extra_6search_MaxViolation; + __pyx_vtable_5thinc_5extra_6search_MaxViolation.check = (int (*)(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *, struct __pyx_obj_5thinc_5extra_6search_Beam *, struct __pyx_obj_5thinc_5extra_6search_Beam *, int __pyx_skip_dispatch))__pyx_f_5thinc_5extra_6search_12MaxViolation_check; + __pyx_vtable_5thinc_5extra_6search_MaxViolation.check_crf = (int (*)(struct __pyx_obj_5thinc_5extra_6search_MaxViolation *, struct __pyx_obj_5thinc_5extra_6search_Beam *, struct __pyx_obj_5thinc_5extra_6search_Beam *, int __pyx_skip_dispatch))__pyx_f_5thinc_5extra_6search_12MaxViolation_check_crf; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_5extra_6search_MaxViolation_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation)) __PYX_ERR(0, 218, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_5extra_6search_MaxViolation_spec, __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation = &__pyx_type_5thinc_5extra_6search_MaxViolation; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation, __pyx_vtabptr_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_MaxViolation, (PyObject *) __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_ptype_5thinc_5extra_6search_MaxViolation) < (0)) __PYX_ERR(0, 218, __pyx_L1_error) + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr)) __PYX_ERR(0, 302, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr_spec, __pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr) < (0)) __PYX_ERR(0, 302, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr = &__pyx_type_5thinc_5extra_6search___pyx_scope_struct__genexpr; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr) < (0)) __PYX_ERR(0, 302, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_5extra_6search___pyx_scope_struct__genexpr->tp_getattro = PyObject_GenericGetAttr; + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule("cymem.cymem"); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyMalloc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc) __PYX_ERR(3, 4, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyMalloc = (struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyMalloc)) __PYX_ERR(3, 4, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyFree", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree) __PYX_ERR(3, 10, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyFree = (struct __pyx_vtabstruct_5cymem_5cymem_PyFree*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyFree)) __PYX_ERR(3, 10, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Pool", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool) __PYX_ERR(3, 16, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_Pool = (struct __pyx_vtabstruct_5cymem_5cymem_Pool*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_Pool)) __PYX_ERR(3, 16, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Address = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Address", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Address) __PYX_ERR(3, 28, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("preshed.maps"); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMap", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap) __PYX_ERR(4, 45, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMap = (struct __pyx_vtabstruct_7preshed_4maps_PreshMap*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMap)) __PYX_ERR(4, 45, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMapArray", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray) __PYX_ERR(4, 53, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMapArray = (struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMapArray)) __PYX_ERR(4, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_search(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_search}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "search", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_search(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_search(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_search(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'search' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "search" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_search", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__extra__search) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.extra.search")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.extra.search", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "thinc/extra/search.pyx":6 + * from libc.string cimport memcpy, memset + * + * import math # <<<<<<<<<<<<<< + * + * from cymem.cymem cimport Pool +*/ + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_math, 0, 0, __pyx_mstate_global->__pyx_kp_u_thinc_extra_math, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) + __pyx_t_2 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_math, __pyx_t_2) < (0)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_4Beam_7__reduce_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_Beam___reduce_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_reduce_cython, __pyx_t_2) < (0)) __PYX_ERR(2, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "self._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for pickling" +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_4Beam_9__setstate_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_Beam___setstate_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 3, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_setstate_cython, __pyx_t_2) < (0)) __PYX_ERR(2, 3, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":231 + * self.g_probs = [] + * + * cpdef int check(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * cdef _State* p = &pred._states[0] + * cdef _State* g = &gold._states[0] +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_12MaxViolation_3check, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_MaxViolation_check, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[3])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation, __pyx_mstate_global->__pyx_n_u_check, __pyx_t_2) < (0)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":253 + * self.gZ += prob + * + * cpdef int check_crf(self, Beam pred, Beam gold) except -1: # <<<<<<<<<<<<<< + * d = pred.score - gold.score + * seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_12MaxViolation_5check_crf, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_MaxViolation_check_crf, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[4])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation, __pyx_mstate_global->__pyx_n_u_check_crf, __pyx_t_2) < (0)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_12MaxViolation_7__reduce_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_MaxViolation___reduce_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[5])); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation, __pyx_mstate_global->__pyx_n_u_reduce_cython, __pyx_t_2) < (0)) __PYX_ERR(2, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_MaxViolation__set_state(self, __pyx_state) +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_12MaxViolation_9__setstate_cython__, __Pyx_CYFUNCTION_CCLASS, __pyx_mstate_global->__pyx_n_u_MaxViolation___setstate_cython, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[6])); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_ptype_5thinc_5extra_6search_MaxViolation, __pyx_mstate_global->__pyx_n_u_setstate_cython, __pyx_t_2) < (0)) __PYX_ERR(2, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":297 + * + * + * def _softmax(nums): # <<<<<<<<<<<<<< + * if not nums: + * return [] +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_1_softmax, 0, __pyx_mstate_global->__pyx_n_u_softmax, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[7])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 297, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_softmax, __pyx_t_2) < (0)) __PYX_ERR(0, 297, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_MaxViolation(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0xe21910b, 0x7f36b37, 0xf4b5e82, b'Z, cost, delta, gZ, g_hist, g_probs, g_score, mem, n, p_hist, p_probs, p_score') +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_5extra_6search_3__pyx_unpickle_MaxViolation, 0, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_MaxViolation, NULL, __pyx_mstate_global->__pyx_n_u_thinc_extra_search, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[8])); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_2); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_MaxViolation, __pyx_t_2) < (0)) __PYX_ERR(2, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/extra/search.pyx":1 + * # cython: experimental_cpp_class_def=True, cdivision=True, infer_types=True # <<<<<<<<<<<<<< + * cimport cython + * from libc.math cimport exp, log +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.extra.search", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.extra.search"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_zip = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_zip); if (!__pyx_builtin_zip) __PYX_ERR(0, 294, __pyx_L1_error) + __pyx_builtin_max = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_max); if (!__pyx_builtin_max) __PYX_ERR(0, 300, __pyx_L1_error) + __pyx_builtin_sum = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_sum); if (!__pyx_builtin_sum) __PYX_ERR(0, 302, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_MaxViolation, (type(self), 0xe21910b, None), state +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_self_g_hist_is_not_None_or_self); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(2, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<1; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 9; } index[] = {{1},{179},{8},{7},{6},{2},{9},{129},{130},{14},{16},{22},{4},{22},{24},{12},{30},{32},{18},{22},{20},{1},{6},{18},{5},{9},{18},{5},{8},{5},{8},{7},{12},{4},{13},{5},{4},{8},{4},{3},{4},{11},{10},{1},{8},{7},{4},{8},{4},{3},{4},{14},{12},{11},{10},{27},{14},{12},{10},{17},{13},{5},{4},{4},{12},{10},{12},{19},{8},{25},{5},{3},{8},{18},{5},{6},{12},{5},{6},{5},{3},{11},{267},{2},{420},{9},{163},{55},{86}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (1411 bytes) */ +const char* const cstring = "BZh91AY&SYH\273%\324\000\000\355\377\377\377\377\377\374\377\277\375\375\277\367\177\221\277\377\377\367\300@@@@@@@@@@@@\000@\000P\004\333\311\025;\032\024\r\315\016\r\"\211\240\320\000a2yL\311\244\006\207\2246Q\2404\320\304\320\304\310\032\003L\232\r\032=M\006\201\005\022~\220\217M\023\321\244\300\020d4h\321\246\2150\021\221\220\320dd\000\310\03144\006\2321\003@\232\0214$\036\241\241\352z\215\r\017S@\r444\000\000\320\000\003@\007\251\247\224\364\324\032 \300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\tA\020\023#I\232h#\023\321Q\352hi\240\323@\006\200\000\000\000\000\365\r4h\036\243G\251\246\004\361\246\305\236\010\357_\300?Xt\211p\251\214A\020t;=\311\001\001!\236\356\370\234\277\300':\363\225R\230\260\241VV\211\034\340\357\t\202u\005P\214H\320\214:\202\332\250\254\023\310\211*\200\252\"\254#Dt`\"\224\312\206\264\003e5\005hP\177\312\024\247aK\034\013\021\244\020Y\310F\2215\236\262\314\000\204\017$\225TR\373\225rs`eV\206\242\375\354\344!{A\272m\361\2773\316i\032\376\0210\031\0067\002x>\246\00418W\223\264\204\224\351\271\r,\303\013\234m0h!\000\207\020\356\207_\311t\210\267o\255:\365\307\265\352\021\376\356\306.3\201\314\316R)\361.\245O\325Z\370\267\005\211u\211h]\013Q\216x\300uf\370=\230L\021\tdw\333>\2101\024\337}\261\223;\342\342\246\202\254KL\023\005|\235\376\236\236\307\032R\265PM\255Pbj\327$+\316K\024\221YX\035\214\273\226\006\003'!\350w\3665\327C\203\327]u\330X\230\\\21249\034\352J\320\235\335\322c\201\303(\221\215\364\323?[\010\206\252\364)\025\240&\200\240\002\234\021\025r.\266\311\365\352\237\273\266\347\001\014\201S\264\030V\212\2754\333\022\005D\202a@2\264\232\246\024\031RkTb\300\0070QD8 \232\231\322nQ\026\330F1*\266\261\177\354\026^'\202B\260s\227\033F\331\2010PTW\206\002\300\200\362H0\000Nj\377\240\212\233M\201\261\r\250\273\372\033\265\021I\030\2639o6\376\237t\nBi\371\265!\030\346\241\t\233Y\277\217t'\337\210&\3046\342]\024UI\014\002\222\314\n0\226f\252\004U\265p\266\274\004-\261v?bw6\323\230\021_\306\375\224\341t\362>Ln\314\316\241Xd\367[h\357\231\026\034gP\220u\324\243\233(\201\252\234\034+\222\200l\347\020 j\350\341a2\323G\177\370\273\222)\302\204\202E\331.""\240"; + PyObject *data = __Pyx_DecompressString(cstring, 1411, 2); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #elif (CYTHON_COMPRESS_STRINGS) != 0 /* compression: zlib (1271 bytes) */ +const char* const cstring = "x\332\215S\315s\323F\024\217\215\t.\204\022C\002!t\246\233\2206\323\026L\r\231Bi\206\216\223\020\332\016elH\313\014\007v\326\322:\022\310\222\254]\031\273\323\217\034u\334\343\036u\334\243\216>\346\310\321G\035\363'\344O\350[\311\311\004\222Bgl\275\217\335\367\366\275\337\357\275\037\237x\234\"n\021\216\326\007\334\362\\d3dR\307n\321\200p\352\014\020\343\201mp\032\350K.j\374p\007l\223\272\314\346\003\360{f\350@\006\270\342\222\016( \351\033\375\351s7\300\331&\270a\207\371\236\357\003l\030\003\0278+\233\205\235\334\2029\014\035\236\353\343z\264\252W,\327B7\233=\372\016\220\371Q\217\353\245\320\257vC\342\034T0\346\007\037c\352\320A\373\300S\266\0020c\260ZfF\333\270\005\320L\332&YQ\207d\342\023\210\305\314ks\r\311XVW\035\317\200-~P\035c\234/\246n\024\004d;:\253\371lr+\360\336\204\276\t\367B\340\353\340\t\330\3240\377\2607\266\311\255?l\177\247\220\226\276V\013""\352\273\2442*\335\033\026\322\362UY\221K\322\212I\334}\317\230\227\013\362n\274\030\327\343fZ>\027\325\242Mq]<\025\241\334\210'cC]TwT+)&K\tI\247\246EY\026dE+\237\344\342,$\"\222\307\367US\221\023\035\347dM\326\337QN\213\246\026\223\202\344\326oP\301\212\354\246\323\227\304\212\350\313n\\\214\277TE\265\220\316\314\311Ii\304\225x)\266\024QAR\371@\300)UKg\256\311;\331\353\367TM\255\301\373Y\206L\234\221\335\250>*\255\016\353iy*\332\020gD g$\264<+\236\351\236\344\027\220\247\024\377\002\357\360d%\t\207\017w+\273K\273Dc\262\022\365\000\221@\316J3{\350\266z\006\210,&\033\200\354\324\254\356\346\212\350\036\325>\334\325g\362Q\\\2137T9)$\027\023\250z^.gM\222\230\253\373I3!\373\223\023\227\347\345\0348\027\343Mu]5\323\351\031\3610c\355\225*\250\031\365<\251\377\237<\037\203\353\343\031\366\246\346d9.\304W\325\242\002\362.\013[\006\361\245lX\246\346\345W\020\275\036wUA\237X\200|w/\233\221RN\364\341\214,\214\305^>\006\373\323\023\347+\371<\310\246l\001\354\353q\250\326U79?l\352\331=\033}#@\331/O\234\3764\332\0223bK^\201[?\251\215\344t\2625\274\n\304\274\0325\266F[/G/[\243\2265\262\234\221\343\217|>\342\177\355OL\374]X+\202X+>\326\342q\261\241E\243\330,\246\245\251\350\221\250\211My#&i\351\314N/z\016\270fspQ,\244e`o\257<'\013\372\014v\343\302\350\302M\265\225T\222\345\341\265\335Gok{\357z\352;\205\275\322\347\262;B\337&\347\206?\354\276\0365\237\246\245KbU\3663P \311\237\342.t_\256\302X\3753t\337\326\323\022\254\330Nv\304\243\032\274\020u\265\353NDr\345v\364L\024DE,\210\357\345\232\334\216_\250\0364<;\204j\313\321)\330\316z~\253\t\001\241X\023\333\362E\334\203\202f\023\362/\020oo^"; + PyObject *data = __Pyx_DecompressString(cstring, 1271, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (2242 bytes) */ +const char* const bytes = "?Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False.add_notedisableenablegcisenabledself._parents,self._states,self.costs,self.del_func,self.is_valid,self.scores cannot be converted to a Python object for picklingself.g_hist is not None or self.g_probs is not None or self.mem is not None or self.p_hist is not None or self.p_probs is not Nonethinc.extra.maththinc/extra/search.pyxBeamBeam.__reduce_cython__Beam.__setstate_cython__MaxViolationMaxViolation.__reduce_cython__MaxViolation.__setstate_cython__MaxViolation.checkMaxViolation.check_crf__Pyx_PyDict_NextRefZappendasyncio.coroutinescheckcheck_crfcline_in_tracebackclose__dict___dict__func__genexpr__getstate__gold_is_coroutineitemsloss__main__mathmaxmax_min_density__module__n__name____new__nextnr_classnumspoppred__pyx_checksum__pyx_result__pyx_state__pyx_type__pyx_unpickle_MaxViolation__pyx_vtable____qualname____reduce____reduce_cython____reduce_ex__scoreselfsend__set_name__setdefault__setstate____setstate_cython___softmax_softmax..genexprstatesum__test__thinc.extra.searchthrowupdateuse_setstatevaluevalueswidthzip\200\001\330\004*\250!\2506\260\021\320\0048\270\001\330\010\031\230\021\230$\230h\240a\240q\330\010\031\230\021\230$\230h\240a\240q\330\010\032\230!\2307\240\"\240A\240Q\330\010\013\2101\210F\220#\220R\220u\230D\240\006\240c\250\022\2503\250b\260\002\260$\260a\330\014\020\220\010\230\001\230\021\330\014\020\220\t\230\021\330\014\020\220\n\230$\230a\230t\240:\250Q\250a\330\014\020\220\n\230$\230a\230t\240:\250Q\250a\330\014\020\220\013\2301\230A\330\014\020\220\013\2301\230A\330\014\020\220\005\220Q\330\014\020\220\006\220a\330\014\020\220\005\220U\230!\2304\230q\330\020\023\2204\220x\230q\240\002\240&\250\002\250!\330\024\030\230\006\230c\240\021\240$\240h\250a\250r\260\021\330\014\020\220\005\220U\230!\2304\230q\330\020\023\2204\220x\230q\240\002\240&\250""\003\2501\330\024\033\2303\230a\230t\2408\2501\250B\250a\330\024\030\230\006\230a\330\024\030\230\007\230q\210A\320\004<\270A\330\010\014\210D\220\007\220r\230\024\230Q\330\010\025\220S\230\001\230\021\230%\230q\240\004\240J\250a\250t\2604\260u\270E\300\021\300$\300a\330\010\013\2104\210v\220R\220r\230\025\230d\240&\250\003\2502\250S\260\002\260\"\260D\270\001\330\014\025\220Q\330\014\027\220q\330\014\025\220Q\330\014\027\220q\330\014\020\220\005\220U\230!\2304\230q\330\020\023\2204\220x\230q\240\002\240&\250\002\250!\330\024\034\230G\2401\240D\250\010\260\001\260\022\2601\330\024\032\230'\240\021\240$\240a\240t\250:\260Q\260a\360\006\000\026\032\230\030\240\021\240\"\240F\250#\250Q\330\020\024\220E\230\021\230$\230j\250\001\250\024\250W\260A\330\024\034\230G\2401\240D\250\010\260\001\260\022\2601\330\024\032\230'\240\021\240$\240a\240t\250:\260Q\260a\330\014\020\220\005\220U\230!\2304\230q\330\020\023\2204\220x\230q\240\002\240&\250\003\2501\330\024\034\230G\2401\240D\250\010\260\001\260\022\2601\330\024\032\230'\240\021\240$\240a\240t\250:\260Q\260a\340\014\030\230\010\240\001\240\031\250\"\250A\330\014\026\220i\230r\240\023\240A\240Q\330\014\032\230)\2401\240C\240q\250\001\330\014\026\220h\230a\230q\340\014\020\220\010\230\004\230A\330\014\020\220\t\230\021\330\014\020\220\n\230!\330\014\020\220\n\230!\340\014\020\220\013\2301\360\020\000\r\021\220\013\2301\230A\230Q\230b\240\004\240C\240u\250C\250q\260\r\270Q\200\001\330\004\n\210+\220Q\200\001\360\010\000\005\016\210T\220\024\220T\230\027\240\004\240H\250D\260\005\260T\270\031\300$\300j\320PT\320T^\320^b\320bh\320hl\320lp\320pt\320t}\360\000\000~\001B\002\360\000\000B\002L\002\360\000\000L\002P\002\360\000\000P\002Q\002\330\004\014\210G\2201\220F\230,\240a\330\004\007\200v\210W\220E\230\024\230Q\330\010\022\220!\330\010\027\220q\340\010\030\230\001\330\004\007\200q\330\010\017\320\017-\250T\260\021\260'\270\033\300G\3101\340\010\017\320\017-\250T\260\021\260'\270\033\300A\200\001\340\004\037\230q\320 0\260\013""\270;\300k\320QR\330\004\023\220<\230x\240q\250\001\330\004\007\200|\2207\230!\330\010.\250a\250\177\270n\310A\330\004\013\2101\200\001\330\004\007\200t\2101\330\010\017\210q\330\004\013\2103\210a\210q\330\004\013\2102\210S\220\001\220\021\220!\2209\230B\230g\240Z\250v\260T\270\025\270a\330\004\010\210\003\2101\210A\330\004\013\2102\210Q\210a\210u\220B\220g\230Z\240v\250T\260\025\260a"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 81; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 12) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 81; i < 89; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 89; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 81; + for (Py_ssize_t i=0; i<8; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 0; + int8_t const cint_constants_1[] = {0}; + int32_t const cint_constants_4[] = {237080843L}; + for (int i = 0; i < 2; i++) { + numbertab[i] = PyLong_FromLong((i < 1 ? cint_constants_1[i - 0] : cint_constants_4[i - 1])); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<2; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 2; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 3; + unsigned int flags : 10; + unsigned int first_line : 9; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS|CO_GENERATOR), 302}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_n}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_extra_search_pyx, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_kp_b_iso88591_A, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 1}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_reduce_cython, __pyx_mstate->__pyx_kp_b_iso88591_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 3}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_pyx_state}; + __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_setstate_cython, __pyx_mstate->__pyx_kp_b_iso88591_Q, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 231}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_pred, __pyx_mstate->__pyx_n_u_gold}; + __pyx_mstate_global->__pyx_codeobj_tab[3] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_extra_search_pyx, __pyx_mstate->__pyx_n_u_check, __pyx_mstate->__pyx_kp_b_iso88591_8_haq_haq_7_AQ_1F_RuD_c_3b_a_at, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[3])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 3, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 253}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_pred, __pyx_mstate->__pyx_n_u_gold}; + __pyx_mstate_global->__pyx_codeobj_tab[4] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_extra_search_pyx, __pyx_mstate->__pyx_n_u_check_crf, __pyx_mstate->__pyx_kp_b_iso88591_A_D_r_Q_S_q_Jat4uE_a_4vRr_d_2S, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[4])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 1}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_state, __pyx_mstate->__pyx_n_u_dict_2, __pyx_mstate->__pyx_n_u_use_setstate}; + __pyx_mstate_global->__pyx_codeobj_tab[5] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_reduce_cython, __pyx_mstate->__pyx_kp_b_iso88591_T_T_HD_T_jPTT_bbhhllpptt_B_B_L, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[5])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 16}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_pyx_state}; + __pyx_mstate_global->__pyx_codeobj_tab[6] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_setstate_cython, __pyx_mstate->__pyx_kp_b_iso88591_6, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[6])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 297}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_nums, __pyx_mstate->__pyx_n_u_max_2, __pyx_mstate->__pyx_n_u_Z, __pyx_mstate->__pyx_n_u_n, __pyx_mstate->__pyx_n_u_genexpr, __pyx_mstate->__pyx_n_u_genexpr}; + __pyx_mstate_global->__pyx_codeobj_tab[7] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_extra_search_pyx, __pyx_mstate->__pyx_n_u_softmax, __pyx_mstate->__pyx_kp_b_iso88591_t1_q_3aq_2S_9BgZvT_a_1A_2QauBgZ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[7])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 4, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 4}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_pyx_type, __pyx_mstate->__pyx_n_u_pyx_checksum, __pyx_mstate->__pyx_n_u_pyx_state, __pyx_mstate->__pyx_n_u_pyx_result}; + __pyx_mstate_global->__pyx_codeobj_tab[8] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_stringsource, __pyx_mstate->__pyx_n_u_pyx_unpickle_MaxViolation, __pyx_mstate->__pyx_kp_b_iso88591_q_0_kQR_xq_7_a_nA_1, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[8])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* Generator.init */ + if (likely(__pyx_Generator_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallOneArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallOneArg (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod0 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* WriteUnraisableException */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); + else state = (PyGILState_STATE)0; + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(0); + } + ctx = PyUnicode_FromString(name); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } + if (nogil) + PyGILState_Release(state); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* PyObjectCall2Args (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetMethod (used by PyObjectCallMethod1) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod1 (used by append) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; +} +#endif +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); +#endif +} + +/* append */ +static CYTHON_INLINE int __Pyx_PyObject_Append(PyObject* L, PyObject* x) { + if (likely(PyList_CheckExact(L))) { + if (unlikely(__Pyx_PyList_Append(L, x) < 0)) return -1; + } else { + PyObject* retval = __Pyx_PyObject_CallMethod1(L, __pyx_mstate_global->__pyx_n_u_append, x); + if (unlikely(!retval)) + return -1; + Py_DECREF(retval); + } + return 0; +} + +/* RejectKeywords */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) { + PyObject *key = NULL; + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) { + key = __Pyx_PySequence_ITEM(kwds, 0); + } else { +#if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; +#else + Py_ssize_t pos = 0; +#endif +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return; +#endif + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); +#endif + } + if (likely(key)) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + Py_DECREF(key); + } +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* PyObjectCallNoArg (used by pyfrozenset_new) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* pyfrozenset_new (used by PySetContains) */ +static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it) { + if (it) { + PyObject* result; +#if CYTHON_COMPILING_IN_PYPY + PyObject* args; + args = PyTuple_Pack(1, it); + if (unlikely(!args)) + return NULL; + result = PyObject_Call((PyObject*)&PyFrozenSet_Type, args, NULL); + Py_DECREF(args); + return result; +#else + if (PyFrozenSet_CheckExact(it)) { + Py_INCREF(it); + return it; + } + result = PyFrozenSet_New(it); + if (unlikely(!result)) + return NULL; + if ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000) +#if CYTHON_COMPILING_IN_LIMITED_API + || __Pyx_get_runtime_version() >= 0x030A0000 +#endif + ) + return result; + { + Py_ssize_t size = __Pyx_PySet_GET_SIZE(result); + if (likely(size > 0)) + return result; +#if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(size < 0)) { + Py_DECREF(result); + return NULL; + } +#endif + } + Py_DECREF(result); +#endif + } + return __Pyx_PyObject_CallNoArg((PyObject*) &PyFrozenSet_Type); +} + +/* PySetContains */ +static int __Pyx_PySet_ContainsUnhashable(PyObject *set, PyObject *key) { + int result = -1; + if (PySet_Check(key) && PyErr_ExceptionMatches(PyExc_TypeError)) { + PyObject *tmpkey; + PyErr_Clear(); + tmpkey = __Pyx_PyFrozenSet_New(key); + if (tmpkey != NULL) { + result = PySet_Contains(set, tmpkey); + Py_DECREF(tmpkey); + } + } + return result; +} +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq) { + int result = PySet_Contains(set, key); + if (unlikely(result < 0)) { + result = __Pyx_PySet_ContainsUnhashable(set, key); + } + return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); +} + +/* SliceObject */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetSlice(PyObject* obj, + Py_ssize_t cstart, Py_ssize_t cstop, + PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, + int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + __Pyx_TypeName obj_type_name; +#if CYTHON_USE_TYPE_SLOTS + PyMappingMethods* mp = Py_TYPE(obj)->tp_as_mapping; + if (likely(mp && mp->mp_subscript)) +#endif + { + PyObject* result; + PyObject *py_slice, *py_start, *py_stop; + if (_py_slice) { + py_slice = *_py_slice; + } else { + PyObject* owned_start = NULL; + PyObject* owned_stop = NULL; + if (_py_start) { + py_start = *_py_start; + } else { + if (has_cstart) { + owned_start = py_start = PyLong_FromSsize_t(cstart); + if (unlikely(!py_start)) goto bad; + } else + py_start = Py_None; + } + if (_py_stop) { + py_stop = *_py_stop; + } else { + if (has_cstop) { + owned_stop = py_stop = PyLong_FromSsize_t(cstop); + if (unlikely(!py_stop)) { + Py_XDECREF(owned_start); + goto bad; + } + } else + py_stop = Py_None; + } + py_slice = PySlice_New(py_start, py_stop, Py_None); + Py_XDECREF(owned_start); + Py_XDECREF(owned_stop); + if (unlikely(!py_slice)) goto bad; + } +#if CYTHON_USE_TYPE_SLOTS + result = mp->mp_subscript(obj, py_slice); +#else + result = PyObject_GetItem(obj, py_slice); +#endif + if (!_py_slice) { + Py_DECREF(py_slice); + } + return result; + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); +bad: + return NULL; +} + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* RaiseTooManyValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* IterFinish */ +static CYTHON_INLINE int __Pyx_IterFinish(void) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return -1; + __Pyx_PyErr_Clear(); + return 0; + } + return 0; +} + +/* UnpackItemEndCheck */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } + return __Pyx_IterFinish(); +} + +/* GetAttr3 */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int res = PyObject_GetOptionalAttr(o, n, &r); + return (res != 0) ? r : __Pyx_NewRef(d); +#else + #if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } + #endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +#endif +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* PyObjectFastCallMethod */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) { + PyObject *result; + PyObject *attr = PyObject_GetAttr(args[0], name); + if (unlikely(!attr)) + return NULL; + result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1); + Py_DECREF(attr); + return result; +} +#endif + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); + return 0; +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#endif + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* GetVTable (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* MergeVTables */ +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; +#if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; +#else + size = PyTuple_GET_SIZE(bases); +#endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; +#if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#else + basei = PyTuple_GET_ITEM(bases, i); +#endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; +bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); +#if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); +#endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +really_bad: // bad has failed! +#endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +other_failure: +#endif + PyMem_Free(base_vtables); + return -1; +} + +/* DelItemOnTypeDict (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_DelItem(tp_dict, k); + if (likely(!result)) PyType_Modified(tp); + return result; +} + +/* SetupReduce */ +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_getstate = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + PyObject *getstate = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate); +#else + getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + if (!getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (getstate) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); +#else + object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + if (!object_getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (object_getstate != getstate) { + goto __PYX_GOOD; + } + } +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); + if (likely(reduce_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + goto __PYX_BAD; + } + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); + if (likely(setstate_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; +__PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; +__PYX_GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); + Py_XDECREF(object_getstate); + Py_XDECREF(getstate); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + +/* TypeImport */ +#ifndef __PYX_HAVE_RT_ImportType_3_2_1 +#define __PYX_HAVE_RT_ImportType_3_2_1 +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size) +{ + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; +#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; +#endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; +#else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_1 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_1 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return NULL; +} +#endif + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* dict_setdefault (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* AddModuleRef (used by FetchSharedCythonModule) */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* FetchCommonType (used by CommonTypesMetaclass) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +#if __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; +} +static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; +} +#endif +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* PyMethodNew (used by CythonFunctionShared) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared (used by CythonFunction) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +#endif +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, +#else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, +#endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#endif +#if CYTHON_METH_FASTCALL +#if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; +#endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); +#elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); +#else + PyObject_ClearManagedDict((PyObject*)m); +#endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); +#else + { + int e = +#if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict +#else + PyObject_VisitManagedDict +#endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } +#endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CheckUnpickleChecksum */ +static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + PyObject *pickle_module = PyImport_ImportModule("pickle"); + if (unlikely(!pickle_module)) return; + PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (unlikely(!pickle_error)) return; + if (checksum2 == checksum1) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))", + checksum, checksum1, members); + } else if (checksum3 == checksum2) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, members); + } else { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, checksum3, members); + } + Py_DECREF(pickle_error); +} +static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + int found = 0; + found |= checksum1 == checksum; + found |= checksum2 == checksum; + found |= checksum3 == checksum; + if (likely(found)) + return 0; + __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members); + return -1; +} + +/* PyObjectVectorCallKwBuilder (used by CIntToPy) */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_uint64_t(uint64_t value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(uint64_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(uint64_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(uint64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(uint64_t), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(uint64_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE uint64_t __Pyx_PyLong_As_uint64_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const uint64_t neg_one = (uint64_t) -1, const_zero = (uint64_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + uint64_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (uint64_t) -1; + val = __Pyx_PyLong_As_uint64_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(uint64_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(uint64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) >= 2 * PyLong_SHIFT)) { + return (uint64_t) (((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(uint64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) >= 3 * PyLong_SHIFT)) { + return (uint64_t) (((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(uint64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) >= 4 * PyLong_SHIFT)) { + return (uint64_t) (((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (uint64_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(uint64_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(uint64_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(uint64_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(uint64_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT)) { + return (uint64_t) (((uint64_t)-1)*(((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(uint64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT)) { + return (uint64_t) ((((((uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(uint64_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT)) { + return (uint64_t) (((uint64_t)-1)*(((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(uint64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT)) { + return (uint64_t) ((((((((uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(uint64_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT)) { + return (uint64_t) (((uint64_t)-1)*(((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(uint64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(uint64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(uint64_t) - 1 > 4 * PyLong_SHIFT)) { + return (uint64_t) ((((((((((uint64_t)digits[3]) << PyLong_SHIFT) | (uint64_t)digits[2]) << PyLong_SHIFT) | (uint64_t)digits[1]) << PyLong_SHIFT) | (uint64_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(uint64_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, long, PyLong_AsLong(x)) + } else if ((sizeof(uint64_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(uint64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + uint64_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (uint64_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (uint64_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (uint64_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (uint64_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(uint64_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((uint64_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(uint64_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((uint64_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((uint64_t) 1) << (sizeof(uint64_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (uint64_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to uint64_t"); + return (uint64_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to uint64_t"); + return (uint64_t) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* UpdateUnpickledDict */ +static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + PyObject *state_dict = __Pyx_PySequence_ITEM(state, index); + if (unlikely(!state_dict)) { + return -1; + } + int non_empty = PyObject_IsTrue(state_dict); + if (non_empty == 0) { + Py_DECREF(state_dict); + return 0; + } else if (unlikely(non_empty == -1)) { + return -1; + } + PyObject *dict; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = PyObject_GetAttrString(obj, "__dict__"); + #else + dict = PyObject_GenericGetDict(obj, NULL); + #endif + if (unlikely(!dict)) { + Py_DECREF(state_dict); + return -1; + } + int result; + if (likely(PyDict_CheckExact(dict))) { + result = PyDict_Update(dict, state_dict); + } else { + PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict); + if (likely(obj_result)) { + Py_DECREF(obj_result); + result = 0; + } else { + result = -1; + } + } + Py_DECREF(state_dict); + Py_DECREF(dict); + return result; +} +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(state_size == -1)) return -1; + #endif + if (state_size <= index) { + return 0; + } + return __Pyx__UpdateUnpickledDict(obj, state, index); +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u_); + } + goto done; +} +#endif + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } +} +#endif +static unsigned long __Pyx_get_runtime_version(void) { +#if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + return __Pyx_cached_runtime_version; +#endif +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* SwapException (used by CoroutineBase) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* IterNextPlain (used by CoroutineBase) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject *__Pyx_GetBuiltinNext_LimitedAPI(void) { + if (unlikely(!__pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache)) + __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache = __Pyx_GetBuiltinName(__pyx_mstate_global->__pyx_n_u_next); + return __pyx_mstate_global->__Pyx_GetBuiltinNext_LimitedAPI_cache; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_PyIter_Next_Plain(PyObject *iterator) { +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + PyObject *result; + PyObject *next = __Pyx_GetBuiltinNext_LimitedAPI(); + if (unlikely(!next)) return NULL; + result = PyObject_CallFunctionObjArgs(next, iterator, NULL); + return result; +#else + (void)__Pyx_GetBuiltinName; // only for early limited API + iternextfunc iternext = __Pyx_PyObject_GetIterNextFunc(iterator); + assert(iternext); + return iternext(iterator); +#endif +} + +/* ReturnWithStopIteration (used by CoroutineBase) */ +static void __Pyx__ReturnWithStopIteration(PyObject* value, int async); +static CYTHON_INLINE void __Pyx_ReturnWithStopIteration(PyObject* value, int async, int iternext) { + if (value == Py_None) { + if (async || !iternext) + PyErr_SetNone(async ? PyExc_StopAsyncIteration : PyExc_StopIteration); + return; + } + __Pyx__ReturnWithStopIteration(value, async); +} +static void __Pyx__ReturnWithStopIteration(PyObject* value, int async) { +#if CYTHON_COMPILING_IN_CPYTHON + __Pyx_PyThreadState_declare +#endif + PyObject *exc; + PyObject *exc_type = async ? PyExc_StopAsyncIteration : PyExc_StopIteration; +#if CYTHON_COMPILING_IN_CPYTHON + if ((PY_VERSION_HEX >= (0x030C00A6)) || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) { + if (PY_VERSION_HEX >= (0x030e00A1)) { + exc = __Pyx_PyObject_CallOneArg(exc_type, value); + } else { + PyObject *args_tuple = PyTuple_New(1); + if (unlikely(!args_tuple)) return; + Py_INCREF(value); + PyTuple_SET_ITEM(args_tuple, 0, value); + exc = PyObject_Call(exc_type, args_tuple, NULL); + Py_DECREF(args_tuple); + } + if (unlikely(!exc)) return; + } else { + Py_INCREF(value); + exc = value; + } + #if CYTHON_FAST_THREAD_STATE + __Pyx_PyThreadState_assign + #if CYTHON_USE_EXC_INFO_STACK + if (!__pyx_tstate->exc_info->exc_value) + #else + if (!__pyx_tstate->exc_type) + #endif + { + Py_INCREF(exc_type); + __Pyx_ErrRestore(exc_type, exc, NULL); + return; + } + #endif +#else + exc = __Pyx_PyObject_CallOneArg(exc_type, value); + if (unlikely(!exc)) return; +#endif + PyErr_SetObject(exc_type, exc); + Py_DECREF(exc); +} + +/* CoroutineBase (used by Generator) */ +#if !CYTHON_COMPILING_IN_LIMITED_API +#include +#if PY_VERSION_HEX >= 0x030b00a6 && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#endif // CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE void +__Pyx_Coroutine_Undelegate(__pyx_CoroutineObject *gen) { +#if CYTHON_USE_AM_SEND + gen->yieldfrom_am_send = NULL; +#endif + Py_CLEAR(gen->yieldfrom); +} +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *__pyx_tstate, PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + CYTHON_UNUSED_VAR(__pyx_tstate); + __Pyx_ErrFetch(&et, &ev, &tb); + if (!et) { + Py_XDECREF(tb); + Py_XDECREF(ev); + Py_INCREF(Py_None); + *pvalue = Py_None; + return 0; + } + if (likely(et == PyExc_StopIteration)) { + if (!ev) { + Py_INCREF(Py_None); + value = Py_None; + } + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { + #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value); + if (unlikely(!value)) goto limited_api_failure; + #else + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + #endif + Py_DECREF(ev); + } + else if (unlikely(PyTuple_Check(ev))) { + Py_ssize_t tuple_size = __Pyx_PyTuple_GET_SIZE(ev); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(tuple_size < 0)) { + Py_XDECREF(tb); + Py_DECREF(ev); + Py_DECREF(et); + return -1; + } + #endif + if (tuple_size >= 1) { +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + value = PyTuple_GET_ITEM(ev, 0); + Py_INCREF(value); +#elif CYTHON_ASSUME_SAFE_MACROS + value = PySequence_ITEM(ev, 0); +#else + value = PySequence_GetItem(ev, 0); + if (!value) goto limited_api_failure; +#endif + } else { + Py_INCREF(Py_None); + value = Py_None; + } + Py_DECREF(ev); + } + else if (!__Pyx_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + value = ev; + } + if (likely(value)) { + Py_XDECREF(tb); + Py_DECREF(et); + *pvalue = value; + return 0; + } + } else if (!__Pyx_PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + PyErr_NormalizeException(&et, &ev, &tb); + if (unlikely(!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration))) { + __Pyx_ErrRestore(et, ev, tb); + return -1; + } + Py_XDECREF(tb); + Py_DECREF(et); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_GetAttr(ev, __pyx_mstate_global->__pyx_n_u_value); +#else + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); +#endif + Py_DECREF(ev); +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!value)) return -1; +#endif + *pvalue = value; + return 0; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL || !CYTHON_ASSUME_SAFE_MACROS + limited_api_failure: + Py_XDECREF(et); + Py_XDECREF(tb); + Py_XDECREF(ev); + return -1; +#endif +} +static CYTHON_INLINE +__Pyx_PySendResult __Pyx_Coroutine_status_from_result(PyObject **retval) { + if (*retval) { + return PYGEN_NEXT; + } else if (likely(__Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, retval) == 0)) { + return PYGEN_RETURN; + } else { + return PYGEN_ERROR; + } +} +static CYTHON_INLINE +void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_CLEAR(exc_state->exc_value); +#else + PyObject *t, *v, *tb; + t = exc_state->exc_type; + v = exc_state->exc_value; + tb = exc_state->exc_traceback; + exc_state->exc_type = NULL; + exc_state->exc_value = NULL; + exc_state->exc_traceback = NULL; + Py_XDECREF(t); + Py_XDECREF(v); + Py_XDECREF(tb); +#endif +} +#define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) { + const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check((PyObject*)gen)) { + msg = "coroutine already executing"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact((PyObject*)gen)) { + msg = "async generator already executing"; + #endif + } else { + msg = "generator already executing"; + } + PyErr_SetString(PyExc_ValueError, msg); +} +static void __Pyx_Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) { + CYTHON_MAYBE_UNUSED_VAR(gen); + CYTHON_MAYBE_UNUSED_VAR(closing); + #ifdef __Pyx_Coroutine_USED + if (!closing && __Pyx_Coroutine_Check(gen)) { + PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine"); + } else + #endif + if (value) { + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + PyErr_SetNone(PyExc_StopAsyncIteration); + else + #endif + PyErr_SetNone(PyExc_StopIteration); + } +} +static +__Pyx_PySendResult __Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, PyObject **result, int closing) { + __Pyx_PyThreadState_declare + PyThreadState *tstate; + __Pyx_ExcInfoStruct *exc_state; + PyObject *retval; + assert(__Pyx_Coroutine_get_is_running(self)); // Callers should ensure is_running + if (unlikely(self->resume_label == -1)) { + __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing); + return PYGEN_ERROR; + } +#if CYTHON_FAST_THREAD_STATE + __Pyx_PyThreadState_assign + tstate = __pyx_tstate; +#else + tstate = __Pyx_PyThreadState_Current; +#endif + exc_state = &self->gi_exc_state; + if (exc_state->exc_value) { + #if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + #else + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #elif PY_VERSION_HEX >= 0x030B00a4 + exc_tb = ((PyBaseExceptionObject*) exc_state->exc_value)->traceback; + #else + exc_tb = exc_state->exc_traceback; + #endif + if (exc_tb) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; + PyFrameObject *f = tb->tb_frame; + assert(f->f_back == NULL); + #if PY_VERSION_HEX >= 0x030B00A1 + f->f_back = PyThreadState_GetFrame(tstate); + #else + Py_XINCREF(tstate->frame); + f->f_back = tstate->frame; + #endif + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_CPYTHON + Py_DECREF(exc_tb); + #endif + } + #endif + } +#if CYTHON_USE_EXC_INFO_STACK + exc_state->previous_item = tstate->exc_info; + tstate->exc_info = exc_state; +#else + if (exc_state->exc_type) { + __Pyx_ExceptionSwap(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); + } else { + __Pyx_Coroutine_ExceptionClear(exc_state); + __Pyx_ExceptionSave(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); + } +#endif + retval = self->body(self, tstate, value); +#if CYTHON_USE_EXC_INFO_STACK + exc_state = &self->gi_exc_state; + tstate->exc_info = exc_state->previous_item; + exc_state->previous_item = NULL; + __Pyx_Coroutine_ResetFrameBackpointer(exc_state); +#endif + *result = retval; + if (self->resume_label == -1) { + return likely(retval) ? PYGEN_RETURN : PYGEN_ERROR; + } + return PYGEN_NEXT; +} +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED_VAR(exc_state); +#else + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 + if (!exc_state->exc_value) return; + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + if (likely(exc_tb)) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif + } +#endif +} +#define __Pyx_Coroutine_MethodReturnFromResult(gen, result, retval, iternext)\ + ((result) == PYGEN_NEXT ? (retval) : __Pyx__Coroutine_MethodReturnFromResult(gen, result, retval, iternext)) +static PyObject * +__Pyx__Coroutine_MethodReturnFromResult(PyObject* gen, __Pyx_PySendResult result, PyObject *retval, int iternext) { + CYTHON_MAYBE_UNUSED_VAR(gen); + if (likely(result == PYGEN_RETURN)) { + int is_async = 0; + #ifdef __Pyx_AsyncGen_USED + is_async = __Pyx_AsyncGen_CheckExact(gen); + #endif + __Pyx_ReturnWithStopIteration(retval, is_async, iternext); + Py_XDECREF(retval); + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE +PyObject *__Pyx_PyGen_Send(PyGenObject *gen, PyObject *arg) { +#if PY_VERSION_HEX <= 0x030A00A1 + return _PyGen_Send(gen, arg); +#else + PyObject *result; + if (PyIter_Send((PyObject*)gen, arg ? arg : Py_None, &result) == PYGEN_RETURN) { + if (PyAsyncGen_CheckExact(gen)) { + assert(result == Py_None); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + else if (result == Py_None) { + PyErr_SetNone(PyExc_StopIteration); + } + else { +#if PY_VERSION_HEX < 0x030d00A1 + _PyGen_SetStopIterationValue(result); +#else + if (!PyTuple_Check(result) && !PyExceptionInstance_Check(result)) { + PyErr_SetObject(PyExc_StopIteration, result); + } else { + PyObject *exc = __Pyx_PyObject_CallOneArg(PyExc_StopIteration, result); + if (likely(exc != NULL)) { + PyErr_SetObject(PyExc_StopIteration, exc); + Py_DECREF(exc); + } + } +#endif + } + Py_DECREF(result); + result = NULL; + } + return result; +#endif +} +#endif +static CYTHON_INLINE __Pyx_PySendResult +__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen, PyObject** retval) { + __Pyx_PySendResult result; + PyObject *val = NULL; + assert(__Pyx_Coroutine_get_is_running(gen)); + __Pyx_Coroutine_Undelegate(gen); + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val); + result = __Pyx_Coroutine_SendEx(gen, val, retval, 0); + Py_XDECREF(val); + return result; +} +#if CYTHON_USE_AM_SEND +static __Pyx_PySendResult +__Pyx_Coroutine_SendToDelegate(__pyx_CoroutineObject *gen, __Pyx_pyiter_sendfunc gen_am_send, PyObject *value, PyObject **retval) { + PyObject *ret = NULL; + __Pyx_PySendResult delegate_result, result; + assert(__Pyx_Coroutine_get_is_running(gen)); + delegate_result = gen_am_send(gen->yieldfrom, value, &ret); + if (delegate_result == PYGEN_NEXT) { + assert (ret != NULL); + *retval = ret; + return PYGEN_NEXT; + } + assert (delegate_result != PYGEN_ERROR || ret == NULL); + __Pyx_Coroutine_Undelegate(gen); + result = __Pyx_Coroutine_SendEx(gen, ret, retval, 0); + Py_XDECREF(ret); + return result; +} +#endif +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_AmSend(self, value, &retval); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0); +} +static __Pyx_PySendResult +__Pyx_Coroutine_AmSend(PyObject *self, PyObject *value, PyObject **retval) { + __Pyx_PySendResult result; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + *retval = __Pyx_Coroutine_AlreadyRunningError(gen); + return PYGEN_ERROR; + } + #if CYTHON_USE_AM_SEND + if (gen->yieldfrom_am_send) { + result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, value, retval); + } else + #endif + if (gen->yieldfrom) { + PyObject *yf = gen->yieldfrom; + PyObject *ret; + #if !CYTHON_USE_AM_SEND + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_Check(yf)) { + ret = __Pyx_Coroutine_Send(yf, value); + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + ret = __Pyx_async_gen_asend_send(yf, value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + if (PyCoro_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif + #endif + { + #if !CYTHON_COMPILING_IN_LIMITED_API || __PYX_LIMITED_VERSION_HEX >= 0x03080000 + if (value == Py_None && PyIter_Check(yf)) + ret = __Pyx_PyIter_Next_Plain(yf); + else + #endif + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_mstate_global->__pyx_n_u_send, value); + } + if (likely(ret)) { + __Pyx_Coroutine_unset_is_running(gen); + *retval = ret; + return PYGEN_NEXT; + } + result = __Pyx_Coroutine_FinishDelegation(gen, retval); + } else { + result = __Pyx_Coroutine_SendEx(gen, value, retval, 0); + } + __Pyx_Coroutine_unset_is_running(gen); + return result; +} +static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { + __Pyx_PySendResult result; + PyObject *retval = NULL; + CYTHON_UNUSED_VAR(gen); + assert(__Pyx_Coroutine_get_is_running(gen)); + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + result = __Pyx_Coroutine_Close(yf, &retval); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_Check(yf)) { + result = __Pyx_Coroutine_Close(yf, &retval); + } else + if (__Pyx_CoroutineAwait_CheckExact(yf)) { + result = __Pyx_CoroutineAwait_Close((__pyx_CoroutineAwaitObject*)yf); + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + retval = __Pyx_async_gen_asend_close(yf, NULL); + result = PYGEN_RETURN; + } else + if (__pyx_PyAsyncGenAThrow_CheckExact(yf)) { + retval = __Pyx_async_gen_athrow_close(yf, NULL); + result = PYGEN_RETURN; + } else + #endif + { + PyObject *meth; + result = PYGEN_RETURN; + meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_close); + if (unlikely(!meth)) { + if (unlikely(PyErr_Occurred())) { + PyErr_WriteUnraisable(yf); + } + } else { + retval = __Pyx_PyObject_CallNoArg(meth); + Py_DECREF(meth); + if (unlikely(!retval)) { + result = PYGEN_ERROR; + } + } + } + Py_XDECREF(retval); + return result == PYGEN_ERROR ? -1 : 0; +} +static PyObject *__Pyx_Generator_Next(PyObject *self) { + __Pyx_PySendResult result; + PyObject *retval = NULL; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + return __Pyx_Coroutine_AlreadyRunningError(gen); + } + #if CYTHON_USE_AM_SEND + if (gen->yieldfrom_am_send) { + result = __Pyx_Coroutine_SendToDelegate(gen, gen->yieldfrom_am_send, Py_None, &retval); + } else + #endif + if (gen->yieldfrom) { + PyObject *yf = gen->yieldfrom; + PyObject *ret; + #ifdef __Pyx_Generator_USED + if (__Pyx_Generator_CheckExact(yf)) { + ret = __Pyx_Generator_Next(yf); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_CheckExact(yf)) { + ret = __Pyx_Coroutine_Send(yf, Py_None); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && (PY_VERSION_HEX < 0x030A00A3 || !CYTHON_USE_AM_SEND) + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, NULL); + } else + #endif + ret = __Pyx_PyIter_Next_Plain(yf); + if (likely(ret)) { + __Pyx_Coroutine_unset_is_running(gen); + return ret; + } + result = __Pyx_Coroutine_FinishDelegation(gen, &retval); + } else { + result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0); + } + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 1); +} +static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) { + PyObject *retval = NULL; + __Pyx_PySendResult result; + CYTHON_UNUSED_VAR(arg); + result = __Pyx_Coroutine_Close(self, &retval); + if (unlikely(result == PYGEN_ERROR)) + return NULL; + Py_XDECREF(retval); + Py_RETURN_NONE; +} +static __Pyx_PySendResult +__Pyx_Coroutine_Close(PyObject *self, PyObject **retval) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PySendResult result; + PyObject *yf; + int err = 0; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + *retval = __Pyx_Coroutine_AlreadyRunningError(gen); + return PYGEN_ERROR; + } + yf = gen->yieldfrom; + if (yf) { + Py_INCREF(yf); + err = __Pyx_Coroutine_CloseIter(gen, yf); + __Pyx_Coroutine_Undelegate(gen); + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); + result = __Pyx_Coroutine_SendEx(gen, NULL, retval, 1); + if (result == PYGEN_ERROR) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_Coroutine_unset_is_running(gen); + if (!__Pyx_PyErr_Occurred()) { + return PYGEN_RETURN; + } else if (likely(__Pyx_PyErr_ExceptionMatches2(PyExc_GeneratorExit, PyExc_StopIteration))) { + __Pyx_PyErr_Clear(); + return PYGEN_RETURN; + } + return PYGEN_ERROR; + } else if (likely(result == PYGEN_RETURN && *retval == Py_None)) { + __Pyx_Coroutine_unset_is_running(gen); + return PYGEN_RETURN; + } else { + const char *msg; + Py_DECREF(*retval); + *retval = NULL; + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check(self)) { + msg = "coroutine ignored GeneratorExit"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(self)) { + msg = "async generator ignored GeneratorExit"; + #endif + } else { + msg = "generator ignored GeneratorExit"; + } + PyErr_SetString(PyExc_RuntimeError, msg); + __Pyx_Coroutine_unset_is_running(gen); + return PYGEN_ERROR; + } +} +static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject *val, PyObject *tb, + PyObject *args, int close_on_genexit) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject *yf; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) + return __Pyx_Coroutine_AlreadyRunningError(gen); + yf = gen->yieldfrom; + if (yf) { + __Pyx_PySendResult result; + PyObject *ret; + Py_INCREF(yf); + if (__Pyx_PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) { + int err = __Pyx_Coroutine_CloseIter(gen, yf); + Py_DECREF(yf); + __Pyx_Coroutine_Undelegate(gen); + if (err < 0) + goto propagate_exception; + goto throw_here; + } + if (0 + #ifdef __Pyx_Generator_USED + || __Pyx_Generator_CheckExact(yf) + #endif + #ifdef __Pyx_Coroutine_USED + || __Pyx_Coroutine_Check(yf) + #endif + ) { + ret = __Pyx__Coroutine_Throw(yf, typ, val, tb, args, close_on_genexit); + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_CoroutineAwait_CheckExact(yf)) { + ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); + #endif + } else { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, __pyx_mstate_global->__pyx_n_u_throw); + if (unlikely(!meth)) { + Py_DECREF(yf); + if (unlikely(PyErr_Occurred())) { + __Pyx_Coroutine_unset_is_running(gen); + return NULL; + } + __Pyx_Coroutine_Undelegate(gen); + goto throw_here; + } + if (likely(args)) { + ret = __Pyx_PyObject_Call(meth, args, NULL); + } else { + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } + Py_DECREF(meth); + } + Py_DECREF(yf); + if (ret) { + __Pyx_Coroutine_unset_is_running(gen); + return ret; + } + result = __Pyx_Coroutine_FinishDelegation(gen, &ret); + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, ret, 0); + } +throw_here: + __Pyx_Raise(typ, val, tb, NULL); +propagate_exception: + { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, NULL, &retval, 0); + __Pyx_Coroutine_unset_is_running(gen); + return __Pyx_Coroutine_MethodReturnFromResult(self, result, retval, 0); + } +} +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + PyObject *typ; + PyObject *val = NULL; + PyObject *tb = NULL; + if (unlikely(!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))) + return NULL; + return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); +} +static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_VISIT(exc_state->exc_value); +#else + Py_VISIT(exc_state->exc_type); + Py_VISIT(exc_state->exc_value); + Py_VISIT(exc_state->exc_traceback); +#endif + return 0; +} +static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) { + { + int e = __Pyx_call_type_traverse((PyObject*)gen, 1, visit, arg); + if (e) return e; + } + Py_VISIT(gen->closure); + Py_VISIT(gen->classobj); + Py_VISIT(gen->yieldfrom); + return __Pyx_Coroutine_traverse_excstate(&gen->gi_exc_state, visit, arg); +} +static int __Pyx_Coroutine_clear(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + __Pyx_Coroutine_Undelegate(gen); + __Pyx_Coroutine_ExceptionClear(&gen->gi_exc_state); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer); + } +#endif + Py_CLEAR(gen->gi_code); + Py_CLEAR(gen->gi_frame); + Py_CLEAR(gen->gi_name); + Py_CLEAR(gen->gi_qualname); + Py_CLEAR(gen->gi_modulename); + return 0; +} +static void __Pyx_Coroutine_dealloc(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + PyObject_GC_UnTrack(gen); + #if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + if (gen->gi_weakreflist != NULL) + #endif + PyObject_ClearWeakRefs(self); + if (gen->resume_label >= 0) { + PyObject_GC_Track(self); +#if CYTHON_USE_TP_FINALIZE + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) +#else + { + destructor del = __Pyx_PyObject_GetSlot(gen, tp_del, destructor); + if (del) del(self); + } + if (unlikely(Py_REFCNT(self) > 0)) +#endif + { + return; + } + PyObject_GC_UnTrack(self); + } +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + /* We have to handle this case for asynchronous generators + right here, because this code has to be between UNTRACK + and GC_Del. */ + Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer); + } +#endif + __Pyx_Coroutine_clear(self); + __Pyx_PyHeapTypeObject_GC_Del(gen); +} +#if CYTHON_USE_TP_FINALIZE +static void __Pyx_Coroutine_del(PyObject *self) { + PyObject *error_type, *error_value, *error_traceback; + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; + __Pyx_PyThreadState_declare + if (gen->resume_label < 0) { + return; + } + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self; + PyObject *finalizer = agen->ag_finalizer; + if (finalizer && !agen->ag_closed) { + PyObject *res = __Pyx_PyObject_CallOneArg(finalizer, self); + if (unlikely(!res)) { + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + __Pyx_ErrRestore(error_type, error_value, error_traceback); + return; + } + } +#endif + if (unlikely(gen->resume_label == 0 && !error_value)) { +#ifdef __Pyx_Coroutine_USED +#ifdef __Pyx_Generator_USED + if (!__Pyx_Generator_CheckExact(self)) +#endif + { + PyObject_GC_UnTrack(self); + if (unlikely(PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname) < 0)) + PyErr_WriteUnraisable(self); + PyObject_GC_Track(self); + } +#endif + } else { + PyObject *retval = NULL; + __Pyx_PySendResult result = __Pyx_Coroutine_Close(self, &retval); + if (result == PYGEN_ERROR) { + PyErr_WriteUnraisable(self); + } else { + Py_XDECREF(retval); + } + } + __Pyx_ErrRestore(error_type, error_value, error_traceback); +} +#endif +static PyObject * +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context) +{ + PyObject *name = self->gi_name; + CYTHON_UNUSED_VAR(context); + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_Py_XDECREF_SET(self->gi_name, value); + return 0; +} +static PyObject * +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context) +{ + PyObject *name = self->gi_qualname; + CYTHON_UNUSED_VAR(context); + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; +} +static int +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); + return 0; +} +static PyObject * +__Pyx__Coroutine_get_frame(__pyx_CoroutineObject *self) +{ +#if !CYTHON_COMPILING_IN_LIMITED_API + PyObject *frame; + #if PY_VERSION_HEX >= 0x030d0000 + Py_BEGIN_CRITICAL_SECTION(self); + #endif + frame = self->gi_frame; + if (!frame) { + if (unlikely(!self->gi_code)) { + Py_RETURN_NONE; + } + PyObject *globals = PyDict_New(); + if (unlikely(!globals)) return NULL; + frame = (PyObject *) PyFrame_New( + PyThreadState_Get(), /*PyThreadState *tstate,*/ + (PyCodeObject*) self->gi_code, /*PyCodeObject *code,*/ + globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + Py_DECREF(globals); + if (unlikely(!frame)) + return NULL; + if (unlikely(self->gi_frame)) { + Py_DECREF(frame); + frame = self->gi_frame; + } else { + self->gi_frame = frame; + } + } + Py_INCREF(frame); + #if PY_VERSION_HEX >= 0x030d0000 + Py_END_CRITICAL_SECTION(); + #endif + return frame; +#else + CYTHON_UNUSED_VAR(self); + Py_RETURN_NONE; +#endif +} +static PyObject * +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) { + CYTHON_UNUSED_VAR(context); + PyObject *frame = self->gi_frame; + if (frame) + return __Pyx_NewRef(frame); + return __Pyx__Coroutine_get_frame(self); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_New( + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); + if (unlikely(!gen)) + return NULL; + return __Pyx__Coroutine_NewInit(gen, body, code, closure, name, qualname, module_name); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + gen->body = body; + gen->closure = closure; + Py_XINCREF(closure); + gen->is_running = 0; + gen->resume_label = 0; + gen->classobj = NULL; + gen->yieldfrom = NULL; + gen->yieldfrom_am_send = NULL; + #if PY_VERSION_HEX >= 0x030B00a4 && !CYTHON_COMPILING_IN_LIMITED_API + gen->gi_exc_state.exc_value = NULL; + #else + gen->gi_exc_state.exc_type = NULL; + gen->gi_exc_state.exc_value = NULL; + gen->gi_exc_state.exc_traceback = NULL; + #endif +#if CYTHON_USE_EXC_INFO_STACK + gen->gi_exc_state.previous_item = NULL; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + gen->gi_weakreflist = NULL; +#endif + Py_XINCREF(qualname); + gen->gi_qualname = qualname; + Py_XINCREF(name); + gen->gi_name = name; + Py_XINCREF(module_name); + gen->gi_modulename = module_name; + Py_XINCREF(code); + gen->gi_code = code; + gen->gi_frame = NULL; + PyObject_GC_Track(gen); + return gen; +} +static char __Pyx_Coroutine_test_and_set_is_running(__pyx_CoroutineObject *gen) { + char result; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + result = gen->is_running; + gen->is_running = 1; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif + return result; +} +static void __Pyx_Coroutine_unset_is_running(__pyx_CoroutineObject *gen) { + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + assert(gen->is_running); + gen->is_running = 0; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif +} +static char __Pyx_Coroutine_get_is_running(__pyx_CoroutineObject *gen) { + char result; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_BEGIN_CRITICAL_SECTION(gen); + #endif + result = gen->is_running; + #if PY_VERSION_HEX >= 0x030d0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_END_CRITICAL_SECTION(); + #endif + return result; +} +static PyObject *__Pyx_Coroutine_get_is_running_getter(PyObject *gen, void *closure) { + CYTHON_UNUSED_VAR(closure); + char result = __Pyx_Coroutine_get_is_running((__pyx_CoroutineObject*)gen); + if (result) Py_RETURN_TRUE; + else Py_RETURN_FALSE; +} +#if __PYX_HAS_PY_AM_SEND == 2 +static void __Pyx_SetBackportTypeAmSend(PyTypeObject *type, __Pyx_PyAsyncMethodsStruct *static_amsend_methods, __Pyx_pyiter_sendfunc am_send) { + Py_ssize_t ptr_offset = (char*)(type->tp_as_async) - (char*)type; + if (ptr_offset < 0 || ptr_offset > type->tp_basicsize) { + return; + } + memcpy((void*)static_amsend_methods, (void*)(type->tp_as_async), sizeof(*type->tp_as_async)); + static_amsend_methods->am_send = am_send; + type->tp_as_async = __Pyx_SlotTpAsAsync(static_amsend_methods); +} +#endif +static PyObject *__Pyx_Coroutine_fail_reduce_ex(PyObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + __Pyx_TypeName self_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE((PyObject*)self)); + PyErr_Format(PyExc_TypeError, "cannot pickle '" __Pyx_FMT_TYPENAME "' object", + self_type_name); + __Pyx_DECREF_TypeName(self_type_name); + return NULL; +} + +/* Generator */ +static PyMethodDef __pyx_Generator_methods[] = { + {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, + PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, + {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, + PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, + {"close", (PyCFunction) __Pyx_Coroutine_Close_Method, METH_NOARGS, + PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, + {"__reduce_ex__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_O, 0}, + {"__reduce__", (PyCFunction) __Pyx_Coroutine_fail_reduce_ex, METH_NOARGS, 0}, + {0, 0, 0, 0} +}; +static PyMemberDef __pyx_Generator_memberlist[] = { + {"gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + PyDoc_STR("object being iterated by 'yield from', or None")}, + {"gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {"__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyGetSetDef __pyx_Generator_getsets[] = { + {"__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + PyDoc_STR("name of the generator"), 0}, + {"__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + PyDoc_STR("qualified name of the generator"), 0}, + {"gi_frame", (getter)__Pyx_Coroutine_get_frame, NULL, + PyDoc_STR("Frame of the generator"), 0}, + {"gi_running", __Pyx_Coroutine_get_is_running_getter, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_GeneratorType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Generator_methods}, + {Py_tp_members, (void *)__pyx_Generator_memberlist}, + {Py_tp_getset, (void *)__pyx_Generator_getsets}, + {Py_tp_getattro, (void *) PyObject_GenericGetAttr}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + {Py_am_send, (void *)__Pyx_Coroutine_AmSend}, +#endif + {0, 0}, +}; +static PyType_Spec __pyx_GeneratorType_spec = { + __PYX_TYPE_MODULE_PREFIX "generator", + sizeof(__pyx_CoroutineObject), + 0, +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_WEAKREF | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | __Pyx_TPFLAGS_HAVE_AM_SEND, + __pyx_GeneratorType_slots +}; +#if __PYX_HAS_PY_AM_SEND == 2 +static __Pyx_PyAsyncMethodsStruct __pyx_Generator_as_async; +#endif +static int __pyx_Generator_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_GeneratorType_spec, NULL); + if (unlikely(!mstate->__pyx_GeneratorType)) { + return -1; + } +#if __PYX_HAS_PY_AM_SEND == 2 + __Pyx_SetBackportTypeAmSend(mstate->__pyx_GeneratorType, &__pyx_Generator_as_async, &__Pyx_Coroutine_AmSend); +#endif + return 0; +} +static PyObject *__Pyx_Generator_GetInlinedResult(PyObject *self) { + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; + PyObject *retval = NULL; + if (unlikely(__Pyx_Coroutine_test_and_set_is_running(gen))) { + return __Pyx_Coroutine_AlreadyRunningError(gen); + } + __Pyx_PySendResult result = __Pyx_Coroutine_SendEx(gen, Py_None, &retval, 0); + __Pyx_Coroutine_unset_is_running(gen); + (void) result; + assert (result == PYGEN_RETURN || result == PYGEN_ERROR); + assert ((result == PYGEN_RETURN && retval != NULL) || (result == PYGEN_ERROR && retval == NULL)); + return retval; +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* DecompressString */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; +import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); +bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pxd b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pxd new file mode 100644 index 0000000000000000000000000000000000000000..ff943b837332fda4beeb265f9c4970b716b51584 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pxd @@ -0,0 +1,90 @@ +from cymem.cymem cimport Pool +from libc.stdint cimport uint32_t, uint64_t +from libcpp.pair cimport pair +from libcpp.queue cimport priority_queue +from libcpp.vector cimport vector + +ctypedef uint64_t hash_t +ctypedef uint64_t class_t +ctypedef float weight_t + + +ctypedef pair[weight_t, size_t] Entry +ctypedef priority_queue[Entry] Queue + + +ctypedef int (*trans_func_t)(void* dest, void* src, class_t clas, void* x) except -1 + +ctypedef void* (*init_func_t)(Pool mem, int n, void* extra_args) except NULL + +ctypedef int (*del_func_t)(Pool mem, void* state, void* extra_args) except -1 + +ctypedef int (*finish_func_t)(void* state, void* extra_args) except -1 + +ctypedef hash_t (*hash_func_t)(void* state, void* x) except 0 + + +cdef struct _State: + void* content + class_t* hist + weight_t score + weight_t loss + int i + int t + bint is_done + + +cdef class Beam: + cdef Pool mem + cdef class_t nr_class + cdef class_t width + cdef class_t size + cdef public weight_t min_density + cdef int t + cdef readonly bint is_done + cdef list histories + cdef list _parent_histories + cdef weight_t** scores + cdef int** is_valid + cdef weight_t** costs + cdef _State* _parents + cdef _State* _states + cdef del_func_t del_func + + cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1 + + cdef inline void* at(self, int i) nogil: + return self._states[i].content + + cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1 + cdef int advance(self, trans_func_t transition_func, hash_func_t hash_func, + void* extra_args) except -1 + cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1 + + + cdef inline void set_cell(self, int i, int j, weight_t score, int is_valid, weight_t cost) nogil: + self.scores[i][j] = score + self.is_valid[i][j] = is_valid + self.costs[i][j] = cost + + cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, + const weight_t* costs) except -1 + cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1 + + +cdef class MaxViolation: + cdef Pool mem + cdef weight_t cost + cdef weight_t delta + cdef readonly weight_t p_score + cdef readonly weight_t g_score + cdef readonly double Z + cdef readonly double gZ + cdef class_t n + cdef readonly list p_hist + cdef readonly list g_hist + cdef readonly list p_probs + cdef readonly list g_probs + + cpdef int check(self, Beam pred, Beam gold) except -1 + cpdef int check_crf(self, Beam pred, Beam gold) except -1 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pyx new file mode 100644 index 0000000000000000000000000000000000000000..ba8e721a751ffe6a7b49f80583b48a1b07fd221e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/search.pyx @@ -0,0 +1,303 @@ +# cython: experimental_cpp_class_def=True, cdivision=True, infer_types=True +cimport cython +from libc.math cimport exp, log +from libc.string cimport memcpy, memset + +import math + +from cymem.cymem cimport Pool +from preshed.maps cimport PreshMap + + +cdef class Beam: + def __init__(self, class_t nr_class, class_t width, weight_t min_density=0.0): + assert nr_class != 0 + assert width != 0 + self.nr_class = nr_class + self.width = width + self.min_density = min_density + self.size = 1 + self.t = 0 + self.mem = Pool() + self._parents = <_State*>self.mem.alloc(self.width, sizeof(_State)) + self._states = <_State*>self.mem.alloc(self.width, sizeof(_State)) + cdef int i + self.histories = [[] for i in range(self.width)] + self._parent_histories = [[] for i in range(self.width)] + + self.scores = self.mem.alloc(self.width, sizeof(weight_t*)) + self.is_valid = self.mem.alloc(self.width, sizeof(weight_t*)) + self.costs = self.mem.alloc(self.width, sizeof(weight_t*)) + for i in range(self.width): + self.scores[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + self.is_valid[i] = self.mem.alloc(self.nr_class, sizeof(int)) + self.costs[i] = self.mem.alloc(self.nr_class, sizeof(weight_t)) + + def __len__(self): + return self.size + + property score: + def __get__(self): + return self._states[0].score + + property min_score: + def __get__(self): + return self._states[self.size-1].score + + property loss: + def __get__(self): + return self._states[0].loss + + property probs: + def __get__(self): + return _softmax([self._states[i].score for i in range(self.size)]) + + property scores: + def __get__(self): + return [self._states[i].score for i in range(self.size)] + + property histories: + def __get__(self): + return self.histories + + cdef int set_row(self, int i, const weight_t* scores, const int* is_valid, + const weight_t* costs) except -1: + cdef int j + for j in range(self.nr_class): + self.scores[i][j] = scores[j] + self.is_valid[i][j] = is_valid[j] + self.costs[i][j] = costs[j] + + cdef int set_table(self, weight_t** scores, int** is_valid, weight_t** costs) except -1: + cdef int i, j + for i in range(self.width): + memcpy(self.scores[i], scores[i], sizeof(weight_t) * self.nr_class) + memcpy(self.is_valid[i], is_valid[i], sizeof(bint) * self.nr_class) + memcpy(self.costs[i], costs[i], sizeof(int) * self.nr_class) + + cdef int initialize(self, init_func_t init_func, del_func_t del_func, int n, void* extra_args) except -1: + for i in range(self.width): + self._states[i].content = init_func(self.mem, n, extra_args) + self._parents[i].content = init_func(self.mem, n, extra_args) + self.del_func = del_func + + def __dealloc__(self): + for i in range(self.width): + self.del_func(self.mem, self._states[i].content, NULL) + self.del_func(self.mem, self._parents[i].content, NULL) + + @cython.cdivision(True) + cdef int advance(self, trans_func_t transition_func, hash_func_t hash_func, + void* extra_args) except -1: + cdef weight_t** scores = self.scores + cdef int** is_valid = self.is_valid + cdef weight_t** costs = self.costs + + cdef Queue* q = new Queue() + self._fill(q, scores, is_valid) + # For a beam of width k, we only ever need 2k state objects. How? + # Each transition takes a parent and a class and produces a new state. + # So, we don't need the whole history --- just the parent. So at + # each step, we take a parent, and apply one or more extensions to + # it. + self._parents, self._states = self._states, self._parents + self._parent_histories, self.histories = self.histories, self._parent_histories + cdef weight_t score + cdef int p_i + cdef int i = 0 + cdef class_t clas + cdef _State* parent + cdef _State* state + cdef hash_t key + cdef PreshMap seen_states = PreshMap(self.width) + cdef uint64_t is_seen + cdef uint64_t one = 1 + while i < self.width and not q.empty(): + data = q.top() + p_i = data.second / self.nr_class + clas = data.second % self.nr_class + score = data.first + q.pop() + parent = &self._parents[p_i] + # Indicates terminal state reached; i.e. state is done + if parent.is_done: + # Now parent will not be changed, so we don't have to copy. + # Once finished, should also be unbranching. + self._states[i], parent[0] = parent[0], self._states[i] + parent.i = self._states[i].i + parent.t = self._states[i].t + parent.is_done = self._states[i].t + self._states[i].score = score + self.histories[i] = list(self._parent_histories[p_i]) + i += 1 + else: + state = &self._states[i] + # The supplied transition function should adjust the destination + # state to be the result of applying the class to the source state + transition_func(state.content, parent.content, clas, extra_args) + key = hash_func(state.content, extra_args) if hash_func is not NULL else 0 + is_seen = seen_states.get(key) + if key == 0 or key == 1 or not is_seen: + if key != 0 and key != 1: + seen_states.set(key, one) + state.score = score + state.loss = parent.loss + costs[p_i][clas] + self.histories[i] = list(self._parent_histories[p_i]) + self.histories[i].append(clas) + i += 1 + del q + self.size = i + assert self.size >= 1 + for i in range(self.width): + memset(self.scores[i], 0, sizeof(weight_t) * self.nr_class) + memset(self.costs[i], 0, sizeof(weight_t) * self.nr_class) + memset(self.is_valid[i], 0, sizeof(int) * self.nr_class) + self.t += 1 + + cdef int check_done(self, finish_func_t finish_func, void* extra_args) except -1: + cdef int i + for i in range(self.size): + if not self._states[i].is_done: + self._states[i].is_done = finish_func(self._states[i].content, extra_args) + for i in range(self.size): + if not self._states[i].is_done: + self.is_done = False + break + else: + self.is_done = True + + @cython.cdivision(True) + cdef int _fill(self, Queue* q, weight_t** scores, int** is_valid) except -1: + """Populate the queue from a k * n matrix of scores, where k is the + beam-width, and n is the number of classes. + """ + cdef Entry entry + cdef weight_t score + cdef _State* s + cdef int i, j, move_id + assert self.size >= 1 + cdef vector[Entry] entries + for i in range(self.size): + s = &self._states[i] + move_id = i * self.nr_class + if s.is_done: + # Update score by path average, following TACL '13 paper. + if self.histories[i]: + entry.first = s.score + (s.score / self.t) + else: + entry.first = s.score + entry.second = move_id + entries.push_back(entry) + else: + for j in range(self.nr_class): + if is_valid[i][j]: + entry.first = s.score + scores[i][j] + entry.second = move_id + j + entries.push_back(entry) + cdef double max_, Z, cutoff + if self.min_density == 0.0: + for i in range(entries.size()): + q.push(entries[i]) + elif not entries.empty(): + max_ = entries[0].first + Z = 0. + cutoff = 0. + # Softmax into probabilities, so we can prune + for i in range(entries.size()): + if entries[i].first > max_: + max_ = entries[i].first + for i in range(entries.size()): + Z += exp(entries[i].first-max_) + cutoff = (1. / Z) * self.min_density + for i in range(entries.size()): + prob = exp(entries[i].first-max_) / Z + if prob >= cutoff: + q.push(entries[i]) + + +cdef class MaxViolation: + def __init__(self): + self.p_score = 0.0 + self.g_score = 0.0 + self.Z = 0.0 + self.gZ = 0.0 + self.delta = -1 + self.cost = 0 + self.p_hist = [] + self.g_hist = [] + self.p_probs = [] + self.g_probs = [] + + cpdef int check(self, Beam pred, Beam gold) except -1: + cdef _State* p = &pred._states[0] + cdef _State* g = &gold._states[0] + cdef weight_t d = p.score - g.score + if p.loss >= 1 and (self.cost == 0 or d > self.delta): + self.cost = p.loss + self.delta = d + self.p_hist = list(pred.histories[0]) + self.g_hist = list(gold.histories[0]) + self.p_score = p.score + self.g_score = g.score + self.Z = 1e-10 + self.gZ = 1e-10 + for i in range(pred.size): + if pred._states[i].loss > 0: + self.Z += exp(pred._states[i].score) + for i in range(gold.size): + if gold._states[i].loss == 0: + prob = exp(gold._states[i].score) + self.Z += prob + self.gZ += prob + + cpdef int check_crf(self, Beam pred, Beam gold) except -1: + d = pred.score - gold.score + seen_golds = set([tuple(gold.histories[i]) for i in range(gold.size)]) + if pred.loss > 0 and (self.cost == 0 or d > self.delta): + p_hist = [] + p_scores = [] + g_hist = [] + g_scores = [] + for i in range(pred.size): + if pred._states[i].loss > 0: + p_scores.append(pred._states[i].score) + p_hist.append(list(pred.histories[i])) + # This can happen from non-monotonic actions + # If we find a better gold analysis this way, be sure to keep it. + elif pred._states[i].loss <= 0 \ + and tuple(pred.histories[i]) not in seen_golds: + g_scores.append(pred._states[i].score) + g_hist.append(list(pred.histories[i])) + for i in range(gold.size): + if gold._states[i].loss == 0: + g_scores.append(gold._states[i].score) + g_hist.append(list(gold.histories[i])) + + all_probs = _softmax(p_scores + g_scores) + p_probs = all_probs[:len(p_scores)] + g_probs_all = all_probs[len(p_scores):] + g_probs = _softmax(g_scores) + + self.cost = pred.loss + self.delta = d + self.p_hist = p_hist + self.g_hist = g_hist + # TODO: These variables are misnamed! These are the gradients of the loss. + self.p_probs = p_probs + # Intuition here: + # The gradient of the loss is: + # P(model) - P(truth) + # Normally, P(truth) is 1 for the gold + # But, if we want to do the "partial credit" scheme, we want + # to create a distribution over the gold, proportional to the scores + # awarded. + self.g_probs = [x-y for x, y in zip(g_probs_all, g_probs)] + + +def _softmax(nums): + if not nums: + return [] + max_ = max(nums) + nums = [(exp(n-max_) if n is not None else None) for n in nums] + Z = sum(n for n in nums if n is not None) + return [(n/Z if n is not None else None) for n in nums] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a9224aa483a051c679e3f63d6eb27d83b4f8bbf Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/c_test_search.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/c_test_search.pyx new file mode 100644 index 0000000000000000000000000000000000000000..dc3dbdf4b5e009d836a71a9eef8f5e7dba966a15 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/extra/tests/c_test_search.pyx @@ -0,0 +1,83 @@ +# cython: profile=False +from cymem.cymem cimport Pool + +from thinc.extra.search cimport Beam +from thinc.typedefs cimport class_t, weight_t + + +cdef struct TestState: + int length + int x + Py_UNICODE* string + + +cdef int transition(void* dest, void* src, class_t clas, void* extra_args) except -1: + dest_state = dest + src_state = src + dest_state.length = src_state.length + dest_state.x = src_state.x + dest_state.x += clas + if extra_args != NULL: + dest_state.string = extra_args + else: + dest_state.string = src_state.string + + +cdef void* initialize(Pool mem, int n, void* extra_args) except NULL: + state = mem.alloc(1, sizeof(TestState)) + state.length = n + state.x = 1 + if extra_args == NULL: + state.string = 'default' + else: + state.string = extra_args + return state + + +cdef int destroy(Pool mem, void* state, void* extra_args) except -1: + state = state + mem.free(state) + + +def test_init(nr_class, beam_width): + b = Beam(nr_class, beam_width) + assert b.size == 1 + assert b.width == beam_width + assert b.nr_class == nr_class + + +def test_initialize(nr_class, beam_width, length): + b = Beam(nr_class, beam_width) + b.initialize(initialize, destroy, length, NULL) + for i in range(b.width): + s = b.at(i) + assert s.length == length, s.length + assert s.string == 'default' + + +def test_initialize_extra(nr_class, beam_width, length, unicode extra): + b = Beam(nr_class, beam_width) + b.initialize(initialize, destroy, length, extra) + for i in range(b.width): + s = b.at(i) + assert s.length == length + + +def test_transition(nr_class=3, beam_width=6, length=3): + b = Beam(nr_class, beam_width) + b.initialize(initialize, destroy, length, NULL) + b.set_cell(0, 2, 30, True, 0) + b.set_cell(0, 1, 42, False, 0) + b.advance(transition, NULL, NULL) + assert b.size == 1, b.size + assert b.score == 30, b.score + s = b.at(0) + assert s.x == 3 + assert b._states[0].score == 30, b._states[0].score + b.set_cell(0, 1, 10, True, 0) + b.set_cell(0, 2, 20, True, 0) + b.advance(transition, NULL, NULL) + assert b._states[0].score == 50, b._states[0].score + assert b._states[1].score == 40 + s = b.at(0) + assert s.x == 5 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..98667a029366b47d8e921824024775974fe21de1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__init__.py @@ -0,0 +1,170 @@ +# Weights layers +# Combinators +from .add import add + +# Array manipulation +from .array_getitem import array_getitem +from .bidirectional import bidirectional +from .cauchysimilarity import CauchySimilarity +from .chain import chain +from .clipped_linear import ClippedLinear, HardSigmoid, HardTanh, ReluK +from .clone import clone +from .concatenate import concatenate +from .dish import Dish +from .dropout import Dropout +from .embed import Embed +from .expand_window import expand_window +from .gelu import Gelu +from .hard_swish import HardSwish +from .hard_swish_mobilenet import HardSwishMobilenet +from .hashembed import HashEmbed +from .layernorm import LayerNorm +from .linear import Linear + +# Data-type transfers +from .list2array import list2array +from .list2padded import list2padded +from .list2ragged import list2ragged +from .logistic import Logistic +from .lstm import LSTM, PyTorchLSTM +from .map_list import map_list +from .maxout import Maxout +from .mish import Mish +from .multisoftmax import MultiSoftmax +from .mxnetwrapper import MXNetWrapper +from .noop import noop +from .padded2list import padded2list +from .parametricattention import ParametricAttention +from .parametricattention_v2 import ParametricAttention_v2 +from .premap_ids import premap_ids +from .pytorchwrapper import ( + PyTorchRNNWrapper, + PyTorchWrapper, + PyTorchWrapper_v2, + PyTorchWrapper_v3, +) +from .ragged2list import ragged2list + +# Pooling +from .reduce_first import reduce_first +from .reduce_last import reduce_last +from .reduce_max import reduce_max +from .reduce_mean import reduce_mean +from .reduce_sum import reduce_sum +from .relu import Relu +from .remap_ids import remap_ids, remap_ids_v2 +from .residual import residual +from .resizable import resizable +from .siamese import siamese +from .sigmoid import Sigmoid +from .sigmoid_activation import sigmoid_activation +from .softmax import Softmax, Softmax_v2 +from .softmax_activation import softmax_activation +from .sparselinear import SparseLinear, SparseLinear_v2 +from .strings2arrays import strings2arrays +from .swish import Swish +from .tensorflowwrapper import TensorFlowWrapper, keras_subclass +from .torchscriptwrapper import TorchScriptWrapper_v1, pytorch_to_torchscript_wrapper +from .tuplify import tuplify +from .uniqued import uniqued +from .with_array import with_array +from .with_array2d import with_array2d +from .with_cpu import with_cpu +from .with_debug import with_debug +from .with_flatten import with_flatten +from .with_flatten_v2 import with_flatten_v2 +from .with_getitem import with_getitem +from .with_list import with_list +from .with_nvtx_range import with_nvtx_range +from .with_padded import with_padded +from .with_ragged import with_ragged +from .with_reshape import with_reshape +from .with_signpost_interval import with_signpost_interval + +# fmt: off +__all__ = [ + "CauchySimilarity", + "Linear", + "Dropout", + "Embed", + "expand_window", + "HashEmbed", + "LayerNorm", + "LSTM", + "Maxout", + "Mish", + "MultiSoftmax", + "ParametricAttention", + "ParametricAttention_v2", + "PyTorchLSTM", + "PyTorchWrapper", + "PyTorchWrapper_v2", + "PyTorchWrapper_v3", + "PyTorchRNNWrapper", + "Relu", + "sigmoid_activation", + "Sigmoid", + "softmax_activation", + "Softmax", + "Softmax_v2", + "SparseLinear", + "SparseLinear_v2", + "TensorFlowWrapper", + "TorchScriptWrapper_v1", + "add", + "bidirectional", + "chain", + "clone", + "concatenate", + "noop", + "residual", + "uniqued", + "siamese", + "reduce_first", + "reduce_last", + "reduce_max", + "reduce_mean", + "reduce_sum", + "resizable", + "list2array", + "list2ragged", + "list2padded", + "ragged2list", + "padded2list", + "with_reshape", + "with_getitem", + "with_array", + "with_array2d", + "with_cpu", + "with_list", + "with_ragged", + "with_padded", + "with_flatten", + "with_flatten_v2", + "with_debug", + "with_nvtx_range", + "with_signpost_interval", + "remap_ids", + "remap_ids_v2", + "premap_ids", + "softmax_activation", + "Logistic", + "Sigmoid", + "ClippedLinear", + "ReluK", + "HardTanh", + "HardSigmoid", + "Dish", + "HardSwish", + "HardSwishMobilenet", + "Swish", + "Gelu", + "keras_subclass", + "MXNetWrapper", + "map_list", + "strings2arrays", + "array_getitem", + "tuplify", + "pytorch_to_torchscript_wrapper", +] +# fmt: on diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..278fd6e489ad9edc46f60e3f369b4221af55c75a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/add.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/add.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29d8f0ecce562291acb82cc8c57b7d9955dc1761 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/add.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/array_getitem.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/array_getitem.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe29a05c7a9095d2d2fe1c38ff915e45430d7e45 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/array_getitem.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/bidirectional.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/bidirectional.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c79288fdfea679379a889d0e57c5538a0144d2b3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/bidirectional.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/cauchysimilarity.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/cauchysimilarity.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f8b0625691a3736bc991942a002407211af40c1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/cauchysimilarity.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/chain.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/chain.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e262e0b4a2b70ac8a00cb333087cc9ba8f151e10 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/chain.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clipped_linear.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clipped_linear.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..551dc8b5298eb83ad74dc6ceba61b63f7ccaba1c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clipped_linear.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clone.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clone.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..610dd049973c58b94aac2555c947662a776b32b8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/clone.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/concatenate.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/concatenate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8bbff76b18ce91632c8610463f469f7e2f38bcd Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/concatenate.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dish.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dish.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b64e2e6cc23c8f65e510de9b34440a21d79e7177 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dish.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dropout.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dropout.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ea0e442cce80a2800c932a6347c5fa7c40c524c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/dropout.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/embed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/embed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c6c0f68fe7add3862db5b150f52881b4d25c66d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/embed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/expand_window.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/expand_window.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad6d489643f7ade945953b556effbf21bf875726 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/expand_window.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/gelu.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/gelu.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08943b0a6ae7a812f9c2bb1c4228ee96f5bf4fd1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/gelu.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f6e38477037e363f837e5187d9968420f4362ef Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish_mobilenet.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish_mobilenet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b224f2f944478d12903dd0b2ee64c80c65d4bed2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hard_swish_mobilenet.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hashembed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hashembed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8069e95b99b5e6f8218ba55fb512d7c3b8a638e2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/hashembed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/layernorm.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/layernorm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5692944fdbfff09846ca42e43e835274d064f82 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/layernorm.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/linear.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/linear.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0c8a27290c4d8ff1ee73d248b7a472c18a265d6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/linear.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0c656999f9e2c48c233407bef2141f2d1fc2a31 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2padded.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2padded.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd754db34e3ea0f118fd896ae86d9c5ba8e9cf2a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2padded.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2ragged.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2ragged.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef65049fed00fc117d8680d0279d974f2c31eb8b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/list2ragged.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/logistic.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/logistic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9fbb1f7fd8e5db1922811255215e568c190f36a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/logistic.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/lstm.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/lstm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..392e7ddc8904b51560ae996f3fd89dbaa5c466bb Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/lstm.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/map_list.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/map_list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ceba4528bfcf6a41c8c1cb58641fbd28acd4cd26 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/map_list.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/maxout.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/maxout.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6175ea6906bd286f72b0183ecf8910f8f1454735 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/maxout.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mish.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mish.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..298495bd75bab2cab7081a55a3d3c6c3c21fae88 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mish.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/multisoftmax.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/multisoftmax.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff5811ef45ce5744ad89a1e0d082743310009d3f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/multisoftmax.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mxnetwrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mxnetwrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61c6b2542ee2f73bf07ed9117cac4c64bd62537d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/mxnetwrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/noop.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/noop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..502cf57c32b466c76f54ac1261a391527341c608 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/noop.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/padded2list.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/padded2list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c085e8d2d182ae81d79e3fe4eab2282dff150b05 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/padded2list.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1466bfb0d57265acd503cbc72dcd265025d143a6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention_v2.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b263337c9a2bd68e4c184f1991546412f58e11ec Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/parametricattention_v2.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/pytorchwrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/pytorchwrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7777095d29cf759b7d9220bbae2ccd7585160707 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/pytorchwrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/ragged2list.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/ragged2list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3dafeb7c952fe4918a3ef36e11c7d49627c4e571 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/ragged2list.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_first.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_first.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53f67d0015cae335530ae05da17165edc06092b8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_first.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_last.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_last.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf00c201687261b204d77b9bdcbcabfbab71470e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_last.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_max.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_max.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88c8bcaffc90aaa4229a49dcac50784972e9d66c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_max.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_mean.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_mean.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3486b6a1ad6be2faa15431655e854fb6776dec1a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_mean.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_sum.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_sum.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37708acf2e0a10e2597a77a36f0815f640bd60b9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/reduce_sum.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/relu.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/relu.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..babcf046c187e96c9fb5368eebac0aee6266fdf5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/relu.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/remap_ids.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/remap_ids.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88cdb7196e0d7cfc06b10506d7786a74f6df1f70 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/remap_ids.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/residual.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/residual.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..300ec59b6eadb39e8fb45d10c40ed65f31f08197 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/residual.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/resizable.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/resizable.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a9bd9bd21c1ed801b6eb979a915b940932eec32 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/resizable.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/siamese.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/siamese.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d910fe5fd50e273fdbc7e94071be154fbb56fecc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/siamese.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1ef8a76f4f70e39cbfb1cd39f4eea0ddef8b54d2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid_activation.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid_activation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..acfb00dcfea5ab5336c62924833e6b98cc29477b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/sigmoid_activation.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa33427eb31e73415b4ad23eab069cb916a94fc3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax_activation.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax_activation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9adfb3ce2f805599457243445346440c347c43f8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/softmax_activation.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/strings2arrays.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/strings2arrays.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..173a9682f7db7d271949cfce53f2971ec6e52b22 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/strings2arrays.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/swish.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/swish.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f35a8752391aa633ff6d85d5058706a7e45494c7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/swish.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tensorflowwrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tensorflowwrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..901cc2ab25e8929dd3a3941dde45aa5fe32af852 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tensorflowwrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/torchscriptwrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/torchscriptwrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cda19312f4ce9ae5066613ab69cbf2f517d0e12a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/torchscriptwrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tuplify.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tuplify.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f2119a7ce7ffa2be5b93e34cc2670186de727d2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/tuplify.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/uniqued.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/uniqued.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..604004879f8d43eda4d8e7f7727c6c44e863c399 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/uniqued.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..854493db43ad7e455016e259dc18eb26fbf2d4e2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array2d.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array2d.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..032d47745b320dab30ee310c5c745d5a97f0fb39 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_array2d.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_cpu.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_cpu.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d681abed59509c0de98886135f135d6f5cb202f0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_cpu.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_debug.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_debug.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d62e242ce6780936e7251ad121572300fb058633 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_debug.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..947f184779c71fdc26fd64e12124f6ed7117daac Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten_v2.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5fb5307639610beaed500a8f8b8e040c42c93df Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_flatten_v2.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_getitem.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_getitem.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b694c488e118bad5cf06ec4a7f27d13f3798759 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_getitem.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_list.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66c74449ebe2140f7126fe770aea178621add021 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_list.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_nvtx_range.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_nvtx_range.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e44077ca21a8b36027c29e3a4d1b289f4f17f6b5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_nvtx_range.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_padded.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_padded.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..caf6c9bd20f9de32c660e8be536a807e9f16c201 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_padded.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_ragged.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_ragged.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..414c47c4564bcfe17d028a3c8743e8d50ad5d10e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_ragged.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_reshape.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_reshape.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7c812b73c5fa2538043e2f39f3b74de8a62cb6e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_reshape.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_signpost_interval.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_signpost_interval.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d22befc0510e80b2fa1182d6e6c3b6817b2394ef Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/__pycache__/with_signpost_interval.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/add.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/add.py new file mode 100644 index 0000000000000000000000000000000000000000..0c3461147105313a0a53049227d6b722075f0104 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/add.py @@ -0,0 +1,68 @@ +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar + +from ..config import registry +from ..model import Model +from ..types import ArrayXd, XY_XY_OutT +from ..util import get_width + +InT = TypeVar("InT", bound=Any) +OutT = TypeVar("OutT", bound=ArrayXd) + + +@registry.layers("add.v1") +def add( + layer1: Model[InT, OutT], layer2: Model[InT, OutT], *layers: Model +) -> Model[InT, XY_XY_OutT]: + """Compose two or more models `f`, `g`, etc, such that their outputs are + added, i.e. `add(f, g)(x)` computes `f(x) + g(x)`. + """ + layers = (layer1, layer2) + layers + if layers[0].name == "add": + layers[0].layers.extend(layers[1:]) + return layers[0] + + # only add an nI dimension if each sub-layer has one + dims: Dict[str, Optional[int]] = {"nO": None} + if all(node.has_dim("nI") in [True, None] for node in layers): + dims = {"nO": None, "nI": None} + + return Model("add", forward, init=init, dims=dims, layers=layers) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + if not model.layers: + return X, lambda dY: dY + Y, first_callback = model.layers[0](X, is_train=is_train) + callbacks = [] + for layer in model.layers[1:]: + layer_Y, layer_callback = layer(X, is_train=is_train) + Y += layer_Y + callbacks.append(layer_callback) + + def backprop(dY: InT) -> OutT: + dX = first_callback(dY) + for callback in callbacks: + dX += callback(dY) + return dX + + return Y, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + if model.has_dim("nI") is not False: + model.set_dim("nI", get_width(X)) + for layer in model.layers: + if layer.has_dim("nI") is not False: + layer.set_dim("nI", get_width(X)) + if Y is not None: + if model.has_dim("nO") is not False: + model.set_dim("nO", get_width(Y)) + for layer in model.layers: + if layer.has_dim("nO") is not False: + layer.set_dim("nO", get_width(Y)) + for layer in model.layers: + layer.initialize(X=X, Y=Y) + model.set_dim("nO", model.layers[0].get_dim("nO")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/array_getitem.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/array_getitem.py new file mode 100644 index 0000000000000000000000000000000000000000..d5f7e9287dba6a922b0715eaccc7c5589282c249 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/array_getitem.py @@ -0,0 +1,51 @@ +from typing import Sequence, Tuple, TypeVar, Union + +from ..model import Model +from ..types import ArrayXd, FloatsXd, IntsXd + +AxisIndex = Union[int, slice, Sequence[int]] +Index = Union[AxisIndex, Tuple[AxisIndex, ...]] +ArrayTXd = TypeVar("ArrayTXd", bound=ArrayXd) + + +def array_getitem(index: Index) -> Model[ArrayTXd, ArrayTXd]: + """Index into input arrays, and return the subarrays. + + index: + A valid numpy-style index. Multi-dimensional indexing can be performed + by passing in a tuple, and slicing can be performed using the slice object. + For instance, X[:, :-1] would be (slice(None, None), slice(None, -1)). + """ + return Model("array-getitem", forward, attrs={"index": index}) + + +def floats_getitem(index: Index) -> Model[FloatsXd, FloatsXd]: + """Index into input arrays, and return the subarrays. + + This delegates to `array_getitem`, but allows type declarations. + """ + return Model("floats-getitem", forward, attrs={"index": index}) + + +def ints_getitem(index: Index) -> Model[IntsXd, IntsXd]: + """Index into input arrays, and return the subarrays. + + This delegates to `array_getitem`, but allows type declarations. + """ + return Model("ints-getitem", forward, attrs={"index": index}) + + +def forward(model, X, is_train): + index = model.attrs["index"] + shape = X.shape + dtype = X.dtype + + def backprop_get_column(dY): + dX = model.ops.alloc(shape, dtype=dtype) + dX[index] = dY + return dX + + if len(X) == 0: + return X, backprop_get_column + Y = X[index] + return Y, backprop_get_column diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/bidirectional.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/bidirectional.py new file mode 100644 index 0000000000000000000000000000000000000000..335c1f7d7c6914c85af88e7db0b91dd11db55608 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/bidirectional.py @@ -0,0 +1,71 @@ +from typing import Callable, Optional, Tuple, cast + +from ..backends import Ops +from ..config import registry +from ..model import Model +from ..types import Padded + +InT = Padded +OutT = Padded + + +@registry.layers("bidirectional.v1") +def bidirectional( + l2r: Model[InT, OutT], r2l: Optional[Model[InT, OutT]] = None +) -> Model[InT, OutT]: + """Stitch two RNN models into a bidirectional layer. Expects squared sequences.""" + if r2l is None: + r2l = l2r.copy() + return Model(f"bi{l2r.name}", forward, layers=[l2r, r2l], init=init) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + l2r, r2l = model.layers + X_rev = _reverse(model.ops, X) + l2r_Z, bp_l2r_Z = l2r(X, is_train) + r2l_Z, bp_r2l_Z = r2l(X_rev, is_train) + Z = _concatenate(model.ops, l2r_Z, r2l_Z) + + def backprop(dZ: OutT) -> InT: + d_l2r_Z, d_r2l_Z = _split(model.ops, dZ) + dX_l2r = bp_l2r_Z(d_l2r_Z) + dX_r2l = bp_r2l_Z(d_r2l_Z) + return _sum(dX_l2r, dX_r2l) + + return Z, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + (Y1, Y2) = _split(model.ops, Y) if Y is not None else (None, None) + model.layers[0].initialize(X=X, Y=Y1) + model.layers[1].initialize(X=X, Y=Y2) + + +def _reverse(ops: Ops, Xp: Padded) -> Padded: + return Padded(Xp.data[::1], Xp.size_at_t, Xp.lengths, Xp.indices) + + +def _concatenate(ops: Ops, l2r: Padded, r2l: Padded) -> Padded: + return Padded( + ops.xp.concatenate((l2r.data, r2l.data), axis=-1), + l2r.size_at_t, + l2r.lengths, + l2r.indices, + ) + + +def _split(ops: Ops, Xp: Padded) -> Tuple[Padded, Padded]: + half = Xp.data.shape[-1] // 2 + # I don't know how to write these ellipsis in the overloads :( + X_l2r = Xp.data[cast(Tuple[slice, slice], (..., slice(None, half)))] + X_r2l = Xp.data[cast(Tuple[slice, slice], (..., slice(half)))] + return ( + Padded(X_l2r, Xp.size_at_t, Xp.lengths, Xp.indices), + Padded(X_r2l, Xp.size_at_t, Xp.lengths, Xp.indices), + ) + + +def _sum(Xp: Padded, Yp: Padded) -> Padded: + return Padded(Xp.data + Yp.data, Xp.size_at_t, Xp.lengths, Xp.indices) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/cauchysimilarity.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/cauchysimilarity.py new file mode 100644 index 0000000000000000000000000000000000000000..021af71ad13521d50b5f3f04ed8b8ef7cb7e699c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/cauchysimilarity.py @@ -0,0 +1,64 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width + +InT = Tuple[Floats2d, Floats2d] +OutT = Floats1d + + +@registry.layers("CauchySimilarity.v1") +def CauchySimilarity(nI: Optional[int] = None) -> Model[InT, OutT]: + """Compare input vectors according to the Cauchy similarity function proposed by + Chen (2013). Primarily used within Siamese neural networks. + """ + return Model( + "cauchy_similarity", + forward, + init=init, + dims={"nI": nI, "nO": 1}, + params={"W": None}, + ) + + +def forward( + model: Model[InT, OutT], X1_X2: InT, is_train: bool +) -> Tuple[OutT, Callable]: + X1, X2 = X1_X2 + W = cast(Floats2d, model.get_param("W")) + diff = X1 - X2 + square_diff = diff**2 + total = (W * square_diff).sum(axis=1) + sim, bp_sim = inverse(total) + + def backprop(d_sim: OutT) -> InT: + d_total = bp_sim(d_sim) + d_total = model.ops.reshape2f(d_total, -1, 1) + model.inc_grad("W", (d_total * square_diff).sum(axis=0)) + d_square_diff = W * d_total + d_diff = 2 * d_square_diff * diff + return (d_diff, -d_diff) + + return sim, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + model.set_dim("nI", get_width(X[0])) + # Initialize weights to 1 + W = model.ops.alloc1f(model.get_dim("nI")) + W += 1 + model.set_param("W", W) + + +def inverse(total: OutT) -> Tuple[OutT, Callable]: + inv = 1.0 / (1 + total) + + def backward(d_inverse: OutT) -> OutT: + return d_inverse * (-1 / (total + 1) ** 2) + + return inv, backward diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/chain.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/chain.py new file mode 100644 index 0000000000000000000000000000000000000000..20483c655e67a8dc6243ad693edb831d34a69104 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/chain.py @@ -0,0 +1,102 @@ +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import XY_YZ_OutT +from ..util import get_width + +InT = TypeVar("InT") +MidT = TypeVar("MidT") +OutT = TypeVar("OutT") + +# Keep this function so we can provide variable arguments via the config +@registry.layers("chain.v1") +def chain_no_types(*layer: Model) -> Model: + return chain(*layer) + + +def chain( + layer1: Model[InT, MidT], layer2: Model[MidT, Any], *layers: Model[Any, Any] +) -> Model[InT, XY_YZ_OutT]: + """Compose two models `f` and `g` such that they become layers of a single + feed-forward model that computes `g(f(x))`. + Also supports chaining more than 2 layers. + Note that the type checking for additional layers is carried out by the Thinc Mypy plugin. + """ + all_layers: List[Model[Any, Any]] = [layer1, layer2] + all_layers.extend(layers) + dims: Dict[str, Optional[int]] = {"nO": None} + # set input dimension only if first layer has one - should be "False" otherwise + if all_layers[0].has_dim("nI") is True: + dims["nI"] = all_layers[0].get_dim("nI") + if all_layers[0].has_dim("nI") is None: + dims["nI"] = None + # set output dimension according to last layer + if all_layers[-1].has_dim("nO") is True: + dims["nO"] = all_layers[-1].get_dim("nO") + + model: Model[InT, XY_YZ_OutT] = Model( + ">>".join(layer.name for layer in all_layers), + forward, + init=init, + dims=dims, + layers=all_layers, + ) + return model + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + """Apply the layers of `model` in sequence, feeding the output from one + layer into the next. + """ + callbacks = [] + for layer in model.layers: + Y, inc_layer_grad = layer(X, is_train=is_train) + callbacks.append(inc_layer_grad) + X = Y + + def backprop(dY: OutT) -> InT: + for callback in reversed(callbacks): + dX = callback(dY) + dY = dX + return dX + + return Y, backprop + + +def init( + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is None and Y is None: + for layer in model.layers: + layer.initialize() + if model.layers[0].has_dim("nI"): + model.set_dim("nI", model.layers[0].get_dim("nI")) + if model.layers[-1].has_dim("nO"): + model.set_dim("nO", model.layers[-1].get_dim("nO")) + + # Try to set nO on each layer, where available. + # Shape inference is tricky, especially for the output. The policy is: + # if a layer has an unset nO, we use the final Y (if provided). For other + # layers, Y=None. + curr_input = X + for layer in model.layers: + if layer.has_dim("nO") is None: + layer.initialize(X=curr_input, Y=Y) + else: + layer.initialize(X=curr_input) + if curr_input is not None: + curr_input = layer.predict(curr_input) + + if model.layers[0].has_dim("nI"): + model.set_dim("nI", model.layers[0].get_dim("nI")) + if model.has_dim("nO") is None: + try: + nO = get_width(curr_input) # type: ignore[arg-type] + model.set_dim("nO", nO) + except ValueError: + if model.layers[-1].has_dim("nO"): + nO = model.layers[-1].get_dim("nO") + model.set_dim("nO", nO) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clipped_linear.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clipped_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..1129d7f1f38cc524d39280fe01e3a599df3849a0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clipped_linear.py @@ -0,0 +1,165 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("ClippedLinear.v1") +def ClippedLinear( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, + slope: float = 1.0, + offset: float = 0.0, + min_val: float = 0.0, + max_val: float = 1.0, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + model_attrs = { + "slope": slope, + "offset": offset, + "min_val": min_val, + "max_val": max_val, + } + model: Model[Floats2d, Floats2d] = Model( + "clipped_linear", + forward=forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + attrs=model_attrs, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], + X: Floats2d, + is_train: bool, +) -> Tuple[Floats2d, Callable]: + slope = model.attrs["slope"] + offset = model.attrs["offset"] + min_val = model.attrs["min_val"] + max_val = model.attrs["max_val"] + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.clipped_linear(Y_preact, slope, offset, min_val, max_val) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_clipped_linear( + dY, Y_preact, slope, offset, min_val, max_val, inplace=False + ) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) + + +@registry.layers("HardSigmoid.v1") +def HardSigmoid( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + return ClippedLinear( + nO=nO, + nI=nI, + init_W=init_W, + dropout=dropout, + normalize=normalize, + slope=0.2, + offset=0.5, + ) + + +@registry.layers("HardTanh.v1") +def HardTanh( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + return ClippedLinear( + nO=nO, + nI=nI, + init_W=init_W, + dropout=dropout, + normalize=normalize, + min_val=-1.0, + max_val=1.0, + ) + + +@registry.layers("ReluK.v1") +def ReluK( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, + k: float = 6.0, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + return ClippedLinear( + nO=nO, + nI=nI, + init_W=init_W, + dropout=dropout, + normalize=normalize, + min_val=0.0, + max_val=k, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clone.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clone.py new file mode 100644 index 0000000000000000000000000000000000000000..2244b01db0f3def37d94bcb0eeee1c269307030c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/clone.py @@ -0,0 +1,24 @@ +from typing import List, TypeVar, cast + +from ..config import registry +from ..model import Model +from .chain import chain +from .noop import noop + +InT = TypeVar("InT") +OutT = TypeVar("OutT") + + +@registry.layers("clone.v1") +def clone(orig: Model[InT, OutT], n: int) -> Model[InT, OutT]: + """Construct `n` copies of a layer, with distinct weights. i.e. + `clone(f, 3)(x)` computes f(f'(f''(x))). + """ + if n == 0: + return cast(Model[InT, OutT], noop()) + elif n == 1: + return orig + layers: List[Model] = [orig] + for i in range(n - 1): + layers.append(orig.copy()) + return cast(Model[InT, OutT], chain(*layers)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/concatenate.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/concatenate.py new file mode 100644 index 0000000000000000000000000000000000000000..80992322c90747fd31c6856f70206a698f73f0e3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/concatenate.py @@ -0,0 +1,155 @@ +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + cast, +) + +from ..backends import NumpyOps +from ..config import registry +from ..model import Model +from ..types import Array2d, Ragged, XY_XY_OutT +from ..util import get_width +from .noop import noop + +NUMPY_OPS = NumpyOps() + + +InT = TypeVar("InT", bound=Any) +OutT = TypeVar("OutT", bound=Union[Array2d, Sequence[Array2d], Ragged]) + + +@registry.layers("concatenate.v1") +def concatenate(*layers: Model) -> Model[InT, XY_XY_OutT]: + """Compose two or more models `f`, `g`, etc, such that their outputs are + concatenated, i.e. `concatenate(f, g)(x)` computes `hstack(f(x), g(x))`. + Also supports chaining more than 2 layers. + """ + if not layers: + return cast(Model[InT, XY_XY_OutT], noop()) + elif len(layers) == 1: + return layers[0] + elif layers[0]._func is forward: + layers[0].layers.extend(layers[1:]) + return layers[0] + + # only add an nI dimension if each sub-layer has one + dims: Dict[str, Optional[int]] = {"nO": None} + if all(node.has_dim("nI") in [True, None] for node in layers): + dims = {"nO": None, "nI": None} + + return Model( + "|".join(layer.name for layer in layers), + forward, + init=init, + dims=dims, + layers=layers, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + Ys, callbacks = zip(*[layer(X, is_train=is_train) for layer in model.layers]) + if isinstance(Ys[0], list): + data_l, backprop = _list_forward(model, X, Ys, callbacks, is_train) + return cast(OutT, data_l), backprop + elif isinstance(Ys[0], Ragged): + data_r, backprop = _ragged_forward(model, X, Ys, callbacks, is_train) + return cast(OutT, data_r), backprop + else: + data_a, backprop = _array_forward(model, X, Ys, callbacks, is_train) + return cast(OutT, data_a), backprop + + +def _array_forward( + model: Model[InT, OutT], X, Ys: List, callbacks, is_train: bool +) -> Tuple[Array2d, Callable]: + widths = [Y.shape[1] for Y in Ys] + output = model.ops.xp.hstack(Ys) + + def backprop(d_output: Array2d) -> InT: + dY = model.ops.as_contig(d_output[:, : widths[0]]) + dX = callbacks[0](dY) + start = widths[0] + add_gradients = hasattr(dX, "__add__") or hasattr(dX, "__iadd__") + add_gradients_data = hasattr(dX, "data") and ( + hasattr(dX.data, "__add__") or hasattr(dX.data, "__iadd__") + ) + for bwd, width in zip(callbacks[1:], widths[1:]): + dY = model.ops.as_contig(d_output[:, start : start + width]) + gradient = bwd(dY) + if add_gradients: + dX += gradient + elif add_gradients_data: + dX.data += gradient.data + start += width + return dX + + return output, backprop + + +def _ragged_forward( + model: Model[InT, OutT], X, Ys: List, callbacks, is_train: bool +) -> Tuple[Ragged, Callable]: + + widths = [Y.dataXd.shape[1] for Y in Ys] + output = Ragged(model.ops.xp.hstack([y.data for y in Ys]), Ys[0].lengths) + + def backprop(d_output: Ragged) -> InT: + d_array = d_output.data + dY = Ragged(model.ops.as_contig(d_array[:, : widths[0]]), d_output.lengths) + dX = callbacks[0](dY) + start = widths[0] + for bwd, width in zip(callbacks[1:], widths[1:]): + dY = Ragged( + model.ops.as_contig(d_array[:, start : start + width]), d_output.lengths + ) + dX += bwd(dY) + start += width + return dX + + return output, backprop + + +def _list_forward( + model: Model[InT, OutT], X, Ys: List, callbacks, is_train: bool +) -> Tuple[Sequence[Array2d], Callable]: + def backprop(d_output: Sequence[Array2d]) -> InT: + d_out_array = model.ops.xp.concatenate(d_output, axis=0) + dY = model.ops.as_contig(d_out_array[:, : widths[0]]) + # We want to generalize unflatten later. + dY = model.ops.unflatten(dY, lengths) + dX = callbacks[0](dY) + start = widths[0] + for bwd, width in zip(callbacks[1:], widths[1:]): + dY = model.ops.as_contig(d_out_array[:, start : start + width]) + dY = model.ops.unflatten(dY, lengths) + dX += bwd(dY) + start += width + return dX + + lengths = NUMPY_OPS.asarray1i([len(x) for x in X]) + Ys = [model.ops.xp.concatenate(Y, axis=0) for Y in Ys] + widths = [Y.shape[1] for Y in Ys] + out_array = model.ops.xp.hstack(Ys) + return model.ops.unflatten(out_array, lengths), backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + if model.has_dim("nI") is not False: + model.set_dim("nI", get_width(X)) + for layer in model.layers: + if layer.has_dim("nI") is not False: + layer.set_dim("nI", get_width(X)) + for layer in model.layers: + layer.initialize(X=X, Y=Y) + if all([layer.has_dim("nO") for layer in model.layers]): + model.set_dim("nO", sum(layer.get_dim("nO") for layer in model.layers)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dish.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dish.py new file mode 100644 index 0000000000000000000000000000000000000000..576626cf54cbc989b6c63733e773d41ae36f329d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dish.py @@ -0,0 +1,70 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import he_normal_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("Dish.v1") +def Dish( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = he_normal_init + if init_b is None: + init_b = zero_init + model: Model[Floats2d, Floats2d] = Model( + "dish", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool +) -> Tuple[Floats2d, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.dish(Y_preact) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_dish(dY, X, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dropout.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dropout.py new file mode 100644 index 0000000000000000000000000000000000000000..e1da64f76ed041d84dabbfd5461d048832aa6672 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/dropout.py @@ -0,0 +1,87 @@ +from typing import Callable, List, Sequence, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..model import Model +from ..types import ArrayXd, Padded, Ragged + +InT = TypeVar("InT", bound=Union[ArrayXd, Sequence[ArrayXd], Ragged, Padded]) + + +@registry.layers("Dropout.v1") +def Dropout(rate: float = 0.0) -> Model[InT, InT]: + """Help prevent overfitting by adding a random distortion to the input data + during training. Specifically, cells of the input are zeroed with + probability determined by the `rate` argument. + """ + return Model("dropout", forward, attrs={"dropout_rate": rate, "is_enabled": True}) + + +def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: + rate = model.attrs["dropout_rate"] + is_enabled = model.attrs["is_enabled"] and is_train + if rate == 0 or not is_enabled: + return X, lambda dY: dY + elif isinstance(X, Ragged): + data_r, backprop = _dropout_ragged(model, X, is_train) + return cast(InT, data_r), backprop + elif isinstance(X, Padded): + data_p, backprop = _dropout_padded(model, X, is_train) + return cast(InT, data_p), backprop + elif isinstance(X, Sequence): + data_l, backprop = _dropout_lists(model, X, is_train) + return cast(InT, data_l), backprop + else: + data_a, backprop = _dropout_array(model, cast(ArrayXd, X), is_train) + return cast(InT, data_a), backprop + + +def _dropout_array( + model: Model[InT, InT], X: ArrayXd, is_train: bool +) -> Tuple[ArrayXd, Callable]: + rate = model.attrs["dropout_rate"] + mask = model.ops.get_dropout_mask(X.shape, rate) + + def backprop(dY: ArrayXd) -> ArrayXd: + return dY * mask + + return cast(ArrayXd, X * mask), backprop + + +def _dropout_padded( + model: Model[InT, InT], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + X = Xp.data + mask = model.ops.get_dropout_mask(X.shape, model.attrs["dropout_rate"]) + Y = X * mask + + def backprop(dYp: Padded) -> Padded: + return Padded(dYp.data * mask, dYp.size_at_t, dYp.lengths, dYp.indices) + + return Padded(Y, Xp.size_at_t, Xp.lengths, Xp.indices), backprop + + +def _dropout_ragged( + model: Model[InT, InT], Xr: Ragged, is_train: bool +) -> Tuple[Ragged, Callable]: + X = Xr.data + lengths = Xr.lengths + mask = model.ops.get_dropout_mask(X.shape, model.attrs["dropout_rate"]) + Y = X * mask + + def backprop(dYr: Ragged) -> Ragged: + return Ragged(dYr.data * mask, dYr.lengths) + + return Ragged(Y, lengths), backprop + + +def _dropout_lists( + model: Model[InT, InT], Xs: Sequence[ArrayXd], is_train: bool +) -> Tuple[Sequence[ArrayXd], Callable]: + rate = model.attrs["dropout_rate"] + masks = [model.ops.get_dropout_mask(X.shape, rate) for X in Xs] + Ys = [X * mask for X, mask in zip(Xs, masks)] + + def backprop(dYs: List[ArrayXd]) -> List[ArrayXd]: + return [dY * mask for dY, mask in zip(dYs, masks)] + + return Ys, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/embed.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/embed.py new file mode 100644 index 0000000000000000000000000000000000000000..60bf951cb9005fbc69ad7a49dfd278a5278274b7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/embed.py @@ -0,0 +1,84 @@ +from typing import Callable, Dict, Optional, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..initializers import uniform_init +from ..model import Model +from ..types import Floats1d, Floats2d, Ints1d, Ints2d +from ..util import get_width, partial +from .array_getitem import ints_getitem +from .chain import chain + +InT = TypeVar("InT", bound=Union[Ints1d, Ints2d]) +OutT = Floats2d + + +@registry.layers("Embed.v1") +def Embed( + nO: Optional[int] = None, + nV: Optional[int] = None, + *, + column: Optional[int] = None, + initializer: Optional[Callable] = None, + dropout: Optional[float] = None +) -> Model[InT, OutT]: + """Map integers to vectors, using a fixed-size lookup table.""" + attrs: Dict[str, Union[None, int, float]] = {} + if initializer is None: + initializer = uniform_init + if dropout is not None: + attrs["dropout_rate"] = dropout + model: Model = Model( + "embed", + forward, + init=partial(init, initializer), + attrs=attrs, + dims={"nO": nO, "nV": nV}, + params={"E": None}, + ) + if column is not None: + # This is equivalent to array[:, column]. What you're actually doing + # there is passing in a tuple: array[(:, column)], except in the context + # of array indexing, the ":" creates an object slice(0, None). + # So array[:, column] is array.__getitem__(slice(0), column). + model = chain(ints_getitem((slice(0, None), column)), model) + model.attrs["column"] = column + return cast(Model[InT, OutT], model) + + +def forward( + model: Model[Ints1d, OutT], ids: Ints1d, is_train: bool +) -> Tuple[OutT, Callable]: + vectors = cast(Floats2d, model.get_param("E")) + nO = vectors.shape[1] + nN = ids.shape[0] + dropout: Optional[float] = model.attrs.get("dropout_rate") + output = vectors[ids] + drop_mask = None + if is_train: + drop_mask = cast(Floats1d, model.ops.get_dropout_mask((nO,), dropout)) + if drop_mask is not None: + output *= drop_mask + + def backprop(d_output: OutT) -> Ints1d: + if drop_mask is not None: + d_output *= drop_mask + d_vectors = model.ops.alloc2f(*vectors.shape) + # Does a loop where we do d_vectors[i] += d_output[ids[i]] + model.ops.scatter_add(d_vectors, ids, d_output) + model.inc_grad("E", d_vectors) + dX = model.ops.alloc1i(nN) + return dX + + return output, backprop + + +def init( + initializer: Callable, + model: Model[Ints1d, OutT], + X: Optional[Ints1d] = None, + Y: Optional[OutT] = None, +) -> None: + if Y is not None: + model.set_dim("nO", get_width(Y)) + shape = (model.get_dim("nV"), model.get_dim("nO")) + model.set_param("E", initializer(model.ops, shape)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/expand_window.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/expand_window.py new file mode 100644 index 0000000000000000000000000000000000000000..2b872470c593cf554e11025514d2c010b9e06607 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/expand_window.py @@ -0,0 +1,57 @@ +from typing import Callable, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged + +InT = TypeVar("InT", Floats2d, Ragged) + + +@registry.layers("expand_window.v1") +def expand_window(window_size: int = 1) -> Model[InT, InT]: + """For each vector in an input, construct an output vector that contains the + input and a window of surrounding vectors. This is one step in a convolution. + """ + return Model("expand_window", forward, attrs={"window_size": window_size}) + + +def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: + if isinstance(X, Ragged): + return _expand_window_ragged(model, X) + else: + return _expand_window_floats(model, X) + + +def _expand_window_floats( + model: Model[InT, InT], X: Floats2d +) -> Tuple[Floats2d, Callable]: + nW = model.attrs["window_size"] + if len(X) > 0: + Y = model.ops.seq2col(X, nW) + else: + assert len(X) == 0 + Y = model.ops.tile(X, (nW * 2) + 1) + + def backprop(dY: Floats2d) -> Floats2d: + return model.ops.backprop_seq2col(dY, nW) + + return Y, backprop + + +def _expand_window_ragged( + model: Model[InT, InT], Xr: Ragged +) -> Tuple[Ragged, Callable]: + nW = model.attrs["window_size"] + Y = Ragged( + model.ops.seq2col(cast(Floats2d, Xr.data), nW, lengths=Xr.lengths), Xr.lengths + ) + + def backprop(dYr: Ragged) -> Ragged: + return Ragged( + model.ops.backprop_seq2col( + cast(Floats2d, dYr.data), nW, lengths=Xr.lengths + ), + Xr.lengths, + ) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/gelu.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/gelu.py new file mode 100644 index 0000000000000000000000000000000000000000..cd6353e8c57cbd4b71dbc31321ce16ada71e50a7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/gelu.py @@ -0,0 +1,70 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import he_normal_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("Gelu.v1") +def Gelu( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = he_normal_init + if init_b is None: + init_b = zero_init + model: Model[Floats2d, Floats2d] = Model( + "gelu", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool +) -> Tuple[Floats2d, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.gelu(Y_preact) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_gelu(dY, Y_preact, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish.py new file mode 100644 index 0000000000000000000000000000000000000000..f83780520adcc0c2172092ab4849d2f6b687c08b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish.py @@ -0,0 +1,70 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import he_normal_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("HardSwish.v1") +def HardSwish( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = he_normal_init + if init_b is None: + init_b = zero_init + model: Model[Floats2d, Floats2d] = Model( + "hardswish", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool +) -> Tuple[Floats2d, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.hard_swish(Y_preact) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_hard_swish(dY, Y_preact, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish_mobilenet.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish_mobilenet.py new file mode 100644 index 0000000000000000000000000000000000000000..a5c029cc2ef429fe8ab8acccc05ef5327f76ec1f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hard_swish_mobilenet.py @@ -0,0 +1,70 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import he_normal_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("HardSwishMobilenet.v1") +def HardSwishMobilenet( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = he_normal_init + if init_b is None: + init_b = zero_init + model: Model[Floats2d, Floats2d] = Model( + "hardswishmobilenet", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool +) -> Tuple[Floats2d, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.hard_swish_mobilenet(Y_preact) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_hard_swish_mobilenet(dY, Y_preact, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hashembed.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hashembed.py new file mode 100644 index 0000000000000000000000000000000000000000..b14586fb2d7fb343c0f7d145c04db792f101271f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/hashembed.py @@ -0,0 +1,101 @@ +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..initializers import uniform_init +from ..model import Model +from ..types import Floats1d, Floats2d, Ints1d, Ints2d +from ..util import partial +from .array_getitem import ints_getitem +from .chain import chain + +InT = TypeVar("InT", bound=Union[Ints1d, Ints2d]) +OutT = Floats2d + + +@registry.layers("HashEmbed.v1") +def HashEmbed( + nO: int, + nV: int, + *, + seed: Optional[int] = None, + column: Optional[int] = None, + initializer: Optional[Callable] = None, + dropout: Optional[float] = None +) -> Model[InT, OutT]: + """ + An embedding layer that uses the “hashing trick” to map keys to distinct values. + The hashing trick involves hashing each key four times with distinct seeds, + to produce four likely differing values. Those values are modded into the + table, and the resulting vectors summed to produce a single result. Because + it’s unlikely that two different keys will collide on all four “buckets”, + most distinct keys will receive a distinct vector under this scheme, even + when the number of vectors in the table is very low. + """ + attrs: Dict[str, Any] = {"column": column, "seed": seed} + if initializer is None: + initializer = uniform_init + if dropout is not None: + attrs["dropout_rate"] = dropout + model: Model = Model( + "hashembed", + forward, + init=partial(init, initializer), + params={"E": None}, + dims={"nO": nO, "nV": nV, "nI": None}, + attrs=attrs, + ) + if seed is None: + model.attrs["seed"] = model.id + if column is not None: + # This is equivalent to array[:, column]. What you're actually doing + # there is passing in a tuple: array[(:, column)], except in the context + # of array indexing, the ":" creates an object slice(0, None). + # So array[:, column] is array.__getitem__(slice(0), column). + model = chain(ints_getitem((slice(0, None), column)), model) + model.attrs["column"] = column + return cast(Model[InT, OutT], model) + + +def forward( + model: Model[Ints1d, OutT], ids: Ints1d, is_train: bool +) -> Tuple[OutT, Callable]: + vectors = cast(Floats2d, model.get_param("E")) + nV = vectors.shape[0] + nO = vectors.shape[1] + if len(ids) == 0: + output: Floats2d = model.ops.alloc2f(0, nO, dtype=vectors.dtype) + else: + ids = model.ops.as_contig(ids, dtype="uint64") + nN = ids.shape[0] + seed: int = model.attrs["seed"] + keys = model.ops.hash(ids, seed) % nV + output = model.ops.gather_add(vectors, keys) + drop_mask = None + if is_train: + dropout: Optional[float] = model.attrs.get("dropout_rate") + drop_mask = cast(Floats1d, model.ops.get_dropout_mask((nO,), dropout)) + if drop_mask is not None: + output *= drop_mask + + def backprop(d_vectors: OutT) -> Ints1d: + if drop_mask is not None: + d_vectors *= drop_mask + dE = model.ops.alloc2f(*vectors.shape) + keysT = model.ops.as_contig(keys.T, dtype="i") + for i in range(keysT.shape[0]): + model.ops.scatter_add(dE, keysT[i], d_vectors) + model.inc_grad("E", dE) + dX = model.ops.alloc1i(nN) + return dX + + return output, backprop + + +def init( + initializer: Callable, + model: Model[Ints1d, OutT], + X: Optional[Ints1d] = None, + Y: Optional[OutT] = None, +) -> None: + E = initializer(model.ops, (model.get_dim("nV"), model.get_dim("nO"))) + model.set_param("E", E) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/layernorm.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/layernorm.py new file mode 100644 index 0000000000000000000000000000000000000000..c75af1fd656ce345513febc8592f80b1d1634f38 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/layernorm.py @@ -0,0 +1,87 @@ +from typing import Callable, Optional, Tuple, cast + +from ..backends import Ops +from ..config import registry +from ..model import Model +from ..types import Floats2d +from ..util import get_width + +InT = Floats2d + + +@registry.layers("LayerNorm.v1") +def LayerNorm(nI: Optional[int] = None) -> Model[InT, InT]: + return Model( + "layernorm", + forward, + init=init, + dims={"nI": nI, "nO": nI}, + params={"G": None, "b": None}, + ) + + +def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: + N, mu, var = _get_moments(model.ops, X) + Xhat = (X - mu) * var ** (-1.0 / 2.0) + Y, backprop_rescale = _begin_update_scale_shift(model, Xhat) + + def backprop(dY: InT) -> InT: + dY = backprop_rescale(dY) + dist, sum_dy, sum_dy_dist = _get_d_moments(model.ops, dY, X, mu) + d_xhat = N * dY - sum_dy - dist * var ** (-1.0) * sum_dy_dist + d_xhat *= var ** (-1.0 / 2) + d_xhat /= N + return d_xhat + + return Y, backprop + + +def init( + model: Model[InT, InT], X: Optional[InT] = None, Y: Optional[InT] = None +) -> None: + if X is not None: + X_width = get_width(X) + model.set_dim("nI", X_width) + model.set_dim("nO", X_width) + elif Y is not None: + Y_width = get_width(Y) + model.set_dim("nI", Y_width) + model.set_dim("nO", Y_width) + nI = model.get_dim("nI") + if not model.has_dim("nO"): + model.set_dim("nO", nI) + model.set_param("G", model.ops.alloc1f(nI) + 1) + model.set_param("b", model.ops.alloc1f(nI)) + assert model.get_dim("nO") is not None + + +def _begin_update_scale_shift(model: Model[InT, InT], X: InT) -> Tuple[InT, Callable]: + G = model.get_param("G") + b = model.get_param("b") + Y = X * G + Y += b + + def finish_update_scale_shift(dY: InT) -> InT: + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("G", (dY * X).sum(axis=0)) + return dY * G + + return Y, finish_update_scale_shift + + +def _get_moments(ops: Ops, X: Floats2d) -> Tuple[Floats2d, Floats2d, Floats2d]: + # TODO: Do mean methods + mu: Floats2d = X.mean(axis=1, keepdims=True) + var: Floats2d = X.var(axis=1, keepdims=True) + 1e-08 + return cast(Floats2d, ops.asarray_f([X.shape[1]])), mu, var + + +def _get_d_moments( + ops: Ops, dy: Floats2d, X: Floats2d, mu: Floats2d +) -> Tuple[Floats2d, Floats2d, Floats2d]: + dist = X - mu + return ( + dist, + ops.xp.sum(dy, axis=1, keepdims=True), + ops.xp.sum(dy * dist, axis=1, keepdims=True), + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/linear.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/linear.py new file mode 100644 index 0000000000000000000000000000000000000000..1a69f276bd26d71dcc8118e5f77e4bf4ead6873c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/linear.py @@ -0,0 +1,61 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Linear.v1") +def Linear( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, +) -> Model[InT, OutT]: + """Multiply inputs by a weights matrix and adds a bias vector.""" + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + return Model( + "linear", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y = model.ops.gemm(X, W, trans2=True) + Y += b + + def backprop(dY: OutT) -> InT: + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2array.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2array.py new file mode 100644 index 0000000000000000000000000000000000000000..8ccc371dec5e1487bdc006fde45e607263b37abe --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2array.py @@ -0,0 +1,30 @@ +from typing import Callable, List, Tuple, TypeVar + +from ..backends import NumpyOps +from ..config import registry +from ..model import Model +from ..types import Array2d + +NUMPY_OPS = NumpyOps() + + +OutT = TypeVar("OutT", bound=Array2d) +InT = List[OutT] + + +@registry.layers("list2array.v1") +def list2array() -> Model[InT, OutT]: + """Transform sequences to ragged arrays if necessary and return the data + from the ragged array. If sequences are already ragged, do nothing. A + ragged array is a tuple (data, lengths), where data is the concatenated data. + """ + return Model("list2array", forward) + + +def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: + lengths = NUMPY_OPS.asarray1i([len(x) for x in Xs]) + + def backprop(dY: OutT) -> InT: + return model.ops.unflatten(dY, lengths) + + return model.ops.flatten(Xs), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2padded.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2padded.py new file mode 100644 index 0000000000000000000000000000000000000000..014ad3ee0087542cd226564a53792640d9e72c04 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2padded.py @@ -0,0 +1,23 @@ +from typing import Callable, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import List2d, Padded + +InT = TypeVar("InT", bound=List2d) +OutT = Padded + + +@registry.layers("list2padded.v1") +def list2padded() -> Model[InT, OutT]: + """Create a layer to convert a list of array inputs into Padded.""" + return Model(f"list2padded", forward) + + +def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: + Yp = model.ops.list2padded(Xs) + + def backprop(dYp: OutT) -> InT: + return cast(InT, model.ops.padded2list(dYp)) + + return Yp, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2ragged.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2ragged.py new file mode 100644 index 0000000000000000000000000000000000000000..c7e33144d4f0d65131b5f19d7a4eb39022e1a4b5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/list2ragged.py @@ -0,0 +1,25 @@ +from typing import Callable, List, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import ArrayXd, ListXd, Ragged + +InT = TypeVar("InT", bound=ListXd) +OutT = Ragged + + +@registry.layers("list2ragged.v1") +def list2ragged() -> Model[InT, OutT]: + """Transform sequences to ragged arrays if necessary and return the ragged + array. If sequences are already ragged, do nothing. A ragged array is a + tuple (data, lengths), where data is the concatenated data. + """ + return Model("list2ragged", forward) + + +def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: + def backprop(dYr: OutT) -> InT: + return cast(InT, model.ops.unflatten(dYr.data, dYr.lengths)) + + lengths = model.ops.asarray1i([len(x) for x in Xs]) + return Ragged(model.ops.flatten(Xs), lengths), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/logistic.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/logistic.py new file mode 100644 index 0000000000000000000000000000000000000000..2bafcd09debeb2e5ffb875c1b28a09e5efb10d20 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/logistic.py @@ -0,0 +1,25 @@ +from typing import Callable, Tuple + +from ..config import registry +from ..model import Model +from ..types import Floats2d + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Logistic.v1") +def Logistic() -> Model[InT, OutT]: + """Deprecated in favor of `sigmoid_activation` layer, for more consistent + naming. + """ + return Model("logistic", forward) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + Y = model.ops.sigmoid(X, inplace=False) + + def backprop(dY: OutT) -> InT: + return dY * model.ops.dsigmoid(Y, inplace=False) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/lstm.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/lstm.py new file mode 100644 index 0000000000000000000000000000000000000000..fb1a8077388fdcf862c78e9b596bc672e95a7ada --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/lstm.py @@ -0,0 +1,182 @@ +from functools import partial +from typing import Callable, Optional, Tuple, cast + +from ..backends import Ops +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d, Floats4d, Padded, Ragged +from ..util import get_width +from .noop import noop + + +@registry.layers("LSTM.v1") +def LSTM( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + bi: bool = False, + depth: int = 1, + dropout: float = 0.0, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, +) -> Model[Padded, Padded]: + if depth == 0: + msg = "LSTM depth must be at least 1. Maybe we should make this a noop?" + raise ValueError(msg) + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + + model: Model[Padded, Padded] = Model( + "lstm", + forward, + dims={"nO": nO, "nI": nI, "depth": depth, "dirs": 1 + int(bi)}, + attrs={"registry_name": "LSTM.v1", "dropout_rate": dropout}, + params={"LSTM": None, "HC0": None}, + init=partial(init, init_W, init_b), + ) + return model + + +@registry.layers("PyTorchLSTM.v1") +def PyTorchLSTM( + nO: int, nI: int, *, bi: bool = False, depth: int = 1, dropout: float = 0.0 +) -> Model[Padded, Padded]: + import torch.nn + + from .pytorchwrapper import PyTorchRNNWrapper + from .with_padded import with_padded + + if depth == 0: + return noop() # type: ignore[misc] + nH = nO + if bi: + nH = nO // 2 + pytorch_rnn = PyTorchRNNWrapper( + torch.nn.LSTM(nI, nH, depth, bidirectional=bi, dropout=dropout) + ) + pytorch_rnn.set_dim("nO", nO) + pytorch_rnn.set_dim("nI", nI) + return with_padded(pytorch_rnn) + + +def init( + init_W: Callable, + init_b: Callable, + model: Model, + X: Optional[Padded] = None, + Y: Optional[Padded] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + nH = int(model.get_dim("nO") / model.get_dim("dirs")) + nI = model.get_dim("nI") + depth = model.get_dim("depth") + dirs = model.get_dim("dirs") + # It's easiest to use the initializer if we alloc the weights separately + # and then stick them all together afterwards. The order matters here: + # we need to keep the same format that CuDNN expects. + params = [] + # Convenience + init_W = partial(init_W, model.ops) + init_b = partial(init_b, model.ops) + layer_nI = nI + for i in range(depth): + for j in range(dirs): + # Input-to-gates weights and biases. + params.append(init_W((nH, layer_nI))) + params.append(init_W((nH, layer_nI))) + params.append(init_W((nH, layer_nI))) + params.append(init_W((nH, layer_nI))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + # Hidden-to-gates weights and biases + params.append(init_W((nH, nH))) + params.append(init_W((nH, nH))) + params.append(init_W((nH, nH))) + params.append(init_W((nH, nH))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + params.append(init_b((nH,))) + layer_nI = nH * dirs + model.set_param("LSTM", model.ops.xp.concatenate([p.ravel() for p in params])) + model.set_param("HC0", zero_init(model.ops, (2, depth, dirs, nH))) + size = model.get_param("LSTM").size + expected = 4 * dirs * nH * (nH + nI) + dirs * (8 * nH) + for _ in range(1, depth): + expected += 4 * dirs * (nH + nH * dirs) * nH + dirs * (8 * nH) + assert size == expected, (size, expected) + + +def forward( + model: Model[Padded, Padded], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + dropout = model.attrs["dropout_rate"] + Xr = _padded_to_packed(model.ops, Xp) + LSTM = cast(Floats1d, model.get_param("LSTM")) + HC0 = cast(Floats4d, model.get_param("HC0")) + H0 = HC0[0] + C0 = HC0[1] + if is_train: + # Apply dropout over *weights*, not *activations*. RNN activations are + # heavily correlated, so dropout over the activations is less effective. + # This trick was explained in Yarin Gal's thesis, and popularised by + # Smerity in the AWD-LSTM. It also means we can do the dropout outside + # of the backend, improving compatibility. + mask = cast(Floats1d, model.ops.get_dropout_mask(LSTM.shape, dropout)) + LSTM = LSTM * mask + Y, fwd_state = model.ops.lstm_forward_training( + LSTM, H0, C0, cast(Floats2d, Xr.data), Xr.lengths + ) + else: + Y = model.ops.lstm_forward_inference( + LSTM, H0, C0, cast(Floats2d, Xr.data), Xr.lengths + ) + fwd_state = tuple() + assert Y.shape == (Xr.data.shape[0], Y.shape[1]), (Xr.data.shape, Y.shape) + Yp = _packed_to_padded(model.ops, Ragged(Y, Xr.lengths), Xp) + + def backprop(dYp: Padded) -> Padded: + assert fwd_state + dYr = _padded_to_packed(model.ops, dYp) + dX, dLSTM = model.ops.backprop_lstm( + cast(Floats2d, dYr.data), dYr.lengths, LSTM, fwd_state + ) + dLSTM *= mask + model.inc_grad("LSTM", dLSTM) + return _packed_to_padded(model.ops, Ragged(dX, dYr.lengths), dYp) + + return Yp, backprop + + +def _padded_to_packed(ops: Ops, Xp: Padded) -> Ragged: + """Strip padding from a padded sequence.""" + assert Xp.lengths.sum() == Xp.size_at_t.sum(), ( + Xp.lengths.sum(), + Xp.size_at_t.sum(), + ) + Y = ops.alloc2f(Xp.lengths.sum(), Xp.data.shape[2]) + start = 0 + for t in range(Xp.size_at_t.shape[0]): + batch_size = Xp.size_at_t[t] + Y[start : start + batch_size] = Xp.data[t, :batch_size] # type: ignore[assignment] + start += batch_size + return Ragged(Y, Xp.size_at_t) + + +def _packed_to_padded(ops: Ops, Xr: Ragged, Xp: Padded) -> Padded: + Y = ops.alloc3f(Xp.data.shape[0], Xp.data.shape[1], Xr.data.shape[1]) + X = cast(Floats2d, Xr.data) + start = 0 + for t in range(Xp.size_at_t.shape[0]): + batch_size = Xp.size_at_t[t] + Y[t, :batch_size] = X[start : start + batch_size] + start += batch_size + return Padded(Y, size_at_t=Xp.size_at_t, lengths=Xp.lengths, indices=Xp.indices) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/map_list.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/map_list.py new file mode 100644 index 0000000000000000000000000000000000000000..b875424574fc8d97f2cfb7126ea3a511606749b6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/map_list.py @@ -0,0 +1,36 @@ +from typing import Callable, List, Optional, Tuple, TypeVar + +from ..model import Model + +InT = TypeVar("InT") +OutT = TypeVar("OutT") + + +def map_list(layer: Model[InT, OutT]) -> Model[List[InT], List[OutT]]: + """Create a model that maps a child layer across list inputs.""" + return Model("map_list", forward, layers=[layer], init=init) + + +def forward( + model: Model[List[InT], List[OutT]], Xs: List[InT], is_train: bool +) -> Tuple[List[OutT], Callable[[List[OutT]], List[InT]]]: + layer = model.layers[0] + Ys = [] + callbacks = [] + for X in Xs: + Y, get_dX = layer(X, is_train) + Ys.append(Y) + callbacks.append(get_dX) + + def backprop_map_list(dYs: List[OutT]) -> List[InT]: + return [callback(dY) for callback, dY in zip(callbacks, dYs)] + + return Ys, backprop_map_list + + +def init( + model: Model[List[InT], List[OutT]], + X: Optional[List[InT]] = None, + Y: Optional[List[OutT]] = None, +) -> None: + model.layers[0].initialize(X=X[0] if X else None, Y=Y[0] if Y else None) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/maxout.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/maxout.py new file mode 100644 index 0000000000000000000000000000000000000000..8703347643d3809f07e75c77c44a2cd849e1222a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/maxout.py @@ -0,0 +1,82 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Maxout.v1") +def Maxout( + nO: Optional[int] = None, + nI: Optional[int] = None, + nP: Optional[int] = 3, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[InT, OutT]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + model: Model[InT, OutT] = Model( + "maxout", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI, "nP": nP}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[InT, OutT], Dropout(dropout))) + return model + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + nO = model.get_dim("nO") + nP = model.get_dim("nP") + nI = model.get_dim("nI") + b = model.get_param("b") + W = model.get_param("W") + W = model.ops.reshape2f(W, nO * nP, nI) + Y = model.ops.gemm(X, W, trans2=True) + Y += model.ops.reshape1f(b, nO * nP) + Z = model.ops.reshape3f(Y, Y.shape[0], nO, nP) + best, which = model.ops.maxout(Z) + + def backprop(d_best: OutT) -> InT: + dZ = model.ops.backprop_maxout(d_best, which, nP) + # TODO: Add sum methods for Floats3d + model.inc_grad("b", dZ.sum(axis=0)) # type: ignore[call-overload] + dY = model.ops.reshape2f(dZ, dZ.shape[0], nO * nP) + dW = model.ops.reshape3f(model.ops.gemm(dY, X, trans1=True), nO, nP, nI) + model.inc_grad("W", dW) + return model.ops.gemm(dY, model.ops.reshape2f(W, nO * nP, nI)) + + return best, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + W_shape = (model.get_dim("nO"), model.get_dim("nP"), model.get_dim("nI")) + model.set_param("W", init_W(model.ops, W_shape)) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"), model.get_dim("nP")))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mish.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mish.py new file mode 100644 index 0000000000000000000000000000000000000000..c66478691573ef32c60d1b80c4c105535debe18d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mish.py @@ -0,0 +1,76 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Mish.v1") +def Mish( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[InT, OutT]: + """Dense layer with mish activation. + https://arxiv.org/pdf/1908.08681.pdf + """ + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + model: Model[InT, OutT] = Model( + "mish", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, cast(Model[InT, OutT], LayerNorm(nI=nO))) + if dropout is not None: + model = chain(model, cast(Model[InT, OutT], Dropout(dropout))) + return model + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_pre_mish = model.ops.gemm(X, W, trans2=True) + Y_pre_mish += b + Y = model.ops.mish(Y_pre_mish) + + def backprop(dY: OutT) -> InT: + dY_pre_mish = model.ops.backprop_mish(dY, Y_pre_mish) + model.inc_grad("W", model.ops.gemm(dY_pre_mish, X, trans1=True)) + model.inc_grad("b", dY_pre_mish.sum(axis=0)) + dX = model.ops.gemm(dY_pre_mish, W) + return dX + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/multisoftmax.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/multisoftmax.py new file mode 100644 index 0000000000000000000000000000000000000000..8f22dbd65decef872a14450584e5acceee04514d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/multisoftmax.py @@ -0,0 +1,56 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("MultiSoftmax.v1") +def MultiSoftmax(nOs: Tuple[int, ...], nI: Optional[int] = None) -> Model[InT, OutT]: + """Neural network layer that predicts several multi-class attributes at once. + For instance, we might predict one class with 6 variables, and another with 5. + We predict the 11 neurons required for this, and then softmax them such + that columns 0-6 make a probability distribution and columns 6-11 make another. + """ + return Model( + "multisoftmax", + forward, + init=init, + dims={"nO": sum(nOs), "nI": nI}, + attrs={"nOs": nOs}, + params={"W": None, "b": None}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + nOs = model.attrs["nOs"] + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + + def backprop(dY: OutT) -> InT: + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + model.inc_grad("b", dY.sum(axis=0)) + return model.ops.gemm(dY, W) + + Y = model.ops.gemm(X, W, trans2=True) + Y += b + i = 0 + for out_size in nOs: + model.ops.softmax(Y[:, i : i + out_size], inplace=True) + i += out_size + return Y, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + nO = model.get_dim("nO") + nI = model.get_dim("nI") + model.set_param("W", model.ops.alloc2f(nO, nI)) + model.set_param("b", model.ops.alloc1f(nO)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mxnetwrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mxnetwrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..2d60a56e9f6c60272504ea651d8fb47510c5eb38 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/mxnetwrapper.py @@ -0,0 +1,116 @@ +from typing import Any, Callable, Optional, Tuple, Type + +from ..config import registry +from ..model import Model +from ..shims import MXNetShim +from ..types import ArgsKwargs +from ..util import convert_recursive, is_mxnet_array, is_xp_array, mxnet2xp, xp2mxnet + + +@registry.layers("MXNetWrapper.v1") +def MXNetWrapper( + mxnet_model, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, + model_class: Type[Model] = Model, + model_name: str = "mxnet", +) -> Model[Any, Any]: + """Wrap a MXNet model, so that it has the same API as Thinc models. + To optimize the model, you'll need to create a MXNet optimizer and call + optimizer.step() after each batch. + + Your MXNet model's forward method can take arbitrary args and kwargs, + but must return either a single tensor as output or a tuple. You may find the + MXNet register_forward_hook helpful if you need to adapt the output. + + The convert functions are used to map inputs and outputs to and from your + MXNet model. Each function should return the converted output, and a callback + to use during the backward pass. So: + + Xmxnet, get_dX = convert_inputs(X) + Ymxnet, mxnet_backprop = model.shims[0](Xmxnet, is_train) + Y, get_dYmxnet = convert_outputs(Ymxnet) + + To allow maximum flexibility, the MXNetShim expects ArgsKwargs objects + on the way into the forward and backward passes. The ArgsKwargs objects + will be passed straight into the model in the forward pass, and straight + into `mxnet.autograd.backward` during the backward pass. + """ + if convert_inputs is None: + convert_inputs = convert_mxnet_default_inputs + if convert_outputs is None: + convert_outputs = convert_mxnet_default_outputs + return model_class( + model_name, + forward, + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + shims=[MXNetShim(mxnet_model)], + ) + + +def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + """Return the output of the wrapped MXNet model for the given input, + along with a callback to handle the backward pass. + """ + convert_inputs = model.attrs["convert_inputs"] + convert_outputs = model.attrs["convert_outputs"] + + Xmxnet, get_dX = convert_inputs(model, X, is_train) + Ymxnet, mxnet_backprop = model.shims[0](Xmxnet, is_train) + Y, get_dYmxnet = convert_outputs(model, (X, Ymxnet), is_train) + + def backprop(dY: Any) -> Any: + dYmxnet = get_dYmxnet(dY) + dXmxnet = mxnet_backprop(dYmxnet) + dX = get_dX(dXmxnet) + return dX + + return Y, backprop + + +# Default conversion functions + + +def convert_mxnet_default_inputs( + model: Model, X: Any, is_train: bool +) -> Tuple[ArgsKwargs, Callable[[ArgsKwargs], Any]]: + xp2mxnet_ = lambda x: xp2mxnet(x, requires_grad=is_train) + converted = convert_recursive(is_xp_array, xp2mxnet_, X) + if isinstance(converted, ArgsKwargs): + + def reverse_conversion(dXmxnet): + return convert_recursive(is_mxnet_array, mxnet2xp, dXmxnet) + + return converted, reverse_conversion + elif isinstance(converted, dict): + + def reverse_conversion(dXmxnet): + dX = convert_recursive(is_mxnet_array, mxnet2xp, dXmxnet) + return dX.kwargs + + return ArgsKwargs(args=tuple(), kwargs=converted), reverse_conversion + elif isinstance(converted, (tuple, list)): + + def reverse_conversion(dXmxnet): + dX = convert_recursive(is_mxnet_array, mxnet2xp, dXmxnet) + return dX.args + + return ArgsKwargs(args=tuple(converted), kwargs={}), reverse_conversion + else: + + def reverse_conversion(dXmxnet): + dX = convert_recursive(is_mxnet_array, mxnet2xp, dXmxnet) + return dX.args[0] + + return ArgsKwargs(args=(converted,), kwargs={}), reverse_conversion + + +def convert_mxnet_default_outputs(model: Model, X_Ymxnet: Any, is_train: bool): + X, Ymxnet = X_Ymxnet + Y = convert_recursive(is_mxnet_array, mxnet2xp, Ymxnet) + + def reverse_conversion(dY: Any) -> ArgsKwargs: + dYmxnet = convert_recursive(is_xp_array, xp2mxnet, dY) + return ArgsKwargs(args=((Ymxnet,),), kwargs={"head_grads": dYmxnet}) + + return Y, reverse_conversion diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/noop.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/noop.py new file mode 100644 index 0000000000000000000000000000000000000000..d8cbdfa8ec0137bcb49e1c1d930b9fffba2b55a8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/noop.py @@ -0,0 +1,21 @@ +from typing import Callable, Tuple, TypeVar + +from ..config import registry +from ..model import Model + +InOutT = TypeVar("InOutT") + + +@registry.layers("noop.v1") +def noop(*layers: Model) -> Model[InOutT, InOutT]: + """Transform a sequences of layers into a null operation.""" + return Model("noop", forward, layers=layers) + + +def forward( + model: Model[InOutT, InOutT], X: InOutT, is_train: bool +) -> Tuple[InOutT, Callable]: + def backprop(dY: InOutT) -> InOutT: + return dY + + return X, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/padded2list.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/padded2list.py new file mode 100644 index 0000000000000000000000000000000000000000..3951cb4e9f4535df15a488e00c97c035567ee740 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/padded2list.py @@ -0,0 +1,27 @@ +from typing import Callable, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import List2d, Padded + +InT = Padded +OutT = TypeVar("OutT", bound=List2d) + + +@registry.layers("padded2list.v1") +def padded2list() -> Model[InT, OutT]: + """Create a layer to convert a Padded input into a list of arrays.""" + return Model(f"padded2list", forward) + + +def forward( + model: Model[InT, OutT], Xp: InT, is_train: bool +) -> Tuple[OutT, Callable[[OutT], InT]]: + Ys = cast(OutT, model.ops.padded2list(Xp)) + + def backprop(dYs: OutT) -> InT: + dYp = model.ops.list2padded(dYs) + assert isinstance(dYp, Padded) + return dYp + + return Ys, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention.py new file mode 100644 index 0000000000000000000000000000000000000000..5b9e51667a2a80593ac952b6a5f2c1dc3be108db --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention.py @@ -0,0 +1,65 @@ +from typing import Callable, Optional, Tuple + +from ..config import registry +from ..model import Model +from ..types import Ragged +from ..util import get_width + +InT = Ragged +OutT = Ragged + + +@registry.layers("ParametricAttention.v1") +def ParametricAttention(nO: Optional[int] = None) -> Model[InT, OutT]: + """Weight inputs by similarity to a learned vector""" + return Model("para-attn", forward, init=init, params={"Q": None}, dims={"nO": nO}) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + Q = model.get_param("Q") + attention, bp_attention = _get_attention(model.ops, Q, Xr.dataXd, Xr.lengths) + output, bp_output = _apply_attention(model.ops, attention, Xr.dataXd, Xr.lengths) + + def backprop(dYr: OutT) -> InT: + dX, d_attention = bp_output(dYr.dataXd) + dQ, dX2 = bp_attention(d_attention) + model.inc_grad("Q", dQ.ravel()) + dX += dX2 + return Ragged(dX, dYr.lengths) + + return Ragged(output, Xr.lengths), backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + model.set_dim("nO", get_width(X)) + # Randomly initialize the parameter, as though it were an embedding. + Q = model.ops.alloc1f(model.get_dim("nO")) + Q += model.ops.xp.random.uniform(-0.1, 0.1, Q.shape) + model.set_param("Q", Q) + + +def _get_attention(ops, Q, X, lengths): + attention = ops.gemm(X, ops.reshape2f(Q, -1, 1)) + attention = ops.softmax_sequences(attention, lengths) + + def get_attention_bwd(d_attention): + d_attention = ops.backprop_softmax_sequences(d_attention, attention, lengths) + dQ = ops.gemm(X, d_attention, trans1=True) + dX = ops.xp.outer(d_attention, Q) + return dQ, dX + + return attention, get_attention_bwd + + +def _apply_attention(ops, attention, X, lengths): + output = X * attention + + def apply_attention_bwd(d_output): + d_attention = (X * d_output).sum(axis=1, keepdims=True) + dX = d_output * attention + return dX, d_attention + + return output, apply_attention_bwd diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention_v2.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..8a933bdd1146738abdad8dcff31d6f41fbc388b4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/parametricattention_v2.py @@ -0,0 +1,100 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import get_width +from .noop import noop + +InT = Ragged +OutT = Ragged + +KEY_TRANSFORM_REF: str = "key_transform" + + +@registry.layers("ParametricAttention.v2") +def ParametricAttention_v2( + *, + key_transform: Optional[Model[Floats2d, Floats2d]] = None, + nO: Optional[int] = None +) -> Model[InT, OutT]: + if key_transform is None: + key_transform = noop() + + """Weight inputs by similarity to a learned vector""" + return Model( + "para-attn", + forward, + init=init, + params={"Q": None}, + dims={"nO": nO}, + refs={KEY_TRANSFORM_REF: key_transform}, + layers=[key_transform], + ) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + Q = model.get_param("Q") + key_transform = model.get_ref(KEY_TRANSFORM_REF) + + attention, bp_attention = _get_attention( + model.ops, Q, key_transform, Xr.dataXd, Xr.lengths, is_train + ) + output, bp_output = _apply_attention(model.ops, attention, Xr.dataXd, Xr.lengths) + + def backprop(dYr: OutT) -> InT: + dX, d_attention = bp_output(dYr.dataXd) + dQ, dX2 = bp_attention(d_attention) + model.inc_grad("Q", dQ.ravel()) + dX += dX2 + return Ragged(dX, dYr.lengths) + + return Ragged(output, Xr.lengths), backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + key_transform = model.get_ref(KEY_TRANSFORM_REF) + width = get_width(X) if X is not None else None + if width: + model.set_dim("nO", width) + if key_transform.has_dim("nO"): + key_transform.set_dim("nO", width) + + # Randomly initialize the parameter, as though it were an embedding. + Q = model.ops.alloc1f(model.get_dim("nO")) + Q += model.ops.xp.random.uniform(-0.1, 0.1, Q.shape) + model.set_param("Q", Q) + + X_array = X.dataXd if X is not None else None + Y_array = Y.dataXd if Y is not None else None + + key_transform.initialize(X_array, Y_array) + + +def _get_attention(ops, Q, key_transform, X, lengths, is_train): + K, K_bp = key_transform(X, is_train=is_train) + + attention = ops.gemm(K, ops.reshape2f(Q, -1, 1)) + attention = ops.softmax_sequences(attention, lengths) + + def get_attention_bwd(d_attention): + d_attention = ops.backprop_softmax_sequences(d_attention, attention, lengths) + dQ = ops.gemm(K, d_attention, trans1=True) + dY = ops.xp.outer(d_attention, Q) + dX = K_bp(dY) + return dQ, dX + + return attention, get_attention_bwd + + +def _apply_attention(ops, attention, X, lengths): + output = X * attention + + def apply_attention_bwd(d_output): + d_attention = (X * d_output).sum(axis=1, keepdims=True) + dX = d_output * attention + return dX, d_attention + + return output, apply_attention_bwd diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.cpp new file mode 100644 index 0000000000000000000000000000000000000000..941858ff4ef5634a935ddfafffe973e70e71b557 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.cpp @@ -0,0 +1,29576 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "include_dirs": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.layers.premap_ids", + "sources": [ + "thinc/layers/premap_ids.pyx" + ] + }, + "module_name": "thinc.layers.premap_ids" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__layers__premap_ids +#define __PYX_HAVE_API__thinc__layers__premap_ids +/* Early includes */ +#include +#include "pythread.h" +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/layers/premap_ids.pyx", + "", + "cymem/cymem.pxd", + "preshed/maps.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* BufferFormatStructs.proto */ +struct __Pyx_StructField_; +#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0) +typedef struct { + const char* name; + const struct __Pyx_StructField_* fields; + size_t size; + size_t arraysize[8]; + int ndim; + char typegroup; + char is_unsigned; + int flags; +} __Pyx_TypeInfo; +typedef struct __Pyx_StructField_ { + const __Pyx_TypeInfo* type; + const char* name; + size_t offset; +} __Pyx_StructField; +typedef struct { + const __Pyx_StructField* field; + size_t parent_offset; +} __Pyx_BufFmt_StackElem; +typedef struct { + __Pyx_StructField root; + __Pyx_BufFmt_StackElem* head; + size_t fmt_offset; + size_t new_count, enc_count; + size_t struct_alignment; + int is_complex; + char enc_type; + char new_packmode; + char enc_packmode; + char is_valid_array; +} __Pyx_BufFmt_Context; + +/* MemviewSliceStruct.proto */ +struct __pyx_memoryview_obj; +typedef struct { + struct __pyx_memoryview_obj *memview; + char *data; + Py_ssize_t shape[8]; + Py_ssize_t strides[8]; + Py_ssize_t suboffsets[8]; +} __Pyx_memviewslice; +#define __Pyx_MemoryView_Len(m) (m.shape[0]) +#define __Pyx_MEMVIEW_DIRECT 1 +#define __Pyx_MEMVIEW_PTR 2 +#define __Pyx_MEMVIEW_FULL 4 +#define __Pyx_MEMVIEW_CONTIG 8 +#define __Pyx_MEMVIEW_STRIDED 16 +#define __Pyx_MEMVIEW_FOLLOW 32 +#define __Pyx_IS_C_CONTIG 1 +#define __Pyx_IS_F_CONTIG 2 +#define __Pyx_MEMSLICE_INIT { 0, 0, { 0 }, { 0 }, { 0 } } +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "preshed/maps.pxd":5 + * + * + * ctypedef uint64_t key_t # <<<<<<<<<<<<<< + * + * +*/ +typedef uint64_t __pyx_t_7preshed_4maps_key_t; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc; +struct __pyx_obj_5cymem_5cymem_PyFree; +struct __pyx_obj_5cymem_5cymem_Pool; +struct __pyx_obj_5cymem_5cymem_Address; +struct __pyx_obj_7preshed_4maps_PreshMap; +struct __pyx_obj_7preshed_4maps_PreshMapArray; +struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward; +struct __pyx_array_obj; +struct __pyx_MemviewEnum_obj; +struct __pyx_memoryview_obj; +struct __pyx_memoryviewslice_obj; + +/* "cymem/cymem.pxd":1 + * ctypedef void* (*malloc_t)(size_t n) # <<<<<<<<<<<<<< + * ctypedef void (*free_t)(void *p) + * +*/ +typedef void *(*__pyx_t_5cymem_5cymem_malloc_t)(size_t); + +/* "cymem/cymem.pxd":2 + * ctypedef void* (*malloc_t)(size_t n) + * ctypedef void (*free_t)(void *p) # <<<<<<<<<<<<<< + * + * cdef class PyMalloc: +*/ +typedef void (*__pyx_t_5cymem_5cymem_free_t)(void *); +struct __pyx_t_7preshed_4maps_Cell; +struct __pyx_t_7preshed_4maps_Result; +struct __pyx_t_7preshed_4maps_MapStruct; + +/* "preshed/maps.pxd":8 + * + * + * cdef struct Cell: # <<<<<<<<<<<<<< + * key_t key + * void* value +*/ +struct __pyx_t_7preshed_4maps_Cell { + __pyx_t_7preshed_4maps_key_t key; + void *value; +}; + +/* "preshed/maps.pxd":13 + * + * + * cdef struct Result: # <<<<<<<<<<<<<< + * int found + * void* value +*/ +struct __pyx_t_7preshed_4maps_Result { + int found; + void *value; +}; + +/* "preshed/maps.pxd":18 + * + * + * cdef struct MapStruct: # <<<<<<<<<<<<<< + * Cell* cells + * void* value_for_empty_key +*/ +struct __pyx_t_7preshed_4maps_MapStruct { + struct __pyx_t_7preshed_4maps_Cell *cells; + void *value_for_empty_key; + void *value_for_del_key; + __pyx_t_7preshed_4maps_key_t length; + __pyx_t_7preshed_4maps_key_t filled; + int is_empty_key_set; + int is_del_key_set; +}; + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ +struct __pyx_obj_5cymem_5cymem_PyMalloc { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtab; + __pyx_t_5cymem_5cymem_malloc_t malloc; +}; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ +struct __pyx_obj_5cymem_5cymem_PyFree { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtab; + __pyx_t_5cymem_5cymem_free_t free; +}; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ +struct __pyx_obj_5cymem_5cymem_Pool { + PyObject_HEAD + struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtab; + size_t size; + PyObject *addresses; + PyObject *refs; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "cymem/cymem.pxd":28 + * + * + * cdef class Address: # <<<<<<<<<<<<<< + * cdef void* ptr + * cdef readonly PyMalloc pymalloc +*/ +struct __pyx_obj_5cymem_5cymem_Address { + PyObject_HEAD + void *ptr; + struct __pyx_obj_5cymem_5cymem_PyMalloc *pymalloc; + struct __pyx_obj_5cymem_5cymem_PyFree *pyfree; +}; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ +struct __pyx_obj_7preshed_4maps_PreshMap { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtab; + struct __pyx_t_7preshed_4maps_MapStruct *c_map; + struct __pyx_obj_5cymem_5cymem_Pool *mem; +}; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ +struct __pyx_obj_7preshed_4maps_PreshMapArray { + PyObject_HEAD + struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtab; + struct __pyx_obj_5cymem_5cymem_Pool *mem; + struct __pyx_t_7preshed_4maps_MapStruct *maps; + size_t length; +}; + + +/* "thinc/layers/premap_ids.pyx":62 + * + * + * def forward( # <<<<<<<<<<<<<< + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: +*/ +struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward { + PyObject_HEAD + PyObject *__pyx_v_model; +}; + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ +struct __pyx_array_obj { + PyObject_HEAD + struct __pyx_vtabstruct_array *__pyx_vtab; + char *data; + Py_ssize_t len; + char *format; + int ndim; + Py_ssize_t *_shape; + Py_ssize_t *_strides; + Py_ssize_t itemsize; + PyObject *mode; + PyObject *_format; + void (*callback_free_data)(void *); + int free_data; + int dtype_is_object; +}; + + +/* "View.MemoryView":299 + * + * + * @cname('__pyx_MemviewEnum') # <<<<<<<<<<<<<< + * cdef class Enum(object): + * cdef object name +*/ +struct __pyx_MemviewEnum_obj { + PyObject_HEAD + PyObject *name; +}; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ +struct __pyx_memoryview_obj { + PyObject_HEAD + struct __pyx_vtabstruct_memoryview *__pyx_vtab; + PyObject *obj; + PyObject *_size; + void *_unused; + PyThread_type_lock lock; + __pyx_atomic_int_type acquisition_count; + Py_buffer view; + int flags; + int dtype_is_object; + __Pyx_TypeInfo const *typeinfo; +}; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ +struct __pyx_memoryviewslice_obj { + struct __pyx_memoryview_obj __pyx_base; + __Pyx_memviewslice from_slice; + PyObject *from_object; + PyObject *(*to_object_func)(char *); + int (*to_dtype_func)(char *, PyObject *); +}; + + + +/* "cymem/cymem.pxd":4 + * ctypedef void (*free_t)(void *p) + * + * cdef class PyMalloc: # <<<<<<<<<<<<<< + * cdef malloc_t malloc + * cdef void _set(self, malloc_t malloc) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyMalloc *, __pyx_t_5cymem_5cymem_malloc_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc *__pyx_vtabptr_5cymem_5cymem_PyMalloc; + + +/* "cymem/cymem.pxd":10 + * cdef PyMalloc WrapMalloc(malloc_t malloc) + * + * cdef class PyFree: # <<<<<<<<<<<<<< + * cdef free_t free + * cdef void _set(self, free_t free) +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_PyFree { + void (*_set)(struct __pyx_obj_5cymem_5cymem_PyFree *, __pyx_t_5cymem_5cymem_free_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_PyFree *__pyx_vtabptr_5cymem_5cymem_PyFree; + + +/* "cymem/cymem.pxd":16 + * cdef PyFree WrapFree(free_t free) + * + * cdef class Pool: # <<<<<<<<<<<<<< + * cdef readonly size_t size + * cdef readonly dict addresses +*/ + +struct __pyx_vtabstruct_5cymem_5cymem_Pool { + void *(*alloc)(struct __pyx_obj_5cymem_5cymem_Pool *, size_t, size_t); + void (*free)(struct __pyx_obj_5cymem_5cymem_Pool *, void *); + void *(*realloc)(struct __pyx_obj_5cymem_5cymem_Pool *, void *, size_t); +}; +static struct __pyx_vtabstruct_5cymem_5cymem_Pool *__pyx_vtabptr_5cymem_5cymem_Pool; + + +/* "preshed/maps.pxd":45 + * + * + * cdef class PreshMap: # <<<<<<<<<<<<<< + * cdef MapStruct* c_map + * cdef Pool mem +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMap { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMap *, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMap *__pyx_vtabptr_7preshed_4maps_PreshMap; + + +/* "preshed/maps.pxd":53 + * + * + * cdef class PreshMapArray: # <<<<<<<<<<<<<< + * cdef Pool mem + * cdef MapStruct* maps +*/ + +struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray { + void *(*get)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t); + void (*set)(struct __pyx_obj_7preshed_4maps_PreshMapArray *, size_t, __pyx_t_7preshed_4maps_key_t, void *); +}; +static struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray *__pyx_vtabptr_7preshed_4maps_PreshMapArray; + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ + +struct __pyx_vtabstruct_array { + PyObject *(*get_memview)(struct __pyx_array_obj *); +}; +static struct __pyx_vtabstruct_array *__pyx_vtabptr_array; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ + +struct __pyx_vtabstruct_memoryview { + char *(*get_item_pointer)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*is_slice)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_slice_assignment)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*setitem_slice_assign_scalar)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_indexed)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*convert_item_to_object)(struct __pyx_memoryview_obj *, char *); + PyObject *(*assign_item_from_object)(struct __pyx_memoryview_obj *, char *, PyObject *); + PyObject *(*_get_base)(struct __pyx_memoryview_obj *); +}; +static struct __pyx_vtabstruct_memoryview *__pyx_vtabptr_memoryview; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ + +struct __pyx_vtabstruct__memoryviewslice { + struct __pyx_vtabstruct_memoryview __pyx_base; +}; +static struct __pyx_vtabstruct__memoryviewslice *__pyx_vtabptr__memoryviewslice; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallCFunction.proto (used by CallUnboundCMethod0) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallOneArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectGetAttrStr.proto (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyValueError_Check.proto */ +#define __Pyx_PyExc_ValueError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ValueError) + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* PyObjectFastCallMethod.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL) +#else +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf); +#endif + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* PyMemoryError_Check.proto */ +#define __Pyx_PyExc_MemoryError_Check(obj) __Pyx_TypeCheck(obj, PyExc_MemoryError) + +/* BuildPyUnicode.proto (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char); + +/* COrdinalToPyUnicode.proto (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value); +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t width, char padding_char); + +/* GCCDiagnostics.proto (used by CIntToPyUnicode) */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* IncludeStdlibH.proto (used by CIntToPyUnicode) */ +#include + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_int(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_int(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_int(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char); + +/* JoinPyUnicode.export */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyObjectFormatSimple.proto */ +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#elif CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) :\ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) :\ + PyObject_Format(s, f)) +#else + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#endif + +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *); /*proto*/ +/* GetAttr.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ObjectGetItem.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key); +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/* RejectKeywords.export */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds); + +/* PyTypeError_Check.proto */ +#define __Pyx_PyExc_TypeError_Check(obj) __Pyx_TypeCheck(obj, PyExc_TypeError) + +/* DivInt[Py_ssize_t].proto */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t, Py_ssize_t, int b_is_constant); + +/* UnaryNegOverflows.proto */ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x)\ + (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* RaiseNoneIterError.proto */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); + +/* ExtTypeTest.proto */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* PySequenceMultiply.proto */ +#define __Pyx_PySequence_Multiply_Left(mul, seq) __Pyx_PySequence_Multiply(seq, mul) +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul); + +/* PyObjectFormatAndDecref.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f); +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f); + +/* PyObjectFormat.proto */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* s, PyObject* f); +#else +#define __Pyx_PyObject_Format(s, f) PyObject_Format(s, f) +#endif + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* PyIndexError_Check.proto */ +#define __Pyx_PyExc_IndexError_Check(obj) __Pyx_TypeCheck(obj, PyExc_IndexError) + +/* DivInt[long].proto */ +static CYTHON_INLINE long __Pyx_div_long(long, long, int b_is_constant); + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* BufferIndexError.proto */ +static void __Pyx_RaiseBufferIndexError(int axis); + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* RaiseClosureNameError.proto */ +static void __Pyx_RaiseClosureNameError(const char *varname); + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto (used by FetchCommonType) */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto (used by CythonFunctionShared) */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CheckTypeForFreelists.proto */ +#if CYTHON_USE_FREELISTS +#if CYTHON_USE_TYPE_SPECS +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t) == (expected_tp))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS Py_TPFLAGS_IS_ABSTRACT +#else +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t)->tp_basicsize == (expected_size))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE) +#endif +#define __PYX_CHECK_TYPE_FOR_FREELISTS(t, expected_tp, expected_size)\ + (__PYX_CHECK_FINAL_TYPE_FOR_FREELISTS((t), (expected_tp), (expected_size)) &\ + (int) (!__Pyx_PyType_HasFeature((t), __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS))) +#endif + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* DefaultPlacementNew.proto */ +#include +template +void __Pyx_default_placement_construct(T* x) { + new (static_cast(x)) T(); +} + +/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* DelItemOnTypeDict.proto (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k); +#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k) + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_1 +#define __PYX_HAVE_RT_ImportType_proto_3_2_1 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_2_1 { + __Pyx_ImportType_CheckSize_Error_3_2_1 = 0, + __Pyx_ImportType_CheckSize_Warn_3_2_1 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_2_1 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size); +#endif + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* BufferStructDeclare.proto */ +typedef struct { + Py_ssize_t shape, strides, suboffsets; +} __Pyx_Buf_DimInfo; +typedef struct { + size_t refcount; + Py_buffer pybuffer; +} __Pyx_Buffer; +typedef struct { + __Pyx_Buffer *rcbuffer; + char *data; + __Pyx_Buf_DimInfo diminfo[8]; +} __Pyx_LocalBuf_ND; + +/* MemviewRefcount.proto */ +static CYTHON_INLINE int __pyx_add_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +#define __pyx_get_slice_count_pointer(memview) (&memview->acquisition_count) +#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) +static CYTHON_INLINE void __Pyx_INC_MEMVIEW(__Pyx_memviewslice *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *, int, int); + +/* MemviewSliceIsContig.proto */ +static int __pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim); + +/* OverlappingSlices.proto */ +static int __pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize); + +/* MemviewSliceInit.proto */ +static int __Pyx_init_memviewslice( + struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference); + +/* MemviewDtypeToObject.proto */ +static CYTHON_INLINE PyObject *__pyx_memview_get_nn_int64_t(const char *itemp); +static CYTHON_INLINE int __pyx_memview_set_nn_int64_t(char *itemp, PyObject *obj); + +/* IsLittleEndian.proto (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void); + +/* BufferFormatCheck.proto (used by MemviewSliceValidateAndInit) */ +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts); +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type); + +/* TypeInfoCompare.proto (used by MemviewSliceValidateAndInit) */ +static int __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b); + +/* MemviewSliceValidateAndInit.export */ +static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_ds_nn_int64_t(PyObject *, int writable_flag); + +/* MemviewSliceCopy.proto */ +static __Pyx_memviewslice +__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int64_t(int64_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int64_t __Pyx_PyLong_As_int64_t(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* PyObjectCall2Args.proto (used by PyObjectCallMethod1) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethod1.proto (used by UpdateUnpickledDict) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* UpdateUnpickledDict.proto */ +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index); + +/* CheckUnpickleChecksum.proto */ +static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self); /* proto*/ +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto*/ +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self); /* proto*/ +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto*/ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cymem.cymem" */ + +/* Module declarations from "preshed.maps" */ + +/* Module declarations from "thinc.layers.premap_ids" */ +static PyObject *__pyx_collections_abc_Sequence = 0; +static PyObject *generic = 0; +static PyObject *strided = 0; +static PyObject *indirect = 0; +static PyObject *contiguous = 0; +static PyObject *indirect_contiguous = 0; +static int __pyx_memoryview_thread_locks_used; +static PyThread_type_lock __pyx_memoryview_thread_locks[8]; +static PyObject *__pyx_f_5thinc_6layers_10premap_ids_lookup(struct __pyx_obj_7preshed_4maps_PreshMap *, __Pyx_memviewslice, int64_t); /*proto*/ +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *); /*proto*/ +static struct __pyx_array_obj *__pyx_array_new(PyObject *, Py_ssize_t, char *, char const *, char *); /*proto*/ +static PyObject *__pyx_memoryview_new(PyObject *, int, int, __Pyx_TypeInfo const *); /*proto*/ +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *); /*proto*/ +static PyObject *_unellipsify(PyObject *, int); /*proto*/ +static int assert_direct_dimensions(Py_ssize_t *, int); /*proto*/ +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *, PyObject *); /*proto*/ +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int, int); /*proto*/ +static char *__pyx_pybuffer_index(Py_buffer *, char *, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memslice_transpose(__Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice, int, PyObject *(*)(char *), int (*)(char *, PyObject *), int); /*proto*/ +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static Py_ssize_t abs_py_ssize_t(Py_ssize_t); /*proto*/ +static char __pyx_get_best_slice_order(__Pyx_memviewslice *, int); /*proto*/ +static void _copy_strided_to_strided(char *, Py_ssize_t *, char *, Py_ssize_t *, Py_ssize_t *, Py_ssize_t *, int, size_t); /*proto*/ +static void copy_strided_to_strided(__Pyx_memviewslice *, __Pyx_memviewslice *, int, size_t); /*proto*/ +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *, int); /*proto*/ +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *, Py_ssize_t *, Py_ssize_t, int, char); /*proto*/ +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *, __Pyx_memviewslice *, char, int); /*proto*/ +static int __pyx_memoryview_err_extents(int, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memoryview_err_dim(PyObject *, PyObject *, int); /*proto*/ +static int __pyx_memoryview_err(PyObject *, PyObject *); /*proto*/ +static int __pyx_memoryview_err_no_memory(void); /*proto*/ +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice, __Pyx_memviewslice, int, int, int); /*proto*/ +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *, int, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *, int, size_t, void *, int); /*proto*/ +static void __pyx_memoryview__slice_assign_scalar(char *, Py_ssize_t *, Py_ssize_t *, int, size_t, void *); /*proto*/ +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +static const __Pyx_TypeInfo __Pyx_TypeInfo_nn_int64_t = { "int64_t", NULL, sizeof(int64_t), { 0 }, 0, __PYX_IS_UNSIGNED(int64_t) ? 'U' : 'I', __PYX_IS_UNSIGNED(int64_t), 0 }; +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.layers.premap_ids" +extern int __pyx_module_is_main_thinc__layers__premap_ids; +int __pyx_module_is_main_thinc__layers__premap_ids = 0; + +/* Implementation of "thinc.layers.premap_ids" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin___import__; +static PyObject *__pyx_builtin_enumerate; +static PyObject *__pyx_builtin_Ellipsis; +static PyObject *__pyx_builtin_id; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object); /* proto */ +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_premap_ids(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_mapping_table, PyObject *__pyx_v_default, PyObject *__pyx_v_column); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_7forward_backprop(PyObject *__pyx_self, PyObject *__pyx_v_dY); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_2forward(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_inputs, CYTHON_UNUSED PyObject *__pyx_v_is_train); /* proto */ +static PyObject *__pyx_tp_new_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyMalloc; + PyTypeObject *__pyx_ptype_5cymem_5cymem_PyFree; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Pool; + PyTypeObject *__pyx_ptype_5cymem_5cymem_Address; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMap; + PyTypeObject *__pyx_ptype_7preshed_4maps_PreshMapArray; + PyObject *__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward; + PyObject *__pyx_type___pyx_array; + PyObject *__pyx_type___pyx_MemviewEnum; + PyObject *__pyx_type___pyx_memoryview; + PyObject *__pyx_type___pyx_memoryviewslice; + PyTypeObject *__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward; + PyTypeObject *__pyx_array_type; + PyTypeObject *__pyx_MemviewEnum_type; + PyTypeObject *__pyx_memoryview_type; + PyTypeObject *__pyx_memoryviewslice_type; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_slice[1]; + PyObject *__pyx_tuple[3]; + PyObject *__pyx_codeobj_tab[3]; + PyObject *__pyx_string_tab[179]; + PyObject *__pyx_number_tab[4]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *__pyx_freelist_5thinc_6layers_10premap_ids___pyx_scope_struct__forward[8]; +int __pyx_freecount_5thinc_6layers_10premap_ids___pyx_scope_struct__forward; +#endif +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_All_dimensions_preceding_dimensi __pyx_string_tab[1] +#define __pyx_kp_u_Buffer_view_does_not_expose_stri __pyx_string_tab[2] +#define __pyx_kp_u_Can_only_create_a_buffer_that_is __pyx_string_tab[3] +#define __pyx_kp_u_Cannot_assign_to_read_only_memor __pyx_string_tab[4] +#define __pyx_kp_u_Cannot_create_writable_memory_vi __pyx_string_tab[5] +#define __pyx_kp_u_Cannot_index_with_type __pyx_string_tab[6] +#define __pyx_kp_u_Cannot_transpose_memoryview_with __pyx_string_tab[7] +#define __pyx_kp_u_Dimension_d_is_not_direct __pyx_string_tab[8] +#define __pyx_kp_u_Empty_shape_tuple_for_cython_arr __pyx_string_tab[9] +#define __pyx_kp_u_Index_out_of_bounds_axis_d __pyx_string_tab[10] +#define __pyx_kp_u_Indirect_dimensions_not_supporte __pyx_string_tab[11] +#define __pyx_kp_u_Invalid_mode_expected_c_or_fortr __pyx_string_tab[12] +#define __pyx_kp_u_Invalid_shape_in_axis __pyx_string_tab[13] +#define __pyx_kp_u_Mapping_int_int __pyx_string_tab[14] +#define __pyx_kp_u_MemoryView_of __pyx_string_tab[15] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[16] +#define __pyx_kp_u_Optional_int __pyx_string_tab[17] +#define __pyx_kp_u_Out_of_bounds_on_buffer_access_a __pyx_string_tab[18] +#define __pyx_kp_u_Step_may_not_be_zero_axis_d __pyx_string_tab[19] +#define __pyx_kp_u_Tuple_OutT_Callable __pyx_string_tab[20] +#define __pyx_kp_u_Unable_to_convert_item_to_object __pyx_string_tab[21] +#define __pyx_kp_u__2 __pyx_string_tab[22] +#define __pyx_kp_u__3 __pyx_string_tab[23] +#define __pyx_kp_u__4 __pyx_string_tab[24] +#define __pyx_kp_u__5 __pyx_string_tab[25] +#define __pyx_kp_u__6 __pyx_string_tab[26] +#define __pyx_kp_u__7 __pyx_string_tab[27] +#define __pyx_kp_u_add_note __pyx_string_tab[28] +#define __pyx_kp_u_and __pyx_string_tab[29] +#define __pyx_kp_u_at_0x __pyx_string_tab[30] +#define __pyx_kp_u_collections_abc __pyx_string_tab[31] +#define __pyx_kp_u_contiguous_and_direct __pyx_string_tab[32] +#define __pyx_kp_u_contiguous_and_indirect __pyx_string_tab[33] +#define __pyx_kp_u_disable __pyx_string_tab[34] +#define __pyx_kp_u_enable __pyx_string_tab[35] +#define __pyx_kp_u_gc __pyx_string_tab[36] +#define __pyx_kp_u_got __pyx_string_tab[37] +#define __pyx_kp_u_got_differing_extents_in_dimensi __pyx_string_tab[38] +#define __pyx_kp_u_isenabled __pyx_string_tab[39] +#define __pyx_kp_u_itemsize_0_for_cython_array __pyx_string_tab[40] +#define __pyx_kp_u_mapping_table_has_to_be_of_type __pyx_string_tab[41] +#define __pyx_kp_u_no_default___reduce___due_to_non __pyx_string_tab[42] +#define __pyx_kp_u_object __pyx_string_tab[43] +#define __pyx_kp_u_premap_ids_v1 __pyx_string_tab[44] +#define __pyx_kp_u_self_name_is_not_None __pyx_string_tab[45] +#define __pyx_kp_u_strided_and_direct __pyx_string_tab[46] +#define __pyx_kp_u_strided_and_direct_or_indirect __pyx_string_tab[47] +#define __pyx_kp_u_strided_and_indirect __pyx_string_tab[48] +#define __pyx_kp_u_thinc_config __pyx_string_tab[49] +#define __pyx_kp_u_thinc_layers_numpy __pyx_string_tab[50] +#define __pyx_kp_u_thinc_layers_premap_ids_pyx __pyx_string_tab[51] +#define __pyx_kp_u_thinc_layers_typing __pyx_string_tab[52] +#define __pyx_kp_u_thinc_model __pyx_string_tab[53] +#define __pyx_kp_u_thinc_types __pyx_string_tab[54] +#define __pyx_kp_u_thinc_util __pyx_string_tab[55] +#define __pyx_kp_u_unable_to_allocate_array_data __pyx_string_tab[56] +#define __pyx_kp_u_unable_to_allocate_shape_and_str __pyx_string_tab[57] +#define __pyx_n_u_ASCII __pyx_string_tab[58] +#define __pyx_n_u_Callable __pyx_string_tab[59] +#define __pyx_n_u_Dict __pyx_string_tab[60] +#define __pyx_n_u_Ellipsis __pyx_string_tab[61] +#define __pyx_n_u_InT __pyx_string_tab[62] +#define __pyx_n_u_Ints1d __pyx_string_tab[63] +#define __pyx_n_u_Ints2d __pyx_string_tab[64] +#define __pyx_n_u_Mapping __pyx_string_tab[65] +#define __pyx_n_u_Model __pyx_string_tab[66] +#define __pyx_n_u_Optional __pyx_string_tab[67] +#define __pyx_n_u_OutT __pyx_string_tab[68] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[69] +#define __pyx_n_u_Sequence __pyx_string_tab[70] +#define __pyx_n_u_Tuple __pyx_string_tab[71] +#define __pyx_n_u_Union __pyx_string_tab[72] +#define __pyx_n_u_View_MemoryView __pyx_string_tab[73] +#define __pyx_n_u_abc __pyx_string_tab[74] +#define __pyx_n_u_allocate_buffer __pyx_string_tab[75] +#define __pyx_n_u_arr __pyx_string_tab[76] +#define __pyx_n_u_asarray2i __pyx_string_tab[77] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[78] +#define __pyx_n_u_attrs __pyx_string_tab[79] +#define __pyx_n_u_backprop __pyx_string_tab[80] +#define __pyx_n_u_base __pyx_string_tab[81] +#define __pyx_n_u_bool __pyx_string_tab[82] +#define __pyx_n_u_c __pyx_string_tab[83] +#define __pyx_n_u_cast __pyx_string_tab[84] +#define __pyx_n_u_class __pyx_string_tab[85] +#define __pyx_n_u_class_getitem __pyx_string_tab[86] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[87] +#define __pyx_n_u_column __pyx_string_tab[88] +#define __pyx_n_u_config __pyx_string_tab[89] +#define __pyx_n_u_count __pyx_string_tab[90] +#define __pyx_n_u_dY __pyx_string_tab[91] +#define __pyx_n_u_default __pyx_string_tab[92] +#define __pyx_n_u_dict __pyx_string_tab[93] +#define __pyx_n_u_dtype __pyx_string_tab[94] +#define __pyx_n_u_dtype_is_object __pyx_string_tab[95] +#define __pyx_n_u_empty __pyx_string_tab[96] +#define __pyx_n_u_encode __pyx_string_tab[97] +#define __pyx_n_u_enumerate __pyx_string_tab[98] +#define __pyx_n_u_error __pyx_string_tab[99] +#define __pyx_n_u_flags __pyx_string_tab[100] +#define __pyx_n_u_format __pyx_string_tab[101] +#define __pyx_n_u_fortran __pyx_string_tab[102] +#define __pyx_n_u_forward __pyx_string_tab[103] +#define __pyx_n_u_forward_locals_backprop __pyx_string_tab[104] +#define __pyx_n_u_func __pyx_string_tab[105] +#define __pyx_n_u_getstate __pyx_string_tab[106] +#define __pyx_n_u_id __pyx_string_tab[107] +#define __pyx_n_u_idx __pyx_string_tab[108] +#define __pyx_n_u_import __pyx_string_tab[109] +#define __pyx_n_u_index __pyx_string_tab[110] +#define __pyx_n_u_initial_size __pyx_string_tab[111] +#define __pyx_n_u_inputs __pyx_string_tab[112] +#define __pyx_n_u_int __pyx_string_tab[113] +#define __pyx_n_u_is_coroutine __pyx_string_tab[114] +#define __pyx_n_u_is_train __pyx_string_tab[115] +#define __pyx_n_u_items __pyx_string_tab[116] +#define __pyx_n_u_itemsize __pyx_string_tab[117] +#define __pyx_n_u_k __pyx_string_tab[118] +#define __pyx_n_u_layers __pyx_string_tab[119] +#define __pyx_n_u_main __pyx_string_tab[120] +#define __pyx_n_u_mapper __pyx_string_tab[121] +#define __pyx_n_u_mapping_table __pyx_string_tab[122] +#define __pyx_n_u_memview __pyx_string_tab[123] +#define __pyx_n_u_mode __pyx_string_tab[124] +#define __pyx_n_u_model __pyx_string_tab[125] +#define __pyx_n_u_module __pyx_string_tab[126] +#define __pyx_n_u_name __pyx_string_tab[127] +#define __pyx_n_u_name_2 __pyx_string_tab[128] +#define __pyx_n_u_ndim __pyx_string_tab[129] +#define __pyx_n_u_new __pyx_string_tab[130] +#define __pyx_n_u_numpy __pyx_string_tab[131] +#define __pyx_n_u_obj __pyx_string_tab[132] +#define __pyx_n_u_ops __pyx_string_tab[133] +#define __pyx_n_u_output __pyx_string_tab[134] +#define __pyx_n_u_pack __pyx_string_tab[135] +#define __pyx_n_u_pop __pyx_string_tab[136] +#define __pyx_n_u_premap_ids __pyx_string_tab[137] +#define __pyx_n_u_pyx_checksum __pyx_string_tab[138] +#define __pyx_n_u_pyx_state __pyx_string_tab[139] +#define __pyx_n_u_pyx_type __pyx_string_tab[140] +#define __pyx_n_u_pyx_unpickle_Enum __pyx_string_tab[141] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[142] +#define __pyx_n_u_qualname __pyx_string_tab[143] +#define __pyx_n_u_reduce __pyx_string_tab[144] +#define __pyx_n_u_reduce_cython __pyx_string_tab[145] +#define __pyx_n_u_reduce_ex __pyx_string_tab[146] +#define __pyx_n_u_register __pyx_string_tab[147] +#define __pyx_n_u_registry __pyx_string_tab[148] +#define __pyx_n_u_reshape2i __pyx_string_tab[149] +#define __pyx_n_u_result __pyx_string_tab[150] +#define __pyx_n_u_return __pyx_string_tab[151] +#define __pyx_n_u_set_name __pyx_string_tab[152] +#define __pyx_n_u_setdefault __pyx_string_tab[153] +#define __pyx_n_u_setstate __pyx_string_tab[154] +#define __pyx_n_u_setstate_cython __pyx_string_tab[155] +#define __pyx_n_u_shape __pyx_string_tab[156] +#define __pyx_n_u_size __pyx_string_tab[157] +#define __pyx_n_u_start __pyx_string_tab[158] +#define __pyx_n_u_step __pyx_string_tab[159] +#define __pyx_n_u_stop __pyx_string_tab[160] +#define __pyx_n_u_struct __pyx_string_tab[161] +#define __pyx_n_u_table __pyx_string_tab[162] +#define __pyx_n_u_test __pyx_string_tab[163] +#define __pyx_n_u_thinc_layers_premap_ids __pyx_string_tab[164] +#define __pyx_n_u_to_numpy __pyx_string_tab[165] +#define __pyx_n_u_types __pyx_string_tab[166] +#define __pyx_n_u_typing __pyx_string_tab[167] +#define __pyx_n_u_unpack __pyx_string_tab[168] +#define __pyx_n_u_update __pyx_string_tab[169] +#define __pyx_n_u_util __pyx_string_tab[170] +#define __pyx_n_u_v __pyx_string_tab[171] +#define __pyx_n_u_values __pyx_string_tab[172] +#define __pyx_n_u_x __pyx_string_tab[173] +#define __pyx_n_u_xp __pyx_string_tab[174] +#define __pyx_kp_b_iso88591_E_q_e6_U_wc_haq_hat1HG4q_V1G5_t __pyx_string_tab[175] +#define __pyx_kp_b_iso88591_Q_A_XQm3aq_5_V1_5_3e4z_Q_A_AT_a __pyx_string_tab[176] +#define __pyx_kp_b_iso88591_Ya_uD_6_A __pyx_string_tab[177] +#define __pyx_n_b_O __pyx_string_tab[178] +#define __pyx_int_0 __pyx_number_tab[0] +#define __pyx_int_neg_1 __pyx_number_tab[1] +#define __pyx_int_1 __pyx_number_tab[2] +#define __pyx_int_136983863 __pyx_number_tab[3] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_CLEAR(clear_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_CLEAR(clear_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward); + Py_CLEAR(clear_module_state->__pyx_array_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_array); + Py_CLEAR(clear_module_state->__pyx_MemviewEnum_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_MemviewEnum); + Py_CLEAR(clear_module_state->__pyx_memoryview_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryview); + Py_CLEAR(clear_module_state->__pyx_memoryviewslice_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_slice[i]); } + for (int i=0; i<3; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<3; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<179; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<4; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyMalloc); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_PyFree); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Pool); + Py_VISIT(traverse_module_state->__pyx_ptype_5cymem_5cymem_Address); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMap); + Py_VISIT(traverse_module_state->__pyx_ptype_7preshed_4maps_PreshMapArray); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward); + Py_VISIT(traverse_module_state->__pyx_array_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_array); + Py_VISIT(traverse_module_state->__pyx_MemviewEnum_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_MemviewEnum); + Py_VISIT(traverse_module_state->__pyx_memoryview_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryview); + Py_VISIT(traverse_module_state->__pyx_memoryviewslice_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_slice[i]); } + for (int i=0; i<3; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<3; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<179; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<4; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + +/* Python wrapper */ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_shape = 0; + Py_ssize_t __pyx_v_itemsize; + PyObject *__pyx_v_format = 0; + PyObject *__pyx_v_mode = 0; + int __pyx_v_allocate_buffer; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_shape,&__pyx_mstate_global->__pyx_n_u_itemsize,&__pyx_mstate_global->__pyx_n_u_format,&__pyx_mstate_global->__pyx_n_u_mode,&__pyx_mstate_global->__pyx_n_u_allocate_buffer,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 129, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 129, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, i); __PYX_ERR(1, 129, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + } + __pyx_v_shape = ((PyObject*)values[0]); + __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_v_format = values[2]; + __pyx_v_mode = values[3]; + if (values[4]) { + __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) + } else { + + /* "View.MemoryView":130 + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, + * mode="c", bint allocate_buffer=True): # <<<<<<<<<<<<<< + * + * cdef int idx +*/ + __pyx_v_allocate_buffer = ((int)1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, __pyx_nargs); __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 129, __pyx_L1_error) + if (unlikely(((PyObject *)__pyx_v_format) == Py_None)) { + PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 129, __pyx_L1_error) + } + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v_shape, __pyx_v_itemsize, __pyx_v_format, __pyx_v_mode, __pyx_v_allocate_buffer); + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer) { + int __pyx_v_idx; + Py_ssize_t __pyx_v_dim; + char __pyx_v_order; + int __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11[5]; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + __Pyx_INCREF(__pyx_v_format); + + /* "View.MemoryView":135 + * cdef Py_ssize_t dim + * + * self.ndim = len(shape) # <<<<<<<<<<<<<< + * self.itemsize = itemsize + * +*/ + if (unlikely(__pyx_v_shape == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 135, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 135, __pyx_L1_error) + __pyx_v_self->ndim = ((int)__pyx_t_1); + + /* "View.MemoryView":136 + * + * self.ndim = len(shape) + * self.itemsize = itemsize # <<<<<<<<<<<<<< + * + * if not self.ndim: +*/ + __pyx_v_self->itemsize = __pyx_v_itemsize; + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + __pyx_t_2 = (!(__pyx_v_self->ndim != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":139 + * + * if not self.ndim: + * raise ValueError, "Empty shape tuple for cython.array" # <<<<<<<<<<<<<< + * + * if itemsize <= 0: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Empty_shape_tuple_for_cython_arr, 0, 0); + __PYX_ERR(1, 139, __pyx_L1_error) + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + } + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + __pyx_t_2 = (__pyx_v_itemsize <= 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":142 + * + * if itemsize <= 0: + * raise ValueError, "itemsize <= 0 for cython.array" # <<<<<<<<<<<<<< + * + * if not isinstance(format, bytes): +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_itemsize_0_for_cython_array, 0, 0); + __PYX_ERR(1, 142, __pyx_L1_error) + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + } + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + __pyx_t_2 = PyBytes_Check(__pyx_v_format); + __pyx_t_3 = (!__pyx_t_2); + if (__pyx_t_3) { + + /* "View.MemoryView":145 + * + * if not isinstance(format, bytes): + * format = format.encode('ASCII') # <<<<<<<<<<<<<< + * self._format = format # keep a reference to the byte string + * self.format = self._format +*/ + __pyx_t_5 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ASCII}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_encode, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF_SET(__pyx_v_format, __pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + } + + /* "View.MemoryView":146 + * if not isinstance(format, bytes): + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string # <<<<<<<<<<<<<< + * self.format = self._format + * +*/ + __pyx_t_4 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_4); + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_v_self->_format); + __Pyx_DECREF(__pyx_v_self->_format); + __pyx_v_self->_format = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":147 + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string + * self.format = self._format # <<<<<<<<<<<<<< + * + * +*/ + if (unlikely(__pyx_v_self->_format == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(1, 147, __pyx_L1_error) + } + __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) + __pyx_v_self->format = __pyx_t_7; + + /* "View.MemoryView":150 + * + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) # <<<<<<<<<<<<<< + * self._strides = self._shape + self.ndim + * +*/ + __pyx_v_self->_shape = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * __pyx_v_self->ndim) * 2))); + + /* "View.MemoryView":151 + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) + * self._strides = self._shape + self.ndim # <<<<<<<<<<<<<< + * + * if not self._shape: +*/ + __pyx_v_self->_strides = (__pyx_v_self->_shape + __pyx_v_self->ndim); + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + __pyx_t_3 = (!(__pyx_v_self->_shape != 0)); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":154 + * + * if not self._shape: + * raise MemoryError, "unable to allocate shape and strides." # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_shape_and_str, 0, 0); + __PYX_ERR(1, 154, __pyx_L1_error) + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + } + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + __pyx_t_8 = 0; + __pyx_t_4 = __pyx_v_shape; __Pyx_INCREF(__pyx_t_4); + __pyx_t_1 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 157, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_4, __pyx_t_1); + #endif + ++__pyx_t_1; + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_9; + __pyx_v_idx = __pyx_t_8; + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + __pyx_t_3 = (__pyx_v_dim <= 0); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":159 + * for idx, dim in enumerate(shape): + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." # <<<<<<<<<<<<<< + * self._shape[idx] = dim + * +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_int(__pyx_v_idx, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11[0] = __pyx_mstate_global->__pyx_kp_u_Invalid_shape_in_axis; + __pyx_t_11[1] = __pyx_t_5; + __pyx_t_11[2] = __pyx_mstate_global->__pyx_kp_u_; + __pyx_t_11[3] = __pyx_t_10; + __pyx_t_11[4] = __pyx_mstate_global->__pyx_kp_u__2; + __pyx_t_12 = __Pyx_PyUnicode_Join(__pyx_t_11, 5, 22 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_10) + 1, 127); + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 159, __pyx_L1_error) + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + } + + /* "View.MemoryView":160 + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim # <<<<<<<<<<<<<< + * + * cdef char order +*/ + (__pyx_v_self->_shape[__pyx_v_idx]) = __pyx_v_dim; + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 163, __pyx_L1_error) + if (__pyx_t_3) { + + /* "View.MemoryView":164 + * cdef char order + * if mode == 'c': + * order = b'C' # <<<<<<<<<<<<<< + * self.mode = u'c' + * elif mode == 'fortran': +*/ + __pyx_v_order = 'C'; + + /* "View.MemoryView":165 + * if mode == 'c': + * order = b'C' + * self.mode = u'c' # <<<<<<<<<<<<<< + * elif mode == 'fortran': + * order = b'F' +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_c; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 166, __pyx_L1_error) + if (likely(__pyx_t_3)) { + + /* "View.MemoryView":167 + * self.mode = u'c' + * elif mode == 'fortran': + * order = b'F' # <<<<<<<<<<<<<< + * self.mode = u'fortran' + * else: +*/ + __pyx_v_order = 'F'; + + /* "View.MemoryView":168 + * elif mode == 'fortran': + * order = b'F' + * self.mode = u'fortran' # <<<<<<<<<<<<<< + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_fortran; + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":170 + * self.mode = u'fortran' + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" # <<<<<<<<<<<<<< + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) +*/ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_mode, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_12 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_Invalid_mode_expected_c_or_fortr, __pyx_t_4); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 170, __pyx_L1_error) + } + __pyx_L11:; + + /* "View.MemoryView":172 + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) # <<<<<<<<<<<<<< + * + * self.free_data = allocate_buffer +*/ + __pyx_v_self->len = __pyx_fill_contig_strides_array(__pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_itemsize, __pyx_v_self->ndim, __pyx_v_order); + + /* "View.MemoryView":174 + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) + * + * self.free_data = allocate_buffer # <<<<<<<<<<<<<< + * self.dtype_is_object = format == b'O' + * +*/ + __pyx_v_self->free_data = __pyx_v_allocate_buffer; + + /* "View.MemoryView":175 + * + * self.free_data = allocate_buffer + * self.dtype_is_object = format == b'O' # <<<<<<<<<<<<<< + * + * if allocate_buffer: +*/ + __pyx_t_12 = PyObject_RichCompare(__pyx_v_format, __pyx_mstate_global->__pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_12); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_12); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 175, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_v_self->dtype_is_object = __pyx_t_3; + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + if (__pyx_v_allocate_buffer) { + + /* "View.MemoryView":178 + * + * if allocate_buffer: + * _allocate_buffer(self) # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_t_8 = __pyx_array_allocate_buffer(__pyx_v_self); if (unlikely(__pyx_t_8 == ((int)-1))) __PYX_ERR(1, 178, __pyx_L1_error) + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + } + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_format); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(((struct __pyx_array_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_bufmode; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + char *__pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":182 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 # <<<<<<<<<<<<<< + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": +*/ + __pyx_v_bufmode = -1; + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_t_1 = ((__pyx_v_flags & ((PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS) | PyBUF_ANY_CONTIGUOUS)) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 184, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":185 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_v_bufmode = (PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 186, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":187 + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." +*/ + __pyx_v_bufmode = (PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + } + __pyx_L4:; + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + __pyx_t_1 = (!((__pyx_v_flags & __pyx_v_bufmode) != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":189 + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." # <<<<<<<<<<<<<< + * info.buf = self.data + * info.len = self.len +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Can_only_create_a_buffer_that_is, 0, 0); + __PYX_ERR(1, 189, __pyx_L1_error) + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + } + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + } + + /* "View.MemoryView":190 + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data # <<<<<<<<<<<<<< + * info.len = self.len + * +*/ + __pyx_t_2 = __pyx_v_self->data; + __pyx_v_info->buf = __pyx_t_2; + + /* "View.MemoryView":191 + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data + * info.len = self.len # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + __pyx_t_3 = __pyx_v_self->len; + __pyx_v_info->len = __pyx_t_3; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":194 + * + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim # <<<<<<<<<<<<<< + * info.shape = self._shape + * info.strides = self._strides +*/ + __pyx_t_4 = __pyx_v_self->ndim; + __pyx_v_info->ndim = __pyx_t_4; + + /* "View.MemoryView":195 + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim + * info.shape = self._shape # <<<<<<<<<<<<<< + * info.strides = self._strides + * else: +*/ + __pyx_t_5 = __pyx_v_self->_shape; + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":196 + * info.ndim = self.ndim + * info.shape = self._shape + * info.strides = self._strides # <<<<<<<<<<<<<< + * else: + * info.ndim = 1 +*/ + __pyx_t_5 = __pyx_v_self->_strides; + __pyx_v_info->strides = __pyx_t_5; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":198 + * info.strides = self._strides + * else: + * info.ndim = 1 # <<<<<<<<<<<<<< + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL +*/ + /*else*/ { + __pyx_v_info->ndim = 1; + + /* "View.MemoryView":199 + * else: + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL # <<<<<<<<<<<<<< + * info.strides = NULL + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + __pyx_t_5 = (&__pyx_v_self->len); + } else { + __pyx_t_5 = NULL; + } + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":200 + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL # <<<<<<<<<<<<<< + * + * info.suboffsets = NULL +*/ + __pyx_v_info->strides = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":202 + * info.strides = NULL + * + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * info.itemsize = self.itemsize + * info.readonly = 0 +*/ + __pyx_v_info->suboffsets = NULL; + + /* "View.MemoryView":203 + * + * info.suboffsets = NULL + * info.itemsize = self.itemsize # <<<<<<<<<<<<<< + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL +*/ + __pyx_t_3 = __pyx_v_self->itemsize; + __pyx_v_info->itemsize = __pyx_t_3; + + /* "View.MemoryView":204 + * info.suboffsets = NULL + * info.itemsize = self.itemsize + * info.readonly = 0 # <<<<<<<<<<<<<< + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self +*/ + __pyx_v_info->readonly = 0; + + /* "View.MemoryView":205 + * info.itemsize = self.itemsize + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + __pyx_t_2 = __pyx_v_self->format; + } else { + __pyx_t_2 = NULL; + } + __pyx_v_info->format = __pyx_t_2; + + /* "View.MemoryView":206 + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self # <<<<<<<<<<<<<< + * + * def __dealloc__(array self): +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + +/* Python wrapper */ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + __pyx_t_1 = (__pyx_v_self->callback_free_data != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":210 + * def __dealloc__(array self): + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) # <<<<<<<<<<<<<< + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: +*/ + __pyx_v_self->callback_free_data(__pyx_v_self->data); + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + if (__pyx_v_self->free_data) { + } else { + __pyx_t_1 = __pyx_v_self->free_data; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_self->data != NULL); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":213 + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) # <<<<<<<<<<<<<< + * free(self.data) + * PyObject_Free(self._shape) +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_self->data, __pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_self->ndim, 0); + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + } + + /* "View.MemoryView":214 + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) # <<<<<<<<<<<<<< + * PyObject_Free(self._shape) + * +*/ + free(__pyx_v_self->data); + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + } + __pyx_L3:; + + /* "View.MemoryView":215 + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) + * PyObject_Free(self._shape) # <<<<<<<<<<<<<< + * + * @property +*/ + PyObject_Free(__pyx_v_self->_shape); + + /* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_5array_7memview___get__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":219 + * @property + * def memview(self): + * return self.get_memview() # <<<<<<<<<<<<<< + * + * @cname('get_memview') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.memview.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_memview", 0); + + /* "View.MemoryView":223 + * @cname('get_memview') + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE # <<<<<<<<<<<<<< + * return memoryview(self, flags, self.dtype_is_object) + * +*/ + __pyx_v_flags = ((PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT) | PyBUF_WRITABLE); + + /* "View.MemoryView":224 + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE + * return memoryview(self, flags, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, ((PyObject *)__pyx_v_self), __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_r = ((PyObject *)__pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array.get_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + + /* "View.MemoryView":227 + * + * def __len__(self): + * return self._shape[0] # <<<<<<<<<<<<<< + * + * def __getattr__(self, attr): +*/ + __pyx_r = (__pyx_v_self->_shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr); /*proto*/ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getattr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_attr)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getattr__", 0); + + /* "View.MemoryView":230 + * + * def __getattr__(self, attr): + * return getattr(self.memview, attr) # <<<<<<<<<<<<<< + * + * def __getitem__(self, item): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getattr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item); /*proto*/ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":233 + * + * def __getitem__(self, item): + * return self.memview[item] # <<<<<<<<<<<<<< + * + * def __setitem__(self, item, value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + +/* Python wrapper */ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + + /* "View.MemoryView":236 + * + * def __setitem__(self, item, value): + * self.memview[item] = value # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely((PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0))) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_array___reduce_cython__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_array_2__setstate_cython__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_v_i; + PyObject **__pyx_v_p; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":252 + * cdef PyObject **p + * + * self.free_data = True # <<<<<<<<<<<<<< + * self.data = malloc(self.len) + * if not self.data: +*/ + __pyx_v_self->free_data = 1; + + /* "View.MemoryView":253 + * + * self.free_data = True + * self.data = malloc(self.len) # <<<<<<<<<<<<<< + * if not self.data: + * raise MemoryError, "unable to allocate array data." +*/ + __pyx_v_self->data = ((char *)malloc(__pyx_v_self->len)); + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + __pyx_t_1 = (!(__pyx_v_self->data != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":255 + * self.data = malloc(self.len) + * if not self.data: + * raise MemoryError, "unable to allocate array data." # <<<<<<<<<<<<<< + * + * if self.dtype_is_object: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_array_data, 0, 0); + __PYX_ERR(1, 255, __pyx_L1_error) + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":258 + * + * if self.dtype_is_object: + * p = self.data # <<<<<<<<<<<<<< + * for i in range(self.len // self.itemsize): + * p[i] = Py_None +*/ + __pyx_v_p = ((PyObject **)__pyx_v_self->data); + + /* "View.MemoryView":259 + * if self.dtype_is_object: + * p = self.data + * for i in range(self.len // self.itemsize): # <<<<<<<<<<<<<< + * p[i] = Py_None + * Py_INCREF(Py_None) +*/ + if (unlikely(__pyx_v_self->itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_self->itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_self->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_div_Py_ssize_t(__pyx_v_self->len, __pyx_v_self->itemsize, 0); + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":260 + * p = self.data + * for i in range(self.len // self.itemsize): + * p[i] = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * return 0 +*/ + (__pyx_v_p[__pyx_v_i]) = Py_None; + + /* "View.MemoryView":261 + * for i in range(self.len // self.itemsize): + * p[i] = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * return 0 + * +*/ + Py_INCREF(Py_None); + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + } + + /* "View.MemoryView":262 + * p[i] = Py_None + * Py_INCREF(Py_None) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._allocate_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + +static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, char *__pyx_v_format, char const *__pyx_v_c_mode, char *__pyx_v_buf) { + struct __pyx_array_obj *__pyx_v_result = 0; + PyObject *__pyx_v_mode = 0; + struct __pyx_array_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("array_cwrapper", 0); + + /* "View.MemoryView":268 + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. # <<<<<<<<<<<<<< + * + * if buf is NULL: +*/ + __pyx_t_2 = ((__pyx_v_c_mode[0]) == 'f'); + if (__pyx_t_2) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_fortran; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_c; + } + __pyx_v_mode = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + __pyx_t_2 = (__pyx_v_buf == NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":271 + * + * if buf is NULL: + * result = array.__new__(array, shape, itemsize, format, mode) # <<<<<<<<<<<<<< + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) +*/ + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_3) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_4, NULL)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":273 + * result = array.__new__(array, shape, itemsize, format, mode) + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) # <<<<<<<<<<<<<< + * result.data = buf + * +*/ + /*else*/ { + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_3) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_allocate_buffer, Py_False) < (0)) __PYX_ERR(1, 273, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_1, __pyx_t_4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":274 + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) + * result.data = buf # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->data = __pyx_v_buf; + } + __pyx_L3:; + + /* "View.MemoryView":276 + * result.data = buf + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_mode); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + +/* Python wrapper */ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_name = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 302, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(1, 302, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, i); __PYX_ERR(1, 302, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + } + __pyx_v_name = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 302, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v_name); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__", 0); + + /* "View.MemoryView":303 + * cdef object name + * def __init__(self, name): + * self.name = name # <<<<<<<<<<<<<< + * def __repr__(self): + * return self.name +*/ + __Pyx_INCREF(__pyx_v_name); + __Pyx_GIVEREF(__pyx_v_name); + __Pyx_GOTREF(__pyx_v_self->name); + __Pyx_DECREF(__pyx_v_self->name); + __pyx_v_self->name = __pyx_v_name; + + /* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":305 + * self.name = name + * def __repr__(self): + * return self.name # <<<<<<<<<<<<<< + * + * cdef generic = Enum("") +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->name); + __pyx_r = __pyx_v_self->name; + goto __pyx_L0; + + /* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_MemviewEnum___reduce_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_v_state = 0; + PyObject *__pyx_v__dict = 0; + int __pyx_v_use_setstate; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":5 + * cdef object _dict + * cdef bint use_setstate + * state = (self.name,) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_self->name); + __Pyx_GIVEREF(__pyx_v_self->name); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->name) != (0)) __PYX_ERR(1, 5, __pyx_L1_error); + __pyx_v_state = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "(tree fragment)":6 + * cdef bint use_setstate + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None and _dict: + * state += (_dict,) +*/ + __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v__dict = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + __pyx_t_3 = (__pyx_v__dict != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_2) { + + /* "(tree fragment)":8 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(1, 8, __pyx_L1_error); + __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "(tree fragment)":9 + * if _dict is not None and _dict: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = ('self.name is not None',) +*/ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + goto __pyx_L3; + } + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + /*else*/ { + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_mstate_global->__pyx_tuple[0]); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_v_use_setstate = __pyx_t_2; + } + __pyx_L3:; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + if (__pyx_v_use_setstate) { + + /* "(tree fragment)":13 + * use_setstate = ('self.name is not None',) + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + } + + /* "(tree fragment)":15 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.Enum.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 16, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 16, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 16, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 16, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_MemviewEnum_2__setstate_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":17 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) # <<<<<<<<<<<<<< +*/ + __pyx_t_1 = __pyx_v___pyx_state; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(1, 17, __pyx_L1_error) + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 17, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + +/* Python wrapper */ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + int __pyx_v_flags; + int __pyx_v_dtype_is_object; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_obj,&__pyx_mstate_global->__pyx_n_u_flags,&__pyx_mstate_global->__pyx_n_u_dtype_is_object,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 348, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 348, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, i); __PYX_ERR(1, 348, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_obj = values[0]; + __pyx_v_flags = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + if (values[2]) { + __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + } else { + __pyx_v_dtype_is_object = ((int)0); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, __pyx_nargs); __PYX_ERR(1, 348, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_obj, __pyx_v_flags, __pyx_v_dtype_is_object); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + Py_intptr_t __pyx_t_4; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "View.MemoryView":349 + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj # <<<<<<<<<<<<<< + * self.flags = flags + * if type(self) is memoryview or obj is not None: +*/ + __Pyx_INCREF(__pyx_v_obj); + __Pyx_GIVEREF(__pyx_v_obj); + __Pyx_GOTREF(__pyx_v_self->obj); + __Pyx_DECREF(__pyx_v_self->obj); + __pyx_v_self->obj = __pyx_v_obj; + + /* "View.MemoryView":350 + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj + * self.flags = flags # <<<<<<<<<<<<<< + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) +*/ + __pyx_v_self->flags = __pyx_v_flags; + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + __pyx_t_2 = (((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))) == ((PyObject *)__pyx_mstate_global->__pyx_memoryview_type)); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_obj != Py_None); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":352 + * self.flags = flags + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) # <<<<<<<<<<<<<< + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None +*/ + __pyx_t_3 = PyObject_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 352, __pyx_L1_error) + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_self->view.obj) == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":354 + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = Py_None; + + /* "View.MemoryView":355 + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + } + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + __pyx_t_1 = (!__PYX_CYTHON_ATOMICS_ENABLED()); + if (__pyx_t_1) { + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + __pyx_t_2 = (__pyx_memoryview_thread_locks_used < 8); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L9_bool_binop_done; + } + + /* "View.MemoryView":361 + * if (__pyx_memoryview_thread_locks_used < 8 and + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 +*/ + __pyx_t_2 = (!__PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()); + __pyx_t_1 = __pyx_t_2; + __pyx_L9_bool_binop_done:; + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":362 + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: +*/ + __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + + /* "View.MemoryView":363 + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":365 + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< + * if self.lock is NULL: + * raise MemoryError +*/ + __pyx_v_self->lock = PyThread_allocate_lock(); + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":367 + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 367, __pyx_L1_error) + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + } + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":370 + * + * if flags & PyBUF_FORMAT: + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') # <<<<<<<<<<<<<< + * else: + * self.dtype_is_object = dtype_is_object +*/ + __pyx_t_2 = ((__pyx_v_self->view.format[0]) == 'O'); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_self->view.format[1]) == '\x00'); + __pyx_t_1 = __pyx_t_2; + __pyx_L14_bool_binop_done:; + __pyx_v_self->dtype_is_object = __pyx_t_1; + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":372 + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: + * self.dtype_is_object = dtype_is_object # <<<<<<<<<<<<<< + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 +*/ + /*else*/ { + __pyx_v_self->dtype_is_object = __pyx_v_dtype_is_object; + } + __pyx_L13:; + + /* "View.MemoryView":374 + * self.dtype_is_object = dtype_is_object + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 # <<<<<<<<<<<<<< + * self.typeinfo = NULL + * +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_4 = ((Py_intptr_t)((void *)(&__pyx_v_self->acquisition_count))); + __pyx_t_5 = (sizeof(__pyx_atomic_int_type)); + if (unlikely(__pyx_t_5 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 374, __pyx_L1_error) + } + __pyx_t_1 = ((__pyx_t_4 % __pyx_t_5) == 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 374, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 374, __pyx_L1_error) + #endif + + /* "View.MemoryView":375 + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 + * self.typeinfo = NULL # <<<<<<<<<<<<<< + * + * def __dealloc__(memoryview self): +*/ + __pyx_v_self->typeinfo = NULL; + + /* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + +/* Python wrapper */ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self) { + int __pyx_v_i; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + PyThread_type_lock __pyx_t_5; + PyThread_type_lock __pyx_t_6; + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + __pyx_t_1 = (__pyx_v_self->obj != Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":379 + * def __dealloc__(memoryview self): + * if self.obj is not None: + * PyBuffer_Release(&self.view) # <<<<<<<<<<<<<< + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * +*/ + PyBuffer_Release((&__pyx_v_self->view)); + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + __pyx_t_1 = (((Py_buffer *)(&__pyx_v_self->view))->obj == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":382 + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * + * (<__pyx_buffer *> &self.view).obj = NULL # <<<<<<<<<<<<<< + * Py_DECREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = NULL; + + /* "View.MemoryView":383 + * + * (<__pyx_buffer *> &self.view).obj = NULL + * Py_DECREF(Py_None) # <<<<<<<<<<<<<< + * + * cdef int i +*/ + Py_DECREF(Py_None); + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + } + __pyx_L3:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + __pyx_t_1 = (__pyx_v_self->lock != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":388 + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): # <<<<<<<<<<<<<< + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 +*/ + __pyx_t_1 = __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING(); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = __pyx_memoryview_thread_locks_used; + } + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + __pyx_t_1 = ((__pyx_memoryview_thread_locks[__pyx_v_i]) == __pyx_v_self->lock); + if (__pyx_t_1) { + + /* "View.MemoryView":390 + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 # <<<<<<<<<<<<<< + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used - 1); + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + __pyx_t_1 = (__pyx_v_i != __pyx_memoryview_thread_locks_used); + if (__pyx_t_1) { + + /* "View.MemoryView":393 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_t_5 = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + __pyx_t_6 = (__pyx_memoryview_thread_locks[__pyx_v_i]); + + /* "View.MemoryView":392 + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break +*/ + (__pyx_memoryview_thread_locks[__pyx_v_i]) = __pyx_t_5; + (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]) = __pyx_t_6; + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + } + + /* "View.MemoryView":394 + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break # <<<<<<<<<<<<<< + * else: + * PyThread_free_lock(self.lock) +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + } + } + /*else*/ { + + /* "View.MemoryView":396 + * break + * else: + * PyThread_free_lock(self.lock) # <<<<<<<<<<<<<< + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: +*/ + PyThread_free_lock(__pyx_v_self->lock); + } + __pyx_L6_break:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + } + + /* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + Py_ssize_t __pyx_v_dim; + char *__pyx_v_itemp; + PyObject *__pyx_v_idx = NULL; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_item_pointer", 0); + + /* "View.MemoryView":400 + * cdef char *get_item_pointer(memoryview self, object index) except NULL: + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf # <<<<<<<<<<<<<< + * + * for dim, idx in enumerate(index): +*/ + __pyx_v_itemp = ((char *)__pyx_v_self->view.buf); + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + __pyx_t_1 = 0; + if (likely(PyList_CheckExact(__pyx_v_index)) || PyTuple_CheckExact(__pyx_v_index)) { + __pyx_t_2 = __pyx_v_index; __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 402, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 402, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_3, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 402, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 402, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_1; + __pyx_t_1 = (__pyx_t_1 + 1); + + /* "View.MemoryView":403 + * + * for dim, idx in enumerate(index): + * itemp = pybuffer_index(&self.view, itemp, idx, dim) # <<<<<<<<<<<<<< + * + * return itemp +*/ + __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_7; + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":405 + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * + * return itemp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_itemp; + goto __pyx_L0; + + /* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.get_item_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index); /*proto*/ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_indices = NULL; + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + char *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + __pyx_t_1 = (__pyx_v_index == __pyx_builtin_Ellipsis); + if (__pyx_t_1) { + + /* "View.MemoryView":410 + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: + * return self # <<<<<<<<<<<<<< + * + * have_slices, indices = _unellipsify(index, self.view.ndim) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_self); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + } + + /* "View.MemoryView":412 + * return self + * + * have_slices, indices = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * cdef char *itemp +*/ + __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (likely(__pyx_t_2 != Py_None)) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 412, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 412, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_indices = __pyx_t_4; + __pyx_t_4 = 0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 415, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":416 + * cdef char *itemp + * if have_slices: + * return memview_slice(self, indices) # <<<<<<<<<<<<<< + * else: + * itemp = self.get_item_pointer(indices) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + } + + /* "View.MemoryView":418 + * return memview_slice(self, indices) + * else: + * itemp = self.get_item_pointer(indices) # <<<<<<<<<<<<<< + * return self.convert_item_to_object(itemp) + * +*/ + /*else*/ { + __pyx_t_5 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_5; + + /* "View.MemoryView":419 + * else: + * itemp = self.get_item_pointer(indices) + * return self.convert_item_to_object(itemp) # <<<<<<<<<<<<<< + * + * def __setitem__(memoryview self, object index, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + +/* Python wrapper */ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_obj = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + __Pyx_INCREF(__pyx_v_index); + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + if (unlikely(__pyx_v_self->view.readonly)) { + + /* "View.MemoryView":423 + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" # <<<<<<<<<<<<<< + * + * have_slices, index = _unellipsify(index, self.view.ndim) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_Cannot_assign_to_read_only_memor, 0, 0); + __PYX_ERR(1, 423, __pyx_L1_error) + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + } + + /* "View.MemoryView":425 + * raise TypeError, "Cannot assign to read-only memoryview" + * + * have_slices, index = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * if have_slices: +*/ + __pyx_t_1 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (likely(__pyx_t_1 != Py_None)) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 425, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 425, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_DECREF_SET(__pyx_v_index, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(1, 427, __pyx_L1_error) + if (__pyx_t_4) { + + /* "View.MemoryView":428 + * + * if have_slices: + * obj = self.is_slice(value) # <<<<<<<<<<<<<< + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 428, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_obj = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + __pyx_t_4 = (__pyx_v_obj != Py_None); + if (__pyx_t_4) { + + /* "View.MemoryView":430 + * obj = self.is_slice(value) + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) # <<<<<<<<<<<<<< + * else: + * self.setitem_slice_assign_scalar(self[index], value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_1, __pyx_v_obj); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":432 + * self.setitem_slice_assignment(self[index], obj) + * else: + * self.setitem_slice_assign_scalar(self[index], value) # <<<<<<<<<<<<<< + * else: + * self.setitem_indexed(index, value) +*/ + /*else*/ { + __pyx_t_3 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 432, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_3), __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L5:; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":434 + * self.setitem_slice_assign_scalar(self[index], value) + * else: + * self.setitem_indexed(index, value) # <<<<<<<<<<<<<< + * + * cdef is_slice(self, obj): +*/ + /*else*/ { + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L4:; + + /* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_slice", 0); + __Pyx_INCREF(__pyx_v_obj); + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_obj, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + /*try:*/ { + + /* "View.MemoryView":439 + * if not isinstance(obj, memoryview): + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< + * self.dtype_is_object) + * except TypeError: +*/ + __pyx_t_7 = NULL; + __pyx_t_8 = __Pyx_PyLong_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + + /* "View.MemoryView":440 + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) # <<<<<<<<<<<<<< + * except TypeError: + * return None +*/ + __pyx_t_9 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 440, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_7, __pyx_v_obj, __pyx_t_8, __pyx_t_9}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_10, (4-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF((PyObject *)__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_obj, ((PyObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L9_try_end; + __pyx_L4_error:; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":441 + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) + * except TypeError: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_11) { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":442 + * self.dtype_is_object) + * except TypeError: + * return None # <<<<<<<<<<<<<< + * + * return obj +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L7_except_return; + } + goto __pyx_L6_except_error; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + __pyx_L6_except_error:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L1_error; + __pyx_L7_except_return:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L0; + __pyx_L9_try_end:; + } + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + } + + /* "View.MemoryView":444 + * return None + * + * return obj # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assignment(self, dst, src): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_obj); + __pyx_r = __pyx_v_obj; + goto __pyx_L0; + + /* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src) { + __Pyx_memviewslice __pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_src_slice; + __Pyx_memviewslice __pyx_v_msrc; + __Pyx_memviewslice __pyx_v_mdst; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assignment", 0); + + /* "View.MemoryView":449 + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * +*/ + if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_v_msrc = (__pyx_t_1[0]); + + /* "View.MemoryView":450 + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] # <<<<<<<<<<<<<< + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) +*/ + if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_v_mdst = (__pyx_t_1[0]); + + /* "View.MemoryView":452 + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __pyx_memoryview_copy_contents(__pyx_v_msrc, __pyx_v_mdst, __pyx_t_3, __pyx_t_4, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 452, __pyx_L1_error) + + /* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assignment", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value) { + int __pyx_v_array[128]; + void *__pyx_v_tmp; + void *__pyx_v_item; + __Pyx_memviewslice *__pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_tmp_slice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + char const *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assign_scalar", 0); + + /* "View.MemoryView":456 + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): + * cdef int array[128] + * cdef void *tmp = NULL # <<<<<<<<<<<<<< + * cdef void *item + * +*/ + __pyx_v_tmp = NULL; + + /* "View.MemoryView":461 + * cdef __Pyx_memviewslice *dst_slice + * cdef __Pyx_memviewslice tmp_slice + * dst_slice = get_slice_from_memview(dst, &tmp_slice) # <<<<<<<<<<<<<< + * + * if self.view.itemsize > sizeof(array): +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 461, __pyx_L1_error) + __pyx_v_dst_slice = __pyx_t_1; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + __pyx_t_2 = (((size_t)__pyx_v_self->view.itemsize) > (sizeof(__pyx_v_array))); + if (__pyx_t_2) { + + /* "View.MemoryView":464 + * + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) # <<<<<<<<<<<<<< + * if tmp == NULL: + * raise MemoryError +*/ + __pyx_v_tmp = PyMem_Malloc(__pyx_v_self->view.itemsize); + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + __pyx_t_2 = (__pyx_v_tmp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":466 + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * item = tmp + * else: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 466, __pyx_L1_error) + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + } + + /* "View.MemoryView":467 + * if tmp == NULL: + * raise MemoryError + * item = tmp # <<<<<<<<<<<<<< + * else: + * item = array +*/ + __pyx_v_item = __pyx_v_tmp; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":469 + * item = tmp + * else: + * item = array # <<<<<<<<<<<<<< + * + * try: +*/ + /*else*/ { + __pyx_v_item = ((void *)__pyx_v_array); + } + __pyx_L3:; + + /* "View.MemoryView":471 + * item = array + * + * try: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * ( item)[0] = value +*/ + /*try:*/ { + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":473 + * try: + * if self.dtype_is_object: + * ( item)[0] = value # <<<<<<<<<<<<<< + * else: + * self.assign_item_from_object( item, value) +*/ + (((PyObject **)__pyx_v_item)[0]) = ((PyObject *)__pyx_v_value); + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":475 + * ( item)[0] = value + * else: + * self.assign_item_from_object( item, value) # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 475, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L8:; + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + __pyx_t_2 = (__pyx_v_self->view.suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":480 + * + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) # <<<<<<<<<<<<<< + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, + * item, self.dtype_is_object) +*/ + __pyx_t_4 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 480, __pyx_L6_error) + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + } + + /* "View.MemoryView":481 + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, # <<<<<<<<<<<<<< + * item, self.dtype_is_object) + * finally: +*/ + __pyx_memoryview_slice_assign_scalar(__pyx_v_dst_slice, __pyx_v_dst->view.ndim, __pyx_v_self->view.itemsize, __pyx_v_item, __pyx_v_self->dtype_is_object); + } + + /* "View.MemoryView":484 + * item, self.dtype_is_object) + * finally: + * PyMem_Free(tmp) # <<<<<<<<<<<<<< + * + * cdef setitem_indexed(self, index, value): +*/ + /*finally:*/ { + /*normal exit:*/{ + PyMem_Free(__pyx_v_tmp); + goto __pyx_L7; + } + __pyx_L6_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_ExceptionSwap(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); + if ( unlikely(__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0)) __Pyx_ErrFetch(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_XGOTREF(__pyx_t_12); + __pyx_t_4 = __pyx_lineno; __pyx_t_5 = __pyx_clineno; __pyx_t_6 = __pyx_filename; + { + PyMem_Free(__pyx_v_tmp); + } + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ErrRestore(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_5; __pyx_filename = __pyx_t_6; + goto __pyx_L1_error; + } + __pyx_L7:; + } + + /* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assign_scalar", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_indexed", 0); + + /* "View.MemoryView":487 + * + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) # <<<<<<<<<<<<<< + * self.assign_item_from_object(itemp, value) + * +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 487, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_1; + + /* "View.MemoryView":488 + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 488, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_indexed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_v_struct = NULL; + PyObject *__pyx_v_bytesitem = 0; + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":493 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef bytes bytesitem + * +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 493, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":496 + * cdef bytes bytesitem + * + * bytesitem = itemp[:self.view.itemsize] # <<<<<<<<<<<<<< + * try: + * result = struct.unpack(self.view.format, bytesitem) +*/ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 496, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_bytesitem = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_2, &__pyx_t_3, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + /*try:*/ { + + /* "View.MemoryView":498 + * bytesitem = itemp[:self.view.itemsize] + * try: + * result = struct.unpack(self.view.format, bytesitem) # <<<<<<<<<<<<<< + * except struct.error: + * raise ValueError, "Unable to convert item to object" +*/ + __pyx_t_5 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_t_6, __pyx_v_bytesitem}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_unpack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_result = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + } + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + /*else:*/ { + __pyx_t_8 = __Pyx_ssize_strlen(__pyx_v_self->view.format); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 502, __pyx_L5_except_error) + __pyx_t_9 = (__pyx_t_8 == 1); + if (__pyx_t_9) { + + /* "View.MemoryView":503 + * else: + * if len(self.view.format) == 1: + * return result[0] # <<<<<<<<<<<<<< + * return result + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 503, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L6_except_return; + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + } + + /* "View.MemoryView":504 + * if len(self.view.format) == 1: + * return result[0] + * return result # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L6_except_return; + } + __pyx_L3_error:; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "View.MemoryView":499 + * try: + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: # <<<<<<<<<<<<<< + * raise ValueError, "Unable to convert item to object" + * else: +*/ + __Pyx_ErrFetch(&__pyx_t_1, &__pyx_t_6, &__pyx_t_5); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_error); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_1, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_ErrRestore(__pyx_t_1, __pyx_t_6, __pyx_t_5); + __pyx_t_1 = 0; __pyx_t_6 = 0; __pyx_t_5 = 0; + if (__pyx_t_11) { + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_1) < 0) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_1); + + /* "View.MemoryView":500 + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: + * raise ValueError, "Unable to convert item to object" # <<<<<<<<<<<<<< + * else: + * if len(self.view.format) == 1: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Unable_to_convert_item_to_object, 0, 0); + __PYX_ERR(1, 500, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L1_error; + __pyx_L6_except_return:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L0; + } + + /* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesitem); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_v_struct = NULL; + char __pyx_v_c; + PyObject *__pyx_v_bytesvalue = 0; + Py_ssize_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + char *__pyx_t_10; + char *__pyx_t_11; + Py_ssize_t __pyx_t_12; + char *__pyx_t_13; + char *__pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":509 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef char c + * cdef bytes bytesvalue +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 509, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + __pyx_t_3 = PyTuple_Check(__pyx_v_value); + if (__pyx_t_3) { + + /* "View.MemoryView":515 + * + * if isinstance(value, tuple): + * bytesvalue = struct.pack(self.view.format, *value) # <<<<<<<<<<<<<< + * else: + * bytesvalue = struct.pack(self.view.format, value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 515, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 515, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":517 + * bytesvalue = struct.pack(self.view.format, *value) + * else: + * bytesvalue = struct.pack(self.view.format, value) # <<<<<<<<<<<<<< + * + * for i, c in enumerate(bytesvalue): +*/ + /*else*/ { + __pyx_t_6 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_t_1, __pyx_v_value}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_pack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 517, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = 0; + if (unlikely(__pyx_v_bytesvalue == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(1, 519, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytesvalue); + __pyx_t_9 = __pyx_v_bytesvalue; + __pyx_t_11 = __Pyx_PyBytes_AsWritableString(__pyx_t_9); if (unlikely(__pyx_t_11 == ((char *)NULL))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyBytes_GET_SIZE(__pyx_t_9); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_13 = (__pyx_t_11 + __pyx_t_12); + for (__pyx_t_14 = __pyx_t_11; __pyx_t_14 < __pyx_t_13; __pyx_t_14++) { + __pyx_t_10 = __pyx_t_14; + __pyx_v_c = (__pyx_t_10[0]); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_v_i = __pyx_t_8; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + (__pyx_v_itemp[__pyx_v_i]) = __pyx_v_c; + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t *__pyx_t_3; + char *__pyx_t_4; + void *__pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + __pyx_t_2 = ((__pyx_v_flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_L4_bool_binop_done:; + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":525 + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: + * raise ValueError, "Cannot create writable memory view from read-only memoryview" # <<<<<<<<<<<<<< + * + * if flags & PyBUF_ND: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Cannot_create_writable_memory_vi, 0, 0); + __PYX_ERR(1, 525, __pyx_L1_error) + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + } + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":528 + * + * if flags & PyBUF_ND: + * info.shape = self.view.shape # <<<<<<<<<<<<<< + * else: + * info.shape = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.shape; + __pyx_v_info->shape = __pyx_t_3; + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":530 + * info.shape = self.view.shape + * else: + * info.shape = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + /*else*/ { + __pyx_v_info->shape = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":533 + * + * if flags & PyBUF_STRIDES: + * info.strides = self.view.strides # <<<<<<<<<<<<<< + * else: + * info.strides = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.strides; + __pyx_v_info->strides = __pyx_t_3; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":535 + * info.strides = self.view.strides + * else: + * info.strides = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_INDIRECT: +*/ + /*else*/ { + __pyx_v_info->strides = NULL; + } + __pyx_L7:; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_INDIRECT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":538 + * + * if flags & PyBUF_INDIRECT: + * info.suboffsets = self.view.suboffsets # <<<<<<<<<<<<<< + * else: + * info.suboffsets = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.suboffsets; + __pyx_v_info->suboffsets = __pyx_t_3; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":540 + * info.suboffsets = self.view.suboffsets + * else: + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + /*else*/ { + __pyx_v_info->suboffsets = NULL; + } + __pyx_L8:; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":543 + * + * if flags & PyBUF_FORMAT: + * info.format = self.view.format # <<<<<<<<<<<<<< + * else: + * info.format = NULL +*/ + __pyx_t_4 = __pyx_v_self->view.format; + __pyx_v_info->format = __pyx_t_4; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":545 + * info.format = self.view.format + * else: + * info.format = NULL # <<<<<<<<<<<<<< + * + * info.buf = self.view.buf +*/ + /*else*/ { + __pyx_v_info->format = NULL; + } + __pyx_L9:; + + /* "View.MemoryView":547 + * info.format = NULL + * + * info.buf = self.view.buf # <<<<<<<<<<<<<< + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize +*/ + __pyx_t_5 = __pyx_v_self->view.buf; + __pyx_v_info->buf = __pyx_t_5; + + /* "View.MemoryView":548 + * + * info.buf = self.view.buf + * info.ndim = self.view.ndim # <<<<<<<<<<<<<< + * info.itemsize = self.view.itemsize + * info.len = self.view.len +*/ + __pyx_t_6 = __pyx_v_self->view.ndim; + __pyx_v_info->ndim = __pyx_t_6; + + /* "View.MemoryView":549 + * info.buf = self.view.buf + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize # <<<<<<<<<<<<<< + * info.len = self.view.len + * info.readonly = self.view.readonly +*/ + __pyx_t_7 = __pyx_v_self->view.itemsize; + __pyx_v_info->itemsize = __pyx_t_7; + + /* "View.MemoryView":550 + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize + * info.len = self.view.len # <<<<<<<<<<<<<< + * info.readonly = self.view.readonly + * info.obj = self +*/ + __pyx_t_7 = __pyx_v_self->view.len; + __pyx_v_info->len = __pyx_t_7; + + /* "View.MemoryView":551 + * info.itemsize = self.view.itemsize + * info.len = self.view.len + * info.readonly = self.view.readonly # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_v_info->readonly = __pyx_t_1; + + /* "View.MemoryView":552 + * info.len = self.view.len + * info.readonly = self.view.readonly + * info.obj = self # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":557 + * @property + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) # <<<<<<<<<<<<<< + * transpose_memslice(&result.from_slice) + * return result +*/ + __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 557, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 557, __pyx_L1_error) + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":558 + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 558, __pyx_L1_error) + + /* "View.MemoryView":559 + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) + * return result # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.T.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":563 + * @property + * def base(self): + * return self._get_base() # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->_get_base(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 563, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.base.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":566 + * + * cdef _get_base(self): + * return self.obj # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->obj); + __pyx_r = __pyx_v_self->obj; + goto __pyx_L0; + + /* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_7genexpr__pyx_v_length; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":570 + * @property + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_7genexpr__pyx_v_length = (__pyx_t_2[0]); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_7genexpr__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } /* exit inner scope */ + __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.shape.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr1__pyx_v_stride; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + __pyx_t_1 = (__pyx_v_self->view.strides == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":576 + * if self.view.strides == NULL: + * + * raise ValueError, "Buffer view does not expose strides" # <<<<<<<<<<<<<< + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Buffer_view_does_not_expose_stri, 0, 0); + __PYX_ERR(1, 576, __pyx_L1_error) + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + } + + /* "View.MemoryView":578 + * raise ValueError, "Buffer view does not expose strides" + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.strides + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.strides; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr1__pyx_v_stride = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr1__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.strides.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr2__pyx_v_suboffset; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + __pyx_t_1 = (__pyx_v_self->view.suboffsets == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PySequence_Multiply(__pyx_mstate_global->__pyx_tuple[1], __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + } + + /* "View.MemoryView":585 + * return (-1,) * self.view.ndim + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.suboffsets + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.suboffsets; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr2__pyx_v_suboffset = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr2__pyx_v_suboffset); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.suboffsets.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":589 + * @property + * def ndim(self): + * return self.view.ndim # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 589, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.ndim.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":593 + * @property + * def itemsize(self): + * return self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 593, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.itemsize.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":597 + * @property + * def nbytes(self): + * return self.size * self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.nbytes.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + __pyx_t_1 = (__pyx_v_self->_size == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":602 + * def size(self): + * if self._size is None: + * result = 1 # <<<<<<<<<<<<<< + * + * for length in self.view.shape[:self.view.ndim]: +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __pyx_v_result = __pyx_mstate_global->__pyx_int_1; + + /* "View.MemoryView":604 + * result = 1 + * + * for length in self.view.shape[:self.view.ndim]: # <<<<<<<<<<<<<< + * result *= length + * +*/ + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_t_5 = PyLong_FromSsize_t((__pyx_t_2[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 604, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_5); + __pyx_t_5 = 0; + + /* "View.MemoryView":605 + * + * for length in self.view.shape[:self.view.ndim]: + * result *= length # <<<<<<<<<<<<<< + * + * self._size = result +*/ + __pyx_t_5 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 605, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_result, __pyx_t_5); + __pyx_t_5 = 0; + } + + /* "View.MemoryView":607 + * result *= length + * + * self._size = result # <<<<<<<<<<<<<< + * + * return self._size +*/ + __Pyx_INCREF(__pyx_v_result); + __Pyx_GIVEREF(__pyx_v_result); + __Pyx_GOTREF(__pyx_v_self->_size); + __Pyx_DECREF(__pyx_v_self->_size); + __pyx_v_self->_size = __pyx_v_result; + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + } + + /* "View.MemoryView":609 + * self._size = result + * + * return self._size # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->_size); + __pyx_r = __pyx_v_self->_size; + goto __pyx_L0; + + /* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.size.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + __pyx_t_1 = (__pyx_v_self->view.ndim >= 1); + if (__pyx_t_1) { + + /* "View.MemoryView":613 + * def __len__(self): + * if self.view.ndim >= 1: + * return self.view.shape[0] # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_r = (__pyx_v_self->view.shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + } + + /* "View.MemoryView":615 + * return self.view.shape[0] + * + * return 0 # <<<<<<<<<<<<<< + * + * def __repr__(self): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[5]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":619 + * def __repr__(self): + * return "" % (self.base.__class__.__name__, + * id(self)) # <<<<<<<<<<<<<< + * + * def __str__(self): +*/ + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_Format(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_x); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_4[1] = __pyx_t_2; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_at_0x; + __pyx_t_4[3] = __pyx_t_3; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u__3; + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_4, 5, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3[3]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__str__", 0); + + /* "View.MemoryView":622 + * + * def __str__(self): + * return "" % (self.base.__class__.__name__,) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_3[1] = __pyx_t_2; + __pyx_t_3[2] = __pyx_mstate_global->__pyx_kp_u_object; + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_3, 3, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 8, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_c_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_c_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_c_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_c_contig", 0); + + /* "View.MemoryView":628 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 628, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":629 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'C', self.view.ndim) # <<<<<<<<<<<<<< + * + * def is_f_contig(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 629, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_c_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_f_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_f_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_f_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_f_contig", 0); + + /* "View.MemoryView":634 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 634, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":635 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'F', self.view.ndim) # <<<<<<<<<<<<<< + * + * def copy(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 635, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_f_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_mslice; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy", 0); + + /* "View.MemoryView":639 + * def copy(self): + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &mslice) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_F_CONTIGUOUS)); + + /* "View.MemoryView":641 + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS + * + * slice_copy(self, &mslice) # <<<<<<<<<<<<<< + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_mslice)); + + /* "View.MemoryView":642 + * + * slice_copy(self, &mslice) + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_C_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char const *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 642, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":647 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &mslice) # <<<<<<<<<<<<<< + * + * def copy_fortran(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 647, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy_fortran (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy_fortran", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy_fortran", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy_fortran", 0); + + /* "View.MemoryView":651 + * def copy_fortran(self): + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &src) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_C_CONTIGUOUS)); + + /* "View.MemoryView":653 + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS + * + * slice_copy(self, &src) # <<<<<<<<<<<<<< + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_src)); + + /* "View.MemoryView":654 + * + * slice_copy(self, &src) + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_F_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char const *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 654, __pyx_L1_error) + __pyx_v_dst = __pyx_t_1; + + /* "View.MemoryView":659 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &dst) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 659, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy_fortran", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryview___reduce_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryview_2__setstate_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + +static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, int __pyx_v_dtype_is_object, __Pyx_TypeInfo const *__pyx_v_typeinfo) { + struct __pyx_memoryview_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_cwrapper", 0); + + /* "View.MemoryView":664 + * @cname('__pyx_memoryview_new') + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) # <<<<<<<<<<<<<< + * result.typeinfo = typeinfo + * return result +*/ + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, __pyx_v_o, __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_v_result = ((struct __pyx_memoryview_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":665 + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_v_result->typeinfo = __pyx_v_typeinfo; + + /* "View.MemoryView":666 + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_check') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { + int __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":670 + * @cname('__pyx_memoryview_check') + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) # <<<<<<<<<<<<<< + * + * cdef tuple _unellipsify(object index, int ndim): +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_o, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + +static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_idx; + PyObject *__pyx_v_tup = NULL; + PyObject *__pyx_v_result = NULL; + int __pyx_v_have_slices; + int __pyx_v_seen_ellipsis; + PyObject *__pyx_v_item = NULL; + Py_ssize_t __pyx_v_nslices; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6[3]; + PyObject *__pyx_t_7 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_unellipsify", 0); + + /* "View.MemoryView":678 + * """ + * cdef Py_ssize_t idx + * tup = index if isinstance(index, tuple) else (index,) # <<<<<<<<<<<<<< + * + * result = [slice(None)] * ndim +*/ + __pyx_t_2 = PyTuple_Check(__pyx_v_index); + if (__pyx_t_2) { + __Pyx_INCREF(((PyObject*)__pyx_v_index)); + __pyx_t_1 = __pyx_v_index; + } else { + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_index); + __Pyx_GIVEREF(__pyx_v_index); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_index) != (0)) __PYX_ERR(1, 678, __pyx_L1_error); + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_v_tup = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_t_1 = PyList_New(1 * ((__pyx_v_ndim<0) ? 0:__pyx_v_ndim)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_ndim; __pyx_temp++) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, __pyx_temp, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(1, 680, __pyx_L1_error); + } + } + __pyx_v_result = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":681 + * + * result = [slice(None)] * ndim + * have_slices = False # <<<<<<<<<<<<<< + * seen_ellipsis = False + * idx = 0 +*/ + __pyx_v_have_slices = 0; + + /* "View.MemoryView":682 + * result = [slice(None)] * ndim + * have_slices = False + * seen_ellipsis = False # <<<<<<<<<<<<<< + * idx = 0 + * for item in tup: +*/ + __pyx_v_seen_ellipsis = 0; + + /* "View.MemoryView":683 + * have_slices = False + * seen_ellipsis = False + * idx = 0 # <<<<<<<<<<<<<< + * for item in tup: + * if item is Ellipsis: +*/ + __pyx_v_idx = 0; + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(1, 684, __pyx_L1_error) + } + __pyx_t_1 = __pyx_v_tup; __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 684, __pyx_L1_error) + #endif + if (__pyx_t_4 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_4); + #endif + ++__pyx_t_4; + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 684, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + __pyx_t_2 = (__pyx_v_item == __pyx_builtin_Ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + __pyx_t_2 = (!__pyx_v_seen_ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":687 + * if item is Ellipsis: + * if not seen_ellipsis: + * idx += ndim - len(tup) # <<<<<<<<<<<<<< + * seen_ellipsis = True + * have_slices = True +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 687, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(__pyx_v_tup); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 687, __pyx_L1_error) + __pyx_v_idx = (__pyx_v_idx + (__pyx_v_ndim - __pyx_t_5)); + + /* "View.MemoryView":688 + * if not seen_ellipsis: + * idx += ndim - len(tup) + * seen_ellipsis = True # <<<<<<<<<<<<<< + * have_slices = True + * else: +*/ + __pyx_v_seen_ellipsis = 1; + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + } + + /* "View.MemoryView":689 + * idx += ndim - len(tup) + * seen_ellipsis = True + * have_slices = True # <<<<<<<<<<<<<< + * else: + * if isinstance(item, slice): +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + /*else*/ { + __pyx_t_2 = PySlice_Check(__pyx_v_item); + if (__pyx_t_2) { + + /* "View.MemoryView":692 + * else: + * if isinstance(item, slice): + * have_slices = True # <<<<<<<<<<<<<< + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + __pyx_t_2 = (!(PyIndex_Check(__pyx_v_item) != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":694 + * have_slices = True + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" # <<<<<<<<<<<<<< + * result[idx] = item + * idx += 1 +*/ + __pyx_t_3 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_item)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6[0] = __pyx_mstate_global->__pyx_kp_u_Cannot_index_with_type; + __pyx_t_6[1] = __pyx_t_3; + __pyx_t_6[2] = __pyx_mstate_global->__pyx_kp_u__4; + __pyx_t_7 = __Pyx_PyUnicode_Join(__pyx_t_6, 3, 24 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_t_7, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 694, __pyx_L1_error) + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + } + __pyx_L7:; + + /* "View.MemoryView":695 + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item # <<<<<<<<<<<<<< + * idx += 1 + * +*/ + if (unlikely((__Pyx_SetItemInt(__pyx_v_result, __pyx_v_idx, __pyx_v_item, Py_ssize_t, 1, PyLong_FromSsize_t, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(1, 695, __pyx_L1_error) + } + __pyx_L5:; + + /* "View.MemoryView":696 + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item + * idx += 1 # <<<<<<<<<<<<<< + * + * nslices = ndim - idx +*/ + __pyx_v_idx = (__pyx_v_idx + 1); + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":698 + * idx += 1 + * + * nslices = ndim - idx # <<<<<<<<<<<<<< + * return have_slices or nslices, tuple(result) + * +*/ + __pyx_v_nslices = (__pyx_v_ndim - __pyx_v_idx); + + /* "View.MemoryView":699 + * + * nslices = ndim - idx + * return have_slices or nslices, tuple(result) # <<<<<<<<<<<<<< + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: +*/ + __Pyx_XDECREF(__pyx_r); + if (!__pyx_v_have_slices) { + } else { + __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_7 = PyLong_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_L9_bool_binop_done:; + __pyx_t_7 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_7 = 0; + __pyx_r = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("View.MemoryView._unellipsify", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_tup); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_item); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + +static int assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_suboffset; + int __pyx_r; + Py_ssize_t *__pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":702 + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" +*/ + __pyx_t_2 = (__pyx_v_suboffsets + __pyx_v_ndim); + for (__pyx_t_3 = __pyx_v_suboffsets; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_suboffset = (__pyx_t_1[0]); + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + __pyx_t_4 = (__pyx_v_suboffset >= 0); + if (unlikely(__pyx_t_4)) { + + /* "View.MemoryView":704 + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" # <<<<<<<<<<<<<< + * return 0 # return type just used as an error flag + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Indirect_dimensions_not_supporte, 0, 0); + __PYX_ERR(1, 704, __pyx_L1_error) + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + } + } + + /* "View.MemoryView":705 + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.assert_direct_dimensions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *__pyx_v_memview, PyObject *__pyx_v_indices) { + int __pyx_v_new_ndim; + int __pyx_v_suboffset_dim; + int __pyx_v_dim; + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + __Pyx_memviewslice *__pyx_v_p_src; + struct __pyx_memoryviewslice_obj *__pyx_v_memviewsliceobj = 0; + __Pyx_memviewslice *__pyx_v_p_dst; + int *__pyx_v_p_suboffset_dim; + Py_ssize_t __pyx_v_start; + Py_ssize_t __pyx_v_stop; + Py_ssize_t __pyx_v_step; + Py_ssize_t __pyx_v_cindex; + int __pyx_v_have_start; + int __pyx_v_have_stop; + int __pyx_v_have_step; + PyObject *__pyx_v_index = NULL; + struct __pyx_memoryview_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + struct __pyx_memoryview_obj *__pyx_t_3; + char *__pyx_t_4; + int __pyx_t_5; + Py_ssize_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memview_slice", 0); + + /* "View.MemoryView":713 + * @cname('__pyx_memview_slice') + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim # <<<<<<<<<<<<<< + * cdef bint negative_step + * cdef __Pyx_memviewslice src, dst +*/ + __pyx_v_new_ndim = 0; + __pyx_v_suboffset_dim = -1; + + /* "View.MemoryView":720 + * + * + * memset(&dst, 0, sizeof(dst)) # <<<<<<<<<<<<<< + * + * cdef _memoryviewslice memviewsliceobj +*/ + (void)(memset((&__pyx_v_dst), 0, (sizeof(__pyx_v_dst)))); + + /* "View.MemoryView":724 + * cdef _memoryviewslice memviewsliceobj + * + * assert memview.view.ndim > 0 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_memview->view.ndim > 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 724, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 724, __pyx_L1_error) + #endif + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":727 + * + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview # <<<<<<<<<<<<<< + * p_src = &memviewsliceobj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 727, __pyx_L1_error) + __pyx_v_memviewsliceobj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":728 + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, &src) +*/ + __pyx_v_p_src = (&__pyx_v_memviewsliceobj->from_slice); + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":730 + * p_src = &memviewsliceobj.from_slice + * else: + * slice_copy(memview, &src) # <<<<<<<<<<<<<< + * p_src = &src + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_src)); + + /* "View.MemoryView":731 + * else: + * slice_copy(memview, &src) + * p_src = &src # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_p_src = (&__pyx_v_src); + } + __pyx_L3:; + + /* "View.MemoryView":737 + * + * + * dst.memview = p_src.memview # <<<<<<<<<<<<<< + * dst.data = p_src.data + * +*/ + __pyx_t_3 = __pyx_v_p_src->memview; + __pyx_v_dst.memview = __pyx_t_3; + + /* "View.MemoryView":738 + * + * dst.memview = p_src.memview + * dst.data = p_src.data # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_v_p_src->data; + __pyx_v_dst.data = __pyx_t_4; + + /* "View.MemoryView":743 + * + * + * cdef __Pyx_memviewslice *p_dst = &dst # <<<<<<<<<<<<<< + * cdef int *p_suboffset_dim = &suboffset_dim + * cdef Py_ssize_t start, stop, step, cindex +*/ + __pyx_v_p_dst = (&__pyx_v_dst); + + /* "View.MemoryView":744 + * + * cdef __Pyx_memviewslice *p_dst = &dst + * cdef int *p_suboffset_dim = &suboffset_dim # <<<<<<<<<<<<<< + * cdef Py_ssize_t start, stop, step, cindex + * cdef bint have_start, have_stop, have_step +*/ + __pyx_v_p_suboffset_dim = (&__pyx_v_suboffset_dim); + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_indices)) || PyTuple_CheckExact(__pyx_v_indices)) { + __pyx_t_2 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 748, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 748, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_6)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 748, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_2); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 748, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_index, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_dim = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + __pyx_t_1 = (PyIndex_Check(__pyx_v_index) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":750 + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): + * cindex = index # <<<<<<<<<<<<<< + * slice_memviewslice( + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], +*/ + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 750, __pyx_L1_error) + __pyx_v_cindex = __pyx_t_9; + + /* "View.MemoryView":751 + * if PyIndex_Check(index): + * cindex = index + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_cindex, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 751, __pyx_L1_error) + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + __pyx_t_1 = (__pyx_v_index == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":758 + * False) + * elif index is None: + * p_dst.shape[new_ndim] = 1 # <<<<<<<<<<<<<< + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 +*/ + (__pyx_v_p_dst->shape[__pyx_v_new_ndim]) = 1; + + /* "View.MemoryView":759 + * elif index is None: + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 # <<<<<<<<<<<<<< + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 +*/ + (__pyx_v_p_dst->strides[__pyx_v_new_ndim]) = 0; + + /* "View.MemoryView":760 + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 # <<<<<<<<<<<<<< + * new_ndim += 1 + * else: +*/ + (__pyx_v_p_dst->suboffsets[__pyx_v_new_ndim]) = -1L; + + /* "View.MemoryView":761 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 # <<<<<<<<<<<<<< + * else: + * start = index.start or 0 +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":763 + * new_ndim += 1 + * else: + * start = index.start or 0 # <<<<<<<<<<<<<< + * stop = index.stop or 0 + * step = index.step or 0 +*/ + /*else*/ { + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 763, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 763, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 763, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L7_bool_binop_done:; + __pyx_v_start = __pyx_t_9; + + /* "View.MemoryView":764 + * else: + * start = index.start or 0 + * stop = index.stop or 0 # <<<<<<<<<<<<<< + * step = index.step or 0 + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 764, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 764, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 764, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L9_bool_binop_done:; + __pyx_v_stop = __pyx_t_9; + + /* "View.MemoryView":765 + * start = index.start or 0 + * stop = index.stop or 0 + * step = index.step or 0 # <<<<<<<<<<<<<< + * + * have_start = index.start is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 765, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 765, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 765, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L11_bool_binop_done:; + __pyx_v_step = __pyx_t_9; + + /* "View.MemoryView":767 + * step = index.step or 0 + * + * have_start = index.start is not None # <<<<<<<<<<<<<< + * have_stop = index.stop is not None + * have_step = index.step is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 767, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_start = __pyx_t_1; + + /* "View.MemoryView":768 + * + * have_start = index.start is not None + * have_stop = index.stop is not None # <<<<<<<<<<<<<< + * have_step = index.step is not None + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 768, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_stop = __pyx_t_1; + + /* "View.MemoryView":769 + * have_start = index.start is not None + * have_stop = index.stop is not None + * have_step = index.step is not None # <<<<<<<<<<<<<< + * + * slice_memviewslice( +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_step = __pyx_t_1; + + /* "View.MemoryView":771 + * have_step = index.step is not None + * + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 771, __pyx_L1_error) + + /* "View.MemoryView":777 + * have_start, have_stop, have_step, + * True) + * new_ndim += 1 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + } + __pyx_L6:; + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":781 + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, # <<<<<<<<<<<<<< + * memviewsliceobj.to_dtype_func, + * memview.dtype_is_object) +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 781, __pyx_L1_error) } + + /* "View.MemoryView":782 + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * else: +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 782, __pyx_L1_error) } + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 780, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + } + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + /*else*/ { + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":786 + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 785, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 785, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("View.MemoryView.memview_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_memviewsliceobj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, Py_ssize_t __pyx_v_shape, Py_ssize_t __pyx_v_stride, Py_ssize_t __pyx_v_suboffset, int __pyx_v_dim, int __pyx_v_new_ndim, int *__pyx_v_suboffset_dim, Py_ssize_t __pyx_v_start, Py_ssize_t __pyx_v_stop, Py_ssize_t __pyx_v_step, int __pyx_v_have_start, int __pyx_v_have_stop, int __pyx_v_have_step, int __pyx_v_is_slice) { + Py_ssize_t __pyx_v_new_shape; + int __pyx_v_negative_step; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + __pyx_t_1 = (!__pyx_v_is_slice); + if (__pyx_t_1) { + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + __pyx_t_1 = (__pyx_v_start < 0); + if (__pyx_t_1) { + + /* "View.MemoryView":817 + * + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + } + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + __pyx_t_1 = (0 <= __pyx_v_start); + if (__pyx_t_1) { + __pyx_t_1 = (__pyx_v_start < __pyx_v_shape); + } + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":819 + * start += shape + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_Index_out_of_bounds_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 819, __pyx_L1_error) + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + /*else*/ { + __pyx_t_2 = (__pyx_v_have_step != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":823 + * + * if have_step: + * negative_step = step < 0 # <<<<<<<<<<<<<< + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) +*/ + __pyx_v_negative_step = (__pyx_v_step < 0); + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + __pyx_t_2 = (__pyx_v_step == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":825 + * negative_step = step < 0 + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * negative_step = False +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Step_may_not_be_zero_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 825, __pyx_L1_error) + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":827 + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: + * negative_step = False # <<<<<<<<<<<<<< + * step = 1 + * +*/ + /*else*/ { + __pyx_v_negative_step = 0; + + /* "View.MemoryView":828 + * else: + * negative_step = False + * step = 1 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_step = 1; + } + __pyx_L6:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + __pyx_t_2 = (__pyx_v_have_start != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":833 + * if have_start: + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if start < 0: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":835 + * start += shape + * if start < 0: + * start = 0 # <<<<<<<<<<<<<< + * elif start >= shape: + * if negative_step: +*/ + __pyx_v_start = 0; + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + } + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + __pyx_t_2 = (__pyx_v_start >= __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + if (__pyx_v_negative_step) { + + /* "View.MemoryView":838 + * elif start >= shape: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = shape +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":840 + * start = shape - 1 + * else: + * start = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + /*else*/ { + __pyx_v_start = __pyx_v_shape; + } + __pyx_L11:; + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + } + __pyx_L9:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":843 + * else: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":845 + * start = shape - 1 + * else: + * start = 0 # <<<<<<<<<<<<<< + * + * if have_stop: +*/ + /*else*/ { + __pyx_v_start = 0; + } + __pyx_L12:; + } + __pyx_L8:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + __pyx_t_2 = (__pyx_v_have_stop != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":849 + * if have_stop: + * if stop < 0: + * stop += shape # <<<<<<<<<<<<<< + * if stop < 0: + * stop = 0 +*/ + __pyx_v_stop = (__pyx_v_stop + __pyx_v_shape); + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":851 + * stop += shape + * if stop < 0: + * stop = 0 # <<<<<<<<<<<<<< + * elif stop > shape: + * stop = shape +*/ + __pyx_v_stop = 0; + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + } + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + goto __pyx_L14; + } + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + __pyx_t_2 = (__pyx_v_stop > __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":853 + * stop = 0 + * elif stop > shape: + * stop = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + __pyx_v_stop = __pyx_v_shape; + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + } + __pyx_L14:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":856 + * else: + * if negative_step: + * stop = -1 # <<<<<<<<<<<<<< + * else: + * stop = shape +*/ + __pyx_v_stop = -1L; + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + goto __pyx_L16; + } + + /* "View.MemoryView":858 + * stop = -1 + * else: + * stop = shape # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_v_stop = __pyx_v_shape; + } + __pyx_L16:; + } + __pyx_L13:; + + /* "View.MemoryView":862 + * + * with cython.cdivision(True): + * new_shape = (stop - start) // step # <<<<<<<<<<<<<< + * + * if (stop - start) - step * new_shape: +*/ + __pyx_v_new_shape = ((__pyx_v_stop - __pyx_v_start) / __pyx_v_step); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + __pyx_t_2 = (((__pyx_v_stop - __pyx_v_start) - (__pyx_v_step * __pyx_v_new_shape)) != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":865 + * + * if (stop - start) - step * new_shape: + * new_shape += 1 # <<<<<<<<<<<<<< + * + * if new_shape < 0: +*/ + __pyx_v_new_shape = (__pyx_v_new_shape + 1); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + } + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + __pyx_t_2 = (__pyx_v_new_shape < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":868 + * + * if new_shape < 0: + * new_shape = 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_new_shape = 0; + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + } + + /* "View.MemoryView":871 + * + * + * dst.strides[new_ndim] = stride * step # <<<<<<<<<<<<<< + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset +*/ + (__pyx_v_dst->strides[__pyx_v_new_ndim]) = (__pyx_v_stride * __pyx_v_step); + + /* "View.MemoryView":872 + * + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape # <<<<<<<<<<<<<< + * dst.suboffsets[new_ndim] = suboffset + * +*/ + (__pyx_v_dst->shape[__pyx_v_new_ndim]) = __pyx_v_new_shape; + + /* "View.MemoryView":873 + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_dst->suboffsets[__pyx_v_new_ndim]) = __pyx_v_suboffset; + } + __pyx_L3:; + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + __pyx_t_2 = ((__pyx_v_suboffset_dim[0]) < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":877 + * + * if suboffset_dim[0] < 0: + * dst.data += start * stride # <<<<<<<<<<<<<< + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride +*/ + __pyx_v_dst->data = (__pyx_v_dst->data + (__pyx_v_start * __pyx_v_stride)); + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + goto __pyx_L19; + } + + /* "View.MemoryView":879 + * dst.data += start * stride + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride # <<<<<<<<<<<<<< + * + * if suboffset >= 0: +*/ + /*else*/ { + __pyx_t_3 = (__pyx_v_suboffset_dim[0]); + (__pyx_v_dst->suboffsets[__pyx_t_3]) = ((__pyx_v_dst->suboffsets[__pyx_t_3]) + (__pyx_v_start * __pyx_v_stride)); + } + __pyx_L19:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + __pyx_t_2 = (!__pyx_v_is_slice); + if (__pyx_t_2) { + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + __pyx_t_2 = (__pyx_v_new_ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":884 + * if not is_slice: + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset # <<<<<<<<<<<<<< + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " +*/ + __pyx_v_dst->data = ((((char **)__pyx_v_dst->data)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + goto __pyx_L22; + } + + /* "View.MemoryView":886 + * dst.data = ( dst.data)[0] + suboffset + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " # <<<<<<<<<<<<<< + * "must be indexed and not sliced", dim) + * else: +*/ + /*else*/ { + + /* "View.MemoryView":887 + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " + * "must be indexed and not sliced", dim) # <<<<<<<<<<<<<< + * else: + * suboffset_dim[0] = new_ndim +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_All_dimensions_preceding_dimensi, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 886, __pyx_L1_error) + } + __pyx_L22:; + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + goto __pyx_L21; + } + + /* "View.MemoryView":889 + * "must be indexed and not sliced", dim) + * else: + * suboffset_dim[0] = new_ndim # <<<<<<<<<<<<<< + * + * return 0 +*/ + /*else*/ { + (__pyx_v_suboffset_dim[0]) = __pyx_v_new_ndim; + } + __pyx_L21:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + } + + /* "View.MemoryView":891 + * suboffset_dim[0] = new_ndim + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.slice_memviewslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + +static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, Py_ssize_t __pyx_v_index, Py_ssize_t __pyx_v_dim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_suboffset; + Py_ssize_t __pyx_v_itemsize; + char *__pyx_v_resultp; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[3]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("pybuffer_index", 0); + + /* "View.MemoryView":899 + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 # <<<<<<<<<<<<<< + * cdef Py_ssize_t itemsize = view.itemsize + * cdef char *resultp +*/ + __pyx_v_suboffset = -1L; + + /* "View.MemoryView":900 + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 + * cdef Py_ssize_t itemsize = view.itemsize # <<<<<<<<<<<<<< + * cdef char *resultp + * +*/ + __pyx_t_1 = __pyx_v_view->itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + __pyx_t_2 = (__pyx_v_view->ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":904 + * + * if view.ndim == 0: + * shape = view.len // itemsize # <<<<<<<<<<<<<< + * stride = itemsize + * else: +*/ + if (unlikely(__pyx_v_itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_view->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + __pyx_v_shape = __Pyx_div_Py_ssize_t(__pyx_v_view->len, __pyx_v_itemsize, 0); + + /* "View.MemoryView":905 + * if view.ndim == 0: + * shape = view.len // itemsize + * stride = itemsize # <<<<<<<<<<<<<< + * else: + * shape = view.shape[dim] +*/ + __pyx_v_stride = __pyx_v_itemsize; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":907 + * stride = itemsize + * else: + * shape = view.shape[dim] # <<<<<<<<<<<<<< + * stride = view.strides[dim] + * if view.suboffsets != NULL: +*/ + /*else*/ { + __pyx_v_shape = (__pyx_v_view->shape[__pyx_v_dim]); + + /* "View.MemoryView":908 + * else: + * shape = view.shape[dim] + * stride = view.strides[dim] # <<<<<<<<<<<<<< + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] +*/ + __pyx_v_stride = (__pyx_v_view->strides[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + __pyx_t_2 = (__pyx_v_view->suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":910 + * stride = view.strides[dim] + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] # <<<<<<<<<<<<<< + * + * if index < 0: +*/ + __pyx_v_suboffset = (__pyx_v_view->suboffsets[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + } + } + __pyx_L3:; + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":913 + * + * if index < 0: + * index += view.shape[dim] # <<<<<<<<<<<<<< + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" +*/ + __pyx_v_index = (__pyx_v_index + (__pyx_v_view->shape[__pyx_v_dim])); + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":915 + * index += view.shape[dim] + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * if index >= shape: +*/ + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_3; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 915, __pyx_L1_error) + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + } + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index >= __pyx_v_shape); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":918 + * + * if index >= shape: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * resultp = bufp + index * stride +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_5; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 1, 127); + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 918, __pyx_L1_error) + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":920 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * resultp = bufp + index * stride # <<<<<<<<<<<<<< + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset +*/ + __pyx_v_resultp = (__pyx_v_bufp + (__pyx_v_index * __pyx_v_stride)); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":922 + * resultp = bufp + index * stride + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset # <<<<<<<<<<<<<< + * + * return resultp +*/ + __pyx_v_resultp = ((((char **)__pyx_v_resultp)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + } + + /* "View.MemoryView":924 + * resultp = ( resultp)[0] + suboffset + * + * return resultp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_resultp; + goto __pyx_L0; + + /* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.pybuffer_index", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + +static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { + int __pyx_v_ndim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + int __pyx_v_i; + int __pyx_v_j; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + long __pyx_t_3; + long __pyx_t_4; + Py_ssize_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":931 + * @cname('__pyx_memslice_transpose') + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim # <<<<<<<<<<<<<< + * + * cdef Py_ssize_t *shape = memslice.shape +*/ + __pyx_t_1 = __pyx_v_memslice->memview->view.ndim; + __pyx_v_ndim = __pyx_t_1; + + /* "View.MemoryView":933 + * cdef int ndim = memslice.memview.view.ndim + * + * cdef Py_ssize_t *shape = memslice.shape # <<<<<<<<<<<<<< + * cdef Py_ssize_t *strides = memslice.strides + * +*/ + __pyx_t_2 = __pyx_v_memslice->shape; + __pyx_v_shape = __pyx_t_2; + + /* "View.MemoryView":934 + * + * cdef Py_ssize_t *shape = memslice.shape + * cdef Py_ssize_t *strides = memslice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_v_memslice->strides; + __pyx_v_strides = __pyx_t_2; + + /* "View.MemoryView":938 + * + * cdef int i, j + * for i in range(ndim // 2): # <<<<<<<<<<<<<< + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] +*/ + __pyx_t_3 = __Pyx_div_long(__pyx_v_ndim, 2, 1); + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_4; __pyx_t_1+=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":939 + * cdef int i, j + * for i in range(ndim // 2): + * j = ndim - 1 - i # <<<<<<<<<<<<<< + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] +*/ + __pyx_v_j = ((__pyx_v_ndim - 1) - __pyx_v_i); + + /* "View.MemoryView":940 + * for i in range(ndim // 2): + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] # <<<<<<<<<<<<<< + * shape[i], shape[j] = shape[j], shape[i] + * +*/ + __pyx_t_5 = (__pyx_v_strides[__pyx_v_j]); + __pyx_t_6 = (__pyx_v_strides[__pyx_v_i]); + (__pyx_v_strides[__pyx_v_i]) = __pyx_t_5; + (__pyx_v_strides[__pyx_v_j]) = __pyx_t_6; + + /* "View.MemoryView":941 + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] # <<<<<<<<<<<<<< + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: +*/ + __pyx_t_6 = (__pyx_v_shape[__pyx_v_j]); + __pyx_t_5 = (__pyx_v_shape[__pyx_v_i]); + (__pyx_v_shape[__pyx_v_i]) = __pyx_t_6; + (__pyx_v_shape[__pyx_v_j]) = __pyx_t_5; + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_i]) >= 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_j]) >= 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L6_bool_binop_done:; + if (__pyx_t_7) { + + /* "View.MemoryView":944 + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_t_9 = __pyx_memoryview_err(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Cannot_transpose_memoryview_with); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 944, __pyx_L1_error) + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + } + } + + /* "View.MemoryView":946 + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.transpose_memslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + +/* Python wrapper */ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + + /* "View.MemoryView":965 + * + * def __dealloc__(self): + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_self->from_slice), 1); + + /* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_object_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":969 + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) # <<<<<<<<<<<<<< + * else: + * return memoryview.convert_item_to_object(self, itemp) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 969, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + } + + /* "View.MemoryView":971 + * return self.to_object_func(itemp) + * else: + * return memoryview.convert_item_to_object(self, itemp) # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 971, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_dtype_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":975 + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) # <<<<<<<<<<<<<< + * else: + * memoryview.assign_item_from_object(self, itemp, value) +*/ + __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 975, __pyx_L1_error) + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":977 + * self.to_dtype_func(itemp, value) + * else: + * memoryview.assign_item_from_object(self, itemp, value) # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + /*else*/ { + __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 977, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":980 + * + * cdef _get_base(self): + * return self.from_object # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->from_object); + __pyx_r = __pyx_v_self->from_object; + goto __pyx_L0; + + /* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryviewslice___reduce_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryviewslice_2__setstate_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewslice, int __pyx_v_ndim, PyObject *(*__pyx_v_to_object_func)(char *), int (*__pyx_v_to_dtype_func)(char *, PyObject *), int __pyx_v_dtype_is_object) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + Py_ssize_t __pyx_v_suboffset; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_TypeInfo const *__pyx_t_4; + Py_buffer __pyx_t_5; + Py_ssize_t *__pyx_t_6; + Py_ssize_t *__pyx_t_7; + Py_ssize_t *__pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_fromslice", 0); + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_memviewslice.memview) == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":1009 + * + * if memviewslice.memview == Py_None: + * return None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + } + + /* "View.MemoryView":1014 + * + * + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) # <<<<<<<<<<<<<< + * + * result.from_slice = memviewslice +*/ + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, Py_None) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_0) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_2 = ((PyObject *)__pyx_tp_new__memoryviewslice(((PyTypeObject *)__pyx_mstate_global->__pyx_memoryviewslice_type), __pyx_t_3, NULL)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1016 + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) + * + * result.from_slice = memviewslice # <<<<<<<<<<<<<< + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * +*/ + __pyx_v_result->from_slice = __pyx_v_memviewslice; + + /* "View.MemoryView":1017 + * + * result.from_slice = memviewslice + * __PYX_INC_MEMVIEW(&memviewslice, 1) # <<<<<<<<<<<<<< + * + * result.from_object = ( memviewslice.memview)._get_base() +*/ + __PYX_INC_MEMVIEW((&__pyx_v_memviewslice), 1); + + /* "View.MemoryView":1019 + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * + * result.from_object = ( memviewslice.memview)._get_base() # <<<<<<<<<<<<<< + * result.typeinfo = memviewslice.memview.typeinfo + * +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->__pyx_vtab)->_get_base(((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1019, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_result->from_object); + __Pyx_DECREF(__pyx_v_result->from_object); + __pyx_v_result->from_object = __pyx_t_2; + __pyx_t_2 = 0; + + /* "View.MemoryView":1020 + * + * result.from_object = ( memviewslice.memview)._get_base() + * result.typeinfo = memviewslice.memview.typeinfo # <<<<<<<<<<<<<< + * + * result.view = memviewslice.memview.view +*/ + __pyx_t_4 = __pyx_v_memviewslice.memview->typeinfo; + __pyx_v_result->__pyx_base.typeinfo = __pyx_t_4; + + /* "View.MemoryView":1022 + * result.typeinfo = memviewslice.memview.typeinfo + * + * result.view = memviewslice.memview.view # <<<<<<<<<<<<<< + * result.view.buf = memviewslice.data + * result.view.ndim = ndim +*/ + __pyx_t_5 = __pyx_v_memviewslice.memview->view; + __pyx_v_result->__pyx_base.view = __pyx_t_5; + + /* "View.MemoryView":1023 + * + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data # <<<<<<<<<<<<<< + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None +*/ + __pyx_v_result->__pyx_base.view.buf = ((void *)__pyx_v_memviewslice.data); + + /* "View.MemoryView":1024 + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data + * result.view.ndim = ndim # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_v_result->__pyx_base.view.ndim = __pyx_v_ndim; + + /* "View.MemoryView":1025 + * result.view.buf = memviewslice.data + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_result->__pyx_base.view))->obj = Py_None; + + /* "View.MemoryView":1026 + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + __pyx_t_1 = ((((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1029 + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: + * result.flags = PyBUF_RECORDS # <<<<<<<<<<<<<< + * else: + * result.flags = PyBUF_RECORDS_RO +*/ + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS; + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1031 + * result.flags = PyBUF_RECORDS + * else: + * result.flags = PyBUF_RECORDS_RO # <<<<<<<<<<<<<< + * + * result.view.shape = result.from_slice.shape +*/ + /*else*/ { + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS_RO; + } + __pyx_L4:; + + /* "View.MemoryView":1033 + * result.flags = PyBUF_RECORDS_RO + * + * result.view.shape = result.from_slice.shape # <<<<<<<<<<<<<< + * result.view.strides = result.from_slice.strides + * +*/ + __pyx_v_result->__pyx_base.view.shape = ((Py_ssize_t *)__pyx_v_result->from_slice.shape); + + /* "View.MemoryView":1034 + * + * result.view.shape = result.from_slice.shape + * result.view.strides = result.from_slice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_result->__pyx_base.view.strides = ((Py_ssize_t *)__pyx_v_result->from_slice.strides); + + /* "View.MemoryView":1037 + * + * + * result.view.suboffsets = NULL # <<<<<<<<<<<<<< + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: +*/ + __pyx_v_result->__pyx_base.view.suboffsets = NULL; + + /* "View.MemoryView":1038 + * + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets +*/ + __pyx_t_7 = (__pyx_v_result->from_slice.suboffsets + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->from_slice.suboffsets; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_v_suboffset = (__pyx_t_6[0]); + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + __pyx_t_1 = (__pyx_v_suboffset >= 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1040 + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_result->__pyx_base.view.suboffsets = ((Py_ssize_t *)__pyx_v_result->from_slice.suboffsets); + + /* "View.MemoryView":1041 + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets + * break # <<<<<<<<<<<<<< + * + * result.view.len = result.view.itemsize +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + } + } + __pyx_L6_break:; + + /* "View.MemoryView":1043 + * break + * + * result.view.len = result.view.itemsize # <<<<<<<<<<<<<< + * for length in result.view.shape[:ndim]: + * result.view.len *= length +*/ + __pyx_t_9 = __pyx_v_result->__pyx_base.view.itemsize; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + + /* "View.MemoryView":1044 + * + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: # <<<<<<<<<<<<<< + * result.view.len *= length + * +*/ + __pyx_t_7 = (__pyx_v_result->__pyx_base.view.shape + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->__pyx_base.view.shape; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_t_2 = PyLong_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1044, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1045 + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: + * result.view.len *= length # <<<<<<<<<<<<<< + * + * result.to_object_func = to_object_func +*/ + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + } + + /* "View.MemoryView":1047 + * result.view.len *= length + * + * result.to_object_func = to_object_func # <<<<<<<<<<<<<< + * result.to_dtype_func = to_dtype_func + * +*/ + __pyx_v_result->to_object_func = __pyx_v_to_object_func; + + /* "View.MemoryView":1048 + * + * result.to_object_func = to_object_func + * result.to_dtype_func = to_dtype_func # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->to_dtype_func = __pyx_v_to_dtype_func; + + /* "View.MemoryView":1050 + * result.to_dtype_func = to_dtype_func + * + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview_fromslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_mslice) { + struct __pyx_memoryviewslice_obj *__pyx_v_obj = 0; + __Pyx_memviewslice *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_slice_from_memview", 0); + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1057 + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): + * obj = memview # <<<<<<<<<<<<<< + * return &obj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 1057, __pyx_L1_error) + __pyx_v_obj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1058 + * if isinstance(memview, _memoryviewslice): + * obj = memview + * return &obj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, mslice) +*/ + __pyx_r = (&__pyx_v_obj->from_slice); + goto __pyx_L0; + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + } + + /* "View.MemoryView":1060 + * return &obj.from_slice + * else: + * slice_copy(memview, mslice) # <<<<<<<<<<<<<< + * return mslice + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, __pyx_v_mslice); + + /* "View.MemoryView":1061 + * else: + * slice_copy(memview, mslice) + * return mslice # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_slice_copy') +*/ + __pyx_r = __pyx_v_mslice; + goto __pyx_L0; + } + + /* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.get_slice_from_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_obj); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_dst) { + int __pyx_v_dim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + Py_ssize_t *__pyx_v_suboffsets; + Py_ssize_t *__pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + + /* "View.MemoryView":1068 + * cdef (Py_ssize_t*) shape, strides, suboffsets + * + * shape = memview.view.shape # <<<<<<<<<<<<<< + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets +*/ + __pyx_t_1 = __pyx_v_memview->view.shape; + __pyx_v_shape = __pyx_t_1; + + /* "View.MemoryView":1069 + * + * shape = memview.view.shape + * strides = memview.view.strides # <<<<<<<<<<<<<< + * suboffsets = memview.view.suboffsets + * +*/ + __pyx_t_1 = __pyx_v_memview->view.strides; + __pyx_v_strides = __pyx_t_1; + + /* "View.MemoryView":1070 + * shape = memview.view.shape + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets # <<<<<<<<<<<<<< + * + * dst.memview = <__pyx_memoryview *> memview +*/ + __pyx_t_1 = __pyx_v_memview->view.suboffsets; + __pyx_v_suboffsets = __pyx_t_1; + + /* "View.MemoryView":1072 + * suboffsets = memview.view.suboffsets + * + * dst.memview = <__pyx_memoryview *> memview # <<<<<<<<<<<<<< + * dst.data = memview.view.buf + * +*/ + __pyx_v_dst->memview = ((struct __pyx_memoryview_obj *)__pyx_v_memview); + + /* "View.MemoryView":1073 + * + * dst.memview = <__pyx_memoryview *> memview + * dst.data = memview.view.buf # <<<<<<<<<<<<<< + * + * for dim in range(memview.view.ndim): +*/ + __pyx_v_dst->data = ((char *)__pyx_v_memview->view.buf); + + /* "View.MemoryView":1075 + * dst.data = memview.view.buf + * + * for dim in range(memview.view.ndim): # <<<<<<<<<<<<<< + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] +*/ + __pyx_t_2 = __pyx_v_memview->view.ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_dim = __pyx_t_4; + + /* "View.MemoryView":1076 + * + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] # <<<<<<<<<<<<<< + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 +*/ + (__pyx_v_dst->shape[__pyx_v_dim]) = (__pyx_v_shape[__pyx_v_dim]); + + /* "View.MemoryView":1077 + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] # <<<<<<<<<<<<<< + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * +*/ + (__pyx_v_dst->strides[__pyx_v_dim]) = (__pyx_v_strides[__pyx_v_dim]); + + /* "View.MemoryView":1078 + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object') +*/ + __pyx_t_6 = (__pyx_v_suboffsets != 0); + if (__pyx_t_6) { + __pyx_t_5 = (__pyx_v_suboffsets[__pyx_v_dim]); + } else { + __pyx_t_5 = -1L; + } + (__pyx_v_dst->suboffsets[__pyx_v_dim]) = __pyx_t_5; + } + + /* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx_v_memview) { + __Pyx_memviewslice __pyx_v_memviewslice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy", 0); + + /* "View.MemoryView":1084 + * "Create a new memoryview object" + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) # <<<<<<<<<<<<<< + * return memoryview_copy_from_slice(memview, &memviewslice) + * +*/ + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_memviewslice)); + + /* "View.MemoryView":1085 + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) + * return memoryview_copy_from_slice(memview, &memviewslice) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object_from_slice') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1085, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_memviewslice) { + PyObject *(*__pyx_v_to_object_func)(char *); + int (*__pyx_v_to_dtype_func)(char *, PyObject *); + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *(*__pyx_t_2)(char *); + int (*__pyx_t_3)(char *, PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy_from_slice", 0); + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1096 + * + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func # <<<<<<<<<<<<<< + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: +*/ + __pyx_t_2 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_object_func; + __pyx_v_to_object_func = __pyx_t_2; + + /* "View.MemoryView":1097 + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func # <<<<<<<<<<<<<< + * else: + * to_object_func = NULL +*/ + __pyx_t_3 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_dtype_func; + __pyx_v_to_dtype_func = __pyx_t_3; + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1099 + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: + * to_object_func = NULL # <<<<<<<<<<<<<< + * to_dtype_func = NULL + * +*/ + /*else*/ { + __pyx_v_to_object_func = NULL; + + /* "View.MemoryView":1100 + * else: + * to_object_func = NULL + * to_dtype_func = NULL # <<<<<<<<<<<<<< + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, +*/ + __pyx_v_to_dtype_func = NULL; + } + __pyx_L3:; + + /* "View.MemoryView":1102 + * to_dtype_func = NULL + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, # <<<<<<<<<<<<<< + * to_object_func, to_dtype_func, + * memview.dtype_is_object) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "View.MemoryView":1104 + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, + * to_object_func, to_dtype_func, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_from_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + +static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":1111 + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: + * return -arg if arg < 0 else arg # <<<<<<<<<<<<<< + * + * @cname('__pyx_get_best_slice_order') +*/ + __pyx_t_2 = (__pyx_v_arg < 0); + if (__pyx_t_2) { + __pyx_t_1 = (-__pyx_v_arg); + } else { + __pyx_t_1 = __pyx_v_arg; + } + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + +static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim) { + int __pyx_v_i; + Py_ssize_t __pyx_v_c_stride; + Py_ssize_t __pyx_v_f_stride; + char __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1119 + * """ + * cdef int i + * cdef Py_ssize_t c_stride = 0 # <<<<<<<<<<<<<< + * cdef Py_ssize_t f_stride = 0 + * +*/ + __pyx_v_c_stride = 0; + + /* "View.MemoryView":1120 + * cdef int i + * cdef Py_ssize_t c_stride = 0 + * cdef Py_ssize_t f_stride = 0 # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_f_stride = 0; + + /* "View.MemoryView":1122 + * cdef Py_ssize_t f_stride = 0 + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1124 + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_c_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1125 + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + goto __pyx_L4_break; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L4_break:; + + /* "View.MemoryView":1127 + * break + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] +*/ + __pyx_t_1 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_1; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1129 + * for i in range(ndim): + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_f_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1130 + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): +*/ + goto __pyx_L7_break; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L7_break:; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + __pyx_t_2 = (abs_py_ssize_t(__pyx_v_c_stride) <= abs_py_ssize_t(__pyx_v_f_stride)); + if (__pyx_t_2) { + + /* "View.MemoryView":1133 + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): + * return 'C' # <<<<<<<<<<<<<< + * else: + * return 'F' +*/ + __pyx_r = 'C'; + goto __pyx_L0; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + } + + /* "View.MemoryView":1135 + * return 'C' + * else: + * return 'F' # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + /*else*/ { + __pyx_r = 'F'; + goto __pyx_L0; + } + + /* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + +static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v_src_strides, char *__pyx_v_dst_data, Py_ssize_t *__pyx_v_dst_strides, Py_ssize_t *__pyx_v_src_shape, Py_ssize_t *__pyx_v_dst_shape, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + CYTHON_UNUSED Py_ssize_t __pyx_v_src_extent; + Py_ssize_t __pyx_v_dst_extent; + Py_ssize_t __pyx_v_src_stride; + Py_ssize_t __pyx_v_dst_stride; + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + + /* "View.MemoryView":1145 + * + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] +*/ + __pyx_v_src_extent = (__pyx_v_src_shape[0]); + + /* "View.MemoryView":1146 + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] +*/ + __pyx_v_dst_extent = (__pyx_v_dst_shape[0]); + + /* "View.MemoryView":1147 + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_stride = dst_strides[0] + * +*/ + __pyx_v_src_stride = (__pyx_v_src_strides[0]); + + /* "View.MemoryView":1148 + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_dst_stride = (__pyx_v_dst_strides[0]); + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + __pyx_t_2 = (__pyx_v_src_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_dst_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + + /* "View.MemoryView":1152 + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: +*/ + __pyx_t_2 = (((size_t)__pyx_v_src_stride) == __pyx_v_itemsize); + if (__pyx_t_2) { + __pyx_t_2 = (__pyx_v_itemsize == ((size_t)__pyx_v_dst_stride)); + } + __pyx_t_1 = __pyx_t_2; + __pyx_L5_bool_binop_done:; + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":1153 + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, (__pyx_v_itemsize * __pyx_v_dst_extent))); + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1155 + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1156 + * else: + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) # <<<<<<<<<<<<<< + * src_data += src_stride + * dst_data += dst_stride +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, __pyx_v_itemsize)); + + /* "View.MemoryView":1157 + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * else: +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1158 + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L4:; + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1160 + * dst_data += dst_stride + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * _copy_strided_to_strided(src_data, src_strides + 1, + * dst_data, dst_strides + 1, +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1161 + * else: + * for i in range(dst_extent): + * _copy_strided_to_strided(src_data, src_strides + 1, # <<<<<<<<<<<<<< + * dst_data, dst_strides + 1, + * src_shape + 1, dst_shape + 1, +*/ + _copy_strided_to_strided(__pyx_v_src_data, (__pyx_v_src_strides + 1), __pyx_v_dst_data, (__pyx_v_dst_strides + 1), (__pyx_v_src_shape + 1), (__pyx_v_dst_shape + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize); + + /* "View.MemoryView":1165 + * src_shape + 1, dst_shape + 1, + * ndim - 1, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1166 + * ndim - 1, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + +static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + + /* "View.MemoryView":1171 + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: + * _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, # <<<<<<<<<<<<<< + * src.shape, dst.shape, ndim, itemsize) + * +*/ + _copy_strided_to_strided(__pyx_v_src->data, __pyx_v_src->strides, __pyx_v_dst->data, __pyx_v_dst->strides, __pyx_v_src->shape, __pyx_v_dst->shape, __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_src, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_size; + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + + /* "View.MemoryView":1177 + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" + * cdef Py_ssize_t shape, size = src.memview.view.itemsize # <<<<<<<<<<<<<< + * + * for shape in src.shape[:ndim]: +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_size = __pyx_t_1; + + /* "View.MemoryView":1179 + * cdef Py_ssize_t shape, size = src.memview.view.itemsize + * + * for shape in src.shape[:ndim]: # <<<<<<<<<<<<<< + * size *= shape + * +*/ + __pyx_t_3 = (__pyx_v_src->shape + __pyx_v_ndim); + for (__pyx_t_4 = __pyx_v_src->shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_v_shape = (__pyx_t_2[0]); + + /* "View.MemoryView":1180 + * + * for shape in src.shape[:ndim]: + * size *= shape # <<<<<<<<<<<<<< + * + * return size +*/ + __pyx_v_size = (__pyx_v_size * __pyx_v_shape); + } + + /* "View.MemoryView":1182 + * size *= shape + * + * return size # <<<<<<<<<<<<<< + * + * @cname('__pyx_fill_contig_strides_array') +*/ + __pyx_r = __pyx_v_size; + goto __pyx_L0; + + /* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, Py_ssize_t __pyx_v_stride, int __pyx_v_ndim, char __pyx_v_order) { + int __pyx_v_idx; + Py_ssize_t __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + __pyx_t_1 = (__pyx_v_order == 'F'); + if (__pyx_t_1) { + + /* "View.MemoryView":1195 + * + * if order == 'F': + * for idx in range(ndim): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + __pyx_t_2 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_idx = __pyx_t_4; + + /* "View.MemoryView":1196 + * if order == 'F': + * for idx in range(ndim): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * else: +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1197 + * for idx in range(ndim): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * else: + * for idx in range(ndim - 1, -1, -1): +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1199 + * stride *= shape[idx] + * else: + * for idx in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + /*else*/ { + for (__pyx_t_2 = (__pyx_v_ndim - 1); __pyx_t_2 > -1; __pyx_t_2-=1) { + __pyx_v_idx = __pyx_t_2; + + /* "View.MemoryView":1200 + * else: + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1201 + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * + * return stride +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + } + __pyx_L3:; + + /* "View.MemoryView":1203 + * stride *= shape[idx] + * + * return stride # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_data_to_temp') +*/ + __pyx_r = __pyx_v_stride; + goto __pyx_L0; + + /* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_tmpslice, char __pyx_v_order, int __pyx_v_ndim) { + int __pyx_v_i; + void *__pyx_v_result; + size_t __pyx_v_itemsize; + size_t __pyx_v_size; + void *__pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + struct __pyx_memoryview_obj *__pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1217 + * cdef void *result + * + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef size_t size = slice_get_size(src, ndim) + * +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1218 + * + * cdef size_t itemsize = src.memview.view.itemsize + * cdef size_t size = slice_get_size(src, ndim) # <<<<<<<<<<<<<< + * + * result = malloc(size) +*/ + __pyx_v_size = __pyx_memoryview_slice_get_size(__pyx_v_src, __pyx_v_ndim); + + /* "View.MemoryView":1220 + * cdef size_t size = slice_get_size(src, ndim) + * + * result = malloc(size) # <<<<<<<<<<<<<< + * if not result: + * _err_no_memory() +*/ + __pyx_v_result = malloc(__pyx_v_size); + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + __pyx_t_2 = (!(__pyx_v_result != 0)); + if (__pyx_t_2) { + + /* "View.MemoryView":1222 + * result = malloc(size) + * if not result: + * _err_no_memory() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __pyx_memoryview_err_no_memory(); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1222, __pyx_L1_error) + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + } + + /* "View.MemoryView":1225 + * + * + * tmpslice.data = result # <<<<<<<<<<<<<< + * tmpslice.memview = src.memview + * for i in range(ndim): +*/ + __pyx_v_tmpslice->data = ((char *)__pyx_v_result); + + /* "View.MemoryView":1226 + * + * tmpslice.data = result + * tmpslice.memview = src.memview # <<<<<<<<<<<<<< + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] +*/ + __pyx_t_4 = __pyx_v_src->memview; + __pyx_v_tmpslice->memview = __pyx_t_4; + + /* "View.MemoryView":1227 + * tmpslice.data = result + * tmpslice.memview = src.memview + * for i in range(ndim): # <<<<<<<<<<<<<< + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1228 + * tmpslice.memview = src.memview + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] # <<<<<<<<<<<<<< + * tmpslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_tmpslice->shape[__pyx_v_i]) = (__pyx_v_src->shape[__pyx_v_i]); + + /* "View.MemoryView":1229 + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) +*/ + (__pyx_v_tmpslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1231 + * tmpslice.suboffsets[i] = -1 + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # <<<<<<<<<<<<<< + * + * +*/ + (void)(__pyx_fill_contig_strides_array((&(__pyx_v_tmpslice->shape[0])), (&(__pyx_v_tmpslice->strides[0])), __pyx_v_itemsize, __pyx_v_ndim, __pyx_v_order)); + + /* "View.MemoryView":1234 + * + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + __pyx_t_2 = ((__pyx_v_tmpslice->shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1236 + * for i in range(ndim): + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 # <<<<<<<<<<<<<< + * + * if slice_is_contig(src[0], order, ndim): +*/ + (__pyx_v_tmpslice->strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + } + } + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig((__pyx_v_src[0]), __pyx_v_order, __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1239 + * + * if slice_is_contig(src[0], order, ndim): + * memcpy(result, src.data, size) # <<<<<<<<<<<<<< + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) +*/ + (void)(memcpy(__pyx_v_result, __pyx_v_src->data, __pyx_v_size)); + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":1241 + * memcpy(result, src.data, size) + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) # <<<<<<<<<<<<<< + * + * return result +*/ + /*else*/ { + copy_strided_to_strided(__pyx_v_src, __pyx_v_tmpslice, __pyx_v_ndim, __pyx_v_itemsize); + } + __pyx_L9:; + + /* "View.MemoryView":1243 + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.copy_data_to_temp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + +static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent1, Py_ssize_t __pyx_v_extent2) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[7]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_extents", 0); + + /* "View.MemoryView":1250 + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_dim') +*/ + __pyx_t_1 = __Pyx_PyUnicode_From_int(__pyx_v_i, 0, ' ', 'd'); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent1, 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent2, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_got_differing_extents_in_dimensi; + __pyx_t_4[1] = __pyx_t_1; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_got; + __pyx_t_4[3] = __pyx_t_2; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_4[5] = __pyx_t_3; + __pyx_t_4[6] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 7, 35 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_1) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 1250, __pyx_L1_error) + + /* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView._err_extents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + +static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, PyObject *__pyx_v_msg, int __pyx_v_dim) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_dim", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1254 + * @cname('__pyx_memoryview_err_dim') + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err') +*/ + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_FormatSafe(__pyx_v_msg, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_t_2, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 1254, __pyx_L1_error) + + /* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._err_dim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + +static int __pyx_memoryview_err(PyObject *__pyx_v_error, PyObject *__pyx_v_msg) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1258 + * @cname('__pyx_memoryview_err') + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_no_memory') +*/ + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_v_msg, 0, 0); + __PYX_ERR(1, 1258, __pyx_L1_error) + + /* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + +static int __pyx_memoryview_err_no_memory(void) { + int __pyx_r; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1262 + * @cname('__pyx_memoryview_err_no_memory') + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError # <<<<<<<<<<<<<< + * + * +*/ + PyErr_NoMemory(); __PYX_ERR(1, 1262, __pyx_L1_error) + + /* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err_no_memory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_memviewslice __pyx_v_dst, int __pyx_v_src_ndim, int __pyx_v_dst_ndim, int __pyx_v_dtype_is_object) { + void *__pyx_v_tmpdata; + size_t __pyx_v_itemsize; + int __pyx_v_i; + char __pyx_v_order; + int __pyx_v_broadcasting; + int __pyx_v_direct_copy; + __Pyx_memviewslice __pyx_v_tmp; + int __pyx_v_ndim; + int __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + void *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1274 + * Check for overlapping memory and verify the shapes. + * """ + * cdef void *tmpdata = NULL # <<<<<<<<<<<<<< + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i +*/ + __pyx_v_tmpdata = NULL; + + /* "View.MemoryView":1275 + * """ + * cdef void *tmpdata = NULL + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) +*/ + __pyx_t_1 = __pyx_v_src.memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1277 + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) # <<<<<<<<<<<<<< + * cdef bint broadcasting = False + * cdef bint direct_copy = False +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_src), __pyx_v_src_ndim); + + /* "View.MemoryView":1278 + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False # <<<<<<<<<<<<<< + * cdef bint direct_copy = False + * cdef __Pyx_memviewslice tmp +*/ + __pyx_v_broadcasting = 0; + + /* "View.MemoryView":1279 + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False + * cdef bint direct_copy = False # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice tmp + * +*/ + __pyx_v_direct_copy = 0; + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + __pyx_t_2 = (__pyx_v_src_ndim < __pyx_v_dst_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1283 + * + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_src), __pyx_v_src_ndim, __pyx_v_dst_ndim); + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + __pyx_t_2 = (__pyx_v_dst_ndim < __pyx_v_src_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1285 + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) # <<<<<<<<<<<<<< + * + * cdef int ndim = max(src_ndim, dst_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_dst), __pyx_v_dst_ndim, __pyx_v_src_ndim); + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + } + __pyx_L3:; + + /* "View.MemoryView":1287 + * broadcast_leading(&dst, dst_ndim, src_ndim) + * + * cdef int ndim = max(src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + __pyx_t_3 = __pyx_v_dst_ndim; + __pyx_t_4 = __pyx_v_src_ndim; + __pyx_t_2 = (__pyx_t_3 > __pyx_t_4); + if (__pyx_t_2) { + __pyx_t_5 = __pyx_t_3; + } else { + __pyx_t_5 = __pyx_t_4; + } + __pyx_v_ndim = __pyx_t_5; + + /* "View.MemoryView":1289 + * cdef int ndim = max(src_ndim, dst_ndim) + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: +*/ + __pyx_t_5 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_5; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) != (__pyx_v_dst.shape[__pyx_v_i])); + if (__pyx_t_2) { + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1292 + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: + * broadcasting = True # <<<<<<<<<<<<<< + * src.strides[i] = 0 + * else: +*/ + __pyx_v_broadcasting = 1; + + /* "View.MemoryView":1293 + * if src.shape[i] == 1: + * broadcasting = True + * src.strides[i] = 0 # <<<<<<<<<<<<<< + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) +*/ + (__pyx_v_src.strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":1295 + * src.strides[i] = 0 + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) # <<<<<<<<<<<<<< + * + * if src.suboffsets[i] >= 0: +*/ + /*else*/ { + __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1295, __pyx_L1_error) + } + __pyx_L7:; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + } + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + __pyx_t_2 = ((__pyx_v_src.suboffsets[__pyx_v_i]) >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":1298 + * + * if src.suboffsets[i] >= 0: + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) # <<<<<<<<<<<<<< + * + * if slices_overlap(&src, &dst, ndim, itemsize): +*/ + __pyx_t_6 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Dimension_d_is_not_direct, __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1298, __pyx_L1_error) + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + } + } + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + __pyx_t_2 = __pyx_slices_overlap((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + if (__pyx_t_2) { + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + __pyx_t_2 = (!__pyx_memviewslice_is_contig(__pyx_v_src, __pyx_v_order, __pyx_v_ndim)); + if (__pyx_t_2) { + + /* "View.MemoryView":1303 + * + * if not slice_is_contig(src, order, ndim): + * order = get_best_order(&dst, ndim) # <<<<<<<<<<<<<< + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim); + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + } + + /* "View.MemoryView":1305 + * order = get_best_order(&dst, ndim) + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) # <<<<<<<<<<<<<< + * src = tmp + * +*/ + __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1305, __pyx_L1_error) + __pyx_v_tmpdata = __pyx_t_7; + + /* "View.MemoryView":1306 + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) + * src = tmp # <<<<<<<<<<<<<< + * + * if not broadcasting: +*/ + __pyx_v_src = __pyx_v_tmp; + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (!__pyx_v_broadcasting); + if (__pyx_t_2) { + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'C', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1312 + * + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) # <<<<<<<<<<<<<< + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'C', __pyx_v_ndim); + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'F', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1314 + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) # <<<<<<<<<<<<<< + * + * if direct_copy: +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'F', __pyx_v_ndim); + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + } + __pyx_L12:; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + if (__pyx_v_direct_copy) { + + /* "View.MemoryView":1318 + * if direct_copy: + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1319 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) +*/ + (void)(memcpy(__pyx_v_dst.data, __pyx_v_src.data, __pyx_memoryview_slice_get_size((&__pyx_v_src), __pyx_v_ndim))); + + /* "View.MemoryView":1320 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * free(tmpdata) + * return 0 +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1321 + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1322 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * if order == 'F' == get_best_order(&dst, ndim): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (__pyx_v_order == 'F'); + if (__pyx_t_2) { + __pyx_t_2 = ('F' == __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim)); + } + if (__pyx_t_2) { + + /* "View.MemoryView":1327 + * + * + * transpose_memslice(&src) # <<<<<<<<<<<<<< + * transpose_memslice(&dst) + * +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1327, __pyx_L1_error) + + /* "View.MemoryView":1328 + * + * transpose_memslice(&src) + * transpose_memslice(&dst) # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1328, __pyx_L1_error) + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1330 + * transpose_memslice(&dst) + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1331 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * +*/ + copy_strided_to_strided((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1332 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * free(tmpdata) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1334 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1335 + * + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_broadcast_leading') +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_contents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim, int __pyx_v_ndim_other) { + int __pyx_v_i; + int __pyx_v_offset; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + + /* "View.MemoryView":1342 + * int ndim_other) noexcept nogil: + * cdef int i + * cdef int offset = ndim_other - ndim # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_offset = (__pyx_v_ndim_other - __pyx_v_ndim); + + /* "View.MemoryView":1344 + * cdef int offset = ndim_other - ndim + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1345 + * + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] # <<<<<<<<<<<<<< + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] +*/ + (__pyx_v_mslice->shape[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->shape[__pyx_v_i]); + + /* "View.MemoryView":1346 + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] # <<<<<<<<<<<<<< + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * +*/ + (__pyx_v_mslice->strides[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1347 + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] # <<<<<<<<<<<<<< + * + * for i in range(offset): +*/ + (__pyx_v_mslice->suboffsets[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->suboffsets[__pyx_v_i]); + } + + /* "View.MemoryView":1349 + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * + * for i in range(offset): # <<<<<<<<<<<<<< + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] +*/ + __pyx_t_1 = __pyx_v_offset; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1350 + * + * for i in range(offset): + * mslice.shape[i] = 1 # <<<<<<<<<<<<<< + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 +*/ + (__pyx_v_mslice->shape[__pyx_v_i]) = 1; + + /* "View.MemoryView":1351 + * for i in range(offset): + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] # <<<<<<<<<<<<<< + * mslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_mslice->strides[__pyx_v_i]) = (__pyx_v_mslice->strides[0]); + + /* "View.MemoryView":1352 + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_mslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_dtype_is_object, int __pyx_v_ndim, int __pyx_v_inc) { + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + if (__pyx_v_dtype_is_object) { + + /* "View.MemoryView":1363 + * + * if dtype_is_object: + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') +*/ + __pyx_memoryview_refcount_objects_in_slice_with_gil(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + } + + /* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1369 + * Py_ssize_t *strides, int ndim, + * bint inc) noexcept with gil: + * refcount_objects_in_slice(data, shape, strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, __pyx_v_shape, __pyx_v_strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ + __Pyx_PyGILState_Release(__pyx_gilstate_save); +} + +/* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + +static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1375 + * Py_ssize_t *strides, int ndim, bint inc) noexcept: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * + * for i in range(shape[0]): +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1377 + * cdef Py_ssize_t stride = strides[0] + * + * for i in range(shape[0]): # <<<<<<<<<<<<<< + * if ndim == 1: + * if inc: +*/ + __pyx_t_1 = (__pyx_v_shape[0]); + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + __pyx_t_4 = (__pyx_v_ndim == 1); + if (__pyx_t_4) { + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + if (__pyx_v_inc) { + + /* "View.MemoryView":1380 + * if ndim == 1: + * if inc: + * Py_INCREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * Py_DECREF(( data)[0]) +*/ + Py_INCREF((((PyObject **)__pyx_v_data)[0])); + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":1382 + * Py_INCREF(( data)[0]) + * else: + * Py_DECREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) +*/ + /*else*/ { + Py_DECREF((((PyObject **)__pyx_v_data)[0])); + } + __pyx_L6:; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":1384 + * Py_DECREF(( data)[0]) + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) # <<<<<<<<<<<<<< + * + * data += stride +*/ + /*else*/ { + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_inc); + } + __pyx_L5:; + + /* "View.MemoryView":1386 + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) + * + * data += stride # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item, int __pyx_v_dtype_is_object) { + + /* "View.MemoryView":1395 + * size_t itemsize, void *item, + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1396 + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) # <<<<<<<<<<<<<< + * refcount_copying(dst, dtype_is_object, ndim, inc=True) + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1397 + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_extent; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + + /* "View.MemoryView":1405 + * size_t itemsize, void *item) noexcept nogil: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t extent = shape[0] + * +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1406 + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] + * cdef Py_ssize_t extent = shape[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_extent = (__pyx_v_shape[0]); + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1409 + * + * if ndim == 1: + * for i in range(extent): # <<<<<<<<<<<<<< + * memcpy(data, item, itemsize) + * data += stride +*/ + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1410 + * if ndim == 1: + * for i in range(extent): + * memcpy(data, item, itemsize) # <<<<<<<<<<<<<< + * data += stride + * else: +*/ + (void)(memcpy(__pyx_v_data, __pyx_v_item, __pyx_v_itemsize)); + + /* "View.MemoryView":1411 + * for i in range(extent): + * memcpy(data, item, itemsize) + * data += stride # <<<<<<<<<<<<<< + * else: + * for i in range(extent): +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1413 + * data += stride + * else: + * for i in range(extent): # <<<<<<<<<<<<<< + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride +*/ + /*else*/ { + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1414 + * else: + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) # <<<<<<<<<<<<<< + * data += stride + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1415 + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride # <<<<<<<<<<<<<< + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ +} + +/* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum = {"__pyx_unpickle_Enum", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 4, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_Enum", 0) < (0)) __PYX_ERR(1, 4, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, i); __PYX_ERR(1, 4, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_v___pyx_state = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, __pyx_nargs); __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_r = __pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum", 0); + + /* "(tree fragment)":6 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') # <<<<<<<<<<<<<< + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: +*/ + __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, ((char const *)"name")); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(1, 6, __pyx_L1_error) + + /* "(tree fragment)":7 + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) +*/ + __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_MemviewEnum_type); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v___pyx_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None)); + if (__pyx_t_5) { + + /* "(tree fragment)":9 + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): +*/ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 9, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(((struct __pyx_MemviewEnum_obj *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + } + + /* "(tree fragment)":10 + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum__set_state", 0); + + /* "(tree fragment)":12 + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] # <<<<<<<<<<<<<< + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->name); + __Pyx_DECREF(__pyx_v___pyx_result->name); + __pyx_v___pyx_result->name = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":13 + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) # <<<<<<<<<<<<<< +*/ + __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 13, __pyx_L1_error) + + /* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/premap_ids.pyx":18 + * + * + * cdef lookup(PreshMap mapping, int64_t[:] keys, int64_t default): # <<<<<<<<<<<<<< + * """ + * Faster dict.get(keys, default) for the case when +*/ + +static PyObject *__pyx_f_5thinc_6layers_10premap_ids_lookup(struct __pyx_obj_7preshed_4maps_PreshMap *__pyx_v_mapping, __Pyx_memviewslice __pyx_v_keys, int64_t __pyx_v_default) { + int __pyx_v_maxi; + PyObject *__pyx_v_result = NULL; + __Pyx_memviewslice __pyx_v_result_view = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_v_i; + PyObject *__pyx_v_v = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + __Pyx_memviewslice __pyx_t_8 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int __pyx_t_9; + int __pyx_t_10; + int __pyx_t_11; + Py_ssize_t __pyx_t_12; + int __pyx_t_13; + int64_t __pyx_t_14; + int __pyx_t_15; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("lookup", 0); + + /* "thinc/layers/premap_ids.pyx":24 + * and the "keys" is a numpy integer vector. + * """ + * cdef int maxi = len(keys) # <<<<<<<<<<<<<< + * result = numpy.empty(maxi, dtype="int") + * cdef int64_t[:] result_view = result +*/ + __pyx_t_1 = __Pyx_MemoryView_Len(__pyx_v_keys); + __pyx_v_maxi = __pyx_t_1; + + /* "thinc/layers/premap_ids.pyx":25 + * """ + * cdef int maxi = len(keys) + * result = numpy.empty(maxi, dtype="int") # <<<<<<<<<<<<<< + * cdef int64_t[:] result_view = result + * for i in range(maxi): +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_numpy); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_empty); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyLong_From_int(__pyx_v_maxi); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_3, __pyx_t_4}; + __pyx_t_7 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_int, __pyx_t_7, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 25, __pyx_L1_error) + __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_7); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":26 + * cdef int maxi = len(keys) + * result = numpy.empty(maxi, dtype="int") + * cdef int64_t[:] result_view = result # <<<<<<<<<<<<<< + * for i in range(maxi): + * v = mapping[keys[i]] +*/ + __pyx_t_8 = __Pyx_PyObject_to_MemoryviewSlice_ds_nn_int64_t(__pyx_v_result, PyBUF_WRITABLE); if (unlikely(!__pyx_t_8.memview)) __PYX_ERR(0, 26, __pyx_L1_error) + __pyx_v_result_view = __pyx_t_8; + __pyx_t_8.memview = NULL; + __pyx_t_8.data = NULL; + + /* "thinc/layers/premap_ids.pyx":27 + * result = numpy.empty(maxi, dtype="int") + * cdef int64_t[:] result_view = result + * for i in range(maxi): # <<<<<<<<<<<<<< + * v = mapping[keys[i]] + * if v is None: +*/ + __pyx_t_9 = __pyx_v_maxi; + __pyx_t_10 = __pyx_t_9; + for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_10; __pyx_t_11+=1) { + __pyx_v_i = __pyx_t_11; + + /* "thinc/layers/premap_ids.pyx":28 + * cdef int64_t[:] result_view = result + * for i in range(maxi): + * v = mapping[keys[i]] # <<<<<<<<<<<<<< + * if v is None: + * result_view[i] = default +*/ + __pyx_t_12 = __pyx_v_i; + __pyx_t_13 = -1; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_keys.shape[0]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_13 = 0; + } else if (unlikely(__pyx_t_12 >= __pyx_v_keys.shape[0])) __pyx_t_13 = 0; + if (unlikely(__pyx_t_13 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_13); + __PYX_ERR(0, 28, __pyx_L1_error) + } + __pyx_t_14 = (*((int64_t *) ( /* dim=0 */ (__pyx_v_keys.data + __pyx_t_12 * __pyx_v_keys.strides[0]) ))); + __pyx_t_2 = __Pyx_GetItemInt(((PyObject *)__pyx_v_mapping), __pyx_t_14, int64_t, 1, __Pyx_PyLong_From_int64_t, 0, 1, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 28, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":29 + * for i in range(maxi): + * v = mapping[keys[i]] + * if v is None: # <<<<<<<<<<<<<< + * result_view[i] = default + * else: +*/ + __pyx_t_15 = (__pyx_v_v == Py_None); + if (__pyx_t_15) { + + /* "thinc/layers/premap_ids.pyx":30 + * v = mapping[keys[i]] + * if v is None: + * result_view[i] = default # <<<<<<<<<<<<<< + * else: + * result_view[i] = v +*/ + __pyx_t_12 = __pyx_v_i; + __pyx_t_13 = -1; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_result_view.shape[0]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_13 = 0; + } else if (unlikely(__pyx_t_12 >= __pyx_v_result_view.shape[0])) __pyx_t_13 = 0; + if (unlikely(__pyx_t_13 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_13); + __PYX_ERR(0, 30, __pyx_L1_error) + } + *((int64_t *) ( /* dim=0 */ (__pyx_v_result_view.data + __pyx_t_12 * __pyx_v_result_view.strides[0]) )) = __pyx_v_default; + + /* "thinc/layers/premap_ids.pyx":29 + * for i in range(maxi): + * v = mapping[keys[i]] + * if v is None: # <<<<<<<<<<<<<< + * result_view[i] = default + * else: +*/ + goto __pyx_L5; + } + + /* "thinc/layers/premap_ids.pyx":32 + * result_view[i] = default + * else: + * result_view[i] = v # <<<<<<<<<<<<<< + * return result + * +*/ + /*else*/ { + __pyx_t_14 = __Pyx_PyLong_As_int64_t(__pyx_v_v); if (unlikely((__pyx_t_14 == ((int64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 32, __pyx_L1_error) + __pyx_t_12 = __pyx_v_i; + __pyx_t_13 = -1; + if (__pyx_t_12 < 0) { + __pyx_t_12 += __pyx_v_result_view.shape[0]; + if (unlikely(__pyx_t_12 < 0)) __pyx_t_13 = 0; + } else if (unlikely(__pyx_t_12 >= __pyx_v_result_view.shape[0])) __pyx_t_13 = 0; + if (unlikely(__pyx_t_13 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_13); + __PYX_ERR(0, 32, __pyx_L1_error) + } + *((int64_t *) ( /* dim=0 */ (__pyx_v_result_view.data + __pyx_t_12 * __pyx_v_result_view.strides[0]) )) = __pyx_t_14; + } + __pyx_L5:; + } + + /* "thinc/layers/premap_ids.pyx":33 + * else: + * result_view[i] = v + * return result # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "thinc/layers/premap_ids.pyx":18 + * + * + * cdef lookup(PreshMap mapping, int64_t[:] keys, int64_t default): # <<<<<<<<<<<<<< + * """ + * Faster dict.get(keys, default) for the case when +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_7); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_8, 1); + __Pyx_AddTraceback("thinc.layers.premap_ids.lookup", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_result); + __PYX_XCLEAR_MEMVIEW(&__pyx_v_result_view, 1); + __Pyx_XDECREF(__pyx_v_v); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_1premap_ids(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_10premap_ids_premap_ids, "premap_ids(mapping_table: Mapping[int, int], default: int = 0, *, column: Optional[int] = None)\n\nRemap integer inputs to integers a mapping table, usually as a\npreprocess before embeddings."); +static PyMethodDef __pyx_mdef_5thinc_6layers_10premap_ids_1premap_ids = {"premap_ids", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_10premap_ids_1premap_ids, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_10premap_ids_premap_ids}; +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_1premap_ids(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_mapping_table = 0; + PyObject *__pyx_v_default = 0; + PyObject *__pyx_v_column = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("premap_ids (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_mapping_table,&__pyx_mstate_global->__pyx_n_u_default,&__pyx_mstate_global->__pyx_n_u_column,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 36, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 36, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 36, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "premap_ids", 0) < (0)) __PYX_ERR(0, 36, __pyx_L3_error) + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_0))); + + /* "thinc/layers/premap_ids.pyx":41 + * default: int = 0, + * *, + * column: Optional[int] = None # <<<<<<<<<<<<<< + * ): + * """Remap integer inputs to integers a mapping table, usually as a +*/ + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("premap_ids", 0, 1, 2, i); __PYX_ERR(0, 36, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 36, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 36, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_0))); + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + } + __pyx_v_mapping_table = values[0]; + __pyx_v_default = values[1]; + __pyx_v_column = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("premap_ids", 0, 1, 2, __pyx_nargs); __PYX_ERR(0, 36, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.premap_ids.premap_ids", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_10premap_ids_premap_ids(__pyx_self, __pyx_v_mapping_table, __pyx_v_default, __pyx_v_column); + + /* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_premap_ids(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_mapping_table, PyObject *__pyx_v_default, PyObject *__pyx_v_column) { + struct __pyx_obj_7preshed_4maps_PreshMap *__pyx_v_mapper = NULL; + PyObject *__pyx_v_k = NULL; + PyObject *__pyx_v_v = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + int __pyx_t_10; + int __pyx_t_11; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14[8]; + PyObject *__pyx_t_15 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("premap_ids", 0); + + /* "thinc/layers/premap_ids.pyx":45 + * """Remap integer inputs to integers a mapping table, usually as a + * preprocess before embeddings.""" + * mapper = PreshMap(initial_size=len(mapping_table)) # <<<<<<<<<<<<<< + * for k, v in mapping_table.items(): + * if not (isinstance(k, int) and isinstance(v, int)): +*/ + __pyx_t_2 = NULL; + __pyx_t_3 = PyObject_Length(__pyx_v_mapping_table); if (unlikely(__pyx_t_3 == ((Py_ssize_t)-1))) __PYX_ERR(0, 45, __pyx_L1_error) + __pyx_t_4 = PyLong_FromSsize_t(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, NULL}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_initial_size, __pyx_t_4, __pyx_t_6, __pyx_callargs+1, 0) < (0)) __PYX_ERR(0, 45, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_ptype_7preshed_4maps_PreshMap, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 45, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_v_mapper = ((struct __pyx_obj_7preshed_4maps_PreshMap *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/premap_ids.pyx":46 + * preprocess before embeddings.""" + * mapper = PreshMap(initial_size=len(mapping_table)) + * for k, v in mapping_table.items(): # <<<<<<<<<<<<<< + * if not (isinstance(k, int) and isinstance(v, int)): + * raise ValueError( +*/ + __pyx_t_6 = __pyx_v_mapping_table; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, NULL}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_items, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) { + __pyx_t_6 = __pyx_t_1; __Pyx_INCREF(__pyx_t_6); + __pyx_t_3 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 46, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 46, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_1 = __Pyx_PyList_GetItemRefFast(__pyx_t_6, __pyx_t_3, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 46, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_1 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_3)); + #else + __pyx_t_1 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 46, __pyx_L1_error) + } else { + __pyx_t_1 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_1)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 46, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_1); + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 46, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_4); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + } else { + __pyx_t_4 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + } + #else + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_8 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 46, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_9 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_8); + index = 0; __pyx_t_4 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_4)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_4); + index = 1; __pyx_t_2 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_2)) goto __pyx_L5_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < (0)) __PYX_ERR(0, 46, __pyx_L1_error) + __pyx_t_9 = NULL; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L6_unpacking_done; + __pyx_L5_unpacking_failed:; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 46, __pyx_L1_error) + __pyx_L6_unpacking_done:; + } + __Pyx_XDECREF_SET(__pyx_v_k, __pyx_t_4); + __pyx_t_4 = 0; + __Pyx_XDECREF_SET(__pyx_v_v, __pyx_t_2); + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":47 + * mapper = PreshMap(initial_size=len(mapping_table)) + * for k, v in mapping_table.items(): + * if not (isinstance(k, int) and isinstance(v, int)): # <<<<<<<<<<<<<< + * raise ValueError( + * "mapping_table has to be of type Mapping[int, int], " +*/ + __pyx_t_11 = PyLong_Check(__pyx_v_k); + if (__pyx_t_11) { + } else { + __pyx_t_10 = __pyx_t_11; + goto __pyx_L8_bool_binop_done; + } + __pyx_t_11 = PyLong_Check(__pyx_v_v); + __pyx_t_10 = __pyx_t_11; + __pyx_L8_bool_binop_done:; + __pyx_t_11 = (!__pyx_t_10); + if (unlikely(__pyx_t_11)) { + + /* "thinc/layers/premap_ids.pyx":48 + * for k, v in mapping_table.items(): + * if not (isinstance(k, int) and isinstance(v, int)): + * raise ValueError( # <<<<<<<<<<<<<< + * "mapping_table has to be of type Mapping[int, int], " + * f"but found {k}, {type(k)} and {v}, {type(v)}" +*/ + __pyx_t_2 = NULL; + + /* "thinc/layers/premap_ids.pyx":50 + * raise ValueError( + * "mapping_table has to be of type Mapping[int, int], " + * f"but found {k}, {type(k)} and {v}, {type(v)}" # <<<<<<<<<<<<<< + * ) + * mapper[k] = v +*/ + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_k, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_k)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_12 = __Pyx_PyObject_FormatSimple(__pyx_v_v, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_13 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_v)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_14[0] = __pyx_mstate_global->__pyx_kp_u_mapping_table_has_to_be_of_type; + __pyx_t_14[1] = __pyx_t_4; + __pyx_t_14[2] = __pyx_mstate_global->__pyx_kp_u__6; + __pyx_t_14[3] = __pyx_t_8; + __pyx_t_14[4] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_14[5] = __pyx_t_12; + __pyx_t_14[6] = __pyx_mstate_global->__pyx_kp_u__6; + __pyx_t_14[7] = __pyx_t_13; + + /* "thinc/layers/premap_ids.pyx":49 + * if not (isinstance(k, int) and isinstance(v, int)): + * raise ValueError( + * "mapping_table has to be of type Mapping[int, int], " # <<<<<<<<<<<<<< + * f"but found {k}, {type(k)} and {v}, {type(v)}" + * ) +*/ + __pyx_t_15 = __Pyx_PyUnicode_Join(__pyx_t_14, 8, 61 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_4) + 2 * 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_8) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_12) + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_13), 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_4) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_8) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_12) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_13)); + if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 49, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_15}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ValueError)), __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(0, 48, __pyx_L1_error) + + /* "thinc/layers/premap_ids.pyx":47 + * mapper = PreshMap(initial_size=len(mapping_table)) + * for k, v in mapping_table.items(): + * if not (isinstance(k, int) and isinstance(v, int)): # <<<<<<<<<<<<<< + * raise ValueError( + * "mapping_table has to be of type Mapping[int, int], " +*/ + } + + /* "thinc/layers/premap_ids.pyx":52 + * f"but found {k}, {type(k)} and {v}, {type(v)}" + * ) + * mapper[k] = v # <<<<<<<<<<<<<< + * return Model( + * "premap_ids", +*/ + if (unlikely((PyObject_SetItem(((PyObject *)__pyx_v_mapper), __pyx_v_k, __pyx_v_v) < 0))) __PYX_ERR(0, 52, __pyx_L1_error) + + /* "thinc/layers/premap_ids.pyx":46 + * preprocess before embeddings.""" + * mapper = PreshMap(initial_size=len(mapping_table)) + * for k, v in mapping_table.items(): # <<<<<<<<<<<<<< + * if not (isinstance(k, int) and isinstance(v, int)): + * raise ValueError( +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "thinc/layers/premap_ids.pyx":53 + * ) + * mapper[k] = v + * return Model( # <<<<<<<<<<<<<< + * "premap_ids", + * forward, +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_15, __pyx_mstate_global->__pyx_n_u_Model); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_15); + + /* "thinc/layers/premap_ids.pyx":55 + * return Model( + * "premap_ids", + * forward, # <<<<<<<<<<<<<< + * attrs={ + * "mapping_table": mapper, "default": default, "column": column +*/ + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_forward); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 55, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + + /* "thinc/layers/premap_ids.pyx":57 + * forward, + * attrs={ + * "mapping_table": mapper, "default": default, "column": column # <<<<<<<<<<<<<< + * } + * ) +*/ + __pyx_t_13 = __Pyx_PyDict_NewPresized(3); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 57, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_mapping_table, ((PyObject *)__pyx_v_mapper)) < (0)) __PYX_ERR(0, 57, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_default, __pyx_v_default) < (0)) __PYX_ERR(0, 57, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_column, __pyx_v_column) < (0)) __PYX_ERR(0, 57, __pyx_L1_error) + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_15))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_15); + assert(__pyx_t_1); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_15); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_15, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_1, __pyx_mstate_global->__pyx_n_u_premap_ids, __pyx_t_2}; + __pyx_t_12 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_attrs, __pyx_t_13, __pyx_t_12, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 53, __pyx_L1_error) + __pyx_t_6 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_15, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_12); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_AddTraceback("thinc.layers.premap_ids.premap_ids", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_mapper); + __Pyx_XDECREF(__pyx_v_k); + __Pyx_XDECREF(__pyx_v_v); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/premap_ids.pyx":62 + * + * + * def forward( # <<<<<<<<<<<<<< + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_3forward(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_10premap_ids_2forward, "forward(model: Model, inputs: InT, is_train: bool) -> Tuple[OutT, Callable]"); +static PyMethodDef __pyx_mdef_5thinc_6layers_10premap_ids_3forward = {"forward", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_10premap_ids_3forward, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_10premap_ids_2forward}; +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_3forward(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_model = 0; + PyObject *__pyx_v_inputs = 0; + CYTHON_UNUSED PyObject *__pyx_v_is_train = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("forward (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_inputs,&__pyx_mstate_global->__pyx_n_u_is_train,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 62, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 62, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 62, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 62, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "forward", 0) < (0)) __PYX_ERR(0, 62, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("forward", 1, 3, 3, i); __PYX_ERR(0, 62, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 62, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 62, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 62, __pyx_L3_error) + } + __pyx_v_model = values[0]; + __pyx_v_inputs = values[1]; + __pyx_v_is_train = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("forward", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 62, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.premap_ids.forward", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_10premap_ids_2forward(__pyx_self, __pyx_v_model, __pyx_v_inputs, __pyx_v_is_train); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/premap_ids.pyx":78 + * output = model.ops.reshape2i(arr, -1, 1) + * + * def backprop(dY: OutT) -> InT: # <<<<<<<<<<<<<< + * return model.ops.xp.empty(dY.shape) # type: ignore + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_7forward_1backprop(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_5thinc_6layers_10premap_ids_7forward_1backprop = {"backprop", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_10premap_ids_7forward_1backprop, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5thinc_6layers_10premap_ids_7forward_1backprop(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_dY = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_dY,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 78, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 78, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop", 0) < (0)) __PYX_ERR(0, 78, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop", 1, 1, 1, i); __PYX_ERR(0, 78, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 78, __pyx_L3_error) + } + __pyx_v_dY = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 78, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.premap_ids.forward.backprop", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_10premap_ids_7forward_backprop(__pyx_self, __pyx_v_dY); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_7forward_backprop(PyObject *__pyx_self, PyObject *__pyx_v_dY) { + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *__pyx_cur_scope; + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop", 0); + __pyx_outer_scope = (struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + + /* "thinc/layers/premap_ids.pyx":79 + * + * def backprop(dY: OutT) -> InT: + * return model.ops.xp.empty(dY.shape) # type: ignore # <<<<<<<<<<<<<< + * + * return output, backprop +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(!__pyx_cur_scope->__pyx_v_model)) { __Pyx_RaiseClosureNameError("model"); __PYX_ERR(0, 79, __pyx_L1_error) } + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_xp); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_2 = __pyx_t_4; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_dY, __pyx_mstate_global->__pyx_n_u_shape); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_empty, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/premap_ids.pyx":78 + * output = model.ops.reshape2i(arr, -1, 1) + * + * def backprop(dY: OutT) -> InT: # <<<<<<<<<<<<<< + * return model.ops.xp.empty(dY.shape) # type: ignore + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("thinc.layers.premap_ids.forward.backprop", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/premap_ids.pyx":62 + * + * + * def forward( # <<<<<<<<<<<<<< + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: +*/ + +static PyObject *__pyx_pf_5thinc_6layers_10premap_ids_2forward(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_inputs, CYTHON_UNUSED PyObject *__pyx_v_is_train) { + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *__pyx_cur_scope; + PyObject *__pyx_v_table = NULL; + PyObject *__pyx_v_default = NULL; + PyObject *__pyx_v_column = NULL; + PyObject *__pyx_v_idx = NULL; + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_v_arr = NULL; + PyObject *__pyx_v_output = NULL; + PyObject *__pyx_v_backprop = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + __Pyx_memviewslice __pyx_t_10 = { 0, 0, { 0 }, { 0 }, { 0 } }; + int64_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("forward", 0); + __pyx_cur_scope = (struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)__pyx_tp_new_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(__pyx_mstate_global->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 62, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_model = __pyx_v_model; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_model); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_model); + + /* "thinc/layers/premap_ids.pyx":65 + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: + * table = model.attrs["mapping_table"] # <<<<<<<<<<<<<< + * default = model.attrs["default"] + * column = model.attrs["column"] +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_attrs); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Dict_GetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_mapping_table); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_table = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":66 + * ) -> Tuple[OutT, Callable]: + * table = model.attrs["mapping_table"] + * default = model.attrs["default"] # <<<<<<<<<<<<<< + * column = model.attrs["column"] + * # Have to convert to numpy anyways, because +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_attrs); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_Dict_GetItem(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_default); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_default = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/layers/premap_ids.pyx":67 + * table = model.attrs["mapping_table"] + * default = model.attrs["default"] + * column = model.attrs["column"] # <<<<<<<<<<<<<< + * # Have to convert to numpy anyways, because + * # cupy ints don't work together with Python ints. +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_attrs); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Dict_GetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_column); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_column = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":70 + * # Have to convert to numpy anyways, because + * # cupy ints don't work together with Python ints. + * if column is None: # <<<<<<<<<<<<<< + * idx = to_numpy(inputs) + * else: +*/ + __pyx_t_3 = (__pyx_v_column == Py_None); + if (__pyx_t_3) { + + /* "thinc/layers/premap_ids.pyx":71 + * # cupy ints don't work together with Python ints. + * if column is None: + * idx = to_numpy(inputs) # <<<<<<<<<<<<<< + * else: + * idx = to_numpy(cast(Ints2d, inputs)[:, column]) +*/ + __pyx_t_1 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_to_numpy); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_1); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_inputs}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_idx = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":70 + * # Have to convert to numpy anyways, because + * # cupy ints don't work together with Python ints. + * if column is None: # <<<<<<<<<<<<<< + * idx = to_numpy(inputs) + * else: +*/ + goto __pyx_L3; + } + + /* "thinc/layers/premap_ids.pyx":73 + * idx = to_numpy(inputs) + * else: + * idx = to_numpy(cast(Ints2d, inputs)[:, column]) # <<<<<<<<<<<<<< + * result = lookup(table, idx, default) + * arr = model.ops.asarray2i(result) +*/ + /*else*/ { + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_to_numpy); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_cast); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_Ints2d); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_8); + assert(__pyx_t_7); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_7); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_8, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[3] = {__pyx_t_7, __pyx_t_9, __pyx_v_inputs}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_8, __pyx_callargs+__pyx_t_5, (3-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(0, 73, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_column); + __Pyx_GIVEREF(__pyx_v_column); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_column) != (0)) __PYX_ERR(0, 73, __pyx_L1_error); + __pyx_t_9 = __Pyx_PyObject_GetItem(__pyx_t_6, __pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_1); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_1, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_t_9}; + __pyx_t_2 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_1, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_idx = __pyx_t_2; + __pyx_t_2 = 0; + } + __pyx_L3:; + + /* "thinc/layers/premap_ids.pyx":74 + * else: + * idx = to_numpy(cast(Ints2d, inputs)[:, column]) + * result = lookup(table, idx, default) # <<<<<<<<<<<<<< + * arr = model.ops.asarray2i(result) + * output = model.ops.reshape2i(arr, -1, 1) +*/ + if (!(likely(((__pyx_v_table) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_table, __pyx_mstate_global->__pyx_ptype_7preshed_4maps_PreshMap))))) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyObject_to_MemoryviewSlice_ds_nn_int64_t(__pyx_v_idx, PyBUF_WRITABLE); if (unlikely(!__pyx_t_10.memview)) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_11 = __Pyx_PyLong_As_int64_t(__pyx_v_default); if (unlikely((__pyx_t_11 == ((int64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5thinc_6layers_10premap_ids_lookup(((struct __pyx_obj_7preshed_4maps_PreshMap *)__pyx_v_table), __pyx_t_10, __pyx_t_11); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 74, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_10, 1); + __pyx_t_10.memview = NULL; __pyx_t_10.data = NULL; + __pyx_v_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":75 + * idx = to_numpy(cast(Ints2d, inputs)[:, column]) + * result = lookup(table, idx, default) + * arr = model.ops.asarray2i(result) # <<<<<<<<<<<<<< + * output = model.ops.reshape2i(arr, -1, 1) + * +*/ + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_1 = __pyx_t_9; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_result}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_asarray2i, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_arr = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":76 + * result = lookup(table, idx, default) + * arr = model.ops.asarray2i(result) + * output = model.ops.reshape2i(arr, -1, 1) # <<<<<<<<<<<<<< + * + * def backprop(dY: OutT) -> InT: +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_9 = __pyx_t_1; + __Pyx_INCREF(__pyx_t_9); + __pyx_t_5 = 0; + { + PyObject *__pyx_callargs[4] = {__pyx_t_9, __pyx_v_arr, __pyx_mstate_global->__pyx_int_neg_1, __pyx_mstate_global->__pyx_int_1}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_reshape2i, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_output = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/premap_ids.pyx":78 + * output = model.ops.reshape2i(arr, -1, 1) + * + * def backprop(dY: OutT) -> InT: # <<<<<<<<<<<<<< + * return model.ops.xp.empty(dY.shape) # type: ignore + * +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_dY, __pyx_mstate_global->__pyx_n_u_OutT) < (0)) __PYX_ERR(0, 78, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_n_u_InT) < (0)) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_10premap_ids_7forward_1backprop, 0, __pyx_mstate_global->__pyx_n_u_forward_locals_backprop, ((PyObject*)__pyx_cur_scope), __pyx_mstate_global->__pyx_n_u_thinc_layers_premap_ids, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_1, __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_backprop = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/layers/premap_ids.pyx":81 + * return model.ops.xp.empty(dY.shape) # type: ignore + * + * return output, backprop # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 81, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_output); + __Pyx_GIVEREF(__pyx_v_output); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_output) != (0)) __PYX_ERR(0, 81, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_backprop); + __Pyx_GIVEREF(__pyx_v_backprop); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_backprop) != (0)) __PYX_ERR(0, 81, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/premap_ids.pyx":62 + * + * + * def forward( # <<<<<<<<<<<<<< + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __PYX_XCLEAR_MEMVIEW(&__pyx_t_10, 1); + __Pyx_AddTraceback("thinc.layers.premap_ids.forward", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_table); + __Pyx_XDECREF(__pyx_v_default); + __Pyx_XDECREF(__pyx_v_column); + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_arr); + __Pyx_XDECREF(__pyx_v_output); + __Pyx_XDECREF(__pyx_v_backprop); + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_6layers_10premap_ids___pyx_scope_struct__forward > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, sizeof(struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_5thinc_6layers_10premap_ids___pyx_scope_struct__forward[--__pyx_mstate_global->__pyx_freecount_5thinc_6layers_10premap_ids___pyx_scope_struct__forward]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(PyObject *o) { + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *p = (struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_6layers_10premap_ids___pyx_scope_struct__forward) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_v_model); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_6layers_10premap_ids___pyx_scope_struct__forward < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, sizeof(struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward)))) + { + __pyx_mstate_global->__pyx_freelist_5thinc_6layers_10premap_ids___pyx_scope_struct__forward[__pyx_mstate_global->__pyx_freecount_5thinc_6layers_10premap_ids___pyx_scope_struct__forward++] = ((struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *p = (struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_v_model) { + e = (*v)(p->__pyx_v_model, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_6layers_10premap_ids___pyx_scope_struct__forward(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *p = (struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward *)o; + tmp = ((PyObject*)p->__pyx_v_model); + p->__pyx_v_model = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_6layers_10premap_ids___pyx_scope_struct__forward}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_6layers_10premap_ids___pyx_scope_struct__forward}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_6layers_10premap_ids___pyx_scope_struct__forward}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_6layers_10premap_ids___pyx_scope_struct__forward}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward_spec = { + "thinc.layers.premap_ids.__pyx_scope_struct__forward", + sizeof(struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.premap_ids.""__pyx_scope_struct__forward", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_6layers_10premap_ids___pyx_scope_struct__forward), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, /*tp_traverse*/ + __pyx_tp_clear_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_6layers_10premap_ids___pyx_scope_struct__forward, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_array __pyx_vtable_array; + +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_array_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_array_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_array; + p->mode = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_format = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (unlikely(__pyx_array___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_array(PyObject *o) { + struct __pyx_array_obj *p = (struct __pyx_array_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_array) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_array___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->mode); + Py_CLEAR(p->_format); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static PyObject *__pyx_sq_item_array(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_array(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_array___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_tp_getattro_array(PyObject *o, PyObject *n) { + PyObject *v = PyObject_GenericGetAttr(o, n); + if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + v = __pyx_array___getattr__(o, n); + } + return v; +} + +static PyObject *__pyx_getprop___pyx_array_memview(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(o); +} + +static PyMethodDef __pyx_methods_array[] = { + {"__getattr__", (PyCFunction)__pyx_array___getattr__, METH_O|METH_COEXIST, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_array[] = { + {"memview", __pyx_getprop___pyx_array_memview, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_array_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_array}, + {Py_sq_length, (void *)__pyx_array___len__}, + {Py_sq_item, (void *)__pyx_sq_item_array}, + {Py_mp_length, (void *)__pyx_array___len__}, + {Py_mp_subscript, (void *)__pyx_array___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_array}, + {Py_tp_getattro, (void *)__pyx_tp_getattro_array}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_array_getbuffer}, + #endif + {Py_tp_methods, (void *)__pyx_methods_array}, + {Py_tp_getset, (void *)__pyx_getsets_array}, + {Py_tp_new, (void *)__pyx_tp_new_array}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_array_spec = { + "thinc.layers.premap_ids.array", + sizeof(struct __pyx_array_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_array_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_array = { + __pyx_array___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_array, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_array = { + __pyx_array___len__, /*mp_length*/ + __pyx_array___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_array, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_array = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.premap_ids.""array", /*tp_name*/ + sizeof(struct __pyx_array_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_array, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_array, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_array, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + __pyx_tp_getattro_array, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_array, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_array, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_array, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_array, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_MemviewEnum_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_MemviewEnum_obj *)o); + p->name = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_Enum(PyObject *o) { + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_Enum) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->name); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_Enum(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->name) { + e = (*v)(p->name, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_Enum(PyObject *o) { + PyObject* tmp; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + tmp = ((PyObject*)p->name); + p->name = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_Enum[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_MemviewEnum_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_Enum}, + {Py_tp_repr, (void *)__pyx_MemviewEnum___repr__}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_Enum}, + {Py_tp_clear, (void *)__pyx_tp_clear_Enum}, + {Py_tp_methods, (void *)__pyx_methods_Enum}, + {Py_tp_init, (void *)__pyx_MemviewEnum___init__}, + {Py_tp_new, (void *)__pyx_tp_new_Enum}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_MemviewEnum_spec = { + "thinc.layers.premap_ids.Enum", + sizeof(struct __pyx_MemviewEnum_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_MemviewEnum_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_MemviewEnum = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.premap_ids.""Enum", /*tp_name*/ + sizeof(struct __pyx_MemviewEnum_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_Enum, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_MemviewEnum___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_Enum, /*tp_traverse*/ + __pyx_tp_clear_Enum, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_Enum, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_MemviewEnum___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_Enum, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_memoryview __pyx_vtable_memoryview; + +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryview_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryview_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_memoryview; + p->obj = Py_None; Py_INCREF(Py_None); + p->_size = Py_None; Py_INCREF(Py_None); + p->view.obj = NULL; + if (unlikely(__pyx_memoryview___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_memoryview(PyObject *o) { + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_memoryview) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryview___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->obj); + Py_CLEAR(p->_size); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_memoryview(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->obj) { + e = (*v)(p->obj, a); if (e) return e; + } + if (p->_size) { + e = (*v)(p->_size, a); if (e) return e; + } + if (p->view.obj) { + e = (*v)(p->view.obj, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_memoryview(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + tmp = ((PyObject*)p->obj); + p->obj = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_size); + p->_size = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + Py_CLEAR(p->view.obj); + return 0; +} + +static PyObject *__pyx_sq_item_memoryview(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_memoryview(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_memoryview___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_getprop___pyx_memoryview_T(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_base(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_shape(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_strides(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_suboffsets(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_ndim(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_itemsize(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_nbytes(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_size(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(o); +} + +static PyMethodDef __pyx_methods_memoryview[] = { + {"is_c_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_c_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"is_f_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_f_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy_fortran", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy_fortran, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_memoryview[] = { + {"T", __pyx_getprop___pyx_memoryview_T, 0, 0, 0}, + {"base", __pyx_getprop___pyx_memoryview_base, 0, 0, 0}, + {"shape", __pyx_getprop___pyx_memoryview_shape, 0, 0, 0}, + {"strides", __pyx_getprop___pyx_memoryview_strides, 0, 0, 0}, + {"suboffsets", __pyx_getprop___pyx_memoryview_suboffsets, 0, 0, 0}, + {"ndim", __pyx_getprop___pyx_memoryview_ndim, 0, 0, 0}, + {"itemsize", __pyx_getprop___pyx_memoryview_itemsize, 0, 0, 0}, + {"nbytes", __pyx_getprop___pyx_memoryview_nbytes, 0, 0, 0}, + {"size", __pyx_getprop___pyx_memoryview_size, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_memoryview_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_memoryview}, + {Py_tp_repr, (void *)__pyx_memoryview___repr__}, + {Py_sq_length, (void *)__pyx_memoryview___len__}, + {Py_sq_item, (void *)__pyx_sq_item_memoryview}, + {Py_mp_length, (void *)__pyx_memoryview___len__}, + {Py_mp_subscript, (void *)__pyx_memoryview___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_memoryview}, + {Py_tp_str, (void *)__pyx_memoryview___str__}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_memoryview_getbuffer}, + #endif + {Py_tp_traverse, (void *)__pyx_tp_traverse_memoryview}, + {Py_tp_clear, (void *)__pyx_tp_clear_memoryview}, + {Py_tp_methods, (void *)__pyx_methods_memoryview}, + {Py_tp_getset, (void *)__pyx_getsets_memoryview}, + {Py_tp_new, (void *)__pyx_tp_new_memoryview}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryview_spec = { + "thinc.layers.premap_ids.memoryview", + sizeof(struct __pyx_memoryview_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_memoryview_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_memoryview = { + __pyx_memoryview___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_memoryview, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_memoryview = { + __pyx_memoryview___len__, /*mp_length*/ + __pyx_memoryview___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_memoryview, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_memoryview = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.premap_ids.""memoryview", /*tp_name*/ + sizeof(struct __pyx_memoryview_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_memoryview, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_memoryview___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_memoryview, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_memoryview, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_memoryview___str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_memoryview, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_memoryview, /*tp_traverse*/ + __pyx_tp_clear_memoryview, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_memoryview, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_memoryview, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_memoryview, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct__memoryviewslice __pyx_vtable__memoryviewslice; + +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryviewslice_obj *p; + PyObject *o = __pyx_tp_new_memoryview(t, a, k); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryviewslice_obj *)o); + p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_memoryview*)__pyx_vtabptr__memoryviewslice; + __Pyx_default_placement_construct(&(p->from_slice)); + p->from_object = Py_None; Py_INCREF(Py_None); + p->from_slice.memview = NULL; + return o; +} + +static void __pyx_tp_dealloc__memoryviewslice(PyObject *o) { + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc__memoryviewslice) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryviewslice___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + __Pyx_call_destructor(p->from_slice); + Py_CLEAR(p->from_object); + PyObject_GC_Track(o); + __pyx_tp_dealloc_memoryview(o); +} + +static int __pyx_tp_traverse__memoryviewslice(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + e = __pyx_tp_traverse_memoryview(o, v, a); if (e) return e; + if (p->from_object) { + e = (*v)(p->from_object, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear__memoryviewslice(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + __pyx_tp_clear_memoryview(o); + tmp = ((PyObject*)p->from_object); + p->from_object = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + __PYX_XCLEAR_MEMVIEW(&p->from_slice, 1); + return 0; +} + +static PyMethodDef __pyx_methods__memoryviewslice[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_memoryviewslice_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc__memoryviewslice}, + {Py_tp_doc, (void *)PyDoc_STR("Internal class for passing memoryview slices to Python")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse__memoryviewslice}, + {Py_tp_clear, (void *)__pyx_tp_clear__memoryviewslice}, + {Py_tp_methods, (void *)__pyx_methods__memoryviewslice}, + {Py_tp_new, (void *)__pyx_tp_new__memoryviewslice}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryviewslice_spec = { + "thinc.layers.premap_ids._memoryviewslice", + sizeof(struct __pyx_memoryviewslice_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_memoryviewslice_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_memoryviewslice = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.premap_ids.""_memoryviewslice", /*tp_name*/ + sizeof(struct __pyx_memoryviewslice_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc__memoryviewslice, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___repr__, /*tp_repr*/ + #else + 0, /*tp_repr*/ + #endif + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___str__, /*tp_str*/ + #else + 0, /*tp_str*/ + #endif + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + PyDoc_STR("Internal class for passing memoryview slices to Python"), /*tp_doc*/ + __pyx_tp_traverse__memoryviewslice, /*tp_traverse*/ + __pyx_tp_clear__memoryviewslice, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods__memoryviewslice, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new__memoryviewslice, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_collections_abc_Sequence = Py_None; Py_INCREF(Py_None); + generic = Py_None; Py_INCREF(Py_None); + strided = Py_None; Py_INCREF(Py_None); + indirect = Py_None; Py_INCREF(Py_None); + contiguous = Py_None; Py_INCREF(Py_None); + indirect_contiguous = Py_None; Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward)) __PYX_ERR(0, 62, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward_spec, __pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward = &__pyx_type_5thinc_6layers_10premap_ids___pyx_scope_struct__forward; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_6layers_10premap_ids___pyx_scope_struct__forward->tp_getattro = PyObject_GenericGetAttr; + } + #endif + __pyx_vtabptr_array = &__pyx_vtable_array; + __pyx_vtable_array.get_memview = (PyObject *(*)(struct __pyx_array_obj *))__pyx_array_get_memview; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_array_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_array_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_array_type)) __PYX_ERR(1, 110, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_array_type->tp_as_buffer = &__pyx_tp_as_buffer_array; + if (!__pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_array_spec, __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #else + __pyx_mstate->__pyx_array_type = &__pyx_type___pyx_array; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_array_type); + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_array_type, __pyx_vtabptr_array) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_MemviewEnum_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_MemviewEnum_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_MemviewEnum_type)) __PYX_ERR(1, 299, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_MemviewEnum_spec, __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #else + __pyx_mstate->__pyx_MemviewEnum_type = &__pyx_type___pyx_MemviewEnum; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_MemviewEnum_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_MemviewEnum_type->tp_dictoffset && __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + __pyx_vtabptr_memoryview = &__pyx_vtable_memoryview; + __pyx_vtable_memoryview.get_item_pointer = (char *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_get_item_pointer; + __pyx_vtable_memoryview.is_slice = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_is_slice; + __pyx_vtable_memoryview.setitem_slice_assignment = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_slice_assignment; + __pyx_vtable_memoryview.setitem_slice_assign_scalar = (PyObject *(*)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_setitem_slice_assign_scalar; + __pyx_vtable_memoryview.setitem_indexed = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_indexed; + __pyx_vtable_memoryview.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryview_convert_item_to_object; + __pyx_vtable_memoryview.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryview_assign_item_from_object; + __pyx_vtable_memoryview._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryview__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_memoryview_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryview_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_memoryview_type)) __PYX_ERR(1, 334, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer = &__pyx_tp_as_buffer_memoryview; + if (!__pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryview_spec, __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryview_type = &__pyx_type___pyx_memoryview; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryview_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryview_type->tp_dictoffset && __pyx_mstate->__pyx_memoryview_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryview_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryview_type, __pyx_vtabptr_memoryview) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + __pyx_vtabptr__memoryviewslice = &__pyx_vtable__memoryviewslice; + __pyx_vtable__memoryviewslice.__pyx_base = *__pyx_vtabptr_memoryview; + __pyx_vtable__memoryviewslice.__pyx_base.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryviewslice_convert_item_to_object; + __pyx_vtable__memoryviewslice.__pyx_base.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryviewslice_assign_item_from_object; + __pyx_vtable__memoryviewslice.__pyx_base._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryviewslice__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_memoryview_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_memoryviewslice_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryviewslice_spec, __pyx_t_1); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_mstate->__pyx_memoryviewslice_type)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryviewslice_spec, __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryviewslice_type = &__pyx_type___pyx_memoryviewslice; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate_global->__pyx_memoryviewslice_type->tp_base = __pyx_mstate_global->__pyx_memoryview_type; + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryviewslice_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryviewslice_type->tp_dictoffset && __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryviewslice_type, __pyx_vtabptr__memoryviewslice) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule("cymem.cymem"); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyMalloc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyMalloc), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyMalloc), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc) __PYX_ERR(2, 4, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyMalloc = (struct __pyx_vtabstruct_5cymem_5cymem_PyMalloc*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyMalloc); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyMalloc)) __PYX_ERR(2, 4, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "PyFree", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_PyFree), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_PyFree), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree) __PYX_ERR(2, 10, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_PyFree = (struct __pyx_vtabstruct_5cymem_5cymem_PyFree*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_PyFree); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_PyFree)) __PYX_ERR(2, 10, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Pool", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Pool), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Pool), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool) __PYX_ERR(2, 16, __pyx_L1_error) + __pyx_vtabptr_5cymem_5cymem_Pool = (struct __pyx_vtabstruct_5cymem_5cymem_Pool*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_5cymem_5cymem_Pool); if (unlikely(!__pyx_vtabptr_5cymem_5cymem_Pool)) __PYX_ERR(2, 16, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5cymem_5cymem_Address = __Pyx_ImportType_3_2_1(__pyx_t_1, "cymem.cymem", "Address", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #else + sizeof(struct __pyx_obj_5cymem_5cymem_Address), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_5cymem_5cymem_Address), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5cymem_5cymem_Address) __PYX_ERR(2, 28, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("preshed.maps"); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 45, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMap", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMap), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMap), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap) __PYX_ERR(3, 45, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMap = (struct __pyx_vtabstruct_7preshed_4maps_PreshMap*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMap); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMap)) __PYX_ERR(3, 45, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray = __Pyx_ImportType_3_2_1(__pyx_t_1, "preshed.maps", "PreshMapArray", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #else + sizeof(struct __pyx_obj_7preshed_4maps_PreshMapArray), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(struct __pyx_obj_7preshed_4maps_PreshMapArray), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray) __PYX_ERR(3, 53, __pyx_L1_error) + __pyx_vtabptr_7preshed_4maps_PreshMapArray = (struct __pyx_vtabstruct_7preshed_4maps_PreshMapArray*)__Pyx_GetVtable(__pyx_mstate->__pyx_ptype_7preshed_4maps_PreshMapArray); if (unlikely(!__pyx_vtabptr_7preshed_4maps_PreshMapArray)) __PYX_ERR(3, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_premap_ids(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_premap_ids}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "premap_ids", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_premap_ids(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_premap_ids(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_premap_ids(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + static PyThread_type_lock __pyx_t_7[8]; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'premap_ids' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "premap_ids" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_premap_ids", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__layers__premap_ids) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.layers.premap_ids")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.layers.premap_ids", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":101 + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence # <<<<<<<<<<<<<< + * except: + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_collections_abc}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin___import__, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_abc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L7_try_end; + __pyx_L2_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":102 + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: # <<<<<<<<<<<<<< + * + * __pyx_collections_abc_Sequence = None +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":104 + * except: + * + * __pyx_collections_abc_Sequence = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, Py_None); + __Pyx_GIVEREF(Py_None); + goto __pyx_L3_exception_handled; + } + __pyx_L3_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L7_try_end:; + } + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":240 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":241 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L15_try_end; + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":242 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L11_exception_handled; + } + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L15_try_end:; + } + + /* "View.MemoryView":307 + * return self.name + * + * cdef generic = Enum("") # <<<<<<<<<<<<<< + * cdef strided = Enum("") # default + * cdef indirect = Enum("") +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct_or_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 307, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(generic); + __Pyx_DECREF_SET(generic, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":308 + * + * cdef generic = Enum("") + * cdef strided = Enum("") # default # <<<<<<<<<<<<<< + * cdef indirect = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 308, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(strided); + __Pyx_DECREF_SET(strided, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":309 + * cdef generic = Enum("") + * cdef strided = Enum("") # default + * cdef indirect = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 309, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect); + __Pyx_DECREF_SET(indirect, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":312 + * + * + * cdef contiguous = Enum("") # <<<<<<<<<<<<<< + * cdef indirect_contiguous = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 312, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(contiguous); + __Pyx_DECREF_SET(contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":313 + * + * cdef contiguous = Enum("") + * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 313, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect_contiguous); + __Pyx_DECREF_SET(indirect_contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":321 + * + * + * cdef int __pyx_memoryview_thread_locks_used = 0 # <<<<<<<<<<<<<< + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ + * PyThread_allocate_lock(), +*/ + __pyx_memoryview_thread_locks_used = 0; + + /* "View.MemoryView":322 + * + * cdef int __pyx_memoryview_thread_locks_used = 0 + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ # <<<<<<<<<<<<<< + * PyThread_allocate_lock(), + * PyThread_allocate_lock(), +*/ + __pyx_t_7[0] = PyThread_allocate_lock(); + __pyx_t_7[1] = PyThread_allocate_lock(); + __pyx_t_7[2] = PyThread_allocate_lock(); + __pyx_t_7[3] = PyThread_allocate_lock(); + __pyx_t_7[4] = PyThread_allocate_lock(); + __pyx_t_7[5] = PyThread_allocate_lock(); + __pyx_t_7[6] = PyThread_allocate_lock(); + __pyx_t_7[7] = PyThread_allocate_lock(); + memcpy(&(__pyx_memoryview_thread_locks[0]), __pyx_t_7, sizeof(__pyx_memoryview_thread_locks[0]) * (8)); + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":984 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":985 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L23_try_end; + __pyx_L18_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":986 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L19_exception_handled; + } + __pyx_L19_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L23_try_end:; + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_collections_abc_Sequence); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(1, 990, __pyx_L26_error) + if (__pyx_t_8) { + + /* "View.MemoryView":994 + * + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence.register(array) + * except: +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_memoryviewslice_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 994, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":995 + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) # <<<<<<<<<<<<<< + * except: + * pass # ignore failure, it's a minor issue +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_array_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 995, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L31_try_end; + __pyx_L26_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":996 + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) + * except: # <<<<<<<<<<<<<< + * pass # ignore failure, it's a minor issue + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L27_exception_handled; + } + __pyx_L27_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L31_try_end:; + } + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + __pyx_t_4 = PyCFunction_NewEx(&__pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum, NULL, __pyx_mstate_global->__pyx_n_u_View_MemoryView); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum, __pyx_t_4) < (0)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":3 + * # cython: binding=True, infer_types=True, profile=False + * from libc.stdint cimport int64_t + * import numpy # <<<<<<<<<<<<<< + * + * from preshed.maps cimport PreshMap +*/ + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_numpy, 0, 0, __pyx_mstate_global->__pyx_kp_u_thinc_layers_numpy, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 3, __pyx_L1_error) + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_numpy, __pyx_t_4) < (0)) __PYX_ERR(0, 3, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":7 + * from preshed.maps cimport PreshMap + * + * from typing import Callable, Dict, Mapping, Optional, Tuple, Union, cast # <<<<<<<<<<<<<< + * + * from ..config import registry +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Callable,__pyx_mstate_global->__pyx_n_u_Dict,__pyx_mstate_global->__pyx_n_u_Mapping,__pyx_mstate_global->__pyx_n_u_Optional,__pyx_mstate_global->__pyx_n_u_Tuple,__pyx_mstate_global->__pyx_n_u_Union,__pyx_mstate_global->__pyx_n_u_cast}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_typing, __pyx_imported_names, 7, __pyx_mstate_global->__pyx_kp_u_thinc_layers_typing, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 7, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Callable,__pyx_mstate_global->__pyx_n_u_Dict,__pyx_mstate_global->__pyx_n_u_Mapping,__pyx_mstate_global->__pyx_n_u_Optional,__pyx_mstate_global->__pyx_n_u_Tuple,__pyx_mstate_global->__pyx_n_u_Union,__pyx_mstate_global->__pyx_n_u_cast}; + for (__pyx_t_9=0; __pyx_t_9 < 7; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":9 + * from typing import Callable, Dict, Mapping, Optional, Tuple, Union, cast + * + * from ..config import registry # <<<<<<<<<<<<<< + * from ..model import Model + * from ..types import Ints1d, Ints2d +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_config, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_config, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 9, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":10 + * + * from ..config import registry + * from ..model import Model # <<<<<<<<<<<<<< + * from ..types import Ints1d, Ints2d + * from ..util import to_numpy +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Model}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_model, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_model, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Model}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":11 + * from ..config import registry + * from ..model import Model + * from ..types import Ints1d, Ints2d # <<<<<<<<<<<<<< + * from ..util import to_numpy + * +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Ints1d,__pyx_mstate_global->__pyx_n_u_Ints2d}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_types, __pyx_imported_names, 2, __pyx_mstate_global->__pyx_kp_u_thinc_types, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Ints1d,__pyx_mstate_global->__pyx_n_u_Ints2d}; + for (__pyx_t_9=0; __pyx_t_9 < 2; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 11, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":12 + * from ..model import Model + * from ..types import Ints1d, Ints2d + * from ..util import to_numpy # <<<<<<<<<<<<<< + * + * InT = Union[Ints1d, Ints2d] +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_to_numpy}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_util, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_util, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 12, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_to_numpy}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":14 + * from ..util import to_numpy + * + * InT = Union[Ints1d, Ints2d] # <<<<<<<<<<<<<< + * OutT = Ints2d + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Union); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Ints1d); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_Ints2d); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = PyTuple_Pack(2, __pyx_t_5, __pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __pyx_t_10 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_11); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_InT, __pyx_t_10) < (0)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/premap_ids.pyx":15 + * + * InT = Union[Ints1d, Ints2d] + * OutT = Ints2d # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_Ints2d); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_OutT, __pyx_t_10) < (0)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + __pyx_t_11 = NULL; + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_12, __pyx_mstate_global->__pyx_n_u_registry); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_13 = __Pyx_PyObject_GetAttrStr(__pyx_t_12, __pyx_mstate_global->__pyx_n_u_layers); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_premap_ids_v1}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_13, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_13 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + + /* "thinc/layers/premap_ids.pyx":41 + * default: int = 0, + * *, + * column: Optional[int] = None # <<<<<<<<<<<<<< + * ): + * """Remap integer inputs to integers a mapping table, usually as a +*/ + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_column, Py_None) < (0)) __PYX_ERR(0, 36, __pyx_L1_error) + + /* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + __pyx_t_5 = __Pyx_PyDict_NewPresized(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_mapping_table, __pyx_mstate_global->__pyx_kp_u_Mapping_int_int) < (0)) __PYX_ERR(0, 36, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_default, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 36, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_column, __pyx_mstate_global->__pyx_kp_u_Optional_int) < (0)) __PYX_ERR(0, 36, __pyx_L1_error) + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_10premap_ids_1premap_ids, 0, __pyx_mstate_global->__pyx_n_u_premap_ids, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_premap_ids, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_12, __pyx_mstate_global->__pyx_tuple[2]); + __Pyx_CyFunction_SetDefaultsKwDict(__pyx_t_12, __pyx_t_13); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_12, __pyx_t_5); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_11, __pyx_t_12}; + __pyx_t_10 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + } + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_premap_ids, __pyx_t_10) < (0)) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/premap_ids.pyx":62 + * + * + * def forward( # <<<<<<<<<<<<<< + * model: Model, inputs: InT, is_train: bool + * ) -> Tuple[OutT, Callable]: +*/ + __pyx_t_10 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_model, __pyx_mstate_global->__pyx_n_u_Model) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_inputs, __pyx_mstate_global->__pyx_n_u_InT) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_is_train, __pyx_mstate_global->__pyx_n_u_bool) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_kp_u_Tuple_OutT_Callable) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + __pyx_t_4 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_10premap_ids_3forward, 0, __pyx_mstate_global->__pyx_n_u_forward, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_premap_ids, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_4); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_4, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_forward, __pyx_t_4) < (0)) __PYX_ERR(0, 62, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/premap_ids.pyx":1 + * # cython: binding=True, infer_types=True, profile=False # <<<<<<<<<<<<<< + * from libc.stdint cimport int64_t + * import numpy +*/ + __pyx_t_4 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_4) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.layers.premap_ids", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.layers.premap_ids"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin___import__ = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_import); if (!__pyx_builtin___import__) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 409, __pyx_L1_error) + __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_id); if (!__pyx_builtin_id) __PYX_ERR(1, 619, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_self_name_is_not_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_New(1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_mstate_global->__pyx_tuple[1], 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(1, 583, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_mstate_global->__pyx_slice[0] = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[0])) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + + /* "thinc/layers/premap_ids.pyx":36 + * + * + * @registry.layers("premap_ids.v1") # <<<<<<<<<<<<<< + * def premap_ids( + * mapping_table: Mapping[int, int], +*/ + __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(1, ((PyObject*)__pyx_mstate_global->__pyx_int_0)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 36, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<3; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_slice; + for (Py_ssize_t i=0; i<1; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 8; } index[] = {{2},{68},{35},{54},{37},{60},{24},{52},{26},{34},{29},{33},{45},{22},{17},{15},{179},{13},{37},{30},{21},{32},{1},{1},{1},{1},{2},{1},{8},{5},{6},{15},{23},{25},{7},{6},{2},{6},{35},{9},{30},{61},{50},{8},{13},{21},{20},{32},{22},{12},{18},{27},{19},{11},{11},{10},{30},{37},{5},{8},{4},{8},{3},{6},{6},{7},{5},{8},{4},{20},{8},{5},{5},{15},{3},{15},{3},{9},{18},{5},{8},{4},{4},{1},{4},{9},{17},{18},{6},{6},{5},{2},{7},{8},{5},{15},{5},{6},{9},{5},{5},{6},{7},{7},{25},{8},{12},{2},{3},{10},{5},{12},{6},{3},{13},{8},{5},{8},{1},{6},{8},{6},{13},{7},{4},{5},{10},{4},{8},{4},{7},{5},{3},{3},{6},{4},{3},{10},{14},{11},{10},{19},{14},{12},{10},{17},{13},{8},{8},{9},{6},{6},{12},{10},{12},{19},{5},{4},{5},{4},{4},{6},{5},{8},{23},{8},{5},{6},{6},{6},{4},{1},{6},{1},{2},{145},{143},{24},{1}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (1599 bytes) */ +const char* const cstring = "BZh91AY&SY42\233B\000\000\271\377\377\357\177\377\026\177\367\377W\277\343\377j\277\377\377\360@@@@@@@@@\000@@@\000@\000P\005\336\347\\\327\000\344\000\0354\0070\325=&\211\242i\2653\022z\233Dc)\352h\000\000\321\2404\000\311\24024\030\203FG\246\241\023@\232OJ~\247\351P\r\006C\324\000\003\020\320h\000\000\000\000\003@\006\203S\004\312i5\032\217Di\241\265\036\220h\000\310i\223@4\310a\000\000\032\006 l\207\000\r4\320\000d\000\0004\000\0004\000\000\310\000\006\206\200\tM\020A\2504\000\220\237\222\noSOR~\251\247\250\3654\006\200\000\000\006\206\2006\243i\242L\247\341f\370=\002\364|\007\242=M\361\241\227\307\361E\233\3442P\024IFT\316\317i\2748\310$\246\250RPt\260=\373\365pg\215ir(t\340R\250\367\376\006\330\371\004\333M\222\372h\376\337\257\242H\375\203h=\213I\262\255j\001\3312\"\035\364\023\314\250\334\250\023`\354\244\367\036\3204U\363\204\356\027\030h\376^D\226\026\232\026\243\244\032\007=)-\005\202\252\026\223\243\345(\022\005\007\330\275\252A\t0\354\240X*\255\256\3441\177\250\360u\301\0328\\4#D\221\020vv[x\207R\r\3748\t\226(v\\\334g\r$\014\304Z\233R*`\006X\0208\355}\305M\221aA\t\243\034\314\262\242)\210\212\311 \031I\003\023f1\026\022\300\317.\016\373)+\021#\367S4\246\220J\021NF\321\342H*\273\321p\363!\362\200\314\222\223\272\322#\272\306\227\214\r\206\314&$\253\327\024\302\013\2671O'v*Ccm`\221\020ag\0258\257\232\223W\021\366N\000Z3LG\206\352\210\316)\013\301\202\251\341\006\024#&D\276b\232\314\270\346id\300\237\330\200\"7\3037\016sm\013]{Q\362\345\204][M\253\316o\002\276\352\312\347wQ\213V\007\022u\200(Swa\321\245yh\354\305\256\322A\204\344\350\306\376\366w&\327\216=S\323\267\341\320\205\240\367\357\312\236\234\005\355\036\023\023o\266\277\304\352\307\315\255\3670'\016\307\304&\302\377l\215\204\253\263i\266\351\355\333\254.\313[\324\345\272b\301\235`K\025\327\353\351c\317\325\26471\220[1\243\023OfZi3'\003\033\343\262\010\231\354\370\242\242k\333\026(\355\301\014\333\016/\343\326q\377\246Z\344&51\027\036\244\220\223\356\316\261m\310\226\t\213\2644\265G@$:\014\323)\2338\031_\342\335\356\030\013M\023n\305\234J\326\030\331\200^\023\316cIx\0144F1;\273\210\225$\035mz\356\321\010l\264\303\2119\272A\332\337\363\273\257~#!B\344\257A\211\232\3063\367 \303\350\270{sg0k\372Xu:\321\360\036\361\030\204\333\212\022\034%\352\203\232\314Eb\261:\361m\002\326l\263\201\312\231\004\304\215s\3160\356h\202\355\202d\006,-X\335\r\207[$\"\304t\204]\257\210\026=\240\354\337yb x\226Q\004O\250\247w\317\216Q\213\370+\371\361(\0023\341\372\205\205\214\245|\340\330E\265\177\000\255fc\330\246\311\275\305\203\266\227\331\006O\022R\331\273\227\322\211\r\213\242\200{\240QK\026R\372\256Z\324v\260^\277\261\314~\345q\034J\242\220\010a\310\342\355~\244\222\204 \3434\003\336\212\326\216\030\200\202|C\353\377\336\236\350n=\026\206*\203\253\337\211\330u\232m\307I\216\032#\007C\337\202\264\316O33\013\343Sb\302i\304\007`k\277\260\333_\0102\365z\010\275\227\272\334PKH\tr\350\034\362\205^\226\221Q\213\307\344\356f@m\264y\236X\026\206\032\342<\2020dq\356x\222J\336D\255\364\004O\360m$\244\260\370\322\263\265\216\327\017V\016El\202\336\212\201\244\035H""\236\302\304MO\225\2045\177\241\304\307H\372r\217\204?\345\352\330\373\351\256\355\n\031\005\310o[t\374:\341\003\320&\220y\232\r\334\316-\277s\353\010\254l\320?\026\354\005\350\267\310\177\211_:Y\373enE\222Oe\202\302Q\221\273\001\211f\026s\313\203\377x\353\335Ix\253\0134h\274\330h6'\272{\204^\333L\022\221\031a\232r\273\211c^\211\351y'\256&\363\204\260L,@\242\r\303\347\203>\376Sj\370\024\265\361'\264_\300n\0162\002'\355\227\022c\311\357\301\314\372(\314\t\244\320;\206`\033\207\375\216\340f #\241\220C\215W\227\220`\270\265\332\264x\264\223i\225\265\270\201\226RI\024q\2033v^\016\303\311\242\003\226$H_13\024\350e\315#\240d\264F\236J?\231\010\025e\3437\225\236\3020&\370aL\374\272G(L\350\305\003t\313b7\3308\340\010\335\305\004Z+\335Nx\307\240\300Sn\253\253\020?\366\270\216\253\217`\215:L\314z0A\036\206\355\\F\204\025Q\032K\335\343\360E\334\307\217\224\356\\\374\244\033\233\304\213:\016\311GBf\2715h\005B4e\004\327X\220\342\310m\225\345v\274v\3020\3057\364\3142\320\307\034\207?7\364[C\202r\242\302 \025\347\t\302 ?L\237hk|\302\036>H\263\310\203\312\014\026F$\031\265\242\262\231r\303\020\245\033F]\210vL\236\372oUk\264t\\\272U.3\021\355`\255M9\211\3539P\304\307n\316\023_|\346\354\351\312_ G6\240O\313\2160\370\303\340?\365@\203\323\366\0354\241\301\211j\260\271\306\034\274\334\253\256p5\235\266\231\322\177d=)\343\316!>q[\343;\310\214\305\376\255\316#[\001\266@\312;f\330\031#Vy\332\274W\235\217\261w\344-\317\320\226@\276\355\341oe\016\246\337\317>\316\215\317])o\355\327\306\247N\217\347\027\207\233\305\345b\267\234\033\317_\030Bq\277\254\225K\343\371\363\303\227\305O\345\\Y\373|\372\304\251\205\217{\303\250\250\215\027.\014\273\005/v\017\253\205-W\312?F\217\367W\367w)\343U\261R<.\357\215\360\244\263\303kC[<,\267J\356\317\272Z\274\037\315\215.\217\314\376\322\341|\255\270T\274)9\035\2758|P\254|\234;\374\346\022\305m\035\"\234\306\347\205\023\247.\014_\027[E:\272;\342#<|axrx\2578_\276\032\255\214\027\316\341\362l\271T\336-a\264:\372pP;\370\361`k\274\370u\361s\331\030_\254\215/~W6""\312\355Qmt\025\223\373\373\273\007\363\007\215\203\355O\265OK\004\234\017\363\2421\236\247C\346\306\013g\252\277\305\357\313\327\243\267\007g>]\371k\345\357\371o\t\336x\341+\014}T\236,\357\343Y?\214\032\317\376\001\221\270\375\t"; + PyObject *data = __Pyx_DecompressString(cstring, 1479, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (2659 bytes) */ +const char* const bytes = ": All dimensions preceding dimension %d must be indexed and not slicedBuffer view does not expose stridesCan only create a buffer that is contiguous in memory.Cannot assign to read-only memoryviewCannot create writable memory view from read-only memoryviewCannot index with type 'Cannot transpose memoryview with indirect dimensionsDimension %d is not directEmpty shape tuple for cython.arrayIndex out of bounds (axis %d)Indirect dimensions not supportedInvalid mode, expected 'c' or 'fortran', got Invalid shape in axis Mapping[int, int]'), ?add_note and at 0xcollections.abcdisableenablegc (got got differing extents in dimension isenableditemsize <= 0 for cython.arraymapping_table has to be of type Mapping[int, int], but found no default __reduce__ due to non-trivial __cinit__ object>premap_ids.v1self.name is not Nonethinc.configthinc.layers.numpythinc/layers/premap_ids.pyxthinc.layers.typingthinc.modelthinc.typesthinc.utilunable to allocate array data.unable to allocate shape and strides.ASCIICallableDictEllipsisInTInts1dInts2dMappingModelOptionalOutT__Pyx_PyDict_NextRefSequenceTupleUnionView.MemoryViewabcallocate_bufferarrasarray2iasyncio.coroutinesattrsbackpropbaseboolccast__class____class_getitem__cline_in_tracebackcolumnconfigcountdYdefault__dict__dtypedtype_is_objectemptyencodeenumerateerrorflagsformatfortranforwardforward..backprop__func____getstate__ididx__import__indexinitial_sizeinputsint_is_coroutineis_trainitemsitemsizeklayers__main__mappermapping_tablememviewmodemodel__module__name__name__ndim__new__numpyob""jopsoutputpackpoppremap_ids__pyx_checksum__pyx_state__pyx_type__pyx_unpickle_Enum__pyx_vtable____qualname____reduce____reduce_cython____reduce_ex__registerregistryreshape2iresultreturn__set_name__setdefault__setstate____setstate_cython__shapesizestartstepstopstructtable__test__thinc.layers.premap_idsto_numpytypestypingunpackupdateutilvvaluesxxp\200\001\330\013\032\230/\250\021\330\005\006\330\004\014\210E\220\026\220q\230\001\330\004\016\210e\2206\230\021\230!\330\004\r\210U\220&\230\001\230\021\360\006\000\005\010\200w\210c\220\021\330\010\016\210h\220a\220q\340\010\016\210h\220a\220t\2301\230H\240G\2504\250q\330\004\r\210V\2201\220G\2305\240\001\330\004\n\210%\210t\220:\230Q\230a\330\004\r\210U\220$\220j\240\001\240\026\240s\250!\340\004\021\220\024\220Y\230a\360\006\000\005\014\2108\2201\200\001\340\023\024\330\004\r\210Q\340\004\014\210A\360\010\000\005\016\210X\220Q\220m\2403\240a\240q\330\004\010\210\003\2105\220\r\230V\2401\330\010\013\2105\220\n\230!\2303\230e\2404\240z\260\021\260#\260Q\330\014\022\220*\230A\330\020\021\330\020\034\230A\230T\240\021\240$\240a\240x\250q\260\004\260A\260T\270\021\270!\340\010\016\210a\210u\220A\330\004\013\2105\220\001\330\010\t\330\010\t\330\010\t\330\014\035\230X\240[\260\t\270\032\3001\320\004\025\220Y\230a\330\010\017\210u\220D\230\003\2306\240\021\240\"\240AO"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 175; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 58) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 175; i < 179; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 179; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 175; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 0; + int8_t const cint_constants_1[] = {0,-1,1}; + int32_t const cint_constants_4[] = {136983863L}; + for (int i = 0; i < 4; i++) { + numbertab[i] = PyLong_FromLong((i < 3 ? cint_constants_1[i - 0] : cint_constants_4[i - 3])); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 2; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 4; + unsigned int flags : 10; + unsigned int first_line : 7; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 78}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_dY}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_premap_ids_pyx, __pyx_mstate->__pyx_n_u_backprop, __pyx_mstate->__pyx_kp_b_iso88591_Ya_uD_6_A, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 1, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 36}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_mapping_table, __pyx_mstate->__pyx_n_u_default, __pyx_mstate->__pyx_n_u_column, __pyx_mstate->__pyx_n_u_mapper, __pyx_mstate->__pyx_n_u_k, __pyx_mstate->__pyx_n_u_v}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_premap_ids_pyx, __pyx_mstate->__pyx_n_u_premap_ids, __pyx_mstate->__pyx_kp_b_iso88591_Q_A_XQm3aq_5_V1_5_3e4z_Q_A_AT_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 12, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 62}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_inputs, __pyx_mstate->__pyx_n_u_is_train, __pyx_mstate->__pyx_n_u_table, __pyx_mstate->__pyx_n_u_default, __pyx_mstate->__pyx_n_u_column, __pyx_mstate->__pyx_n_u_idx, __pyx_mstate->__pyx_n_u_result, __pyx_mstate->__pyx_n_u_arr, __pyx_mstate->__pyx_n_u_output, __pyx_mstate->__pyx_n_u_backprop, __pyx_mstate->__pyx_n_u_backprop}; + __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_premap_ids_pyx, __pyx_mstate->__pyx_n_u_forward, __pyx_mstate->__pyx_kp_b_iso88591_E_q_e6_U_wc_haq_hat1HG4q_V1G5_t, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallOneArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallOneArg (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetAttrStr (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod0 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* PyObjectFastCallMethod */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) { + PyObject *result; + PyObject *attr = PyObject_GetAttr(args[0], name); + if (unlikely(!attr)) + return NULL; + result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1); + Py_DECREF(attr); + return result; +} +#endif + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* CIntToDigits (used by CIntToPyUnicode) */ +static const char DIGIT_PAIRS_10[2*10*10+1] = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; +static const char DIGIT_PAIRS_8[2*8*8+1] = { + "0001020304050607" + "1011121314151617" + "2021222324252627" + "3031323334353637" + "4041424344454647" + "5051525354555657" + "6061626364656667" + "7071727374757677" +}; +static const char DIGITS_HEX[2*16+1] = { + "0123456789abcdef" + "0123456789ABCDEF" +}; + +/* BuildPyUnicode (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char) { + PyObject *uval; + Py_ssize_t uoffset = ulength - clength; +#if CYTHON_USE_UNICODE_INTERNALS + Py_ssize_t i; + void *udata; + uval = PyUnicode_New(ulength, 127); + if (unlikely(!uval)) return NULL; + udata = PyUnicode_DATA(uval); + if (uoffset > 0) { + i = 0; + if (prepend_sign) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); + i++; + } + for (; i < uoffset; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); + } + } + for (i=0; i < clength; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); + } +#else + { + PyObject *sign = NULL, *padding = NULL; + uval = NULL; + if (uoffset > 0) { + prepend_sign = !!prepend_sign; + if (uoffset > prepend_sign) { + padding = PyUnicode_FromOrdinal(padding_char); + if (likely(padding) && uoffset > prepend_sign + 1) { + PyObject *tmp = PySequence_Repeat(padding, uoffset - prepend_sign); + Py_DECREF(padding); + padding = tmp; + } + if (unlikely(!padding)) goto done_or_error; + } + if (prepend_sign) { + sign = PyUnicode_FromOrdinal('-'); + if (unlikely(!sign)) goto done_or_error; + } + } + uval = PyUnicode_DecodeASCII(chars, clength, NULL); + if (likely(uval) && padding) { + PyObject *tmp = PyUnicode_Concat(padding, uval); + Py_DECREF(uval); + uval = tmp; + } + if (likely(uval) && sign) { + PyObject *tmp = PyUnicode_Concat(sign, uval); + Py_DECREF(uval); + uval = tmp; + } +done_or_error: + Py_XDECREF(padding); + Py_XDECREF(sign); + } +#endif + return uval; +} + +/* COrdinalToPyUnicode (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value) { + return value <= 1114111; +} +static PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t ulength, char padding_char) { + Py_ssize_t padding_length = ulength - 1; + if (likely((padding_length <= 250) && (value < 0xD800 || value > 0xDFFF))) { + char chars[256]; + if (value <= 255) { + memset(chars, padding_char, (size_t) padding_length); + chars[ulength-1] = (char) value; + return PyUnicode_DecodeLatin1(chars, ulength, NULL); + } + char *cpos = chars + sizeof(chars); + if (value < 0x800) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xc0 | (value & 0x1f)); + } else if (value < 0x10000) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xe0 | (value & 0x0f)); + } else { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xf0 | (value & 0x07)); + } + cpos -= padding_length; + memset(cpos, padding_char, (size_t) padding_length); + return PyUnicode_DecodeUTF8(cpos, chars + sizeof(chars) - cpos, NULL); + } + if (value <= 127 && CYTHON_USE_UNICODE_INTERNALS) { + const char chars[1] = {(char) value}; + return __Pyx_PyUnicode_BuildFromAscii(ulength, chars, 1, 0, padding_char); + } + { + PyObject *uchar, *padding_uchar, *padding, *result; + padding_uchar = PyUnicode_FromOrdinal(padding_char); + if (unlikely(!padding_uchar)) return NULL; + padding = PySequence_Repeat(padding_uchar, padding_length); + Py_DECREF(padding_uchar); + if (unlikely(!padding)) return NULL; + uchar = PyUnicode_FromOrdinal(value); + if (unlikely(!uchar)) { + Py_DECREF(padding); + return NULL; + } + result = PyUnicode_Concat(padding, uchar); + Py_DECREF(padding); + Py_DECREF(uchar); + return result; + } +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (int) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(int)*3+2]; + char *dpos, *end = digits + sizeof(int)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + int remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (int) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (int) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (int) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (Py_ssize_t) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(Py_ssize_t)*3+2]; + char *dpos, *end = digits + sizeof(Py_ssize_t)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + Py_ssize_t remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (Py_ssize_t) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (Py_ssize_t) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (Py_ssize_t) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* GetAttr */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) + return __Pyx_PyObject_GetAttrStr(o, n); +#endif + return PyObject_GetAttr(o, n); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* ObjectGetItem */ +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + PyObject *runerr = NULL; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1, 1); + } + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, __pyx_mstate_global->__pyx_n_u_class_getitem); + if (!meth) { + PyErr_Clear(); + } else { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + +/* RejectKeywords */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) { + PyObject *key = NULL; + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) { + key = __Pyx_PySequence_ITEM(kwds, 0); + } else { +#if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; +#else + Py_ssize_t pos = 0; +#endif +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return; +#endif + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); +#endif + } + if (likely(key)) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + Py_DECREF(key); + } +} + +/* DivInt[Py_ssize_t] */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t a, Py_ssize_t b, int b_is_constant) { + Py_ssize_t q = a / b; + Py_ssize_t r = a - q*b; + Py_ssize_t adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* GetAttr3 */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int res = PyObject_GetOptionalAttr(o, n, &r); + return (res != 0) ? r : __Pyx_NewRef(d); +#else + #if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } + #endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +#endif +} + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* RaiseTooManyValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* RaiseNoneIterError */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); +} + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); + return 0; +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* SwapException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; itp_as_sequence && type->tp_as_sequence->sq_repeat)) { + return type->tp_as_sequence->sq_repeat(seq, mul); + } else +#endif + { + return __Pyx_PySequence_Multiply_Generic(seq, mul); + } +} + +/* PyObjectFormatAndDecref */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f) { + if (unlikely(!s)) return NULL; + if (likely(PyUnicode_CheckExact(s))) return s; + return __Pyx_PyObject_FormatAndDecref(s, f); +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f) { + PyObject *result; + if (unlikely(!s)) return NULL; + result = PyObject_Format(s, f); + Py_DECREF(s); + return result; +} + +/* PyObjectFormat */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { + int ret; + _PyUnicodeWriter writer; + if (likely(PyFloat_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else if (likely(PyLong_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyLong_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else { + return PyObject_Format(obj, format_spec); + } + if (unlikely(ret == -1)) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} +#endif + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* DivInt[long] */ +static CYTHON_INLINE long __Pyx_div_long(long a, long b, int b_is_constant) { + long q = a / b; + long r = a - q*b; + long adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* BufferIndexError */ +static void __Pyx_RaiseBufferIndexError(int axis) { + PyErr_Format(PyExc_IndexError, + "Out of bounds on buffer access (axis %d)", axis); +} + +/* IterFinish */ +static CYTHON_INLINE int __Pyx_IterFinish(void) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return -1; + __Pyx_PyErr_Clear(); + return 0; + } + return 0; +} + +/* UnpackItemEndCheck */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } + return __Pyx_IterFinish(); +} + +/* RaiseClosureNameError */ +static void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + +/* DictGetItem */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { + PyObject *value; + if (unlikely(__Pyx_PyDict_GetItemRef(d, key, &value) == 0)) { // no value, no error + if (unlikely(PyTuple_Check(key))) { + PyObject* args = PyTuple_Pack(1, key); + if (likely(args)) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + } else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + return value; +} +#endif + +/* dict_setdefault (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType (used by FetchCommonType) */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* AddModuleRef (used by FetchSharedCythonModule) */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* FetchCommonType (used by CommonTypesMetaclass) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +#if __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; +} +static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; +} +#endif +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse (used by CythonFunctionShared) */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew (used by CythonFunctionShared) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared (used by CythonFunction) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +#endif +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, +#else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, +#endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#endif +#if CYTHON_METH_FASTCALL +#if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; +#endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); +#elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); +#else + PyObject_ClearManagedDict((PyObject*)m); +#endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); +#else + { + int e = +#if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict +#else + PyObject_VisitManagedDict +#endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } +#endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* PyObjectCallNoArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#endif + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* GetVTable (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* MergeVTables */ +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; +#if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; +#else + size = PyTuple_GET_SIZE(bases); +#endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; +#if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#else + basei = PyTuple_GET_ITEM(bases, i); +#endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; +bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); +#if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); +#endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +really_bad: // bad has failed! +#endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +other_failure: +#endif + PyMem_Free(base_vtables); + return -1; +} + +/* DelItemOnTypeDict (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_DelItem(tp_dict, k); + if (likely(!result)) PyType_Modified(tp); + return result; +} + +/* SetupReduce */ +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name_2); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_getstate = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + PyObject *getstate = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate); +#else + getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + if (!getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (getstate) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); +#else + object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + if (!object_getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (object_getstate != getstate) { + goto __PYX_GOOD; + } + } +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); + if (likely(reduce_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + goto __PYX_BAD; + } + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); + if (likely(setstate_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; +__PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; +__PYX_GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); + Py_XDECREF(object_getstate); + Py_XDECREF(getstate); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + +/* TypeImport */ +#ifndef __PYX_HAVE_RT_ImportType_3_2_1 +#define __PYX_HAVE_RT_ImportType_3_2_1 +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size) +{ + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; +#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; +#endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; +#else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_1 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_1 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return NULL; +} +#endif + +/* ImportFrom */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u__2); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* MemviewRefcount */ +#include +#ifndef _Py_NO_RETURN +#define _Py_NO_RETURN +#endif +_Py_NO_RETURN +static void __pyx_fatalerror(const char *fmt, ...) { + va_list vargs; + char msg[200]; +#if PY_VERSION_HEX >= 0x030A0000 || defined(HAVE_STDARG_PROTOTYPES) + va_start(vargs, fmt); +#else + va_start(vargs); +#endif + vsnprintf(msg, 200, fmt, vargs); + va_end(vargs); + Py_FatalError(msg); +} +static CYTHON_INLINE int +__pyx_add_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)++; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE int +__pyx_sub_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)--; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE void +__Pyx_INC_MEMVIEW(__Pyx_memviewslice *memslice, int have_gil, int lineno) +{ + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + return; + } + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count+1, lineno); + } + } +} +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *memslice, + int have_gil, int lineno) { + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + memslice->memview = NULL; + return; + } + old_acquisition_count = __pyx_sub_acquisition_count(memview); + memslice->data = NULL; + if (likely(old_acquisition_count > 1)) { + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + if (have_gil) { + Py_CLEAR(memslice->memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_CLEAR(memslice->memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count-1, lineno); + } +} + +/* MemviewSliceIsContig */ +static int +__pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim) +{ + int i, index, step, start; + Py_ssize_t itemsize = mvs.memview->view.itemsize; + if (order == 'F') { + step = 1; + start = 0; + } else { + step = -1; + start = ndim - 1; + } + for (i = 0; i < ndim; i++) { + index = start + step * i; + if (mvs.suboffsets[index] >= 0 || mvs.strides[index] != itemsize) + return 0; + itemsize *= mvs.shape[index]; + } + return 1; +} + +/* OverlappingSlices */ +static void +__pyx_get_array_memory_extents(__Pyx_memviewslice *slice, + void **out_start, void **out_end, + int ndim, size_t itemsize) +{ + char *start, *end; + int i; + start = end = slice->data; + for (i = 0; i < ndim; i++) { + Py_ssize_t stride = slice->strides[i]; + Py_ssize_t extent = slice->shape[i]; + if (extent == 0) { + *out_start = *out_end = start; + return; + } else { + if (stride > 0) + end += stride * (extent - 1); + else + start += stride * (extent - 1); + } + } + *out_start = start; + *out_end = end + itemsize; +} +static int +__pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize) +{ + void *start1, *end1, *start2, *end2; + __pyx_get_array_memory_extents(slice1, &start1, &end1, ndim, itemsize); + __pyx_get_array_memory_extents(slice2, &start2, &end2, ndim, itemsize); + return (start1 < end2) && (start2 < end1); +} + +/* MemviewSliceInit */ +static int +__Pyx_init_memviewslice(struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference) +{ + __Pyx_RefNannyDeclarations + int i, retval=-1; + Py_buffer *buf = &memview->view; + __Pyx_RefNannySetupContext("init_memviewslice", 0); + if (unlikely(memviewslice->memview || memviewslice->data)) { + PyErr_SetString(PyExc_ValueError, + "memviewslice is already initialized!"); + goto fail; + } + if (buf->strides) { + for (i = 0; i < ndim; i++) { + memviewslice->strides[i] = buf->strides[i]; + } + } else { + Py_ssize_t stride = buf->itemsize; + for (i = ndim - 1; i >= 0; i--) { + memviewslice->strides[i] = stride; + stride *= buf->shape[i]; + } + } + for (i = 0; i < ndim; i++) { + memviewslice->shape[i] = buf->shape[i]; + if (buf->suboffsets) { + memviewslice->suboffsets[i] = buf->suboffsets[i]; + } else { + memviewslice->suboffsets[i] = -1; + } + } + memviewslice->memview = memview; + memviewslice->data = (char *)buf->buf; + if (__pyx_add_acquisition_count(memview) == 0 && !memview_is_new_reference) { + Py_INCREF((PyObject*)memview); + } + retval = 0; + goto no_fail; +fail: + memviewslice->memview = 0; + memviewslice->data = 0; + retval = -1; +no_fail: + __Pyx_RefNannyFinishContext(); + return retval; +} + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* MemviewDtypeToObject */ +static CYTHON_INLINE PyObject *__pyx_memview_get_nn_int64_t(const char *itemp) { + return (PyObject *) __Pyx_PyLong_From_int64_t(*(int64_t const *) itemp); +} +static CYTHON_INLINE int __pyx_memview_set_nn_int64_t(char *itemp, PyObject *obj) { + int64_t value = __Pyx_PyLong_As_int64_t(obj); + if (unlikely((value == ((int64_t)-1)) && PyErr_Occurred())) + return 0; + *(int64_t *) itemp = value; + return 1; +} + +/* IsLittleEndian (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } S; + S.u32 = 0x01020304; + return S.u8[0] == 4; +} + +/* BufferFormatCheck (used by MemviewSliceValidateAndInit) */ +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type) { + stack[0].field = &ctx->root; + stack[0].parent_offset = 0; + ctx->root.type = type; + ctx->root.name = "buffer dtype"; + ctx->root.offset = 0; + ctx->head = stack; + ctx->head->field = &ctx->root; + ctx->fmt_offset = 0; + ctx->head->parent_offset = 0; + ctx->new_packmode = '@'; + ctx->enc_packmode = '@'; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->is_complex = 0; + ctx->is_valid_array = 0; + ctx->struct_alignment = 0; + while (type->typegroup == 'S') { + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = 0; + type = type->fields->type; + } +} +static int __Pyx_BufFmt_ParseNumber(const char** ts) { + int count; + const char* t = *ts; + if (*t < '0' || *t > '9') { + return -1; + } else { + count = *t++ - '0'; + while (*t >= '0' && *t <= '9') { + count *= 10; + count += *t++ - '0'; + } + } + *ts = t; + return count; +} +static int __Pyx_BufFmt_ExpectNumber(const char **ts) { + int number = __Pyx_BufFmt_ParseNumber(ts); + if (number == -1) + PyErr_Format(PyExc_ValueError,\ + "Does not understand character buffer dtype format string ('%c')", **ts); + return number; +} +static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) { + PyErr_Format(PyExc_ValueError, + "Unexpected format string character: '%c'", ch); +} +static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { + switch (ch) { + case '?': return "'bool'"; + case 'c': return "'char'"; + case 'b': return "'signed char'"; + case 'B': return "'unsigned char'"; + case 'h': return "'short'"; + case 'H': return "'unsigned short'"; + case 'i': return "'int'"; + case 'I': return "'unsigned int'"; + case 'l': return "'long'"; + case 'L': return "'unsigned long'"; + case 'q': return "'long long'"; + case 'Q': return "'unsigned long long'"; + case 'f': return (is_complex ? "'complex float'" : "'float'"); + case 'd': return (is_complex ? "'complex double'" : "'double'"); + case 'g': return (is_complex ? "'complex long double'" : "'long double'"); + case 'T': return "a struct"; + case 'O': return "Python object"; + case 'P': return "a pointer"; + case 's': case 'p': return "a string"; + case 0: return "end"; + default: return "unparsable format string"; + } +} +static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return 2; + case 'i': case 'I': case 'l': case 'L': return 4; + case 'q': case 'Q': return 8; + case 'f': return (is_complex ? 8 : 4); + case 'd': return (is_complex ? 16 : 8); + case 'g': { + PyErr_SetString(PyExc_ValueError, "Python does not define a standard format string size for long double ('g').."); + return 0; + } + case 'O': case 'P': return sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(short); + case 'i': case 'I': return sizeof(int); + case 'l': case 'L': return sizeof(long); + case 'q': case 'Q': return sizeof(PY_LONG_LONG); + case 'f': return sizeof(float) * (is_complex ? 2 : 1); + case 'd': return sizeof(double) * (is_complex ? 2 : 1); + case 'g': return sizeof(long double) * (is_complex ? 2 : 1); + case 'O': case 'P': return sizeof(void*); + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +typedef struct { char c; short x; } __Pyx_st_short; +typedef struct { char c; int x; } __Pyx_st_int; +typedef struct { char c; long x; } __Pyx_st_long; +typedef struct { char c; float x; } __Pyx_st_float; +typedef struct { char c; double x; } __Pyx_st_double; +typedef struct { char c; long double x; } __Pyx_st_longdouble; +typedef struct { char c; void *x; } __Pyx_st_void_p; +typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_st_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_st_float) - sizeof(float); + case 'd': return sizeof(__Pyx_st_double) - sizeof(double); + case 'g': return sizeof(__Pyx_st_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_st_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +/* These are for computing the padding at the end of the struct to align + on the first member of the struct. This will probably the same as above, + but we don't have any guarantees. + */ +typedef struct { short x; char c; } __Pyx_pad_short; +typedef struct { int x; char c; } __Pyx_pad_int; +typedef struct { long x; char c; } __Pyx_pad_long; +typedef struct { float x; char c; } __Pyx_pad_float; +typedef struct { double x; char c; } __Pyx_pad_double; +typedef struct { long double x; char c; } __Pyx_pad_longdouble; +typedef struct { void *x; char c; } __Pyx_pad_void_p; +typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_pad_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_pad_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_pad_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_pad_float) - sizeof(float); + case 'd': return sizeof(__Pyx_pad_double) - sizeof(double); + case 'g': return sizeof(__Pyx_pad_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_pad_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) { + switch (ch) { + case 'c': + return 'H'; + case 'b': case 'h': case 'i': + case 'l': case 'q': case 's': case 'p': + return 'I'; + case '?': case 'B': case 'H': case 'I': case 'L': case 'Q': + return 'U'; + case 'f': case 'd': case 'g': + return (is_complex ? 'C' : 'R'); + case 'O': + return 'O'; + case 'P': + return 'P'; + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) { + if (ctx->head == NULL || ctx->head->field == &ctx->root) { + const char* expected; + const char* quote; + if (ctx->head == NULL) { + expected = "end"; + quote = ""; + } else { + expected = ctx->head->field->type->name; + quote = "'"; + } + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected %s%s%s but got %s", + quote, expected, quote, + __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex)); + } else { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_StructField* parent = (ctx->head - 1)->field; + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected '%s' but got %s in '%s.%s'", + field->type->name, __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex), + parent->type->name, field->name); + } +} +static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { + char group; + size_t size, offset, arraysize = 1; + if (ctx->enc_type == 0) return 0; + if (ctx->head->field->type->arraysize[0]) { + int i, ndim = 0; + if (ctx->enc_type == 's' || ctx->enc_type == 'p') { + ctx->is_valid_array = ctx->head->field->type->ndim == 1; + ndim = 1; + if (ctx->enc_count != ctx->head->field->type->arraysize[0]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %zu", + ctx->head->field->type->arraysize[0], ctx->enc_count); + return -1; + } + } + if (!ctx->is_valid_array) { + PyErr_Format(PyExc_ValueError, "Expected %d dimensions, got %d", + ctx->head->field->type->ndim, ndim); + return -1; + } + for (i = 0; i < ctx->head->field->type->ndim; i++) { + arraysize *= ctx->head->field->type->arraysize[i]; + } + ctx->is_valid_array = 0; + ctx->enc_count = 1; + } + group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex); + do { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_TypeInfo* type = field->type; + if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') { + size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex); + } else { + size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); + } + if (ctx->enc_packmode == '@') { + size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); + size_t align_mod_offset; + if (align_at == 0) return -1; + align_mod_offset = ctx->fmt_offset % align_at; + if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset; + if (ctx->struct_alignment == 0) + ctx->struct_alignment = __Pyx_BufFmt_TypeCharToPadding(ctx->enc_type, + ctx->is_complex); + } + if (type->size != size || type->typegroup != group) { + if (type->typegroup == 'C' && type->fields != NULL) { + size_t parent_offset = ctx->head->parent_offset + field->offset; + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = parent_offset; + continue; + } + if ((type->typegroup == 'H' || group == 'H') && type->size == size) { + } else { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + } + offset = ctx->head->parent_offset + field->offset; + if (ctx->fmt_offset != offset) { + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch; next field is at offset %" CYTHON_FORMAT_SSIZE_T "d but %" CYTHON_FORMAT_SSIZE_T "d expected", + (Py_ssize_t)ctx->fmt_offset, (Py_ssize_t)offset); + return -1; + } + ctx->fmt_offset += size; + if (arraysize) + ctx->fmt_offset += (arraysize - 1) * size; + --ctx->enc_count; + while (1) { + if (field == &ctx->root) { + ctx->head = NULL; + if (ctx->enc_count != 0) { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + break; + } + ctx->head->field = ++field; + if (field->type == NULL) { + --ctx->head; + field = ctx->head->field; + continue; + } else if (field->type->typegroup == 'S') { + size_t parent_offset = ctx->head->parent_offset + field->offset; + if (field->type->fields->type == NULL) continue; + field = field->type->fields; + ++ctx->head; + ctx->head->field = field; + ctx->head->parent_offset = parent_offset; + break; + } else { + break; + } + } + } while (ctx->enc_count); + ctx->enc_type = 0; + ctx->is_complex = 0; + return 0; +} +static int +__pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp) +{ + const char *ts = *tsp; + int i = 0, number, ndim; + ++ts; + if (ctx->new_count != 1) { + PyErr_SetString(PyExc_ValueError, + "Cannot handle repeated arrays in format string"); + return -1; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return -1; + ndim = ctx->head->field->type->ndim; + while (*ts && *ts != ')') { + switch (*ts) { + case ' ': case '\f': case '\r': case '\n': case '\t': case '\v': continue; + default: break; + } + number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return -1; + if (i < ndim && (size_t) number != ctx->head->field->type->arraysize[i]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %d", + ctx->head->field->type->arraysize[i], number); + return -1; + } + if (*ts != ',' && *ts != ')') { + PyErr_Format(PyExc_ValueError, + "Expected a comma in format string, got '%c'", *ts); + return -1; + } + if (*ts == ',') ts++; + i++; + } + if (i != ndim) { + PyErr_Format(PyExc_ValueError, "Expected %d dimension(s), got %d", + ctx->head->field->type->ndim, i); + return -1; + } + if (!*ts) { + PyErr_SetString(PyExc_ValueError, + "Unexpected end of format string, expected ')'"); + return -1; + } + ctx->is_valid_array = 1; + ctx->new_count = 1; + *tsp = ++ts; + return 0; +} +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { + int got_Z = 0; + while (1) { + switch(*ts) { + case 0: + if (ctx->enc_type != 0 && ctx->head == NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + if (ctx->head != NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + return ts; + case ' ': + case '\r': + case '\n': + ++ts; + break; + case '<': + if (!__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '>': + case '!': + if (__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '=': + case '@': + case '^': + ctx->new_packmode = *ts++; + break; + case 'T': + { + const char* ts_after_sub; + size_t i, struct_count = ctx->new_count; + size_t struct_alignment = ctx->struct_alignment; + ctx->new_count = 1; + ++ts; + if (*ts != '{') { + PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'"); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + ctx->enc_count = 0; + ctx->struct_alignment = 0; + ++ts; + ts_after_sub = ts; + for (i = 0; i != struct_count; ++i) { + ts_after_sub = __Pyx_BufFmt_CheckString(ctx, ts); + if (!ts_after_sub) return NULL; + } + ts = ts_after_sub; + if (struct_alignment) ctx->struct_alignment = struct_alignment; + } + break; + case '}': + { + size_t alignment = ctx->struct_alignment; + ++ts; + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + if (alignment && ctx->fmt_offset % alignment) { + ctx->fmt_offset += alignment - (ctx->fmt_offset % alignment); + } + } + return ts; + case 'x': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->fmt_offset += ctx->new_count; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->enc_packmode = ctx->new_packmode; + ++ts; + break; + case 'Z': + got_Z = 1; + ++ts; + if (*ts != 'f' && *ts != 'd' && *ts != 'g') { + __Pyx_BufFmt_RaiseUnexpectedChar('Z'); + return NULL; + } + CYTHON_FALLTHROUGH; + case '?': case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': + case 'l': case 'L': case 'q': case 'Q': + case 'f': case 'd': case 'g': + case 'O': case 'p': + if ((ctx->enc_type == *ts) && (got_Z == ctx->is_complex) && + (ctx->enc_packmode == ctx->new_packmode) && (!ctx->is_valid_array)) { + ctx->enc_count += ctx->new_count; + ctx->new_count = 1; + got_Z = 0; + ++ts; + break; + } + CYTHON_FALLTHROUGH; + case 's': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_count = ctx->new_count; + ctx->enc_packmode = ctx->new_packmode; + ctx->enc_type = *ts; + ctx->is_complex = got_Z; + ++ts; + ctx->new_count = 1; + got_Z = 0; + break; + case ':': + ++ts; + while(*ts != ':') ++ts; + ++ts; + break; + case '(': + if (__pyx_buffmt_parse_array(ctx, &ts) < 0) return NULL; + break; + default: + { + int number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return NULL; + ctx->new_count = (size_t)number; + } + } + } +} + +/* TypeInfoCompare (used by MemviewSliceValidateAndInit) */ + static int + __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b) + { + int i; + if (!a || !b) + return 0; + if (a == b) + return 1; + if (a->size != b->size || a->typegroup != b->typegroup || + a->is_unsigned != b->is_unsigned || a->ndim != b->ndim) { + if (a->typegroup == 'H' || b->typegroup == 'H') { + return a->size == b->size; + } else { + return 0; + } + } + if (a->ndim) { + for (i = 0; i < a->ndim; i++) + if (a->arraysize[i] != b->arraysize[i]) + return 0; + } + if (a->typegroup == 'S') { + if (a->flags != b->flags) + return 0; + if (a->fields || b->fields) { + if (!(a->fields && b->fields)) + return 0; + for (i = 0; a->fields[i].type && b->fields[i].type; i++) { + const __Pyx_StructField *field_a = a->fields + i; + const __Pyx_StructField *field_b = b->fields + i; + if (field_a->offset != field_b->offset || + !__pyx_typeinfo_cmp(field_a->type, field_b->type)) + return 0; + } + return !a->fields[i].type && !b->fields[i].type; + } + } + return 1; + } + +/* MemviewSliceValidateAndInit (used by ObjectToMemviewSlice) */ + static int + __pyx_check_strides(Py_buffer *buf, int dim, int ndim, int spec) + { + if (buf->shape[dim] <= 1) + return 1; + if (buf->strides) { + if (spec & __Pyx_MEMVIEW_CONTIG) { + if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL)) { + if (unlikely(buf->strides[dim] != sizeof(void *))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly contiguous " + "in dimension %d.", dim); + goto fail; + } + } else if (unlikely(buf->strides[dim] != buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_FOLLOW) { + Py_ssize_t stride = buf->strides[dim]; + if (stride < 0) + stride = -stride; + if (unlikely(stride < buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + } else { + if (unlikely(spec & __Pyx_MEMVIEW_CONTIG && dim != ndim - 1)) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not contiguous in " + "dimension %d", dim); + goto fail; + } else if (unlikely(spec & (__Pyx_MEMVIEW_PTR))) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not indirect in " + "dimension %d", dim); + goto fail; + } else if (unlikely(buf->suboffsets)) { + PyErr_SetString(PyExc_ValueError, + "Buffer exposes suboffsets but no strides"); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) + { + CYTHON_UNUSED_VAR(ndim); + if (spec & __Pyx_MEMVIEW_DIRECT) { + if (unlikely(buf->suboffsets && buf->suboffsets[dim] >= 0)) { + PyErr_Format(PyExc_ValueError, + "Buffer not compatible with direct access " + "in dimension %d.", dim); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_PTR) { + if (unlikely(!buf->suboffsets || (buf->suboffsets[dim] < 0))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly accessible " + "in dimension %d.", dim); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_verify_contig(Py_buffer *buf, int ndim, int c_or_f_flag) + { + int i; + if (c_or_f_flag & __Pyx_IS_F_CONTIG) { + Py_ssize_t stride = 1; + for (i = 0; i < ndim; i++) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not fortran contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } else if (c_or_f_flag & __Pyx_IS_C_CONTIG) { + Py_ssize_t stride = 1; + for (i = ndim - 1; i >- 1; i--) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not C contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } + return 1; + fail: + return 0; + } + static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj) + { + struct __pyx_memoryview_obj *memview, *new_memview; + __Pyx_RefNannyDeclarations + Py_buffer *buf; + int i, spec = 0, retval = -1; + __Pyx_BufFmt_Context ctx; + int from_memoryview = __pyx_memoryview_check(original_obj); + __Pyx_RefNannySetupContext("ValidateAndInit_memviewslice", 0); + if (from_memoryview && __pyx_typeinfo_cmp(dtype, ((struct __pyx_memoryview_obj *) + original_obj)->typeinfo)) { + memview = (struct __pyx_memoryview_obj *) original_obj; + new_memview = NULL; + } else { + memview = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + original_obj, buf_flags, 0, dtype); + new_memview = memview; + if (unlikely(!memview)) + goto fail; + } + buf = &memview->view; + if (unlikely(buf->ndim != ndim)) { + PyErr_Format(PyExc_ValueError, + "Buffer has wrong number of dimensions (expected %d, got %d)", + ndim, buf->ndim); + goto fail; + } + if (new_memview) { + __Pyx_BufFmt_Init(&ctx, stack, dtype); + if (unlikely(!__Pyx_BufFmt_CheckString(&ctx, buf->format))) goto fail; + } + if (unlikely((unsigned) buf->itemsize != dtype->size)) { + PyErr_Format(PyExc_ValueError, + "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "u byte%s) " + "does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "u byte%s)", + buf->itemsize, + (buf->itemsize > 1) ? "s" : "", + dtype->name, + dtype->size, + (dtype->size > 1) ? "s" : ""); + goto fail; + } + if (buf->len > 0) { + for (i = 0; i < ndim; i++) { + spec = axes_specs[i]; + if (unlikely(!__pyx_check_strides(buf, i, ndim, spec))) + goto fail; + if (unlikely(!__pyx_check_suboffsets(buf, i, ndim, spec))) + goto fail; + } + if (unlikely(buf->strides && !__pyx_verify_contig(buf, ndim, c_or_f_flag))) + goto fail; + } + if (unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice, + new_memview != NULL) == -1)) { + goto fail; + } + retval = 0; + goto no_fail; + fail: + Py_XDECREF((PyObject*)new_memview); + retval = -1; + no_fail: + __Pyx_RefNannyFinishContext(); + return retval; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_ds_nn_int64_t(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_STRIDED) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, 0, + PyBUF_RECORDS_RO | writable_flag, 1, + &__Pyx_TypeInfo_nn_int64_t, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* MemviewSliceCopy */ + static __Pyx_memviewslice + __pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object) + { + __Pyx_RefNannyDeclarations + int i; + __Pyx_memviewslice new_mvs = __Pyx_MEMSLICE_INIT; + struct __pyx_memoryview_obj *from_memview = from_mvs->memview; + Py_buffer *buf = &from_memview->view; + PyObject *shape_tuple = NULL; + PyObject *temp_int = NULL; + struct __pyx_array_obj *array_obj = NULL; + struct __pyx_memoryview_obj *memview_obj = NULL; + __Pyx_RefNannySetupContext("__pyx_memoryview_copy_new_contig", 0); + for (i = 0; i < ndim; i++) { + if (unlikely(from_mvs->suboffsets[i] >= 0)) { + PyErr_Format(PyExc_ValueError, "Cannot copy memoryview slice with " + "indirect dimensions (axis %d)", i); + goto fail; + } + } + shape_tuple = PyTuple_New(ndim); + if (unlikely(!shape_tuple)) { + goto fail; + } + __Pyx_GOTREF(shape_tuple); + for(i = 0; i < ndim; i++) { + temp_int = PyLong_FromSsize_t(from_mvs->shape[i]); + if(unlikely(!temp_int)) { + goto fail; + } else { + #if CYTHON_ASSUME_SAFE_MACROS + PyTuple_SET_ITEM(shape_tuple, i, temp_int); + #else + if (PyTuple_SetItem(shape_tuple, i, temp_int) < 0) { + goto fail; + } + #endif + temp_int = NULL; + } + } + array_obj = __pyx_array_new(shape_tuple, sizeof_dtype, buf->format, mode, NULL); + if (unlikely(!array_obj)) { + goto fail; + } + __Pyx_GOTREF(array_obj); + memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + (PyObject *) array_obj, contig_flag, + dtype_is_object, + from_mvs->memview->typeinfo); + if (unlikely(!memview_obj)) + goto fail; + if (unlikely(__Pyx_init_memviewslice(memview_obj, ndim, &new_mvs, 1) < 0)) + goto fail; + if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim, + dtype_is_object) < 0)) + goto fail; + goto no_fail; + fail: + __Pyx_XDECREF((PyObject *) new_mvs.memview); + new_mvs.memview = NULL; + new_mvs.data = NULL; + no_fail: + __Pyx_XDECREF(shape_tuple); + __Pyx_XDECREF(temp_int); + __Pyx_XDECREF((PyObject *) array_obj); + __Pyx_RefNannyFinishContext(); + return new_mvs; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int64_t(int64_t value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int64_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int64_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int64_t), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int64_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE int64_t __Pyx_PyLong_As_int64_t(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int64_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int64_t) -1; + val = __Pyx_PyLong_As_int64_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int64_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 2 * PyLong_SHIFT)) { + return (int64_t) (((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 3 * PyLong_SHIFT)) { + return (int64_t) (((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 4 * PyLong_SHIFT)) { + return (int64_t) (((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int64_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int64_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int64_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int64_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + return (int64_t) ((((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + return (int64_t) ((((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT)) { + return (int64_t) ((((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int64_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, long, PyLong_AsLong(x)) + } else if ((sizeof(int64_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int64_t val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int64_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int64_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int64_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int64_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int64_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int64_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int64_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int64_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int64_t) 1) << (sizeof(int64_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int64_t) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int64_t"); + return (int64_t) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int64_t"); + return (int64_t) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; + } + +/* PyObjectCall2Args (used by PyObjectCallMethod1) */ + static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } + +/* PyObjectCallMethod1 (used by UpdateUnpickledDict) */ + #if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) + static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; + } + #endif + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + #if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + #else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); + #endif + } + +/* UpdateUnpickledDict */ + static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + PyObject *state_dict = __Pyx_PySequence_ITEM(state, index); + if (unlikely(!state_dict)) { + return -1; + } + int non_empty = PyObject_IsTrue(state_dict); + if (non_empty == 0) { + Py_DECREF(state_dict); + return 0; + } else if (unlikely(non_empty == -1)) { + return -1; + } + PyObject *dict; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = PyObject_GetAttrString(obj, "__dict__"); + #else + dict = PyObject_GenericGetDict(obj, NULL); + #endif + if (unlikely(!dict)) { + Py_DECREF(state_dict); + return -1; + } + int result; + if (likely(PyDict_CheckExact(dict))) { + result = PyDict_Update(dict, state_dict); + } else { + PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict); + if (likely(obj_result)) { + Py_DECREF(obj_result); + result = 0; + } else { + result = -1; + } + } + Py_DECREF(state_dict); + Py_DECREF(dict); + return result; + } + static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(state_size == -1)) return -1; + #endif + if (state_size <= index) { + return 0; + } + return __Pyx__UpdateUnpickledDict(obj, state, index); + } + +/* CheckUnpickleChecksum */ + static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + PyObject *pickle_module = PyImport_ImportModule("pickle"); + if (unlikely(!pickle_module)) return; + PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (unlikely(!pickle_error)) return; + if (checksum2 == checksum1) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))", + checksum, checksum1, members); + } else if (checksum3 == checksum2) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, members); + } else { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, checksum3, members); + } + Py_DECREF(pickle_error); + } + static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + int found = 0; + found |= checksum1 == checksum; + found |= checksum2 == checksum; + found |= checksum3 == checksum; + if (likely(found)) + return 0; + __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members); + return -1; + } + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (long) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const char neg_one = (char) -1, const_zero = (char) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + char val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (char) -1; + val = __Pyx_PyLong_As_char(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 2 * PyLong_SHIFT)) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 3 * PyLong_SHIFT)) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 4 * PyLong_SHIFT)) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(char) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(char) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(char) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(char) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) + } else if ((sizeof(char) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + char val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (char) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (char) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (char) -1; + } else { + stepval = v; + } + v = NULL; + val = (char) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(char) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((char) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(char) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((char) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((char) 1) << (sizeof(char) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (char) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; + } + +/* FormatTypeName */ + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 + static __Pyx_TypeName + __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) + { + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__7); + } + goto done; + } + #endif + +/* GetRuntimeVersion */ + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + void __Pyx_init_runtime_version(void) { + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + } + #endif + static unsigned long __Pyx_get_runtime_version(void) { + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; + #else + return __Pyx_cached_runtime_version; + #endif + } + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } + } + +/* NewCodeObj */ + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } + #elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } + #elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #endif + static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map + ) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); + done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; + } + +/* DecompressString */ + static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; + import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); + bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; + } + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + + /* MultiPhaseInitModuleState */ + #if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE + #ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 + #else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 + #endif + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS + #error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" + #endif + #if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #define __Pyx_ModuleStateLookup_Lock() + #define __Pyx_ModuleStateLookup_Unlock() + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; + #define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(__cplusplus) && __cplusplus >= 201103L + #include + static std::mutex __Pyx_ModuleStateLookup_mutex; + #define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() + #define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() + #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) + #include + static mtx_t __Pyx_ModuleStateLookup_mutex; + static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; + static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); + } + #define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(HAVE_PTHREAD_H) + #include + static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; + #define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(_WIN32) + #include // synchapi.h on its own doesn't work + static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; + #define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #else + #error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." + #endif + typedef struct { + int64_t id; + PyObject *module; + } __Pyx_InterpreterIdAndModule; + typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; + } __Pyx_ModuleStateLookupData; + #define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; + #else + static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; + #endif + static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; + } + static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; + #else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + #endif + return result; + } + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); + } + #else + #define __Pyx_ModuleStateLookup_wait_until_no_readers() + #endif + static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; + } + static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; + } + static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); + #else + __Pyx_ModuleStateLookup_data = new_data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return result; + } + static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; + #endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); + #else + __Pyx_ModuleStateLookup_data = data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; + } + #endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.pyx new file mode 100644 index 0000000000000000000000000000000000000000..d741b4d87e488cb59c0078c10197f73c03ea3cd2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/premap_ids.pyx @@ -0,0 +1,81 @@ +# cython: binding=True, infer_types=True, profile=False +from libc.stdint cimport int64_t +import numpy + +from preshed.maps cimport PreshMap + +from typing import Callable, Dict, Mapping, Optional, Tuple, Union, cast + +from ..config import registry +from ..model import Model +from ..types import Ints1d, Ints2d +from ..util import to_numpy + +InT = Union[Ints1d, Ints2d] +OutT = Ints2d + + +cdef lookup(PreshMap mapping, int64_t[:] keys, int64_t default): + """ + Faster dict.get(keys, default) for the case when + the "dict" is a Dict[int, int] converted to PreshMap + and the "keys" is a numpy integer vector. + """ + cdef int maxi = len(keys) + result = numpy.empty(maxi, dtype="int") + cdef int64_t[:] result_view = result + for i in range(maxi): + v = mapping[keys[i]] + if v is None: + result_view[i] = default + else: + result_view[i] = v + return result + + +@registry.layers("premap_ids.v1") +def premap_ids( + mapping_table: Mapping[int, int], + default: int = 0, + *, + column: Optional[int] = None +): + """Remap integer inputs to integers a mapping table, usually as a + preprocess before embeddings.""" + mapper = PreshMap(initial_size=len(mapping_table)) + for k, v in mapping_table.items(): + if not (isinstance(k, int) and isinstance(v, int)): + raise ValueError( + "mapping_table has to be of type Mapping[int, int], " + f"but found {k}, {type(k)} and {v}, {type(v)}" + ) + mapper[k] = v + return Model( + "premap_ids", + forward, + attrs={ + "mapping_table": mapper, "default": default, "column": column + } + ) + + +def forward( + model: Model, inputs: InT, is_train: bool +) -> Tuple[OutT, Callable]: + table = model.attrs["mapping_table"] + default = model.attrs["default"] + column = model.attrs["column"] + # Have to convert to numpy anyways, because + # cupy ints don't work together with Python ints. + if column is None: + idx = to_numpy(inputs) + else: + idx = to_numpy(cast(Ints2d, inputs)[:, column]) + result = lookup(table, idx, default) + arr = model.ops.asarray2i(result) + output = model.ops.reshape2i(arr, -1, 1) + + def backprop(dY: OutT) -> InT: + return model.ops.xp.empty(dY.shape) # type: ignore + + return output, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/pytorchwrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/pytorchwrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..4e09f7f59a1c0eec05c5c7ebe536ac7954d6f3f8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/pytorchwrapper.py @@ -0,0 +1,319 @@ +from typing import Any, Callable, Dict, Optional, Tuple, cast + +from ..compat import torch +from ..config import registry +from ..model import Model +from ..shims import PyTorchGradScaler, PyTorchShim +from ..types import ArgsKwargs, Floats3d, Padded +from ..util import ( + convert_recursive, + is_torch_array, + is_xp_array, + partial, + torch2xp, + xp2torch, +) + + +@registry.layers("PyTorchRNNWrapper.v1") +def PyTorchRNNWrapper( + pytorch_model: Any, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, +) -> Model[Padded, Padded]: + """Wrap a PyTorch RNN model for use in Thinc.""" + if convert_inputs is None: + convert_inputs = convert_rnn_inputs + if convert_outputs is None: + convert_outputs = convert_rnn_outputs + return cast( + Model[Padded, Padded], + PyTorchWrapper( + pytorch_model, + convert_inputs=convert_inputs, + convert_outputs=convert_outputs, + ), + ) + + +@registry.layers("PyTorchWrapper.v1") +def PyTorchWrapper( + pytorch_model: Any, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, +) -> Model[Any, Any]: + """Wrap a PyTorch model, so that it has the same API as Thinc models. + To optimize the model, you'll need to create a PyTorch optimizer and call + optimizer.step() after each batch. See examples/wrap_pytorch.py + + Your PyTorch model's forward method can take arbitrary args and kwargs, + but must return either a single tensor as output or a tuple. You may find the + PyTorch register_forward_hook helpful if you need to adapt the output. + + The convert functions are used to map inputs and outputs to and from your + PyTorch model. Each function should return the converted output, and a callback + to use during the backward pass. So: + + Xtorch, get_dX = convert_inputs(X) + Ytorch, torch_backprop = model.shims[0](Xtorch, is_train) + Y, get_dYtorch = convert_outputs(Ytorch) + + To allow maximum flexibility, the PyTorchShim expects ArgsKwargs objects + on the way into the forward and backward passed. The ArgsKwargs objects + will be passed straight into the model in the forward pass, and straight + into `torch.autograd.backward` during the backward pass. + """ + if convert_inputs is None: + convert_inputs = convert_pytorch_default_inputs + if convert_outputs is None: + convert_outputs = convert_pytorch_default_outputs + return Model( + "pytorch", + forward, + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + shims=[PyTorchShim(pytorch_model)], + dims={"nI": None, "nO": None}, + ) + + +@registry.layers("PyTorchWrapper.v2") +def PyTorchWrapper_v2( + pytorch_model: Any, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, + mixed_precision: bool = False, + grad_scaler: Optional[PyTorchGradScaler] = None, + device: Optional["torch.device"] = None, +) -> Model[Any, Any]: + """Wrap a PyTorch model, so that it has the same API as Thinc models. + To optimize the model, you'll need to create a PyTorch optimizer and call + optimizer.step() after each batch. See examples/wrap_pytorch.py + + Your PyTorch model's forward method can take arbitrary args and kwargs, + but must return either a single tensor as output or a tuple. You may find the + PyTorch register_forward_hook helpful if you need to adapt the output. + + The convert functions are used to map inputs and outputs to and from your + PyTorch model. Each function should return the converted output, and a callback + to use during the backward pass. So: + + Xtorch, get_dX = convert_inputs(X) + Ytorch, torch_backprop = model.shims[0](Xtorch, is_train) + Y, get_dYtorch = convert_outputs(Ytorch) + + To allow maximum flexibility, the PyTorchShim expects ArgsKwargs objects + on the way into the forward and backward passed. The ArgsKwargs objects + will be passed straight into the model in the forward pass, and straight + into `torch.autograd.backward` during the backward pass. + + mixed_precision: + Enable mixed-precision. This changes whitelisted ops to run + in half precision for better performance and lower memory use. + grad_scaler: + The gradient scaler to use for mixed-precision training. If this + argument is set to "None" and mixed precision is enabled, a gradient + scaler with the default configuration is used. + device: + The PyTorch device to run the model on. When this argument is + set to "None", the default device for the currently active Thinc + ops is used. + """ + if convert_inputs is None: + convert_inputs = convert_pytorch_default_inputs + if convert_outputs is None: + convert_outputs = convert_pytorch_default_outputs + return Model( + "pytorch", + forward, + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + shims=[ + PyTorchShim( + pytorch_model, + mixed_precision=mixed_precision, + grad_scaler=grad_scaler, + device=device, + ) + ], + dims={"nI": None, "nO": None}, + ) + + +@registry.layers("PyTorchWrapper.v3") +def PyTorchWrapper_v3( + pytorch_model: "torch.nn.Module", + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, + mixed_precision: bool = False, + grad_scaler: Optional[PyTorchGradScaler] = None, + device: Optional["torch.device"] = None, + serialize_model: Optional[Callable[[Any], bytes]] = None, + deserialize_model: Optional[Callable[[Any, bytes, "torch.device"], Any]] = None, +) -> Model[Any, Any]: + """Wrap a PyTorch model, so that it has the same API as Thinc models. + To optimize the model, you'll need to create a PyTorch optimizer and call + optimizer.step() after each batch. See examples/wrap_pytorch.py + + Your PyTorch model's forward method can take arbitrary args and kwargs, + but must return either a single tensor or a tuple. You may find the + PyTorch register_forward_hook helpful if you need to adapt the output. + + The convert functions are used to map inputs and outputs to and from your + PyTorch model. Each function should return the converted output, and a callback + to use during the backward pass. So: + + Xtorch, get_dX = convert_inputs(X) + Ytorch, torch_backprop = model.shims[0](Xtorch, is_train) + Y, get_dYtorch = convert_outputs(Ytorch) + + To allow maximum flexibility, the PyTorchShim expects ArgsKwargs objects + on the way into the forward and backward passed. The ArgsKwargs objects + will be passed straight into the model in the forward pass, and straight + into `torch.autograd.backward` during the backward pass. + + mixed_precision: + Enable mixed-precision. This changes whitelisted ops to run + in half precision for better performance and lower memory use. + grad_scaler: + The gradient scaler to use for mixed-precision training. If this + argument is set to "None" and mixed precision is enabled, a gradient + scaler with the default configuration is used. + device: + The PyTorch device to run the model on. When this argument is + set to "None", the default device for the currently active Thinc + ops is used. + serialize_model: + Callback that receives the wrapped PyTorch model as its argument and + returns a "bytes" representation of the same. The representation should + contain all the necessary information to fully deserialize the model. + When set to "None", the default serializer serializes the model's parameters. + deserialize_model: + Callback that receives the default PyTorch model (passed to the constructor), the + serialized "bytes" representation and a PyTorch device. It should return a + fully deserialized model on the target device as its result. + When set to "None", the default deserializer deserializes the model's parameters. + """ + if convert_inputs is None: + convert_inputs = convert_pytorch_default_inputs + if convert_outputs is None: + convert_outputs = convert_pytorch_default_outputs + return Model( + "pytorch", + forward, + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + shims=[ + PyTorchShim( + pytorch_model, + mixed_precision=mixed_precision, + grad_scaler=grad_scaler, + device=device, + serialize_model=serialize_model, + deserialize_model=deserialize_model, + ) + ], + dims={"nI": None, "nO": None}, + ) + + +def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + """Return the output of the wrapped PyTorch model for the given input, + along with a callback to handle the backward pass. + """ + convert_inputs = model.attrs["convert_inputs"] + convert_outputs = model.attrs["convert_outputs"] + + Xtorch, get_dX = convert_inputs(model, X, is_train) + Ytorch, torch_backprop = model.shims[0](Xtorch, is_train) + Y, get_dYtorch = convert_outputs(model, (X, Ytorch), is_train) + + def backprop(dY: Any) -> Any: + dYtorch = get_dYtorch(dY) + dXtorch = torch_backprop(dYtorch) + dX = get_dX(dXtorch) + return dX + + return Y, backprop + + +# Default conversion functions + + +def convert_pytorch_default_inputs( + model: Model, X: Any, is_train: bool +) -> Tuple[ArgsKwargs, Callable[[ArgsKwargs], Any]]: + shim = cast(PyTorchShim, model.shims[0]) + xp2torch_ = lambda x: xp2torch(x, requires_grad=is_train, device=shim.device) + converted = convert_recursive(is_xp_array, xp2torch_, X) + if isinstance(converted, ArgsKwargs): + + def reverse_conversion(dXtorch): + return convert_recursive(is_torch_array, torch2xp, dXtorch) + + return converted, reverse_conversion + elif isinstance(converted, dict): + + def reverse_conversion(dXtorch): + dX = convert_recursive(is_torch_array, torch2xp, dXtorch) + return dX.kwargs + + return ArgsKwargs(args=tuple(), kwargs=converted), reverse_conversion + elif isinstance(converted, (tuple, list)): + + def reverse_conversion(dXtorch): + dX = convert_recursive(is_torch_array, torch2xp, dXtorch) + return dX.args + + return ArgsKwargs(args=tuple(converted), kwargs={}), reverse_conversion + else: + + def reverse_conversion(dXtorch): + dX = convert_recursive(is_torch_array, torch2xp, dXtorch) + return dX.args[0] + + return ArgsKwargs(args=(converted,), kwargs={}), reverse_conversion + + +def convert_pytorch_default_outputs(model: Model, X_Ytorch: Any, is_train: bool): + shim = cast(PyTorchShim, model.shims[0]) + X, Ytorch = X_Ytorch + Y = convert_recursive(is_torch_array, torch2xp, Ytorch) + + def reverse_conversion(dY: Any) -> ArgsKwargs: + dYtorch = convert_recursive( + is_xp_array, partial(xp2torch, device=shim.device), dY + ) + return ArgsKwargs(args=((Ytorch,),), kwargs={"grad_tensors": dYtorch}) + + return Y, reverse_conversion + + +# BiLSTM conversion functions + + +def convert_rnn_inputs(model: Model, Xp: Padded, is_train: bool): + shim = cast(PyTorchShim, model.shims[0]) + size_at_t = Xp.size_at_t + lengths = Xp.lengths + indices = Xp.indices + + def convert_from_torch_backward(d_inputs: ArgsKwargs) -> Padded: + dX = torch2xp(d_inputs.args[0]) + return Padded(dX, size_at_t, lengths, indices) # type: ignore + + output = ArgsKwargs( + args=(xp2torch(Xp.data, requires_grad=True, device=shim.device), None), + kwargs={}, + ) + return output, convert_from_torch_backward + + +def convert_rnn_outputs(model: Model, inputs_outputs: Tuple, is_train): + shim = cast(PyTorchShim, model.shims[0]) + Xp, (Ytorch, _) = inputs_outputs + + def convert_for_torch_backward(dYp: Padded) -> ArgsKwargs: + dYtorch = xp2torch(dYp.data, requires_grad=True, device=shim.device) + return ArgsKwargs(args=(Ytorch,), kwargs={"grad_tensors": dYtorch}) + + Y = cast(Floats3d, torch2xp(Ytorch)) + Yp = Padded(Y, Xp.size_at_t, Xp.lengths, Xp.indices) + return Yp, convert_for_torch_backward diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/ragged2list.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/ragged2list.py new file mode 100644 index 0000000000000000000000000000000000000000..dd6ec5cb77119d42e3c05ffffbb99988d047c7a9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/ragged2list.py @@ -0,0 +1,25 @@ +from typing import Callable, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import ListXd, Ragged + +InT = Ragged +OutT = TypeVar("OutT", bound=ListXd) + + +@registry.layers("ragged2list.v1") +def ragged2list() -> Model[InT, OutT]: + """Transform sequences from a ragged format into lists.""" + return Model("ragged2list", forward) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + lengths = Xr.lengths + + def backprop(dXs: OutT) -> InT: + return Ragged(model.ops.flatten(dXs, pad=0), lengths) # type:ignore[arg-type] + # type ignore necessary for older versions of Mypy/Pydantic + + data = cast(OutT, model.ops.unflatten(Xr.dataXd, Xr.lengths)) + return data, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_first.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_first.py new file mode 100644 index 0000000000000000000000000000000000000000..a282cc6a827283c5c79e68476681b7c01db3b302 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_first.py @@ -0,0 +1,30 @@ +from typing import Callable, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import ArrayInfo + +InT = Ragged +OutT = Floats2d + + +@registry.layers("reduce_first.v1") +def reduce_first() -> Model[InT, OutT]: + """Reduce ragged-formatted sequences to their first element.""" + return Model("reduce_first", forward) + + +def forward( + model: Model[InT, OutT], Xr: InT, is_train: bool +) -> Tuple[OutT, Callable[[OutT], InT]]: + Y, starts_ends = model.ops.reduce_first(cast(Floats2d, Xr.data), Xr.lengths) + + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: OutT) -> InT: + array_info.check_consistency(dY) + dX = model.ops.backprop_reduce_first(dY, starts_ends) + return Ragged(dX, Xr.lengths) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_last.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_last.py new file mode 100644 index 0000000000000000000000000000000000000000..f4b7adb8f4212e2d646112de904132e0fc7fe852 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_last.py @@ -0,0 +1,29 @@ +from typing import Callable, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import ArrayInfo + +InT = Ragged +OutT = Floats2d + + +@registry.layers("reduce_last.v1") +def reduce_last() -> Model[InT, OutT]: + """Reduce ragged-formatted sequences to their last element.""" + return Model("reduce_last", forward) + + +def forward( + model: Model[InT, OutT], Xr: InT, is_train: bool +) -> Tuple[OutT, Callable[[OutT], InT]]: + Y, lasts = model.ops.reduce_last(cast(Floats2d, Xr.data), Xr.lengths) + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: OutT) -> InT: + array_info.check_consistency(dY) + dX = model.ops.backprop_reduce_last(dY, lasts) + return Ragged(dX, Xr.lengths) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_max.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_max.py new file mode 100644 index 0000000000000000000000000000000000000000..d21d5cbfe1489160fd219bdc576bea1691405be3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_max.py @@ -0,0 +1,26 @@ +from typing import Callable, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import ArrayInfo + +InT = Ragged +OutT = Floats2d + + +@registry.layers("reduce_max.v1") +def reduce_max() -> Model[InT, OutT]: + return Model("reduce_max", forward) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + Y, which = model.ops.reduce_max(cast(Floats2d, Xr.data), Xr.lengths) + lengths = Xr.lengths + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: OutT) -> InT: + array_info.check_consistency(dY) + return Ragged(model.ops.backprop_reduce_max(dY, which, lengths), lengths) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_mean.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_mean.py new file mode 100644 index 0000000000000000000000000000000000000000..395a5cc9b0e766f785d95612e20a5afb8fcf83dd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_mean.py @@ -0,0 +1,27 @@ +from typing import Callable, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import ArrayInfo + +InT = Ragged +OutT = Floats2d + + +@registry.layers("reduce_mean.v1") +def reduce_mean() -> Model[InT, OutT]: + return Model("reduce_mean", forward) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + Y = model.ops.reduce_mean(cast(Floats2d, Xr.data), Xr.lengths) + lengths = Xr.lengths + + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: OutT) -> InT: + array_info.check_consistency(dY) + return Ragged(model.ops.backprop_reduce_mean(dY, lengths), lengths) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_sum.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_sum.py new file mode 100644 index 0000000000000000000000000000000000000000..07e465d0c83cbb0ed497adca5895bf94dadad60c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/reduce_sum.py @@ -0,0 +1,26 @@ +from typing import Callable, Tuple, cast + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ragged +from ..util import ArrayInfo + +InT = Ragged +OutT = Floats2d + + +@registry.layers("reduce_sum.v1") +def reduce_sum() -> Model[InT, OutT]: + return Model("reduce_sum", forward) + + +def forward(model: Model[InT, OutT], Xr: InT, is_train: bool) -> Tuple[OutT, Callable]: + Y = model.ops.reduce_sum(cast(Floats2d, Xr.data), Xr.lengths) + lengths = Xr.lengths + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: OutT) -> InT: + array_info.check_consistency(dY) + return Ragged(model.ops.backprop_reduce_sum(dY, lengths), lengths) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/relu.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/relu.py new file mode 100644 index 0000000000000000000000000000000000000000..664f121f4f666bb17d2e4b8e34bd7c4d5ef52acf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/relu.py @@ -0,0 +1,71 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import glorot_uniform_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Relu.v1") +def Relu( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[InT, OutT]: + if init_W is None: + init_W = glorot_uniform_init + if init_b is None: + init_b = zero_init + model: Model[InT, OutT] = Model( + "relu", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y = model.ops.affine(X, W, b) + Y = model.ops.relu(Y) + + def backprop(dY: OutT) -> InT: + dY = model.ops.backprop_relu(dY, Y) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/remap_ids.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/remap_ids.py new file mode 100644 index 0000000000000000000000000000000000000000..0c0704bbac5f16bb8f15b0aa44e397f0216bb48d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/remap_ids.py @@ -0,0 +1,97 @@ +from typing import Any, Callable, Dict, Hashable, Optional, Sequence, Tuple, Union, cast + +from ..config import registry +from ..model import Model +from ..types import DTypes, Ints1d, Ints2d +from ..util import is_xp_array, to_numpy + +InT = Union[Sequence[Hashable], Ints1d, Ints2d] +OutT = Ints2d + +InT_v1 = Sequence[Any] +OutT_v1 = Ints2d + + +@registry.layers("remap_ids.v1") +def remap_ids( + mapping_table: Dict[Any, int] = {}, default: int = 0, dtype: DTypes = "i" +) -> Model[InT_v1, OutT_v1]: + """Remap string or integer inputs using a mapping table, usually as a + preprocess before embeddings. The mapping table can be passed in on input, + or updated after the layer has been created. The mapping table is stored in + the "mapping_table" attribute. + """ + return Model( + "remap_ids", + forward, + attrs={"mapping_table": mapping_table, "dtype": dtype, "default": default}, + ) + + +def forward( + model: Model[InT_v1, OutT_v1], inputs: InT_v1, is_train: bool +) -> Tuple[OutT, Callable]: + table = model.attrs["mapping_table"] + default = model.attrs["default"] + dtype = model.attrs["dtype"] + values = [table.get(x, default) for x in inputs] + arr = model.ops.asarray2i(values, dtype=dtype) + output = model.ops.reshape2i(arr, -1, 1) + + def backprop(dY: OutT_v1) -> InT: + return [] + + return output, backprop + + +@registry.layers("remap_ids.v2") +def remap_ids_v2( + mapping_table: Optional[Union[Dict[int, int], Dict[str, int]]] = None, + default: int = 0, + *, + column: Optional[int] = None +) -> Model[InT, OutT]: + """Remap string or integer inputs using a mapping table, + usually as a preprocessing step before embeddings. + The mapping table can be passed in on input, + or updated after the layer has been created. + The mapping table is stored in the "mapping_table" attribute. + Two dimensional arrays can be provided as input in which case + the 'column' chooses which column to process. This is useful + to work together with FeatureExtractor in spaCy. + """ + return Model( + "remap_ids", + forward_v2, + attrs={"mapping_table": mapping_table, "default": default, "column": column}, + ) + + +def forward_v2( + model: Model[InT, OutT], inputs: InT, is_train: bool +) -> Tuple[OutT, Callable]: + table = model.attrs["mapping_table"] + if table is None: + raise ValueError("'mapping table' not set") + default = model.attrs["default"] + column = model.attrs["column"] + if is_xp_array(inputs): + xp_input = True + if column is not None: + idx = to_numpy(cast(Ints2d, inputs)[:, column]) + else: + idx = to_numpy(inputs) + else: + xp_input = False + idx = inputs + values = [table.get(x, default) for x in idx] + arr = model.ops.asarray2i(values, dtype="i") + output = model.ops.reshape2i(arr, -1, 1) + + def backprop(dY: OutT) -> InT: + if xp_input: + return model.ops.xp.empty(dY.shape) # type: ignore + else: + return [] + + return output, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/residual.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/residual.py new file mode 100644 index 0000000000000000000000000000000000000000..ad471de237c138a25aae98e6f697e3f5b5c4ffb4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/residual.py @@ -0,0 +1,64 @@ +from typing import Callable, List, Optional, Tuple, TypeVar + +from ..config import registry +from ..model import Model +from ..types import Floats1d, Floats2d, Floats3d, Floats4d, FloatsXd, Padded, Ragged + +# fmt: off +InT = TypeVar( + "InT", List[Floats1d], List[Floats2d], List[Floats3d], List[Floats4d], + Ragged, Padded, FloatsXd, Floats1d, Floats2d, Floats3d, Floats4d) +# fmt: on + + +@registry.layers("residual.v1") +def residual(layer: Model[InT, InT]) -> Model[InT, InT]: + return Model( + f"residual({layer.name})", + forward, + init=init, + layers=[layer], + dims={ + "nO": layer.get_dim("nO") if layer.has_dim("nO") else None, + "nI": layer.get_dim("nI") if layer.has_dim("nI") else None, + }, + ) + + +def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: + def backprop(d_output: InT) -> InT: + dX = backprop_layer(d_output) + if isinstance(d_output, list): + return [d_output[i] + dX[i] for i in range(len(d_output))] + elif isinstance(d_output, Ragged): + return Ragged(d_output.data + dX.data, dX.lengths) + elif isinstance(X, Padded): + dX.data += d_output.data + return dX + else: + return d_output + dX + + Y, backprop_layer = model.layers[0](X, is_train) + if isinstance(X, list): + return [X[i] + Y[i] for i in range(len(X))], backprop + elif isinstance(X, Ragged): + return Ragged(X.data + Y.data, X.lengths), backprop + elif isinstance(X, Padded): + Y.data += X.data + return Y, backprop + else: + return X + Y, backprop + + +def init( + model: Model[InT, InT], X: Optional[InT] = None, Y: Optional[InT] = None +) -> None: + first_layer = model.layers[0] + if first_layer.has_dim("nO") is None: + first_layer.initialize(X=X, Y=Y) + else: + first_layer.initialize(X=X) + if first_layer.has_dim("nO"): + model.set_dim("nO", first_layer.get_dim("nO")) + if first_layer.has_dim("nI"): + model.set_dim("nI", first_layer.get_dim("nI")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/resizable.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/resizable.py new file mode 100644 index 0000000000000000000000000000000000000000..2ed3142de1c46491b706ceb8df79a4aab0ce4610 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/resizable.py @@ -0,0 +1,93 @@ +from typing import Callable, Optional, TypeVar + +from ..config import registry +from ..model import Model +from ..types import Floats2d + +InT = TypeVar("InT") +OutT = TypeVar("OutT") + + +@registry.layers("resizable.v1") +def resizable(layer, resize_layer: Callable) -> Model[InT, OutT]: + """Container that holds one layer that can change dimensions.""" + return Model( + f"resizable({layer.name})", + forward, + init=init, + layers=[layer], + attrs={"resize_layer": resize_layer}, + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool): + layer = model.layers[0] + Y, callback = layer(X, is_train=is_train) + + def backprop(dY: OutT) -> InT: + return callback(dY) + + return Y, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + layer = model.layers[0] + layer.initialize(X, Y) + + +def resize_model(model: Model[InT, OutT], new_nO): + old_layer = model.layers[0] + new_layer = model.attrs["resize_layer"](old_layer, new_nO) + model.layers[0] = new_layer + return model + + +def resize_linear_weighted( + layer: Model[Floats2d, Floats2d], new_nO, *, fill_defaults=None +) -> Model[Floats2d, Floats2d]: + """Create a resized copy of a layer that has parameters W and b and dimensions nO and nI.""" + assert not layer.layers + assert not layer.ref_names + assert not layer.shims + + # return the original layer if it wasn't initialized or if nO didn't change + if layer.has_dim("nO") is None: + layer.set_dim("nO", new_nO) + return layer + elif new_nO == layer.get_dim("nO"): + return layer + elif layer.has_dim("nI") is None: + layer.set_dim("nO", new_nO, force=True) + return layer + + dims = {name: layer.maybe_get_dim(name) for name in layer.dim_names} + dims["nO"] = new_nO + new_layer: Model[Floats2d, Floats2d] = Model( + layer.name, + layer._func, + dims=dims, + params={name: None for name in layer.param_names}, + init=layer.init, + attrs=layer.attrs, + refs={}, + ops=layer.ops, + ) + new_layer.initialize() + for name in layer.param_names: + if layer.has_param(name): + filler = 0 if not fill_defaults else fill_defaults.get(name, 0) + _resize_parameter(name, layer, new_layer, filler=filler) + return new_layer + + +def _resize_parameter(name, layer, new_layer, filler=0): + larger = new_layer.get_param(name) + smaller = layer.get_param(name) + # copy the original weights + larger[: len(smaller)] = smaller + # set the new weights + larger[len(smaller) :] = filler + new_layer.set_param(name, larger) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/siamese.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/siamese.py new file mode 100644 index 0000000000000000000000000000000000000000..806829e6bd074c144998f7024d3a3c919aad1adf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/siamese.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional, Tuple, TypeVar + +from ..config import registry +from ..model import Model +from ..types import ArrayXd +from ..util import get_width + +LayerT = TypeVar("LayerT") +SimT = TypeVar("SimT") +InT = Tuple[LayerT, LayerT] +OutT = TypeVar("OutT", bound=ArrayXd) + + +@registry.layers("siamese.v1") +def siamese( + layer: Model[LayerT, SimT], similarity: Model[Tuple[SimT, SimT], OutT] +) -> Model[InT, OutT]: + return Model( + f"siamese({layer.name}, {similarity.name})", + forward, + init=init, + layers=[layer, similarity], + dims={"nI": layer.get_dim("nI"), "nO": similarity.get_dim("nO")}, + ) + + +def forward( + model: Model[InT, OutT], X1_X2: InT, is_train: bool +) -> Tuple[OutT, Callable]: + X1, X2 = X1_X2 + vec1, bp_vec1 = model.layers[0](X1, is_train) + vec2, bp_vec2 = model.layers[0](X2, is_train) + output, bp_output = model.layers[1]((vec1, vec2), is_train) + + def finish_update(d_output: OutT) -> InT: + d_vec1, d_vec2 = bp_output(d_output) + d_input1 = bp_vec1(d_vec1) + d_input2 = bp_vec2(d_vec2) + return (d_input1, d_input2) + + return output, finish_update + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is not None: + model.layers[0].set_dim("nI", get_width(X[1])) + model.layers[0].initialize(X=X[0]) + X = (model.layers[0].predict(X[0]), model.layers[0].predict(X[1])) + model.layers[1].initialize(X=X, Y=Y) + model.set_dim("nI", model.layers[0].get_dim("nI")) + model.set_dim("nO", model.layers[1].get_dim("nO")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid.py new file mode 100644 index 0000000000000000000000000000000000000000..53e4f9dd1ff0cb54cad916f5aadb677600c670ca --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid.py @@ -0,0 +1,65 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Sigmoid.v1") +def Sigmoid( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, +) -> Model[InT, OutT]: + """A dense layer, followed by a sigmoid (logistic) activation function. This + is usually used instead of the Softmax layer as an output for multi-label + classification. + """ + if init_W is None: + init_W = zero_init + if init_b is None: + init_b = zero_init + return Model( + "sigmoid", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y = model.ops.affine(X, W, b) + Y = model.ops.sigmoid(Y) + + def backprop(dY: InT) -> OutT: + dY = model.ops.backprop_sigmoid(dY, Y, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None and model.has_dim("nI") is None: + model.set_dim("nI", get_width(X)) + if Y is not None and model.has_dim("nO") is None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid_activation.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid_activation.py new file mode 100644 index 0000000000000000000000000000000000000000..c9ae487e488025dbde3b5e5acdb0ea92184e8b41 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sigmoid_activation.py @@ -0,0 +1,24 @@ +from typing import Callable, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import FloatsXdT + + +@registry.layers("sigmoid_activation.v1") +def sigmoid_activation() -> Model[FloatsXdT, FloatsXdT]: + return Model("sigmoid_activation", forward) + + +def forward( + model: Model[FloatsXdT, FloatsXdT], X: FloatsXdT, is_train: bool +) -> Tuple[FloatsXdT, Callable]: + Y = model.ops.sigmoid(X, inplace=False) + + def backprop(dY: FloatsXdT) -> FloatsXdT: + return cast( + FloatsXdT, + dY * model.ops.dsigmoid(Y, inplace=False), # type:ignore[operator] + ) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax.py new file mode 100644 index 0000000000000000000000000000000000000000..0ed263fc7344e9088c4da813d4953793ae654f7e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax.py @@ -0,0 +1,115 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import ArrayInfo, get_width, partial + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("Softmax.v1") +def Softmax( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, +) -> Model[InT, OutT]: + if init_W is None: + init_W = zero_init + if init_b is None: + init_b = zero_init + return Model( + "softmax", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + attrs={"softmax_normalize": True, "softmax_temperature": 1.0}, + ) + + +@registry.layers("Softmax.v2") +def Softmax_v2( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + normalize_outputs: bool = True, + temperature: float = 1.0, +) -> Model[InT, OutT]: + if init_W is None: + init_W = zero_init + if init_b is None: + init_b = zero_init + validate_temperature(temperature) + return Model( + "softmax", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + attrs={ + "softmax_normalize": normalize_outputs, + "softmax_temperature": temperature, + }, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + normalize = model.attrs["softmax_normalize"] or is_train + + temperature = model.attrs["softmax_temperature"] + validate_temperature(temperature) + + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y = model.ops.affine(X, W, b) + + if normalize: + Y = model.ops.softmax(Y, temperature=temperature) + + array_info = ArrayInfo.from_array(Y) + + def backprop(dY: InT) -> OutT: + array_info.check_consistency(dY) + if temperature != 1.0: + dY = dY / temperature + + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + def backprop_unnormalized(dY: InT): + msg = "backprop is not supported for an unnormalized Softmax layer" + raise ValueError(msg) + + if normalize: + return Y, backprop + else: + return Y, backprop_unnormalized + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[InT, OutT], + X: Optional[InT] = None, + Y: Optional[OutT] = None, +) -> None: + if X is not None and model.has_dim("nI") is None: + model.set_dim("nI", get_width(X)) + if Y is not None and model.has_dim("nO") is None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) + + +def validate_temperature(temperature): + if temperature <= 0.0: + msg = "softmax temperature must not be zero or negative" + raise ValueError(msg) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax_activation.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax_activation.py new file mode 100644 index 0000000000000000000000000000000000000000..6a8762696466a5fd48d5b077be2f7d27e6457bbe --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/softmax_activation.py @@ -0,0 +1,22 @@ +from typing import Callable, Tuple + +from ..config import registry +from ..model import Model +from ..types import Floats2d + +InT = Floats2d +OutT = Floats2d + + +@registry.layers("softmax_activation.v1") +def softmax_activation() -> Model[InT, OutT]: + return Model("softmax_activation", forward) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + Y = model.ops.softmax(X, inplace=False) + + def backprop(dY: OutT) -> InT: + return model.ops.backprop_softmax(Y, dY, axis=-1) + + return Y, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.cpp b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fdca7222f5805a6854a33ba57635802bebfe3ce1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.cpp @@ -0,0 +1,34826 @@ +/* Generated by Cython 3.2.1 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\arrayobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\arrayscalars.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ndarrayobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ndarraytypes.h", + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include\\numpy\\ufuncobject.h", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include\\Python.h" + ], + "include_dirs": [ + "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\build-env-kzdm41oa\\Lib\\site-packages\\numpy\\_core\\include", + "C:\\Users\\runneradmin\\AppData\\Local\\pypa\\cibuildwheel\\Cache\\nuget-cpython\\python.3.12.10\\tools\\Include" + ], + "language": "c++", + "name": "thinc.layers.sparselinear", + "sources": [ + "thinc/layers/sparselinear.pyx" + ] + }, + "module_name": "thinc.layers.sparselinear" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_2_1" +#define CYTHON_HEX_VERSION 0x030201F0 +#define CYTHON_FUTURE_DIVISION 0 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 1 + #endif + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif + #if defined(CYTHON_IMMORTAL_CONSTANTS) && PY_VERSION_HEX < 0x030C0000 + #undef CYTHON_IMMORTAL_CONSTANTS + #define CYTHON_IMMORTAL_CONSTANTS 0 // definitely won't work + #elif !defined(CYTHON_IMMORTAL_CONSTANTS) + #define CYTHON_IMMORTAL_CONSTANTS (PY_VERSION_HEX >= 0x030C0000 && !CYTHON_USE_MODULE_STATE && CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_COMPRESS_STRINGS + #define CYTHON_COMPRESS_STRINGS 1 +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL) +#endif +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#include +typedef uintptr_t __pyx_uintptr_t; +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef Py_TPFLAGS_IMMUTABLETYPE + #define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) +#endif +#ifndef Py_TPFLAGS_DISALLOW_INSTANTIATION + #define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if PY_VERSION_HEX < 0x03090000 || (CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) GraalPyFrame_SetLineNumber((frame), (lineno)) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && PyUnstable_Object_IsUniquelyReferenced(obj)) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#elif CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +enum __Pyx_ReferenceSharing { + __Pyx_ReferenceSharing_DefinitelyUnique, // We created it so we know it's unshared - no need to check + __Pyx_ReferenceSharing_OwnStrongReference, + __Pyx_ReferenceSharing_FunctionArgument, + __Pyx_ReferenceSharing_SharedReference, // Never trust it to be unshared because it's a global or similar +}; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && PY_VERSION_HEX >= 0x030E0000 +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing)\ + (sharing == __Pyx_ReferenceSharing_DefinitelyUnique ? 1 :\ + (sharing == __Pyx_ReferenceSharing_FunctionArgument ? PyUnstable_Object_IsUniqueReferencedTemporary(o) :\ + (sharing == __Pyx_ReferenceSharing_OwnStrongReference ? PyUnstable_Object_IsUniquelyReferenced(o) : 0))) +#elif (CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)sharing), Py_REFCNT(o) == 1) +#else +#define __Pyx_IS_UNIQUELY_REFERENCED(o, sharing) (((void)o), ((void)sharing), 0) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !CYTHON_COMPILING_IN_LIMITED_API && CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) (__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared) ?\ + __Pyx_NewRef(PyList_GET_ITEM(o, i)) : __Pyx_PyList_GetItemRef(o, i)) +#else + #define __Pyx_PyList_GetItemRefFast(o, i, unsafe_shared) __Pyx_PyList_GetItemRef(o, i) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__thinc__layers__sparselinear +#define __PYX_HAVE_API__thinc__layers__sparselinear +/* Early includes */ +#include +#include + + /* Using NumPy API declarations from "numpy/__init__.cython-30.pxd" */ + +#include "numpy/arrayobject.h" +#include "numpy/ndarrayobject.h" +#include "numpy/ndarraytypes.h" +#include "numpy/arrayscalars.h" +#include "numpy/ufuncobject.h" +#include +#include "pythread.h" +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* Header.proto */ +#if !defined(CYTHON_CCOMPLEX) + #if defined(__cplusplus) + #define CYTHON_CCOMPLEX 1 + #elif (defined(_Complex_I) && !defined(_MSC_VER)) || ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_COMPLEX__) && !defined(_MSC_VER)) + #define CYTHON_CCOMPLEX 1 + #else + #define CYTHON_CCOMPLEX 0 + #endif +#endif +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #include + #else + #include + #endif +#endif +#if CYTHON_CCOMPLEX && !defined(__cplusplus) && defined(__sun__) && defined(__GNUC__) + #undef _Complex_I + #define _Complex_I 1.0fj +#endif + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "thinc/layers/sparselinear.pyx", + "", + "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd", + "cpython/type.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* Atomics.proto (used by UnpackUnboundCMethod) */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #define __pyx_atomic_pointer_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_nonatomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_nonatomic_ptr_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #define __pyx_nonatomic_ptr_type void* + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + static CYTHON_INLINE int __pyx_atomic_pointer_cmp_exchange(__pyx_atomic_ptr_type* value, __pyx_nonatomic_ptr_type* expected, __pyx_nonatomic_ptr_type desired) { + __pyx_atomic_ptr_type old = _InterlockedCompareExchangePointer(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif + +/* CriticalSectionsDefinition.proto (used by CriticalSections) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_End(cs) +#define __Pyx_PyCriticalSection2_End(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_End PyCriticalSection_End +#define __Pyx_PyCriticalSection2_End PyCriticalSection2_End +#endif + +/* CriticalSections.proto (used by ParseKeywordsImpl) */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection_Begin(cs, arg) (void)(cs) +#define __Pyx_PyCriticalSection2_Begin(cs, arg1, arg2) (void)(cs) +#else +#define __Pyx_PyCriticalSection_Begin PyCriticalSection_Begin +#define __Pyx_PyCriticalSection2_Begin PyCriticalSection2_Begin +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* IncludeStructmemberH.proto (used by FixUpExtensionType) */ +#include + +/* BufferFormatStructs.proto */ +struct __Pyx_StructField_; +#define __PYX_BUF_FLAGS_PACKED_STRUCT (1 << 0) +typedef struct { + const char* name; + const struct __Pyx_StructField_* fields; + size_t size; + size_t arraysize[8]; + int ndim; + char typegroup; + char is_unsigned; + int flags; +} __Pyx_TypeInfo; +typedef struct __Pyx_StructField_ { + const __Pyx_TypeInfo* type; + const char* name; + size_t offset; +} __Pyx_StructField; +typedef struct { + const __Pyx_StructField* field; + size_t parent_offset; +} __Pyx_BufFmt_StackElem; +typedef struct { + __Pyx_StructField root; + __Pyx_BufFmt_StackElem* head; + size_t fmt_offset; + size_t new_count, enc_count; + size_t struct_alignment; + int is_complex; + char enc_type; + char new_packmode; + char enc_packmode; + char is_valid_array; +} __Pyx_BufFmt_Context; + +/* MemviewSliceStruct.proto */ +struct __pyx_memoryview_obj; +typedef struct { + struct __pyx_memoryview_obj *memview; + char *data; + Py_ssize_t shape[8]; + Py_ssize_t strides[8]; + Py_ssize_t suboffsets[8]; +} __Pyx_memviewslice; +#define __Pyx_MemoryView_Len(m) (m.shape[0]) +#define __Pyx_MEMVIEW_DIRECT 1 +#define __Pyx_MEMVIEW_PTR 2 +#define __Pyx_MEMVIEW_FULL 4 +#define __Pyx_MEMVIEW_CONTIG 8 +#define __Pyx_MEMVIEW_STRIDED 16 +#define __Pyx_MEMVIEW_FOLLOW 32 +#define __Pyx_IS_C_CONTIG 1 +#define __Pyx_IS_F_CONTIG 2 +#define __Pyx_MEMSLICE_INIT { 0, 0, { 0 }, { 0 }, { 0 } } +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":743 + * # in Cython to enable them only on the right systems. + * + * ctypedef npy_int8 int8_t # <<<<<<<<<<<<<< + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t +*/ +typedef npy_int8 __pyx_t_5numpy_int8_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":744 + * + * ctypedef npy_int8 int8_t + * ctypedef npy_int16 int16_t # <<<<<<<<<<<<<< + * ctypedef npy_int32 int32_t + * ctypedef npy_int64 int64_t +*/ +typedef npy_int16 __pyx_t_5numpy_int16_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":745 + * ctypedef npy_int8 int8_t + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t # <<<<<<<<<<<<<< + * ctypedef npy_int64 int64_t + * +*/ +typedef npy_int32 __pyx_t_5numpy_int32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":746 + * ctypedef npy_int16 int16_t + * ctypedef npy_int32 int32_t + * ctypedef npy_int64 int64_t # <<<<<<<<<<<<<< + * + * ctypedef npy_uint8 uint8_t +*/ +typedef npy_int64 __pyx_t_5numpy_int64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":748 + * ctypedef npy_int64 int64_t + * + * ctypedef npy_uint8 uint8_t # <<<<<<<<<<<<<< + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t +*/ +typedef npy_uint8 __pyx_t_5numpy_uint8_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":749 + * + * ctypedef npy_uint8 uint8_t + * ctypedef npy_uint16 uint16_t # <<<<<<<<<<<<<< + * ctypedef npy_uint32 uint32_t + * ctypedef npy_uint64 uint64_t +*/ +typedef npy_uint16 __pyx_t_5numpy_uint16_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":750 + * ctypedef npy_uint8 uint8_t + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t # <<<<<<<<<<<<<< + * ctypedef npy_uint64 uint64_t + * +*/ +typedef npy_uint32 __pyx_t_5numpy_uint32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":751 + * ctypedef npy_uint16 uint16_t + * ctypedef npy_uint32 uint32_t + * ctypedef npy_uint64 uint64_t # <<<<<<<<<<<<<< + * + * ctypedef npy_float32 float32_t +*/ +typedef npy_uint64 __pyx_t_5numpy_uint64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":753 + * ctypedef npy_uint64 uint64_t + * + * ctypedef npy_float32 float32_t # <<<<<<<<<<<<<< + * ctypedef npy_float64 float64_t + * #ctypedef npy_float80 float80_t +*/ +typedef npy_float32 __pyx_t_5numpy_float32_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":754 + * + * ctypedef npy_float32 float32_t + * ctypedef npy_float64 float64_t # <<<<<<<<<<<<<< + * #ctypedef npy_float80 float80_t + * #ctypedef npy_float128 float128_t +*/ +typedef npy_float64 __pyx_t_5numpy_float64_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":761 + * ctypedef double complex complex128_t + * + * ctypedef npy_longlong longlong_t # <<<<<<<<<<<<<< + * ctypedef npy_ulonglong ulonglong_t + * +*/ +typedef npy_longlong __pyx_t_5numpy_longlong_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":762 + * + * ctypedef npy_longlong longlong_t + * ctypedef npy_ulonglong ulonglong_t # <<<<<<<<<<<<<< + * + * ctypedef npy_intp intp_t +*/ +typedef npy_ulonglong __pyx_t_5numpy_ulonglong_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":764 + * ctypedef npy_ulonglong ulonglong_t + * + * ctypedef npy_intp intp_t # <<<<<<<<<<<<<< + * ctypedef npy_uintp uintp_t + * +*/ +typedef npy_intp __pyx_t_5numpy_intp_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":765 + * + * ctypedef npy_intp intp_t + * ctypedef npy_uintp uintp_t # <<<<<<<<<<<<<< + * + * ctypedef npy_double float_t +*/ +typedef npy_uintp __pyx_t_5numpy_uintp_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":767 + * ctypedef npy_uintp uintp_t + * + * ctypedef npy_double float_t # <<<<<<<<<<<<<< + * ctypedef npy_double double_t + * ctypedef npy_longdouble longdouble_t +*/ +typedef npy_double __pyx_t_5numpy_float_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":768 + * + * ctypedef npy_double float_t + * ctypedef npy_double double_t # <<<<<<<<<<<<<< + * ctypedef npy_longdouble longdouble_t + * +*/ +typedef npy_double __pyx_t_5numpy_double_t; + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":769 + * ctypedef npy_double float_t + * ctypedef npy_double double_t + * ctypedef npy_longdouble longdouble_t # <<<<<<<<<<<<<< + * + * ctypedef float complex cfloat_t +*/ +typedef npy_longdouble __pyx_t_5numpy_longdouble_t; +/* #### Code section: complex_type_declarations ### */ +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< float > __pyx_t_float_complex; + #else + typedef float _Complex __pyx_t_float_complex; + #endif +#else + typedef struct { float real, imag; } __pyx_t_float_complex; +#endif +static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float, float); + +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< double > __pyx_t_double_complex; + #else + typedef double _Complex __pyx_t_double_complex; + #endif +#else + typedef struct { double real, imag; } __pyx_t_double_complex; +#endif +static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double, double); + +/* Declarations.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + typedef ::std::complex< long double > __pyx_t_long_double_complex; + #else + typedef long double _Complex __pyx_t_long_double_complex; + #endif +#else + typedef struct { long double real, imag; } __pyx_t_long_double_complex; +#endif +static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double, long double); + +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update; +struct __pyx_array_obj; +struct __pyx_MemviewEnum_obj; +struct __pyx_memoryview_obj; +struct __pyx_memoryviewslice_obj; + +/* "thinc/layers/sparselinear.pyx":71 + * + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: # <<<<<<<<<<<<<< + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) +*/ +struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update { + PyObject_HEAD + PyObject *__pyx_v_callback; + PyObject *__pyx_v_keys; + PyObject *__pyx_v_lengths; + PyObject *__pyx_v_values; +}; + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ +struct __pyx_array_obj { + PyObject_HEAD + struct __pyx_vtabstruct_array *__pyx_vtab; + char *data; + Py_ssize_t len; + char *format; + int ndim; + Py_ssize_t *_shape; + Py_ssize_t *_strides; + Py_ssize_t itemsize; + PyObject *mode; + PyObject *_format; + void (*callback_free_data)(void *); + int free_data; + int dtype_is_object; +}; + + +/* "View.MemoryView":299 + * + * + * @cname('__pyx_MemviewEnum') # <<<<<<<<<<<<<< + * cdef class Enum(object): + * cdef object name +*/ +struct __pyx_MemviewEnum_obj { + PyObject_HEAD + PyObject *name; +}; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ +struct __pyx_memoryview_obj { + PyObject_HEAD + struct __pyx_vtabstruct_memoryview *__pyx_vtab; + PyObject *obj; + PyObject *_size; + void *_unused; + PyThread_type_lock lock; + __pyx_atomic_int_type acquisition_count; + Py_buffer view; + int flags; + int dtype_is_object; + __Pyx_TypeInfo const *typeinfo; +}; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ +struct __pyx_memoryviewslice_obj { + struct __pyx_memoryview_obj __pyx_base; + __Pyx_memviewslice from_slice; + PyObject *from_object; + PyObject *(*to_object_func)(char *); + int (*to_dtype_func)(char *, PyObject *); +}; + + + +/* "View.MemoryView":110 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname("__pyx_array") + * cdef class array: +*/ + +struct __pyx_vtabstruct_array { + PyObject *(*get_memview)(struct __pyx_array_obj *); +}; +static struct __pyx_vtabstruct_array *__pyx_vtabptr_array; + + +/* "View.MemoryView":334 + * + * + * @cname('__pyx_memoryview') # <<<<<<<<<<<<<< + * cdef class memoryview: + * +*/ + +struct __pyx_vtabstruct_memoryview { + char *(*get_item_pointer)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*is_slice)(struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_slice_assignment)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*setitem_slice_assign_scalar)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *); + PyObject *(*setitem_indexed)(struct __pyx_memoryview_obj *, PyObject *, PyObject *); + PyObject *(*convert_item_to_object)(struct __pyx_memoryview_obj *, char *); + PyObject *(*assign_item_from_object)(struct __pyx_memoryview_obj *, char *, PyObject *); + PyObject *(*_get_base)(struct __pyx_memoryview_obj *); +}; +static struct __pyx_vtabstruct_memoryview *__pyx_vtabptr_memoryview; + + +/* "View.MemoryView":951 + * + * + * @cython.collection_type("sequence") # <<<<<<<<<<<<<< + * @cname('__pyx_memoryviewslice') + * cdef class _memoryviewslice(memoryview): +*/ + +struct __pyx_vtabstruct__memoryviewslice { + struct __pyx_vtabstruct_memoryview __pyx_base; +}; +static struct __pyx_vtabstruct__memoryviewslice *__pyx_vtabptr__memoryviewslice; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* TupleAndListFromArray.proto (used by fastcall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto (used by BytesEquals) */ +#include + +/* BytesEquals.proto (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* py_dict_items.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d); + +/* CallCFunction.proto (used by CallUnboundCMethod0) */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* PyObjectCall.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto (used by PyObjectCallOneArg) */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectCallOneArg.proto (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectGetAttrStr.proto (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* UnpackUnboundCMethod.proto (used by CallUnboundCMethod0) */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod0.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); +#else +#define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) +#endif + +/* py_dict_values.proto (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d); + +/* OwnedDictNext.proto (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue); +#else +CYTHON_INLINE +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue); +#endif + +/* RaiseDoubleKeywords.proto (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywordsImpl.export */ +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name +); +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTestFunc.export */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) + +/* PyErrExceptionMatches.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto (used by PyErrFetchRestore) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStrNoError.proto (used by GetBuiltinName) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyValueError_Check.proto */ +#define __Pyx_PyExc_ValueError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ValueError) + +/* RaiseException.export */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* PyObjectFastCallMethod.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyObject_FastCallMethod(name, args, nargsf) PyObject_VectorcallMethod(name, args, nargsf, NULL) +#else +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf); +#endif + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* PyMemoryError_Check.proto */ +#define __Pyx_PyExc_MemoryError_Check(obj) __Pyx_TypeCheck(obj, PyExc_MemoryError) + +/* BuildPyUnicode.proto (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char); + +/* COrdinalToPyUnicode.proto (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value); +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t width, char padding_char); + +/* GCCDiagnostics.proto (used by CIntToPyUnicode) */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* IncludeStdlibH.proto (used by CIntToPyUnicode) */ +#include + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_int(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_int(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_int(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); + +/* CIntToPyUnicode.proto */ +#define __Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char) (\ + ((format_char) == ('c')) ?\ + __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char) :\ + __Pyx____Pyx_PyUnicode_From_Py_ssize_t(value, width, padding_char, format_char)\ + ) +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char); +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char); + +/* JoinPyUnicode.export */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyObjectFormatSimple.proto */ +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#elif CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) :\ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) :\ + PyObject_Format(s, f)) +#else + #define __Pyx_PyObject_FormatSimple(s, f) (\ + likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) :\ + PyObject_Format(s, f)) +#endif + +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *); /*proto*/ +/* GetAttr.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *, PyObject *); + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck, unsafe_shared) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* ObjectGetItem.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key); +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/* RejectKeywords.export */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds); + +/* PyTypeError_Check.proto */ +#define __Pyx_PyExc_TypeError_Check(obj) __Pyx_TypeCheck(obj, PyExc_TypeError) + +/* DivInt[Py_ssize_t].proto */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t, Py_ssize_t, int b_is_constant); + +/* UnaryNegOverflows.proto */ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x)\ + (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) + +/* GetAttr3.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); + +/* PyDictVersioning.proto (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __Pyx_XNewRef(__pyx_dict_cached_value);\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000 + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* PyAssertionError_Check.proto */ +#define __Pyx_PyExc_AssertionError_Check(obj) __Pyx_TypeCheck(obj, PyExc_AssertionError) + +/* RaiseTooManyValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); + +/* RaiseNeedMoreValuesToUnpack.proto */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +/* RaiseNoneIterError.proto */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void); + +/* ExtTypeTest.proto */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); + +/* GetTopmostException.proto (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* HasAttr.proto (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_HasAttr(o, n) PyObject_HasAttrWithError(o, n) +#else +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); +#endif + +/* ImportImpl.export */ +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level); + +/* Import.proto */ +static CYTHON_INLINE PyObject *__Pyx_Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, int level); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 + L->ob_item[len] = x; + #else + PyList_SET_ITEM(list, len, x); + #endif + __Pyx_SET_SIZE(list, len + 1); + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* PySequenceMultiply.proto */ +#define __Pyx_PySequence_Multiply_Left(mul, seq) __Pyx_PySequence_Multiply(seq, mul) +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul); + +/* PyObjectFormatAndDecref.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f); +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f); + +/* PyObjectFormat.proto */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* s, PyObject* f); +#else +#define __Pyx_PyObject_Format(s, f) PyObject_Format(s, f) +#endif + +/* SetItemInt.proto */ +#define __Pyx_SetItemInt(o, i, v, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil, unsafe_shared)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_SetItemInt_Fast(o, (Py_ssize_t)i, v, is_list, wraparound, boundscheck, unsafe_shared) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) :\ + __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, + int is_list, int wraparound, int boundscheck, int unsafe_shared); + +/* RaiseUnboundLocalError.proto */ +static void __Pyx_RaiseUnboundLocalError(const char *varname); + +/* PyIndexError_Check.proto */ +#define __Pyx_PyExc_IndexError_Check(obj) __Pyx_TypeCheck(obj, PyExc_IndexError) + +/* DivInt[long].proto */ +static CYTHON_INLINE long __Pyx_div_long(long, long, int b_is_constant); + +/* PyImportError_Check.proto */ +#define __Pyx_PyExc_ImportError_Check(obj) __Pyx_TypeCheck(obj, PyExc_ImportError) + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* IterFinish.proto */ +static CYTHON_INLINE int __Pyx_IterFinish(void); + +/* UnpackItemEndCheck.proto */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); + +/* PyObjectVectorCallMethodKwBuilder.proto */ +#if CYTHON_VECTORCALL && PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_VectorcallMethod_CallFromBuilder PyObject_VectorcallMethod +#else +static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#endif + +/* RaiseClosureNameError.proto */ +static void __Pyx_RaiseClosureNameError(const char *varname); + +/* dict_setdefault.proto (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value); + +/* LimitedApiGetTypeDict.proto (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto (used by FetchCommonType) */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* AddModuleRef.proto (used by FetchSharedCythonModule) */ +#if ((CYTHON_COMPILING_IN_CPYTHON_FREETHREADING ) ||\ + __PYX_LIMITED_VERSION_HEX < 0x030d0000) + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name); +#else + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#endif + +/* FetchSharedCythonModule.proto (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* FetchCommonType.proto (used by CommonTypesMetaclass) */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto (used by CythonFunctionShared) */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto (used by CythonFunctionShared) */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto (used by CythonFunctionShared) */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto (used by CythonFunction) */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_dict; +#endif + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* PyObjectDelAttr.proto (used by PyObjectSetAttrStr) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +#define __Pyx_PyObject_DelAttr(o, n) PyObject_SetAttr(o, n, NULL) +#else +#define __Pyx_PyObject_DelAttr(o, n) PyObject_DelAttr(o, n) +#endif + +/* PyObjectSetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value); +#else +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_DelAttr(o,n) +#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) +#endif + +/* BufferIndexError.proto */ +static void __Pyx_RaiseBufferIndexError(int axis); + +/* CheckTypeForFreelists.proto */ +#if CYTHON_USE_FREELISTS +#if CYTHON_USE_TYPE_SPECS +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t) == (expected_tp))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS Py_TPFLAGS_IS_ABSTRACT +#else +#define __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, expected_tp, expected_size) ((int) ((t)->tp_basicsize == (expected_size))) +#define __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE) +#endif +#define __PYX_CHECK_TYPE_FOR_FREELISTS(t, expected_tp, expected_size)\ + (__PYX_CHECK_FINAL_TYPE_FOR_FREELISTS((t), (expected_tp), (expected_size)) &\ + (int) (!__Pyx_PyType_HasFeature((t), __PYX_CHECK_TYPE_FOR_FREELIST_FLAGS))) +#endif + +/* AllocateExtensionType.proto */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final); + +/* DefaultPlacementNew.proto */ +#include +template +void __Pyx_default_placement_construct(T* x) { + new (static_cast(x)) T(); +} + +/* PyObjectCallNoArg.proto (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); + +/* PyObjectGetMethod.proto (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +#endif + +/* PyObjectCallMethod0.proto (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); + +/* ValidateBasesTuple.proto (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); +#endif + +/* PyType_Ready.proto */ +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t); + +/* SetVTable.proto */ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); + +/* GetVTable.proto (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type); + +/* MergeVTables.proto */ +static int __Pyx_MergeVtables(PyTypeObject *type); + +/* DelItemOnTypeDict.proto (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k); +#define __Pyx_DelItemOnTypeDict(tp, k) __Pyx__DelItemOnTypeDict((PyTypeObject*)tp, k) + +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_2_1 +#define __PYX_HAVE_RT_ImportType_proto_3_2_1 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_2_1(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_2_1 { + __Pyx_ImportType_CheckSize_Error_3_2_1 = 0, + __Pyx_ImportType_CheckSize_Warn_3_2_1 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_2_1 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size); +#endif + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* SetNameInClass.proto */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030d0000 +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? _PyDict_SetItem_KnownHash(ns, name, value, ((PyASCIIObject *) name)->hash) : PyObject_SetItem(ns, name, value)) +#elif CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_SetNameInClass(ns, name, value)\ + (likely(PyDict_CheckExact(ns)) ? PyDict_SetItem(ns, name, value) : PyObject_SetItem(ns, name, value)) +#else +#define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) +#endif + +/* CalculateMetaclass.proto (used by Py3ClassCreate) */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases); + +/* PyObjectCall2Args.proto (used by Py3ClassCreate) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectLookupSpecial.proto (used by Py3ClassCreate) */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* Py3ClassCreate.proto */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, + PyObject *mkw, PyObject *modname, PyObject *doc); +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, + PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); + +/* CLineInTraceback.proto (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto (used by AddTraceback) */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* BufferStructDeclare.proto */ +typedef struct { + Py_ssize_t shape, strides, suboffsets; +} __Pyx_Buf_DimInfo; +typedef struct { + size_t refcount; + Py_buffer pybuffer; +} __Pyx_Buffer; +typedef struct { + __Pyx_Buffer *rcbuffer; + char *data; + __Pyx_Buf_DimInfo diminfo[8]; +} __Pyx_LocalBuf_ND; + +/* MemviewRefcount.proto */ +static CYTHON_INLINE int __pyx_add_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( + __pyx_atomic_int_type *acquisition_count, PyThread_type_lock lock); +#define __pyx_get_slice_count_pointer(memview) (&memview->acquisition_count) +#define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) +static CYTHON_INLINE void __Pyx_INC_MEMVIEW(__Pyx_memviewslice *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *, int, int); + +/* MemviewSliceIsContig.proto */ +static int __pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim); + +/* OverlappingSlices.proto */ +static int __pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize); + +/* MemviewSliceInit.proto */ +static int __Pyx_init_memviewslice( + struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference); + +/* IsLittleEndian.proto (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void); + +/* BufferFormatCheck.proto (used by MemviewSliceValidateAndInit) */ +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts); +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type); + +/* TypeInfoCompare.proto (used by MemviewSliceValidateAndInit) */ +static int __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b); + +/* MemviewSliceValidateAndInit.export */ +static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj); + +/* ObjectToMemviewSlice.proto */ +static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(PyObject *, int writable_flag); + +/* RealImag.proto */ +#if CYTHON_CCOMPLEX + #ifdef __cplusplus + #define __Pyx_CREAL(z) ((z).real()) + #define __Pyx_CIMAG(z) ((z).imag()) + #else + #define __Pyx_CREAL(z) (__real__(z)) + #define __Pyx_CIMAG(z) (__imag__(z)) + #endif +#else + #define __Pyx_CREAL(z) ((z).real) + #define __Pyx_CIMAG(z) ((z).imag) +#endif +#if defined(__cplusplus) && CYTHON_CCOMPLEX\ + && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103) + #define __Pyx_SET_CREAL(z,x) ((z).real(x)) + #define __Pyx_SET_CIMAG(z,y) ((z).imag(y)) +#else + #define __Pyx_SET_CREAL(z,x) __Pyx_CREAL(z) = (x) + #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y) +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_float(a, b) ((a)==(b)) + #define __Pyx_c_sum_float(a, b) ((a)+(b)) + #define __Pyx_c_diff_float(a, b) ((a)-(b)) + #define __Pyx_c_prod_float(a, b) ((a)*(b)) + #define __Pyx_c_quot_float(a, b) ((a)/(b)) + #define __Pyx_c_neg_float(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_float(z) ((z)==(float)0) + #define __Pyx_c_conj_float(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_float(z) (::std::abs(z)) + #define __Pyx_c_pow_float(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_float(z) ((z)==0) + #define __Pyx_c_conj_float(z) (conjf(z)) + #if 1 + #define __Pyx_c_abs_float(z) (cabsf(z)) + #define __Pyx_c_pow_float(a, b) (cpowf(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sum_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_diff_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prod_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex, __pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_neg_float(__pyx_t_float_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_float(__pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conj_float(__pyx_t_float_complex); + #if 1 + static CYTHON_INLINE float __Pyx_c_abs_float(__pyx_t_float_complex); + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_pow_float(__pyx_t_float_complex, __pyx_t_float_complex); + #endif +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_double(a, b) ((a)==(b)) + #define __Pyx_c_sum_double(a, b) ((a)+(b)) + #define __Pyx_c_diff_double(a, b) ((a)-(b)) + #define __Pyx_c_prod_double(a, b) ((a)*(b)) + #define __Pyx_c_quot_double(a, b) ((a)/(b)) + #define __Pyx_c_neg_double(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_double(z) ((z)==(double)0) + #define __Pyx_c_conj_double(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (::std::abs(z)) + #define __Pyx_c_pow_double(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_double(z) ((z)==0) + #define __Pyx_c_conj_double(z) (conj(z)) + #if 1 + #define __Pyx_c_abs_double(z) (cabs(z)) + #define __Pyx_c_pow_double(a, b) (cpow(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex, __pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex); + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex); + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex, __pyx_t_double_complex); + #endif +#endif + +/* Arithmetic.proto */ +#if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #define __Pyx_c_eq_long__double(a, b) ((a)==(b)) + #define __Pyx_c_sum_long__double(a, b) ((a)+(b)) + #define __Pyx_c_diff_long__double(a, b) ((a)-(b)) + #define __Pyx_c_prod_long__double(a, b) ((a)*(b)) + #define __Pyx_c_quot_long__double(a, b) ((a)/(b)) + #define __Pyx_c_neg_long__double(a) (-(a)) + #ifdef __cplusplus + #define __Pyx_c_is_zero_long__double(z) ((z)==(long double)0) + #define __Pyx_c_conj_long__double(z) (::std::conj(z)) + #if 1 + #define __Pyx_c_abs_long__double(z) (::std::abs(z)) + #define __Pyx_c_pow_long__double(a, b) (::std::pow(a, b)) + #endif + #else + #define __Pyx_c_is_zero_long__double(z) ((z)==0) + #define __Pyx_c_conj_long__double(z) (conjl(z)) + #if 1 + #define __Pyx_c_abs_long__double(z) (cabsl(z)) + #define __Pyx_c_pow_long__double(a, b) (cpowl(a, b)) + #endif + #endif +#else + static CYTHON_INLINE int __Pyx_c_eq_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_sum_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_diff_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_prod_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_neg_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE int __Pyx_c_is_zero_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_conj_long__double(__pyx_t_long_double_complex); + #if 1 + static CYTHON_INLINE long double __Pyx_c_abs_long__double(__pyx_t_long_double_complex); + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_pow_long__double(__pyx_t_long_double_complex, __pyx_t_long_double_complex); + #endif +#endif + +/* MemviewSliceCopy.proto */ +static __Pyx_memviewslice +__pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *); + +/* PyObjectCallMethod1.proto (used by UpdateUnpickledDict) */ +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); + +/* UpdateUnpickledDict.proto */ +static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index); + +/* CheckUnpickleChecksum.proto */ +static CYTHON_INLINE int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GetRuntimeVersion.proto */ +#if __PYX_LIMITED_VERSION_HEX < 0x030b0000 +static unsigned long __Pyx_cached_runtime_version = 0; +static void __Pyx_init_runtime_version(void); +#else +#define __Pyx_init_runtime_version() +#endif +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* DecompressString.proto */ +static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self); /* proto*/ +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto*/ +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src); /* proto*/ +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self); /* proto*/ +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp); /* proto*/ +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value); /* proto*/ +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_8itemsize_itemsize(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_9alignment_alignment(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_6fields_fields(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_5names_names(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyArray_ArrayDescr *__pyx_f_5numpy_5dtype_8subarray_subarray(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_uint64 __pyx_f_5numpy_5dtype_5flags_flags(PyArray_Descr *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_7numiter_numiter(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_4size_size(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_5index_index(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_2nd_nd(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_9broadcast_10dimensions_dimensions(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE void **__pyx_f_5numpy_9broadcast_5iters_iters(PyArrayMultiIterObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyObject *__pyx_f_5numpy_7ndarray_4base_base(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE PyArray_Descr *__pyx_f_5numpy_7ndarray_5descr_descr(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE int __pyx_f_5numpy_7ndarray_4ndim_ndim(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_5shape_shape(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_7strides_strides(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE npy_intp __pyx_f_5numpy_7ndarray_4size_size(PyArrayObject *__pyx_v_self); /* proto*/ +static CYTHON_INLINE char *__pyx_f_5numpy_7ndarray_4data_data(PyArrayObject *__pyx_v_self); /* proto*/ + +/* Module declarations from "cython.view" */ + +/* Module declarations from "cython.dataclasses" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "libc.stdio" */ + +/* Module declarations from "__builtin__" */ + +/* Module declarations from "cpython.type" */ + +/* Module declarations from "cpython" */ + +/* Module declarations from "cpython.object" */ + +/* Module declarations from "cpython.ref" */ + +/* Module declarations from "numpy" */ + +/* Module declarations from "numpy" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "thinc.layers.sparselinear" */ +static PyObject *__pyx_collections_abc_Sequence = 0; +static PyObject *generic = 0; +static PyObject *strided = 0; +static PyObject *indirect = 0; +static PyObject *contiguous = 0; +static PyObject *indirect_contiguous = 0; +static int __pyx_memoryview_thread_locks_used; +static PyThread_type_lock __pyx_memoryview_thread_locks[8]; +static void __pyx_f_5thinc_6layers_12sparselinear_set_scoresC(float *, uint64_t const *, float const *, int32_t const *, int, int, float const *, int, int); /*proto*/ +static void __pyx_f_5thinc_6layers_12sparselinear_set_gradientC(float *, uint64_t const *, float const *, int32_t const *, int, int, float const *, int, int); /*proto*/ +static uint32_t __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64(uint64_t, uint32_t); /*proto*/ +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *); /*proto*/ +static struct __pyx_array_obj *__pyx_array_new(PyObject *, Py_ssize_t, char *, char const *, char *); /*proto*/ +static PyObject *__pyx_memoryview_new(PyObject *, int, int, __Pyx_TypeInfo const *); /*proto*/ +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *); /*proto*/ +static PyObject *_unellipsify(PyObject *, int); /*proto*/ +static int assert_direct_dimensions(Py_ssize_t *, int); /*proto*/ +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *, PyObject *); /*proto*/ +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int *, Py_ssize_t, Py_ssize_t, Py_ssize_t, int, int, int, int); /*proto*/ +static char *__pyx_pybuffer_index(Py_buffer *, char *, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memslice_transpose(__Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice, int, PyObject *(*)(char *), int (*)(char *, PyObject *), int); /*proto*/ +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *); /*proto*/ +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *, __Pyx_memviewslice *); /*proto*/ +static Py_ssize_t abs_py_ssize_t(Py_ssize_t); /*proto*/ +static char __pyx_get_best_slice_order(__Pyx_memviewslice *, int); /*proto*/ +static void _copy_strided_to_strided(char *, Py_ssize_t *, char *, Py_ssize_t *, Py_ssize_t *, Py_ssize_t *, int, size_t); /*proto*/ +static void copy_strided_to_strided(__Pyx_memviewslice *, __Pyx_memviewslice *, int, size_t); /*proto*/ +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *, int); /*proto*/ +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *, Py_ssize_t *, Py_ssize_t, int, char); /*proto*/ +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *, __Pyx_memviewslice *, char, int); /*proto*/ +static int __pyx_memoryview_err_extents(int, Py_ssize_t, Py_ssize_t); /*proto*/ +static int __pyx_memoryview_err_dim(PyObject *, PyObject *, int); /*proto*/ +static int __pyx_memoryview_err(PyObject *, PyObject *); /*proto*/ +static int __pyx_memoryview_err_no_memory(void); /*proto*/ +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice, __Pyx_memviewslice, int, int, int); /*proto*/ +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *, int, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_refcount_objects_in_slice(char *, Py_ssize_t *, Py_ssize_t *, int, int); /*proto*/ +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *, int, size_t, void *, int); /*proto*/ +static void __pyx_memoryview__slice_assign_scalar(char *, Py_ssize_t *, Py_ssize_t *, int, size_t, void *); /*proto*/ +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *, PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +static const __Pyx_TypeInfo __Pyx_TypeInfo_float = { "float", NULL, sizeof(float), { 0 }, 0, 'R', 0, 0 }; +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "thinc.layers.sparselinear" +extern int __pyx_module_is_main_thinc__layers__sparselinear; +int __pyx_module_is_main_thinc__layers__sparselinear = 0; + +/* Implementation of "thinc.layers.sparselinear" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin___import__; +static PyObject *__pyx_builtin_enumerate; +static PyObject *__pyx_builtin_Ellipsis; +static PyObject *__pyx_builtin_id; +/* #### Code section: string_decls ### */ +/* #### Code section: decls ### */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr); /* proto */ +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item); /* proto */ +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object); /* proto */ +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /* proto */ +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_SparseLinear(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nO, PyObject *__pyx_v_length); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_2SparseLinear_v2(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nO, PyObject *__pyx_v_length); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_4forward(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys_values_lengths, CYTHON_UNUSED PyObject *__pyx_v_is_train); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_6init(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, CYTHON_UNUSED PyObject *__pyx_v_X, PyObject *__pyx_v_Y); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_17_begin_gpu_update_backprop_gpu_update(PyObject *__pyx_self, PyObject *__pyx_v_d_scores); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_8_begin_gpu_update(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys, PyObject *__pyx_v_values, PyObject *__pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_10_begin_cpu_update(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyArrayObject *__pyx_v_keys, PyArrayObject *__pyx_v_values, PyArrayObject *__pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys, PyObject *__pyx_v_values, PyObject *__pyx_v_lengths); /* proto */ +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update_2__call__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __Pyx_memviewslice __pyx_v_d_scores); /* proto */ +static PyObject *__pyx_tp_new_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + PyTypeObject *__pyx_ptype_7cpython_4type_type; + PyTypeObject *__pyx_ptype_5numpy_dtype; + PyTypeObject *__pyx_ptype_5numpy_flatiter; + PyTypeObject *__pyx_ptype_5numpy_broadcast; + PyTypeObject *__pyx_ptype_5numpy_ndarray; + PyTypeObject *__pyx_ptype_5numpy_generic; + PyTypeObject *__pyx_ptype_5numpy_number; + PyTypeObject *__pyx_ptype_5numpy_integer; + PyTypeObject *__pyx_ptype_5numpy_signedinteger; + PyTypeObject *__pyx_ptype_5numpy_unsignedinteger; + PyTypeObject *__pyx_ptype_5numpy_inexact; + PyTypeObject *__pyx_ptype_5numpy_floating; + PyTypeObject *__pyx_ptype_5numpy_complexfloating; + PyTypeObject *__pyx_ptype_5numpy_flexible; + PyTypeObject *__pyx_ptype_5numpy_character; + PyTypeObject *__pyx_ptype_5numpy_ufunc; + PyObject *__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update; + PyObject *__pyx_type___pyx_array; + PyObject *__pyx_type___pyx_MemviewEnum; + PyObject *__pyx_type___pyx_memoryview; + PyObject *__pyx_type___pyx_memoryviewslice; + PyTypeObject *__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update; + PyTypeObject *__pyx_array_type; + PyTypeObject *__pyx_MemviewEnum_type; + PyTypeObject *__pyx_memoryview_type; + PyTypeObject *__pyx_memoryviewslice_type; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_items; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_values; + PyObject *__pyx_slice[1]; + PyObject *__pyx_tuple[4]; + PyObject *__pyx_codeobj_tab[9]; + PyObject *__pyx_string_tab[217]; + PyObject *__pyx_number_tab[5]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CythonFunctionShared.module_state_decls */ +PyTypeObject *__pyx_CyFunctionType; + + +#if CYTHON_USE_FREELISTS +struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *__pyx_freelist_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update[8]; +int __pyx_freecount_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update; +#endif +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_All_dimensions_preceding_dimensi __pyx_string_tab[1] +#define __pyx_kp_u_Buffer_view_does_not_expose_stri __pyx_string_tab[2] +#define __pyx_kp_u_Can_only_create_a_buffer_that_is __pyx_string_tab[3] +#define __pyx_kp_u_Cannot_assign_to_read_only_memor __pyx_string_tab[4] +#define __pyx_kp_u_Cannot_create_writable_memory_vi __pyx_string_tab[5] +#define __pyx_kp_u_Cannot_index_with_type __pyx_string_tab[6] +#define __pyx_kp_u_Cannot_transpose_memoryview_with __pyx_string_tab[7] +#define __pyx_kp_u_Dimension_d_is_not_direct __pyx_string_tab[8] +#define __pyx_kp_u_Empty_shape_tuple_for_cython_arr __pyx_string_tab[9] +#define __pyx_kp_u_Index_out_of_bounds_axis_d __pyx_string_tab[10] +#define __pyx_kp_u_Indirect_dimensions_not_supporte __pyx_string_tab[11] +#define __pyx_kp_u_Invalid_mode_expected_c_or_fortr __pyx_string_tab[12] +#define __pyx_kp_u_Invalid_shape_in_axis __pyx_string_tab[13] +#define __pyx_kp_u_MemoryView_of __pyx_string_tab[14] +#define __pyx_kp_u_Model_InT_OutT __pyx_string_tab[15] +#define __pyx_kp_u_Move_this_out_of_a_closure_into __pyx_string_tab[16] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[17] +#define __pyx_kp_u_Optional_InT __pyx_string_tab[18] +#define __pyx_kp_u_Optional_OutT __pyx_string_tab[19] +#define __pyx_kp_u_Optional_int __pyx_string_tab[20] +#define __pyx_kp_u_Out_of_bounds_on_buffer_access_a __pyx_string_tab[21] +#define __pyx_kp_u_SparseLinear_v1 __pyx_string_tab[22] +#define __pyx_kp_u_SparseLinear_v2 __pyx_string_tab[23] +#define __pyx_kp_u_Step_may_not_be_zero_axis_d __pyx_string_tab[24] +#define __pyx_kp_u_Tuple_ArrayXd_ArrayXd_ArrayXd __pyx_string_tab[25] +#define __pyx_kp_u_Tuple_ArrayXd_Callable __pyx_string_tab[26] +#define __pyx_kp_u_Tuple_OutT_Callable __pyx_string_tab[27] +#define __pyx_kp_u_Unable_to_convert_item_to_object __pyx_string_tab[28] +#define __pyx_kp_u__2 __pyx_string_tab[29] +#define __pyx_kp_u__3 __pyx_string_tab[30] +#define __pyx_kp_u__4 __pyx_string_tab[31] +#define __pyx_kp_u__5 __pyx_string_tab[32] +#define __pyx_kp_u__6 __pyx_string_tab[33] +#define __pyx_kp_u_add_note __pyx_string_tab[34] +#define __pyx_kp_u_and __pyx_string_tab[35] +#define __pyx_kp_u_at_0x __pyx_string_tab[36] +#define __pyx_kp_u_collections_abc __pyx_string_tab[37] +#define __pyx_kp_u_contiguous_and_direct __pyx_string_tab[38] +#define __pyx_kp_u_contiguous_and_indirect __pyx_string_tab[39] +#define __pyx_kp_u_disable __pyx_string_tab[40] +#define __pyx_kp_u_enable __pyx_string_tab[41] +#define __pyx_kp_u_gc __pyx_string_tab[42] +#define __pyx_kp_u_got __pyx_string_tab[43] +#define __pyx_kp_u_got_differing_extents_in_dimensi __pyx_string_tab[44] +#define __pyx_kp_u_isenabled __pyx_string_tab[45] +#define __pyx_kp_u_itemsize_0_for_cython_array __pyx_string_tab[46] +#define __pyx_kp_u_no_default___reduce___due_to_non __pyx_string_tab[47] +#define __pyx_kp_u_numpy__core_multiarray_failed_to __pyx_string_tab[48] +#define __pyx_kp_u_numpy__core_umath_failed_to_impo __pyx_string_tab[49] +#define __pyx_kp_u_object __pyx_string_tab[50] +#define __pyx_kp_u_self_name_is_not_None __pyx_string_tab[51] +#define __pyx_kp_u_strided_and_direct __pyx_string_tab[52] +#define __pyx_kp_u_strided_and_direct_or_indirect __pyx_string_tab[53] +#define __pyx_kp_u_strided_and_indirect __pyx_string_tab[54] +#define __pyx_kp_u_thinc_backends __pyx_string_tab[55] +#define __pyx_kp_u_thinc_config __pyx_string_tab[56] +#define __pyx_kp_u_thinc_layers_sparselinear_pyx __pyx_string_tab[57] +#define __pyx_kp_u_thinc_layers_typing __pyx_string_tab[58] +#define __pyx_kp_u_thinc_model __pyx_string_tab[59] +#define __pyx_kp_u_thinc_types __pyx_string_tab[60] +#define __pyx_kp_u_thinc_util __pyx_string_tab[61] +#define __pyx_kp_u_unable_to_allocate_array_data __pyx_string_tab[62] +#define __pyx_kp_u_unable_to_allocate_shape_and_str __pyx_string_tab[63] +#define __pyx_n_u_ASCII __pyx_string_tab[64] +#define __pyx_n_u_ArrayXd __pyx_string_tab[65] +#define __pyx_n_u_Callable __pyx_string_tab[66] +#define __pyx_n_u_CupyOps __pyx_string_tab[67] +#define __pyx_n_u_Ellipsis __pyx_string_tab[68] +#define __pyx_n_u_InT __pyx_string_tab[69] +#define __pyx_n_u_Model __pyx_string_tab[70] +#define __pyx_n_u_NumpyOps __pyx_string_tab[71] +#define __pyx_n_u_Optional __pyx_string_tab[72] +#define __pyx_n_u_OutT __pyx_string_tab[73] +#define __pyx_n_u_Pyx_PyDict_NextRef __pyx_string_tab[74] +#define __pyx_n_u_Sequence __pyx_string_tab[75] +#define __pyx_n_u_SparseLinear __pyx_string_tab[76] +#define __pyx_n_u_SparseLinear_v2_2 __pyx_string_tab[77] +#define __pyx_n_u_Tuple __pyx_string_tab[78] +#define __pyx_n_u_View_MemoryView __pyx_string_tab[79] +#define __pyx_n_u_W __pyx_string_tab[80] +#define __pyx_n_u_X __pyx_string_tab[81] +#define __pyx_n_u_Y __pyx_string_tab[82] +#define __pyx_n_u_abc __pyx_string_tab[83] +#define __pyx_n_u_alloc __pyx_string_tab[84] +#define __pyx_n_u_allocate_buffer __pyx_string_tab[85] +#define __pyx_n_u_asarray __pyx_string_tab[86] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[87] +#define __pyx_n_u_attrs __pyx_string_tab[88] +#define __pyx_n_u_b __pyx_string_tab[89] +#define __pyx_n_u_backends __pyx_string_tab[90] +#define __pyx_n_u_backprop_gpu_update __pyx_string_tab[91] +#define __pyx_n_u_base __pyx_string_tab[92] +#define __pyx_n_u_begin_cpu_update __pyx_string_tab[93] +#define __pyx_n_u_begin_gpu_update __pyx_string_tab[94] +#define __pyx_n_u_begin_gpu_update_locals_backpro __pyx_string_tab[95] +#define __pyx_n_u_bool __pyx_string_tab[96] +#define __pyx_n_u_c __pyx_string_tab[97] +#define __pyx_n_u_call __pyx_string_tab[98] +#define __pyx_n_u_callback __pyx_string_tab[99] +#define __pyx_n_u_class __pyx_string_tab[100] +#define __pyx_n_u_class_getitem __pyx_string_tab[101] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[102] +#define __pyx_n_u_config __pyx_string_tab[103] +#define __pyx_n_u_count __pyx_string_tab[104] +#define __pyx_n_u_d_bias __pyx_string_tab[105] +#define __pyx_n_u_d_scores __pyx_string_tab[106] +#define __pyx_n_u_d_weights __pyx_string_tab[107] +#define __pyx_n_u_dict __pyx_string_tab[108] +#define __pyx_n_u_dims __pyx_string_tab[109] +#define __pyx_n_u_doc __pyx_string_tab[110] +#define __pyx_n_u_dtype __pyx_string_tab[111] +#define __pyx_n_u_dtype_is_object __pyx_string_tab[112] +#define __pyx_n_u_encode __pyx_string_tab[113] +#define __pyx_n_u_enumerate __pyx_string_tab[114] +#define __pyx_n_u_error __pyx_string_tab[115] +#define __pyx_n_u_f __pyx_string_tab[116] +#define __pyx_n_u_finish_linear_update __pyx_string_tab[117] +#define __pyx_n_u_finish_linear_update___call __pyx_string_tab[118] +#define __pyx_n_u_finish_linear_update___init __pyx_string_tab[119] +#define __pyx_n_u_flags __pyx_string_tab[120] +#define __pyx_n_u_format __pyx_string_tab[121] +#define __pyx_n_u_fortran __pyx_string_tab[122] +#define __pyx_n_u_forward __pyx_string_tab[123] +#define __pyx_n_u_func __pyx_string_tab[124] +#define __pyx_n_u_get __pyx_string_tab[125] +#define __pyx_n_u_get_array_module __pyx_string_tab[126] +#define __pyx_n_u_get_dim __pyx_string_tab[127] +#define __pyx_n_u_get_param __pyx_string_tab[128] +#define __pyx_n_u_get_width __pyx_string_tab[129] +#define __pyx_n_u_getstate __pyx_string_tab[130] +#define __pyx_n_u_i __pyx_string_tab[131] +#define __pyx_n_u_id __pyx_string_tab[132] +#define __pyx_n_u_import __pyx_string_tab[133] +#define __pyx_n_u_inc_grad __pyx_string_tab[134] +#define __pyx_n_u_index __pyx_string_tab[135] +#define __pyx_n_u_init __pyx_string_tab[136] +#define __pyx_n_u_init_2 __pyx_string_tab[137] +#define __pyx_n_u_int __pyx_string_tab[138] +#define __pyx_n_u_is_coroutine __pyx_string_tab[139] +#define __pyx_n_u_is_cupy_array __pyx_string_tab[140] +#define __pyx_n_u_is_numpy_array __pyx_string_tab[141] +#define __pyx_n_u_is_train __pyx_string_tab[142] +#define __pyx_n_u_items __pyx_string_tab[143] +#define __pyx_n_u_itemsize __pyx_string_tab[144] +#define __pyx_n_u_j __pyx_string_tab[145] +#define __pyx_n_u_keys __pyx_string_tab[146] +#define __pyx_n_u_keys_values_lengths __pyx_string_tab[147] +#define __pyx_n_u_layers __pyx_string_tab[148] +#define __pyx_n_u_length __pyx_string_tab[149] +#define __pyx_n_u_lengths __pyx_string_tab[150] +#define __pyx_n_u_main __pyx_string_tab[151] +#define __pyx_n_u_memview __pyx_string_tab[152] +#define __pyx_n_u_metaclass __pyx_string_tab[153] +#define __pyx_n_u_mode __pyx_string_tab[154] +#define __pyx_n_u_model __pyx_string_tab[155] +#define __pyx_n_u_module __pyx_string_tab[156] +#define __pyx_n_u_nO __pyx_string_tab[157] +#define __pyx_n_u_name __pyx_string_tab[158] +#define __pyx_n_u_name_2 __pyx_string_tab[159] +#define __pyx_n_u_ndim __pyx_string_tab[160] +#define __pyx_n_u_new __pyx_string_tab[161] +#define __pyx_n_u_obj __pyx_string_tab[162] +#define __pyx_n_u_ops __pyx_string_tab[163] +#define __pyx_n_u_pack __pyx_string_tab[164] +#define __pyx_n_u_params __pyx_string_tab[165] +#define __pyx_n_u_pop __pyx_string_tab[166] +#define __pyx_n_u_prepare __pyx_string_tab[167] +#define __pyx_n_u_pyx_checksum __pyx_string_tab[168] +#define __pyx_n_u_pyx_state __pyx_string_tab[169] +#define __pyx_n_u_pyx_type __pyx_string_tab[170] +#define __pyx_n_u_pyx_unpickle_Enum __pyx_string_tab[171] +#define __pyx_n_u_pyx_vtable __pyx_string_tab[172] +#define __pyx_n_u_qualname __pyx_string_tab[173] +#define __pyx_n_u_reduce __pyx_string_tab[174] +#define __pyx_n_u_reduce_cython __pyx_string_tab[175] +#define __pyx_n_u_reduce_ex __pyx_string_tab[176] +#define __pyx_n_u_register __pyx_string_tab[177] +#define __pyx_n_u_registry __pyx_string_tab[178] +#define __pyx_n_u_return __pyx_string_tab[179] +#define __pyx_n_u_scores __pyx_string_tab[180] +#define __pyx_n_u_scores_cpu __pyx_string_tab[181] +#define __pyx_n_u_self __pyx_string_tab[182] +#define __pyx_n_u_set_dim __pyx_string_tab[183] +#define __pyx_n_u_set_name __pyx_string_tab[184] +#define __pyx_n_u_set_param __pyx_string_tab[185] +#define __pyx_n_u_setdefault __pyx_string_tab[186] +#define __pyx_n_u_setstate __pyx_string_tab[187] +#define __pyx_n_u_setstate_cython __pyx_string_tab[188] +#define __pyx_n_u_shape __pyx_string_tab[189] +#define __pyx_n_u_size __pyx_string_tab[190] +#define __pyx_n_u_sparse_linear __pyx_string_tab[191] +#define __pyx_n_u_start __pyx_string_tab[192] +#define __pyx_n_u_step __pyx_string_tab[193] +#define __pyx_n_u_stop __pyx_string_tab[194] +#define __pyx_n_u_struct __pyx_string_tab[195] +#define __pyx_n_u_test __pyx_string_tab[196] +#define __pyx_n_u_thinc_layers_sparselinear __pyx_string_tab[197] +#define __pyx_n_u_types __pyx_string_tab[198] +#define __pyx_n_u_typing __pyx_string_tab[199] +#define __pyx_n_u_unpack __pyx_string_tab[200] +#define __pyx_n_u_update __pyx_string_tab[201] +#define __pyx_n_u_util __pyx_string_tab[202] +#define __pyx_n_u_v1_indexing __pyx_string_tab[203] +#define __pyx_n_u_values __pyx_string_tab[204] +#define __pyx_n_u_x __pyx_string_tab[205] +#define __pyx_n_u_xp __pyx_string_tab[206] +#define __pyx_kp_b_iso88591_3_XXY_r_XQfIQa_haq_U_1_1E_d_Rz __pyx_string_tab[207] +#define __pyx_kp_b_iso88591_55FFXXdde_AWD_D_d_gUYYZ_Jl_2XQm __pyx_string_tab[208] +#define __pyx_kp_b_iso88591_81_5_Q_V5_fD_a __pyx_string_tab[209] +#define __pyx_kp_b_iso88591_A_IQ_HA_Ja_Kq __pyx_string_tab[210] +#define __pyx_kp_b_iso88591_A_T_xq_V81A_D_d_AQ_V4vRq_t1_Q_a __pyx_string_tab[211] +#define __pyx_kp_b_iso88591_ha_5_Q_V5_fD_a __pyx_string_tab[212] +#define __pyx_kp_b_iso88591_iq_A_AQ_ha_ha __pyx_string_tab[213] +#define __pyx_kp_b_iso88591_l_Q_ha __pyx_string_tab[214] +#define __pyx_kp_b_iso88591_xq_e81A_Zq_Zq_U_fBc_A_E_q_a_q_a __pyx_string_tab[215] +#define __pyx_n_b_O __pyx_string_tab[216] +#define __pyx_int_0 __pyx_number_tab[0] +#define __pyx_int_neg_1 __pyx_number_tab[1] +#define __pyx_int_1 __pyx_number_tab[2] +#define __pyx_int_262144 __pyx_number_tab[3] +#define __pyx_int_136983863 __pyx_number_tab[4] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_7cpython_4type_type); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_dtype); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_flatiter); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_broadcast); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_ndarray); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_generic); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_number); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_integer); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_signedinteger); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_unsignedinteger); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_inexact); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_floating); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_complexfloating); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_flexible); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_character); + Py_CLEAR(clear_module_state->__pyx_ptype_5numpy_ufunc); + Py_CLEAR(clear_module_state->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update); + Py_CLEAR(clear_module_state->__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update); + Py_CLEAR(clear_module_state->__pyx_array_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_array); + Py_CLEAR(clear_module_state->__pyx_MemviewEnum_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_MemviewEnum); + Py_CLEAR(clear_module_state->__pyx_memoryview_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryview); + Py_CLEAR(clear_module_state->__pyx_memoryviewslice_type); + Py_CLEAR(clear_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_slice[i]); } + for (int i=0; i<4; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<9; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<217; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + for (int i=0; i<5; ++i) { Py_CLEAR(clear_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_clear_contents ### */ +/* CommonTypesMetaclass.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_clear */ +Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_clear_end ### */ +return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + Py_VISIT(traverse_module_state->__pyx_ptype_7cpython_4type_type); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_dtype); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_flatiter); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_broadcast); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_ndarray); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_generic); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_number); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_integer); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_signedinteger); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_unsignedinteger); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_inexact); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_floating); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_complexfloating); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_flexible); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_character); + Py_VISIT(traverse_module_state->__pyx_ptype_5numpy_ufunc); + Py_VISIT(traverse_module_state->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update); + Py_VISIT(traverse_module_state->__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update); + Py_VISIT(traverse_module_state->__pyx_array_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_array); + Py_VISIT(traverse_module_state->__pyx_MemviewEnum_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_MemviewEnum); + Py_VISIT(traverse_module_state->__pyx_memoryview_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryview); + Py_VISIT(traverse_module_state->__pyx_memoryviewslice_type); + Py_VISIT(traverse_module_state->__pyx_type___pyx_memoryviewslice); + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_slice[i]); } + for (int i=0; i<4; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<9; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<217; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + for (int i=0; i<5; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_number_tab[i]); } +/* #### Code section: module_state_traverse_contents ### */ +/* CommonTypesMetaclass.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CommonTypesMetaclassType); + +/* CythonFunctionShared.module_state_traverse */ +Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + +/* #### Code section: module_state_traverse_end ### */ +return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + +/* Python wrapper */ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_shape = 0; + Py_ssize_t __pyx_v_itemsize; + PyObject *__pyx_v_format = 0; + PyObject *__pyx_v_mode = 0; + int __pyx_v_allocate_buffer; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_shape,&__pyx_mstate_global->__pyx_n_u_itemsize,&__pyx_mstate_global->__pyx_n_u_format,&__pyx_mstate_global->__pyx_n_u_mode,&__pyx_mstate_global->__pyx_n_u_allocate_buffer,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 129, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 129, __pyx_L3_error) + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, i); __PYX_ERR(1, 129, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_VARARGS(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_VARARGS(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(1, 129, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 129, __pyx_L3_error) + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 129, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 129, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[3]) values[3] = __Pyx_NewRef(((PyObject *)__pyx_mstate_global->__pyx_n_u_c)); + } + __pyx_v_shape = ((PyObject*)values[0]); + __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_v_format = values[2]; + __pyx_v_mode = values[3]; + if (values[4]) { + __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) + } else { + + /* "View.MemoryView":130 + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, + * mode="c", bint allocate_buffer=True): # <<<<<<<<<<<<<< + * + * cdef int idx +*/ + __pyx_v_allocate_buffer = ((int)1); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, __pyx_nargs); __PYX_ERR(1, 129, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 129, __pyx_L1_error) + if (unlikely(((PyObject *)__pyx_v_format) == Py_None)) { + PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 129, __pyx_L1_error) + } + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v_shape, __pyx_v_itemsize, __pyx_v_format, __pyx_v_mode, __pyx_v_allocate_buffer); + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = -1; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, PyObject *__pyx_v_format, PyObject *__pyx_v_mode, int __pyx_v_allocate_buffer) { + int __pyx_v_idx; + Py_ssize_t __pyx_v_dim; + char __pyx_v_order; + int __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11[5]; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + __Pyx_INCREF(__pyx_v_format); + + /* "View.MemoryView":135 + * cdef Py_ssize_t dim + * + * self.ndim = len(shape) # <<<<<<<<<<<<<< + * self.itemsize = itemsize + * +*/ + if (unlikely(__pyx_v_shape == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 135, __pyx_L1_error) + } + __pyx_t_1 = __Pyx_PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 135, __pyx_L1_error) + __pyx_v_self->ndim = ((int)__pyx_t_1); + + /* "View.MemoryView":136 + * + * self.ndim = len(shape) + * self.itemsize = itemsize # <<<<<<<<<<<<<< + * + * if not self.ndim: +*/ + __pyx_v_self->itemsize = __pyx_v_itemsize; + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + __pyx_t_2 = (!(__pyx_v_self->ndim != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":139 + * + * if not self.ndim: + * raise ValueError, "Empty shape tuple for cython.array" # <<<<<<<<<<<<<< + * + * if itemsize <= 0: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Empty_shape_tuple_for_cython_arr, 0, 0); + __PYX_ERR(1, 139, __pyx_L1_error) + + /* "View.MemoryView":138 + * self.itemsize = itemsize + * + * if not self.ndim: # <<<<<<<<<<<<<< + * raise ValueError, "Empty shape tuple for cython.array" + * +*/ + } + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + __pyx_t_2 = (__pyx_v_itemsize <= 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":142 + * + * if itemsize <= 0: + * raise ValueError, "itemsize <= 0 for cython.array" # <<<<<<<<<<<<<< + * + * if not isinstance(format, bytes): +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_itemsize_0_for_cython_array, 0, 0); + __PYX_ERR(1, 142, __pyx_L1_error) + + /* "View.MemoryView":141 + * raise ValueError, "Empty shape tuple for cython.array" + * + * if itemsize <= 0: # <<<<<<<<<<<<<< + * raise ValueError, "itemsize <= 0 for cython.array" + * +*/ + } + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + __pyx_t_2 = PyBytes_Check(__pyx_v_format); + __pyx_t_3 = (!__pyx_t_2); + if (__pyx_t_3) { + + /* "View.MemoryView":145 + * + * if not isinstance(format, bytes): + * format = format.encode('ASCII') # <<<<<<<<<<<<<< + * self._format = format # keep a reference to the byte string + * self.format = self._format +*/ + __pyx_t_5 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ASCII}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_encode, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF_SET(__pyx_v_format, __pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":144 + * raise ValueError, "itemsize <= 0 for cython.array" + * + * if not isinstance(format, bytes): # <<<<<<<<<<<<<< + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string +*/ + } + + /* "View.MemoryView":146 + * if not isinstance(format, bytes): + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string # <<<<<<<<<<<<<< + * self.format = self._format + * +*/ + __pyx_t_4 = __pyx_v_format; + __Pyx_INCREF(__pyx_t_4); + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_v_self->_format); + __Pyx_DECREF(__pyx_v_self->_format); + __pyx_v_self->_format = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":147 + * format = format.encode('ASCII') + * self._format = format # keep a reference to the byte string + * self.format = self._format # <<<<<<<<<<<<<< + * + * +*/ + if (unlikely(__pyx_v_self->_format == Py_None)) { + PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); + __PYX_ERR(1, 147, __pyx_L1_error) + } + __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L1_error) + __pyx_v_self->format = __pyx_t_7; + + /* "View.MemoryView":150 + * + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) # <<<<<<<<<<<<<< + * self._strides = self._shape + self.ndim + * +*/ + __pyx_v_self->_shape = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * __pyx_v_self->ndim) * 2))); + + /* "View.MemoryView":151 + * + * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) + * self._strides = self._shape + self.ndim # <<<<<<<<<<<<<< + * + * if not self._shape: +*/ + __pyx_v_self->_strides = (__pyx_v_self->_shape + __pyx_v_self->ndim); + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + __pyx_t_3 = (!(__pyx_v_self->_shape != 0)); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":154 + * + * if not self._shape: + * raise MemoryError, "unable to allocate shape and strides." # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_shape_and_str, 0, 0); + __PYX_ERR(1, 154, __pyx_L1_error) + + /* "View.MemoryView":153 + * self._strides = self._shape + self.ndim + * + * if not self._shape: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate shape and strides." + * +*/ + } + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + __pyx_t_8 = 0; + __pyx_t_4 = __pyx_v_shape; __Pyx_INCREF(__pyx_t_4); + __pyx_t_1 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_4); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 157, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_4, __pyx_t_1); + #endif + ++__pyx_t_1; + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 157, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_9; + __pyx_v_idx = __pyx_t_8; + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + __pyx_t_3 = (__pyx_v_dim <= 0); + if (unlikely(__pyx_t_3)) { + + /* "View.MemoryView":159 + * for idx, dim in enumerate(shape): + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." # <<<<<<<<<<<<<< + * self._shape[idx] = dim + * +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_int(__pyx_v_idx, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_10 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11[0] = __pyx_mstate_global->__pyx_kp_u_Invalid_shape_in_axis; + __pyx_t_11[1] = __pyx_t_5; + __pyx_t_11[2] = __pyx_mstate_global->__pyx_kp_u_; + __pyx_t_11[3] = __pyx_t_10; + __pyx_t_11[4] = __pyx_mstate_global->__pyx_kp_u__2; + __pyx_t_12 = __Pyx_PyUnicode_Join(__pyx_t_11, 5, 22 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 2 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_10) + 1, 127); + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 159, __pyx_L1_error) + + /* "View.MemoryView":158 + * + * for idx, dim in enumerate(shape): + * if dim <= 0: # <<<<<<<<<<<<<< + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim +*/ + } + + /* "View.MemoryView":160 + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." + * self._shape[idx] = dim # <<<<<<<<<<<<<< + * + * cdef char order +*/ + (__pyx_v_self->_shape[__pyx_v_idx]) = __pyx_v_dim; + + /* "View.MemoryView":157 + * + * + * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< + * if dim <= 0: + * raise ValueError, f"Invalid shape in axis {idx}: {dim}." +*/ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 163, __pyx_L1_error) + if (__pyx_t_3) { + + /* "View.MemoryView":164 + * cdef char order + * if mode == 'c': + * order = b'C' # <<<<<<<<<<<<<< + * self.mode = u'c' + * elif mode == 'fortran': +*/ + __pyx_v_order = 'C'; + + /* "View.MemoryView":165 + * if mode == 'c': + * order = b'C' + * self.mode = u'c' # <<<<<<<<<<<<<< + * elif mode == 'fortran': + * order = b'F' +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_c); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_c; + + /* "View.MemoryView":163 + * + * cdef char order + * if mode == 'c': # <<<<<<<<<<<<<< + * order = b'C' + * self.mode = u'c' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + __pyx_t_3 = (__Pyx_PyUnicode_Equals(__pyx_v_mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 166, __pyx_L1_error) + if (likely(__pyx_t_3)) { + + /* "View.MemoryView":167 + * self.mode = u'c' + * elif mode == 'fortran': + * order = b'F' # <<<<<<<<<<<<<< + * self.mode = u'fortran' + * else: +*/ + __pyx_v_order = 'F'; + + /* "View.MemoryView":168 + * elif mode == 'fortran': + * order = b'F' + * self.mode = u'fortran' # <<<<<<<<<<<<<< + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_n_u_fortran); + __Pyx_GOTREF(__pyx_v_self->mode); + __Pyx_DECREF(__pyx_v_self->mode); + __pyx_v_self->mode = __pyx_mstate_global->__pyx_n_u_fortran; + + /* "View.MemoryView":166 + * order = b'C' + * self.mode = u'c' + * elif mode == 'fortran': # <<<<<<<<<<<<<< + * order = b'F' + * self.mode = u'fortran' +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":170 + * self.mode = u'fortran' + * else: + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" # <<<<<<<<<<<<<< + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) +*/ + /*else*/ { + __pyx_t_4 = __Pyx_PyObject_FormatSimple(__pyx_v_mode, __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_12 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_Invalid_mode_expected_c_or_fortr, __pyx_t_4); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_12, 0, 0); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(1, 170, __pyx_L1_error) + } + __pyx_L11:; + + /* "View.MemoryView":172 + * raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" + * + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) # <<<<<<<<<<<<<< + * + * self.free_data = allocate_buffer +*/ + __pyx_v_self->len = __pyx_fill_contig_strides_array(__pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_itemsize, __pyx_v_self->ndim, __pyx_v_order); + + /* "View.MemoryView":174 + * self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) + * + * self.free_data = allocate_buffer # <<<<<<<<<<<<<< + * self.dtype_is_object = format == b'O' + * +*/ + __pyx_v_self->free_data = __pyx_v_allocate_buffer; + + /* "View.MemoryView":175 + * + * self.free_data = allocate_buffer + * self.dtype_is_object = format == b'O' # <<<<<<<<<<<<<< + * + * if allocate_buffer: +*/ + __pyx_t_12 = PyObject_RichCompare(__pyx_v_format, __pyx_mstate_global->__pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_12); if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_12); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 175, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __pyx_v_self->dtype_is_object = __pyx_t_3; + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + if (__pyx_v_allocate_buffer) { + + /* "View.MemoryView":178 + * + * if allocate_buffer: + * _allocate_buffer(self) # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_t_8 = __pyx_array_allocate_buffer(__pyx_v_self); if (unlikely(__pyx_t_8 == ((int)-1))) __PYX_ERR(1, 178, __pyx_L1_error) + + /* "View.MemoryView":177 + * self.dtype_is_object = format == b'O' + * + * if allocate_buffer: # <<<<<<<<<<<<<< + * _allocate_buffer(self) + * +*/ + } + + /* "View.MemoryView":129 + * cdef bint dtype_is_object + * + * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< + * mode="c", bint allocate_buffer=True): + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_format); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_array_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(((struct __pyx_array_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(struct __pyx_array_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_v_bufmode; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + char *__pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + Py_ssize_t *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":182 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 # <<<<<<<<<<<<<< + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": +*/ + __pyx_v_bufmode = -1; + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_t_1 = ((__pyx_v_flags & ((PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS) | PyBUF_ANY_CONTIGUOUS)) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_c, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 184, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":185 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + __pyx_v_bufmode = (PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":184 + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + * if self.mode == u"c": # <<<<<<<<<<<<<< + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_mstate_global->__pyx_n_u_fortran, Py_EQ)); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 186, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":187 + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." +*/ + __pyx_v_bufmode = (PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); + + /* "View.MemoryView":186 + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * elif self.mode == u"fortran": # <<<<<<<<<<<<<< + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): +*/ + } + __pyx_L4:; + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + __pyx_t_1 = (!((__pyx_v_flags & __pyx_v_bufmode) != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":189 + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." # <<<<<<<<<<<<<< + * info.buf = self.data + * info.len = self.len +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Can_only_create_a_buffer_that_is, 0, 0); + __PYX_ERR(1, 189, __pyx_L1_error) + + /* "View.MemoryView":188 + * elif self.mode == u"fortran": + * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + * if not (flags & bufmode): # <<<<<<<<<<<<<< + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data +*/ + } + + /* "View.MemoryView":183 + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 + * if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): # <<<<<<<<<<<<<< + * if self.mode == u"c": + * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS +*/ + } + + /* "View.MemoryView":190 + * if not (flags & bufmode): + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data # <<<<<<<<<<<<<< + * info.len = self.len + * +*/ + __pyx_t_2 = __pyx_v_self->data; + __pyx_v_info->buf = __pyx_t_2; + + /* "View.MemoryView":191 + * raise ValueError, "Can only create a buffer that is contiguous in memory." + * info.buf = self.data + * info.len = self.len # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + __pyx_t_3 = __pyx_v_self->len; + __pyx_v_info->len = __pyx_t_3; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":194 + * + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim # <<<<<<<<<<<<<< + * info.shape = self._shape + * info.strides = self._strides +*/ + __pyx_t_4 = __pyx_v_self->ndim; + __pyx_v_info->ndim = __pyx_t_4; + + /* "View.MemoryView":195 + * if flags & PyBUF_STRIDES: + * info.ndim = self.ndim + * info.shape = self._shape # <<<<<<<<<<<<<< + * info.strides = self._strides + * else: +*/ + __pyx_t_5 = __pyx_v_self->_shape; + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":196 + * info.ndim = self.ndim + * info.shape = self._shape + * info.strides = self._strides # <<<<<<<<<<<<<< + * else: + * info.ndim = 1 +*/ + __pyx_t_5 = __pyx_v_self->_strides; + __pyx_v_info->strides = __pyx_t_5; + + /* "View.MemoryView":193 + * info.len = self.len + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.ndim = self.ndim + * info.shape = self._shape +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":198 + * info.strides = self._strides + * else: + * info.ndim = 1 # <<<<<<<<<<<<<< + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL +*/ + /*else*/ { + __pyx_v_info->ndim = 1; + + /* "View.MemoryView":199 + * else: + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL # <<<<<<<<<<<<<< + * info.strides = NULL + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + __pyx_t_5 = (&__pyx_v_self->len); + } else { + __pyx_t_5 = NULL; + } + __pyx_v_info->shape = __pyx_t_5; + + /* "View.MemoryView":200 + * info.ndim = 1 + * info.shape = &self.len if flags & PyBUF_ND else NULL + * info.strides = NULL # <<<<<<<<<<<<<< + * + * info.suboffsets = NULL +*/ + __pyx_v_info->strides = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":202 + * info.strides = NULL + * + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * info.itemsize = self.itemsize + * info.readonly = 0 +*/ + __pyx_v_info->suboffsets = NULL; + + /* "View.MemoryView":203 + * + * info.suboffsets = NULL + * info.itemsize = self.itemsize # <<<<<<<<<<<<<< + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL +*/ + __pyx_t_3 = __pyx_v_self->itemsize; + __pyx_v_info->itemsize = __pyx_t_3; + + /* "View.MemoryView":204 + * info.suboffsets = NULL + * info.itemsize = self.itemsize + * info.readonly = 0 # <<<<<<<<<<<<<< + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self +*/ + __pyx_v_info->readonly = 0; + + /* "View.MemoryView":205 + * info.itemsize = self.itemsize + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + __pyx_t_2 = __pyx_v_self->format; + } else { + __pyx_t_2 = NULL; + } + __pyx_v_info->format = __pyx_t_2; + + /* "View.MemoryView":206 + * info.readonly = 0 + * info.format = self.format if flags & PyBUF_FORMAT else NULL + * info.obj = self # <<<<<<<<<<<<<< + * + * def __dealloc__(array self): +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":180 + * _allocate_buffer(self) + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * cdef int bufmode = -1 +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + +/* Python wrapper */ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_array___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + __pyx_t_1 = (__pyx_v_self->callback_free_data != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":210 + * def __dealloc__(array self): + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) # <<<<<<<<<<<<<< + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: +*/ + __pyx_v_self->callback_free_data(__pyx_v_self->data); + + /* "View.MemoryView":209 + * + * def __dealloc__(array self): + * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + if (__pyx_v_self->free_data) { + } else { + __pyx_t_1 = __pyx_v_self->free_data; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_self->data != NULL); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":213 + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) # <<<<<<<<<<<<<< + * free(self.data) + * PyObject_Free(self._shape) +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_self->data, __pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_self->ndim, 0); + + /* "View.MemoryView":212 + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) +*/ + } + + /* "View.MemoryView":214 + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) # <<<<<<<<<<<<<< + * PyObject_Free(self._shape) + * +*/ + free(__pyx_v_self->data); + + /* "View.MemoryView":211 + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) + * elif self.free_data and self.data is not NULL: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) +*/ + } + __pyx_L3:; + + /* "View.MemoryView":215 + * refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) + * free(self.data) + * PyObject_Free(self._shape) # <<<<<<<<<<<<<< + * + * @property +*/ + PyObject_Free(__pyx_v_self->_shape); + + /* "View.MemoryView":208 + * info.obj = self + * + * def __dealloc__(array self): # <<<<<<<<<<<<<< + * if self.callback_free_data != NULL: + * self.callback_free_data(self.data) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_5array_7memview___get__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":219 + * @property + * def memview(self): + * return self.get_memview() # <<<<<<<<<<<<<< + * + * @cname('get_memview') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":217 + * PyObject_Free(self._shape) + * + * @property # <<<<<<<<<<<<<< + * def memview(self): + * return self.get_memview() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.memview.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + +static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_memview", 0); + + /* "View.MemoryView":223 + * @cname('get_memview') + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE # <<<<<<<<<<<<<< + * return memoryview(self, flags, self.dtype_is_object) + * +*/ + __pyx_v_flags = ((PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT) | PyBUF_WRITABLE); + + /* "View.MemoryView":224 + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE + * return memoryview(self, flags, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, ((PyObject *)__pyx_v_self), __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 224, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_r = ((PyObject *)__pyx_t_1); + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":221 + * return self.get_memview() + * + * @cname('get_memview') # <<<<<<<<<<<<<< + * cdef get_memview(self): + * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array.get_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_array___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + + /* "View.MemoryView":227 + * + * def __len__(self): + * return self._shape[0] # <<<<<<<<<<<<<< + * + * def __getattr__(self, attr): +*/ + __pyx_r = (__pyx_v_self->_shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":226 + * return memoryview(self, flags, self.dtype_is_object) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self._shape[0] + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr); /*proto*/ +static PyObject *__pyx_array___getattr__(PyObject *__pyx_v_self, PyObject *__pyx_v_attr) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getattr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_attr)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_attr) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getattr__", 0); + + /* "View.MemoryView":230 + * + * def __getattr__(self, attr): + * return getattr(self.memview, attr) # <<<<<<<<<<<<<< + * + * def __getitem__(self, item): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 230, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":229 + * return self._shape[0] + * + * def __getattr__(self, attr): # <<<<<<<<<<<<<< + * return getattr(self.memview, attr) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getattr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item); /*proto*/ +static PyObject *__pyx_array___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":233 + * + * def __getitem__(self, item): + * return self.memview[item] # <<<<<<<<<<<<<< + * + * def __setitem__(self, item, value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":232 + * return getattr(self.memview, attr) + * + * def __getitem__(self, item): # <<<<<<<<<<<<<< + * return self.memview[item] + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.array.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + +/* Python wrapper */ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_array___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(((struct __pyx_array_obj *)__pyx_v_self), ((PyObject *)__pyx_v_item), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struct __pyx_array_obj *__pyx_v_self, PyObject *__pyx_v_item, PyObject *__pyx_v_value) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + + /* "View.MemoryView":236 + * + * def __setitem__(self, item, value): + * self.memview[item] = value # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely((PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0))) __PYX_ERR(1, 236, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":235 + * return self.memview[item] + * + * def __setitem__(self, item, value): # <<<<<<<<<<<<<< + * self.memview[item] = value + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.array.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_array___reduce_cython__(((struct __pyx_array_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array___reduce_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_array_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_array_2__setstate_cython__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct __pyx_array_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.array.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + +static int __pyx_array_allocate_buffer(struct __pyx_array_obj *__pyx_v_self) { + Py_ssize_t __pyx_v_i; + PyObject **__pyx_v_p; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":252 + * cdef PyObject **p + * + * self.free_data = True # <<<<<<<<<<<<<< + * self.data = malloc(self.len) + * if not self.data: +*/ + __pyx_v_self->free_data = 1; + + /* "View.MemoryView":253 + * + * self.free_data = True + * self.data = malloc(self.len) # <<<<<<<<<<<<<< + * if not self.data: + * raise MemoryError, "unable to allocate array data." +*/ + __pyx_v_self->data = ((char *)malloc(__pyx_v_self->len)); + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + __pyx_t_1 = (!(__pyx_v_self->data != 0)); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":255 + * self.data = malloc(self.len) + * if not self.data: + * raise MemoryError, "unable to allocate array data." # <<<<<<<<<<<<<< + * + * if self.dtype_is_object: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_MemoryError))), __pyx_mstate_global->__pyx_kp_u_unable_to_allocate_array_data, 0, 0); + __PYX_ERR(1, 255, __pyx_L1_error) + + /* "View.MemoryView":254 + * self.free_data = True + * self.data = malloc(self.len) + * if not self.data: # <<<<<<<<<<<<<< + * raise MemoryError, "unable to allocate array data." + * +*/ + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":258 + * + * if self.dtype_is_object: + * p = self.data # <<<<<<<<<<<<<< + * for i in range(self.len // self.itemsize): + * p[i] = Py_None +*/ + __pyx_v_p = ((PyObject **)__pyx_v_self->data); + + /* "View.MemoryView":259 + * if self.dtype_is_object: + * p = self.data + * for i in range(self.len // self.itemsize): # <<<<<<<<<<<<<< + * p[i] = Py_None + * Py_INCREF(Py_None) +*/ + if (unlikely(__pyx_v_self->itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_self->itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_self->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 259, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_div_Py_ssize_t(__pyx_v_self->len, __pyx_v_self->itemsize, 0); + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":260 + * p = self.data + * for i in range(self.len // self.itemsize): + * p[i] = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * return 0 +*/ + (__pyx_v_p[__pyx_v_i]) = Py_None; + + /* "View.MemoryView":261 + * for i in range(self.len // self.itemsize): + * p[i] = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * return 0 + * +*/ + Py_INCREF(Py_None); + } + + /* "View.MemoryView":257 + * raise MemoryError, "unable to allocate array data." + * + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * p = self.data + * for i in range(self.len // self.itemsize): +*/ + } + + /* "View.MemoryView":262 + * p[i] = Py_None + * Py_INCREF(Py_None) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":245 + * pass + * + * @cname("__pyx_array_allocate_buffer") # <<<<<<<<<<<<<< + * cdef int _allocate_buffer(array self) except -1: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._allocate_buffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + +static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize_t __pyx_v_itemsize, char *__pyx_v_format, char const *__pyx_v_c_mode, char *__pyx_v_buf) { + struct __pyx_array_obj *__pyx_v_result = 0; + PyObject *__pyx_v_mode = 0; + struct __pyx_array_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("array_cwrapper", 0); + + /* "View.MemoryView":268 + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. # <<<<<<<<<<<<<< + * + * if buf is NULL: +*/ + __pyx_t_2 = ((__pyx_v_c_mode[0]) == 'f'); + if (__pyx_t_2) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_fortran); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_fortran; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_n_u_c); + __pyx_t_1 = __pyx_mstate_global->__pyx_n_u_c; + } + __pyx_v_mode = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + __pyx_t_2 = (__pyx_v_buf == NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":271 + * + * if buf is NULL: + * result = array.__new__(array, shape, itemsize, format, mode) # <<<<<<<<<<<<<< + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) +*/ + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_3) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 271, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_4, NULL)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 271, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":270 + * cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. + * + * if buf is NULL: # <<<<<<<<<<<<<< + * result = array.__new__(array, shape, itemsize, format, mode) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":273 + * result = array.__new__(array, shape, itemsize, format, mode) + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) # <<<<<<<<<<<<<< + * result.data = buf + * +*/ + /*else*/ { + __pyx_t_3 = PyLong_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_shape); + __Pyx_GIVEREF(__pyx_v_shape); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_shape) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_3) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_mode); + __Pyx_GIVEREF(__pyx_v_mode); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_mode) != (0)) __PYX_ERR(1, 273, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_allocate_buffer, Py_False) < (0)) __PYX_ERR(1, 273, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_tp_new_array(((PyTypeObject *)__pyx_mstate_global->__pyx_array_type), __pyx_t_1, __pyx_t_4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 273, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":274 + * else: + * result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) + * result.data = buf # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->data = __pyx_v_buf; + } + __pyx_L3:; + + /* "View.MemoryView":276 + * result.data = buf + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":265 + * + * + * @cname("__pyx_array_new") # <<<<<<<<<<<<<< + * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, const char *c_mode, char *buf): + * cdef array result +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.array_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_mode); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + +/* Python wrapper */ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_name = 0; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 302, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(1, 302, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, i); __PYX_ERR(1, 302, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 302, __pyx_L3_error) + } + __pyx_v_name = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 302, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v_name); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v_name) { + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__", 0); + + /* "View.MemoryView":303 + * cdef object name + * def __init__(self, name): + * self.name = name # <<<<<<<<<<<<<< + * def __repr__(self): + * return self.name +*/ + __Pyx_INCREF(__pyx_v_name); + __Pyx_GIVEREF(__pyx_v_name); + __Pyx_GOTREF(__pyx_v_self->name); + __Pyx_DECREF(__pyx_v_self->name); + __pyx_v_self->name = __pyx_v_name; + + /* "View.MemoryView":302 + * cdef class Enum(object): + * cdef object name + * def __init__(self, name): # <<<<<<<<<<<<<< + * self.name = name + * def __repr__(self): +*/ + + /* function exit code */ + __pyx_r = 0; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_MemviewEnum___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":305 + * self.name = name + * def __repr__(self): + * return self.name # <<<<<<<<<<<<<< + * + * cdef generic = Enum("") +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->name); + __pyx_r = __pyx_v_self->name; + goto __pyx_L0; + + /* "View.MemoryView":304 + * def __init__(self, name): + * self.name = name + * def __repr__(self): # <<<<<<<<<<<<<< + * return self.name + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_MemviewEnum___reduce_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum___reduce_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self) { + PyObject *__pyx_v_state = 0; + PyObject *__pyx_v__dict = 0; + int __pyx_v_use_setstate; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":5 + * cdef object _dict + * cdef bint use_setstate + * state = (self.name,) # <<<<<<<<<<<<<< + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_self->name); + __Pyx_GIVEREF(__pyx_v_self->name); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->name) != (0)) __PYX_ERR(1, 5, __pyx_L1_error); + __pyx_v_state = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "(tree fragment)":6 + * cdef bint use_setstate + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) # <<<<<<<<<<<<<< + * if _dict is not None and _dict: + * state += (_dict,) +*/ + __pyx_t_1 = __Pyx_GetAttr3(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_dict, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v__dict = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + __pyx_t_3 = (__pyx_v__dict != Py_None); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v__dict); if (unlikely((__pyx_t_3 < 0))) __PYX_ERR(1, 7, __pyx_L1_error) + __pyx_t_2 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + if (__pyx_t_2) { + + /* "(tree fragment)":8 + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: + * state += (_dict,) # <<<<<<<<<<<<<< + * use_setstate = True + * else: +*/ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v__dict); + __Pyx_GIVEREF(__pyx_v__dict); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v__dict) != (0)) __PYX_ERR(1, 8, __pyx_L1_error); + __pyx_t_4 = PyNumber_InPlaceAdd(__pyx_v_state, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF_SET(__pyx_v_state, ((PyObject*)__pyx_t_4)); + __pyx_t_4 = 0; + + /* "(tree fragment)":9 + * if _dict is not None and _dict: + * state += (_dict,) + * use_setstate = True # <<<<<<<<<<<<<< + * else: + * use_setstate = ('self.name is not None',) +*/ + __pyx_v_use_setstate = 1; + + /* "(tree fragment)":7 + * state = (self.name,) + * _dict = getattr(self, '__dict__', None) + * if _dict is not None and _dict: # <<<<<<<<<<<<<< + * state += (_dict,) + * use_setstate = True +*/ + goto __pyx_L3; + } + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + /*else*/ { + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_mstate_global->__pyx_tuple[0]); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 11, __pyx_L1_error) + __pyx_v_use_setstate = __pyx_t_2; + } + __pyx_L3:; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + if (__pyx_v_use_setstate) { + + /* "(tree fragment)":13 + * use_setstate = ('self.name is not None',) + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state # <<<<<<<<<<<<<< + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, Py_None) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 13, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "(tree fragment)":12 + * else: + * use_setstate = ('self.name is not None',) + * if use_setstate: # <<<<<<<<<<<<<< + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: +*/ + } + + /* "(tree fragment)":15 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + __Pyx_GIVEREF(((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self)))) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_136983863); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_136983863); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_mstate_global->__pyx_int_136983863) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_state); + __Pyx_GIVEREF(__pyx_v_state); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_state) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_5) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_1) != (0)) __PYX_ERR(1, 15, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + } + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * cdef tuple state + * cdef object _dict +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.Enum.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_state); + __Pyx_XDECREF(__pyx_v__dict); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_MemviewEnum_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 16, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 16, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 16, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 16, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 16, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_MemviewEnum_2__setstate_cython__(((struct __pyx_MemviewEnum_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_MemviewEnum_obj *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":17 + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): + * __pyx_unpickle_Enum__set_state(self, __pyx_state) # <<<<<<<<<<<<<< +*/ + __pyx_t_1 = __pyx_v___pyx_state; + __Pyx_INCREF(__pyx_t_1); + if (!(likely(PyTuple_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None) || __Pyx_RaiseUnexpectedTypeError("tuple", __pyx_t_1))) __PYX_ERR(1, 17, __pyx_L1_error) + if (unlikely(__pyx_t_1 == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 17, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(__pyx_v_self, ((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 17, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":16 + * else: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, state) + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state(self, __pyx_state) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.Enum.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + +/* Python wrapper */ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + int __pyx_v_flags; + int __pyx_v_dtype_is_object; + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return -1; + #endif + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_obj,&__pyx_mstate_global->__pyx_n_u_flags,&__pyx_mstate_global->__pyx_n_u_dtype_is_object,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_VARARGS(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 348, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__cinit__", 0) < (0)) __PYX_ERR(1, 348, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, i); __PYX_ERR(1, 348, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_VARARGS(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 348, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_VARARGS(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 348, __pyx_L3_error) + values[0] = __Pyx_ArgRef_VARARGS(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 348, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_obj = values[0]; + __pyx_v_flags = __Pyx_PyLong_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + if (values[2]) { + __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 348, __pyx_L3_error) + } else { + __pyx_v_dtype_is_object = ((int)0); + } + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, __pyx_nargs); __PYX_ERR(1, 348, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_obj, __pyx_v_flags, __pyx_v_dtype_is_object); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj, int __pyx_v_flags, int __pyx_v_dtype_is_object) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + Py_intptr_t __pyx_t_4; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__cinit__", 0); + + /* "View.MemoryView":349 + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj # <<<<<<<<<<<<<< + * self.flags = flags + * if type(self) is memoryview or obj is not None: +*/ + __Pyx_INCREF(__pyx_v_obj); + __Pyx_GIVEREF(__pyx_v_obj); + __Pyx_GOTREF(__pyx_v_self->obj); + __Pyx_DECREF(__pyx_v_self->obj); + __pyx_v_self->obj = __pyx_v_obj; + + /* "View.MemoryView":350 + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): + * self.obj = obj + * self.flags = flags # <<<<<<<<<<<<<< + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) +*/ + __pyx_v_self->flags = __pyx_v_flags; + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + __pyx_t_2 = (((PyObject *)Py_TYPE(((PyObject *)__pyx_v_self))) == ((PyObject *)__pyx_mstate_global->__pyx_memoryview_type)); + if (!__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_obj != Py_None); + __pyx_t_1 = __pyx_t_2; + __pyx_L4_bool_binop_done:; + if (__pyx_t_1) { + + /* "View.MemoryView":352 + * self.flags = flags + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) # <<<<<<<<<<<<<< + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None +*/ + __pyx_t_3 = PyObject_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 352, __pyx_L1_error) + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_self->view.obj) == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":354 + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = Py_None; + + /* "View.MemoryView":355 + * if self.view.obj == NULL: + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":353 + * if type(self) is memoryview or obj is not None: + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &self.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + } + + /* "View.MemoryView":351 + * self.obj = obj + * self.flags = flags + * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< + * PyObject_GetBuffer(obj, &self.view, flags) + * if self.view.obj == NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + __pyx_t_1 = (!__PYX_CYTHON_ATOMICS_ENABLED()); + if (__pyx_t_1) { + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + __pyx_t_2 = (__pyx_memoryview_thread_locks_used < 8); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L9_bool_binop_done; + } + + /* "View.MemoryView":361 + * if (__pyx_memoryview_thread_locks_used < 8 and + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 +*/ + __pyx_t_2 = (!__PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()); + __pyx_t_1 = __pyx_t_2; + __pyx_L9_bool_binop_done:; + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":362 + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: +*/ + __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + + /* "View.MemoryView":363 + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); + + /* "View.MemoryView":359 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and # <<<<<<<<<<<<<< + * + * not __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING()): +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":365 + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< + * if self.lock is NULL: + * raise MemoryError +*/ + __pyx_v_self->lock = PyThread_allocate_lock(); + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + __pyx_t_1 = (__pyx_v_self->lock == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":367 + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 367, __pyx_L1_error) + + /* "View.MemoryView":366 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * +*/ + } + + /* "View.MemoryView":364 + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: +*/ + } + + /* "View.MemoryView":357 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if (__pyx_memoryview_thread_locks_used < 8 and +*/ + } + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":370 + * + * if flags & PyBUF_FORMAT: + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') # <<<<<<<<<<<<<< + * else: + * self.dtype_is_object = dtype_is_object +*/ + __pyx_t_2 = ((__pyx_v_self->view.format[0]) == 'O'); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_self->view.format[1]) == '\x00'); + __pyx_t_1 = __pyx_t_2; + __pyx_L14_bool_binop_done:; + __pyx_v_self->dtype_is_object = __pyx_t_1; + + /* "View.MemoryView":369 + * raise MemoryError + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":372 + * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') + * else: + * self.dtype_is_object = dtype_is_object # <<<<<<<<<<<<<< + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 +*/ + /*else*/ { + __pyx_v_self->dtype_is_object = __pyx_v_dtype_is_object; + } + __pyx_L13:; + + /* "View.MemoryView":374 + * self.dtype_is_object = dtype_is_object + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 # <<<<<<<<<<<<<< + * self.typeinfo = NULL + * +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_4 = ((Py_intptr_t)((void *)(&__pyx_v_self->acquisition_count))); + __pyx_t_5 = (sizeof(__pyx_atomic_int_type)); + if (unlikely(__pyx_t_5 == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 374, __pyx_L1_error) + } + __pyx_t_1 = ((__pyx_t_4 % __pyx_t_5) == 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 374, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 374, __pyx_L1_error) + #endif + + /* "View.MemoryView":375 + * + * assert (&self.acquisition_count) % sizeof(__pyx_atomic_int_type) == 0 + * self.typeinfo = NULL # <<<<<<<<<<<<<< + * + * def __dealloc__(memoryview self): +*/ + __pyx_v_self->typeinfo = NULL; + + /* "View.MemoryView":348 + * cdef const __Pyx_TypeInfo *typeinfo + * + * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< + * self.obj = obj + * self.flags = flags +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + +/* Python wrapper */ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryview___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__dealloc__(struct __pyx_memoryview_obj *__pyx_v_self) { + int __pyx_v_i; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + PyThread_type_lock __pyx_t_5; + PyThread_type_lock __pyx_t_6; + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + __pyx_t_1 = (__pyx_v_self->obj != Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":379 + * def __dealloc__(memoryview self): + * if self.obj is not None: + * PyBuffer_Release(&self.view) # <<<<<<<<<<<<<< + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * +*/ + PyBuffer_Release((&__pyx_v_self->view)); + + /* "View.MemoryView":378 + * + * def __dealloc__(memoryview self): + * if self.obj is not None: # <<<<<<<<<<<<<< + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + __pyx_t_1 = (((Py_buffer *)(&__pyx_v_self->view))->obj == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":382 + * elif (<__pyx_buffer *> &self.view).obj == Py_None: + * + * (<__pyx_buffer *> &self.view).obj = NULL # <<<<<<<<<<<<<< + * Py_DECREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_self->view))->obj = NULL; + + /* "View.MemoryView":383 + * + * (<__pyx_buffer *> &self.view).obj = NULL + * Py_DECREF(Py_None) # <<<<<<<<<<<<<< + * + * cdef int i +*/ + Py_DECREF(Py_None); + + /* "View.MemoryView":380 + * if self.obj is not None: + * PyBuffer_Release(&self.view) + * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< + * + * (<__pyx_buffer *> &self.view).obj = NULL +*/ + } + __pyx_L3:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + __pyx_t_1 = (__pyx_v_self->lock != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":388 + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): # <<<<<<<<<<<<<< + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 +*/ + __pyx_t_1 = __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING(); + if (__pyx_t_1) { + __pyx_t_2 = 0; + } else { + __pyx_t_2 = __pyx_memoryview_thread_locks_used; + } + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + __pyx_t_1 = ((__pyx_memoryview_thread_locks[__pyx_v_i]) == __pyx_v_self->lock); + if (__pyx_t_1) { + + /* "View.MemoryView":390 + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 # <<<<<<<<<<<<<< + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( +*/ + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used - 1); + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + __pyx_t_1 = (__pyx_v_i != __pyx_memoryview_thread_locks_used); + if (__pyx_t_1) { + + /* "View.MemoryView":393 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) # <<<<<<<<<<<<<< + * break + * else: +*/ + __pyx_t_5 = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); + __pyx_t_6 = (__pyx_memoryview_thread_locks[__pyx_v_i]); + + /* "View.MemoryView":392 + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break +*/ + (__pyx_memoryview_thread_locks[__pyx_v_i]) = __pyx_t_5; + (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]) = __pyx_t_6; + + /* "View.MemoryView":391 + * if __pyx_memoryview_thread_locks[i] is self.lock: + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) +*/ + } + + /* "View.MemoryView":394 + * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( + * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) + * break # <<<<<<<<<<<<<< + * else: + * PyThread_free_lock(self.lock) +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":389 + * if self.lock != NULL: + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used -= 1 + * if i != __pyx_memoryview_thread_locks_used: +*/ + } + } + /*else*/ { + + /* "View.MemoryView":396 + * break + * else: + * PyThread_free_lock(self.lock) # <<<<<<<<<<<<<< + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: +*/ + PyThread_free_lock(__pyx_v_self->lock); + } + __pyx_L6_break:; + + /* "View.MemoryView":387 + * cdef int i + * global __pyx_memoryview_thread_locks_used + * if self.lock != NULL: # <<<<<<<<<<<<<< + * for i in range(0 if __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() else __pyx_memoryview_thread_locks_used): + * if __pyx_memoryview_thread_locks[i] is self.lock: +*/ + } + + /* "View.MemoryView":377 + * self.typeinfo = NULL + * + * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< + * if self.obj is not None: + * PyBuffer_Release(&self.view) +*/ + + /* function exit code */ +} + +/* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + +static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + Py_ssize_t __pyx_v_dim; + char *__pyx_v_itemp; + PyObject *__pyx_v_idx = NULL; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + char *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_item_pointer", 0); + + /* "View.MemoryView":400 + * cdef char *get_item_pointer(memoryview self, object index) except NULL: + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf # <<<<<<<<<<<<<< + * + * for dim, idx in enumerate(index): +*/ + __pyx_v_itemp = ((char *)__pyx_v_self->view.buf); + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + __pyx_t_1 = 0; + if (likely(PyList_CheckExact(__pyx_v_index)) || PyTuple_CheckExact(__pyx_v_index)) { + __pyx_t_2 = __pyx_v_index; __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 402, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 402, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_3, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 402, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 402, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 402, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_v_dim = __pyx_t_1; + __pyx_t_1 = (__pyx_t_1 + 1); + + /* "View.MemoryView":403 + * + * for dim, idx in enumerate(index): + * itemp = pybuffer_index(&self.view, itemp, idx, dim) # <<<<<<<<<<<<<< + * + * return itemp +*/ + __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_7; + + /* "View.MemoryView":402 + * cdef char *itemp = self.view.buf + * + * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":405 + * itemp = pybuffer_index(&self.view, itemp, idx, dim) + * + * return itemp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_itemp; + goto __pyx_L0; + + /* "View.MemoryView":398 + * PyThread_free_lock(self.lock) + * + * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< + * cdef Py_ssize_t dim + * cdef char *itemp = self.view.buf +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.get_item_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index); /*proto*/ +static PyObject *__pyx_memoryview___getitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4__getitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_indices = NULL; + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + char *__pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__getitem__", 0); + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + __pyx_t_1 = (__pyx_v_index == __pyx_builtin_Ellipsis); + if (__pyx_t_1) { + + /* "View.MemoryView":410 + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: + * return self # <<<<<<<<<<<<<< + * + * have_slices, indices = _unellipsify(index, self.view.ndim) +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_self); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "View.MemoryView":409 + * + * def __getitem__(memoryview self, object index): + * if index is Ellipsis: # <<<<<<<<<<<<<< + * return self + * +*/ + } + + /* "View.MemoryView":412 + * return self + * + * have_slices, indices = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * cdef char *itemp +*/ + __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (likely(__pyx_t_2 != Py_None)) { + PyObject* sequence = __pyx_t_2; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 412, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_4); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + #endif + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 412, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_3; + __pyx_t_3 = 0; + __pyx_v_indices = __pyx_t_4; + __pyx_t_4 = 0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 415, __pyx_L1_error) + if (__pyx_t_1) { + + /* "View.MemoryView":416 + * cdef char *itemp + * if have_slices: + * return memview_slice(self, indices) # <<<<<<<<<<<<<< + * else: + * itemp = self.get_item_pointer(indices) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":415 + * + * cdef char *itemp + * if have_slices: # <<<<<<<<<<<<<< + * return memview_slice(self, indices) + * else: +*/ + } + + /* "View.MemoryView":418 + * return memview_slice(self, indices) + * else: + * itemp = self.get_item_pointer(indices) # <<<<<<<<<<<<<< + * return self.convert_item_to_object(itemp) + * +*/ + /*else*/ { + __pyx_t_5 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_5 == ((void *)NULL))) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_5; + + /* "View.MemoryView":419 + * else: + * itemp = self.get_item_pointer(indices) + * return self.convert_item_to_object(itemp) # <<<<<<<<<<<<<< + * + * def __setitem__(memoryview self, object index, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":408 + * + * + * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< + * if index is Ellipsis: + * return self +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_indices); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + +/* Python wrapper */ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value); /*proto*/ +static int __pyx_memoryview___setitem__(PyObject *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((PyObject *)__pyx_v_index), ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setitem__(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + PyObject *__pyx_v_have_slices = NULL; + PyObject *__pyx_v_obj = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setitem__", 0); + __Pyx_INCREF(__pyx_v_index); + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + if (unlikely(__pyx_v_self->view.readonly)) { + + /* "View.MemoryView":423 + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" # <<<<<<<<<<<<<< + * + * have_slices, index = _unellipsify(index, self.view.ndim) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_Cannot_assign_to_read_only_memor, 0, 0); + __PYX_ERR(1, 423, __pyx_L1_error) + + /* "View.MemoryView":422 + * + * def __setitem__(memoryview self, object index, object value): + * if self.view.readonly: # <<<<<<<<<<<<<< + * raise TypeError, "Cannot assign to read-only memoryview" + * +*/ + } + + /* "View.MemoryView":425 + * raise TypeError, "Cannot assign to read-only memoryview" + * + * have_slices, index = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< + * + * if have_slices: +*/ + __pyx_t_1 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (likely(__pyx_t_1 != Py_None)) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PyTuple_GET_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(1, 425, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_3); + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 425, __pyx_L1_error) + } + __pyx_v_have_slices = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_DECREF_SET(__pyx_v_index, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely((__pyx_t_4 < 0))) __PYX_ERR(1, 427, __pyx_L1_error) + if (__pyx_t_4) { + + /* "View.MemoryView":428 + * + * if have_slices: + * obj = self.is_slice(value) # <<<<<<<<<<<<<< + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 428, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_obj = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + __pyx_t_4 = (__pyx_v_obj != Py_None); + if (__pyx_t_4) { + + /* "View.MemoryView":430 + * obj = self.is_slice(value) + * if obj is not None: + * self.setitem_slice_assignment(self[index], obj) # <<<<<<<<<<<<<< + * else: + * self.setitem_slice_assign_scalar(self[index], value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_1, __pyx_v_obj); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 430, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "View.MemoryView":429 + * if have_slices: + * obj = self.is_slice(value) + * if obj is not None: # <<<<<<<<<<<<<< + * self.setitem_slice_assignment(self[index], obj) + * else: +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":432 + * self.setitem_slice_assignment(self[index], obj) + * else: + * self.setitem_slice_assign_scalar(self[index], value) # <<<<<<<<<<<<<< + * else: + * self.setitem_indexed(index, value) +*/ + /*else*/ { + __pyx_t_3 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 432, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_3), __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L5:; + + /* "View.MemoryView":427 + * have_slices, index = _unellipsify(index, self.view.ndim) + * + * if have_slices: # <<<<<<<<<<<<<< + * obj = self.is_slice(value) + * if obj is not None: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":434 + * self.setitem_slice_assign_scalar(self[index], value) + * else: + * self.setitem_indexed(index, value) # <<<<<<<<<<<<<< + * + * cdef is_slice(self, obj): +*/ + /*else*/ { + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __pyx_L4:; + + /* "View.MemoryView":421 + * return self.convert_item_to_object(itemp) + * + * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< + * if self.view.readonly: + * raise TypeError, "Cannot assign to read-only memoryview" +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_have_slices); + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + +static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_obj) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_slice", 0); + __Pyx_INCREF(__pyx_v_obj); + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_obj, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + /*try:*/ { + + /* "View.MemoryView":439 + * if not isinstance(obj, memoryview): + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< + * self.dtype_is_object) + * except TypeError: +*/ + __pyx_t_7 = NULL; + __pyx_t_8 = __Pyx_PyLong_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + + /* "View.MemoryView":440 + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) # <<<<<<<<<<<<<< + * except TypeError: + * return None +*/ + __pyx_t_9 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 440, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_7, __pyx_v_obj, __pyx_t_8, __pyx_t_9}; + __pyx_t_6 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_10, (4-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 439, __pyx_L4_error) + __Pyx_GOTREF((PyObject *)__pyx_t_6); + } + __Pyx_DECREF_SET(__pyx_v_obj, ((PyObject *)__pyx_t_6)); + __pyx_t_6 = 0; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L9_try_end; + __pyx_L4_error:; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":441 + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) + * except TypeError: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_11 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_TypeError)))); + if (__pyx_t_11) { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":442 + * self.dtype_is_object) + * except TypeError: + * return None # <<<<<<<<<<<<<< + * + * return obj +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L7_except_return; + } + goto __pyx_L6_except_error; + + /* "View.MemoryView":438 + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): + * try: # <<<<<<<<<<<<<< + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, + * self.dtype_is_object) +*/ + __pyx_L6_except_error:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L1_error; + __pyx_L7_except_return:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + goto __pyx_L0; + __pyx_L9_try_end:; + } + + /* "View.MemoryView":437 + * + * cdef is_slice(self, obj): + * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< + * try: + * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, +*/ + } + + /* "View.MemoryView":444 + * return None + * + * return obj # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assignment(self, dst, src): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_obj); + __pyx_r = __pyx_v_obj; + goto __pyx_L0; + + /* "View.MemoryView":436 + * self.setitem_indexed(index, value) + * + * cdef is_slice(self, obj): # <<<<<<<<<<<<<< + * if not isinstance(obj, memoryview): + * try: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_dst, PyObject *__pyx_v_src) { + __Pyx_memviewslice __pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_src_slice; + __Pyx_memviewslice __pyx_v_msrc; + __Pyx_memviewslice __pyx_v_mdst; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assignment", 0); + + /* "View.MemoryView":449 + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * +*/ + if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 449, __pyx_L1_error) + __pyx_v_msrc = (__pyx_t_1[0]); + + /* "View.MemoryView":450 + * cdef __Pyx_memviewslice src_slice + * cdef __Pyx_memviewslice msrc = get_slice_from_memview(src, &src_slice)[0] + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] # <<<<<<<<<<<<<< + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) +*/ + if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 450, __pyx_L1_error) + __pyx_v_mdst = (__pyx_t_1[0]); + + /* "View.MemoryView":452 + * cdef __Pyx_memviewslice mdst = get_slice_from_memview(dst, &dst_slice)[0] + * + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) # <<<<<<<<<<<<<< + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_mstate_global->__pyx_n_u_ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_2); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 452, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __pyx_memoryview_copy_contents(__pyx_v_msrc, __pyx_v_mdst, __pyx_t_3, __pyx_t_4, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 452, __pyx_L1_error) + + /* "View.MemoryView":446 + * return obj + * + * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice dst_slice + * cdef __Pyx_memviewslice src_slice +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assignment", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + +static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memoryview_obj *__pyx_v_self, struct __pyx_memoryview_obj *__pyx_v_dst, PyObject *__pyx_v_value) { + int __pyx_v_array[128]; + void *__pyx_v_tmp; + void *__pyx_v_item; + __Pyx_memviewslice *__pyx_v_dst_slice; + __Pyx_memviewslice __pyx_v_tmp_slice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + char const *__pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_slice_assign_scalar", 0); + + /* "View.MemoryView":456 + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): + * cdef int array[128] + * cdef void *tmp = NULL # <<<<<<<<<<<<<< + * cdef void *item + * +*/ + __pyx_v_tmp = NULL; + + /* "View.MemoryView":461 + * cdef __Pyx_memviewslice *dst_slice + * cdef __Pyx_memviewslice tmp_slice + * dst_slice = get_slice_from_memview(dst, &tmp_slice) # <<<<<<<<<<<<<< + * + * if self.view.itemsize > sizeof(array): +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 461, __pyx_L1_error) + __pyx_v_dst_slice = __pyx_t_1; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + __pyx_t_2 = (((size_t)__pyx_v_self->view.itemsize) > (sizeof(__pyx_v_array))); + if (__pyx_t_2) { + + /* "View.MemoryView":464 + * + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) # <<<<<<<<<<<<<< + * if tmp == NULL: + * raise MemoryError +*/ + __pyx_v_tmp = PyMem_Malloc(__pyx_v_self->view.itemsize); + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + __pyx_t_2 = (__pyx_v_tmp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":466 + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: + * raise MemoryError # <<<<<<<<<<<<<< + * item = tmp + * else: +*/ + PyErr_NoMemory(); __PYX_ERR(1, 466, __pyx_L1_error) + + /* "View.MemoryView":465 + * if self.view.itemsize > sizeof(array): + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * item = tmp +*/ + } + + /* "View.MemoryView":467 + * if tmp == NULL: + * raise MemoryError + * item = tmp # <<<<<<<<<<<<<< + * else: + * item = array +*/ + __pyx_v_item = __pyx_v_tmp; + + /* "View.MemoryView":463 + * dst_slice = get_slice_from_memview(dst, &tmp_slice) + * + * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< + * tmp = PyMem_Malloc(self.view.itemsize) + * if tmp == NULL: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":469 + * item = tmp + * else: + * item = array # <<<<<<<<<<<<<< + * + * try: +*/ + /*else*/ { + __pyx_v_item = ((void *)__pyx_v_array); + } + __pyx_L3:; + + /* "View.MemoryView":471 + * item = array + * + * try: # <<<<<<<<<<<<<< + * if self.dtype_is_object: + * ( item)[0] = value +*/ + /*try:*/ { + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + if (__pyx_v_self->dtype_is_object) { + + /* "View.MemoryView":473 + * try: + * if self.dtype_is_object: + * ( item)[0] = value # <<<<<<<<<<<<<< + * else: + * self.assign_item_from_object( item, value) +*/ + (((PyObject **)__pyx_v_item)[0]) = ((PyObject *)__pyx_v_value); + + /* "View.MemoryView":472 + * + * try: + * if self.dtype_is_object: # <<<<<<<<<<<<<< + * ( item)[0] = value + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":475 + * ( item)[0] = value + * else: + * self.assign_item_from_object( item, value) # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 475, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L8:; + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + __pyx_t_2 = (__pyx_v_self->view.suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":480 + * + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) # <<<<<<<<<<<<<< + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, + * item, self.dtype_is_object) +*/ + __pyx_t_4 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 480, __pyx_L6_error) + + /* "View.MemoryView":479 + * + * + * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, +*/ + } + + /* "View.MemoryView":481 + * if self.view.suboffsets != NULL: + * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) + * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, # <<<<<<<<<<<<<< + * item, self.dtype_is_object) + * finally: +*/ + __pyx_memoryview_slice_assign_scalar(__pyx_v_dst_slice, __pyx_v_dst->view.ndim, __pyx_v_self->view.itemsize, __pyx_v_item, __pyx_v_self->dtype_is_object); + } + + /* "View.MemoryView":484 + * item, self.dtype_is_object) + * finally: + * PyMem_Free(tmp) # <<<<<<<<<<<<<< + * + * cdef setitem_indexed(self, index, value): +*/ + /*finally:*/ { + /*normal exit:*/{ + PyMem_Free(__pyx_v_tmp); + goto __pyx_L7; + } + __pyx_L6_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_ExceptionSwap(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); + if ( unlikely(__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9) < 0)) __Pyx_ErrFetch(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_XGOTREF(__pyx_t_12); + __pyx_t_4 = __pyx_lineno; __pyx_t_5 = __pyx_clineno; __pyx_t_6 = __pyx_filename; + { + PyMem_Free(__pyx_v_tmp); + } + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ErrRestore(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_t_7 = 0; __pyx_t_8 = 0; __pyx_t_9 = 0; __pyx_t_10 = 0; __pyx_t_11 = 0; __pyx_t_12 = 0; + __pyx_lineno = __pyx_t_4; __pyx_clineno = __pyx_t_5; __pyx_filename = __pyx_t_6; + goto __pyx_L1_error; + } + __pyx_L7:; + } + + /* "View.MemoryView":454 + * memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) + * + * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< + * cdef int array[128] + * cdef void *tmp = NULL +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_slice_assign_scalar", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + +static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *__pyx_v_self, PyObject *__pyx_v_index, PyObject *__pyx_v_value) { + char *__pyx_v_itemp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("setitem_indexed", 0); + + /* "View.MemoryView":487 + * + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) # <<<<<<<<<<<<<< + * self.assign_item_from_object(itemp, value) + * +*/ + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 487, __pyx_L1_error) + __pyx_v_itemp = __pyx_t_1; + + /* "View.MemoryView":488 + * cdef setitem_indexed(self, index, value): + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 488, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":486 + * PyMem_Free(tmp) + * + * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< + * cdef char *itemp = self.get_item_pointer(index) + * self.assign_item_from_object(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.setitem_indexed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_v_struct = NULL; + PyObject *__pyx_v_bytesitem = 0; + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + int __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":493 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef bytes bytesitem + * +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 493, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":496 + * cdef bytes bytesitem + * + * bytesitem = itemp[:self.view.itemsize] # <<<<<<<<<<<<<< + * try: + * result = struct.unpack(self.view.format, bytesitem) +*/ + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 496, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_bytesitem = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_2, &__pyx_t_3, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + /*try:*/ { + + /* "View.MemoryView":498 + * bytesitem = itemp[:self.view.itemsize] + * try: + * result = struct.unpack(self.view.format, bytesitem) # <<<<<<<<<<<<<< + * except struct.error: + * raise ValueError, "Unable to convert item to object" +*/ + __pyx_t_5 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_5, __pyx_t_6, __pyx_v_bytesitem}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_unpack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 498, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_result = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + } + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + /*else:*/ { + __pyx_t_8 = __Pyx_ssize_strlen(__pyx_v_self->view.format); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 502, __pyx_L5_except_error) + __pyx_t_9 = (__pyx_t_8 == 1); + if (__pyx_t_9) { + + /* "View.MemoryView":503 + * else: + * if len(self.view.format) == 1: + * return result[0] # <<<<<<<<<<<<<< + * return result + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 503, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L6_except_return; + + /* "View.MemoryView":502 + * raise ValueError, "Unable to convert item to object" + * else: + * if len(self.view.format) == 1: # <<<<<<<<<<<<<< + * return result[0] + * return result +*/ + } + + /* "View.MemoryView":504 + * if len(self.view.format) == 1: + * return result[0] + * return result # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_result); + __pyx_r = __pyx_v_result; + goto __pyx_L6_except_return; + } + __pyx_L3_error:; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "View.MemoryView":499 + * try: + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: # <<<<<<<<<<<<<< + * raise ValueError, "Unable to convert item to object" + * else: +*/ + __Pyx_ErrFetch(&__pyx_t_1, &__pyx_t_6, &__pyx_t_5); + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_error); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_1, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_ErrRestore(__pyx_t_1, __pyx_t_6, __pyx_t_5); + __pyx_t_1 = 0; __pyx_t_6 = 0; __pyx_t_5 = 0; + if (__pyx_t_11) { + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_1) < 0) __PYX_ERR(1, 499, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_1); + + /* "View.MemoryView":500 + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: + * raise ValueError, "Unable to convert item to object" # <<<<<<<<<<<<<< + * else: + * if len(self.view.format) == 1: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Unable_to_convert_item_to_object, 0, 0); + __PYX_ERR(1, 500, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "View.MemoryView":497 + * + * bytesitem = itemp[:self.view.itemsize] + * try: # <<<<<<<<<<<<<< + * result = struct.unpack(self.view.format, bytesitem) + * except struct.error: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L1_error; + __pyx_L6_except_return:; + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); + goto __pyx_L0; + } + + /* "View.MemoryView":490 + * self.assign_item_from_object(itemp, value) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesitem); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + +static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryview_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_v_struct = NULL; + char __pyx_v_c; + PyObject *__pyx_v_bytesvalue = 0; + Py_ssize_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + Py_ssize_t __pyx_t_8; + PyObject *__pyx_t_9 = NULL; + char *__pyx_t_10; + char *__pyx_t_11; + Py_ssize_t __pyx_t_12; + char *__pyx_t_13; + char *__pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":509 + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" + * import struct # <<<<<<<<<<<<<< + * cdef char c + * cdef bytes bytesvalue +*/ + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_struct, 0, 0, NULL, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 509, __pyx_L1_error) + __pyx_t_1 = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_struct = __pyx_t_1; + __pyx_t_1 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + __pyx_t_3 = PyTuple_Check(__pyx_v_value); + if (__pyx_t_3) { + + /* "View.MemoryView":515 + * + * if isinstance(value, tuple): + * bytesvalue = struct.pack(self.view.format, *value) # <<<<<<<<<<<<<< + * else: + * bytesvalue = struct.pack(self.view.format, value) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_mstate_global->__pyx_n_u_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4) != (0)) __PYX_ERR(1, 515, __pyx_L1_error); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 515, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":514 + * cdef Py_ssize_t i + * + * if isinstance(value, tuple): # <<<<<<<<<<<<<< + * bytesvalue = struct.pack(self.view.format, *value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":517 + * bytesvalue = struct.pack(self.view.format, *value) + * else: + * bytesvalue = struct.pack(self.view.format, value) # <<<<<<<<<<<<<< + * + * for i, c in enumerate(bytesvalue): +*/ + /*else*/ { + __pyx_t_6 = __pyx_v_struct; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_6, __pyx_t_1, __pyx_v_value}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_pack, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None) || __Pyx_RaiseUnexpectedTypeError("bytes", __pyx_t_4))) __PYX_ERR(1, 517, __pyx_L1_error) + __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = 0; + if (unlikely(__pyx_v_bytesvalue == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); + __PYX_ERR(1, 519, __pyx_L1_error) + } + __Pyx_INCREF(__pyx_v_bytesvalue); + __pyx_t_9 = __pyx_v_bytesvalue; + __pyx_t_11 = __Pyx_PyBytes_AsWritableString(__pyx_t_9); if (unlikely(__pyx_t_11 == ((char *)NULL))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyBytes_GET_SIZE(__pyx_t_9); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(1, 519, __pyx_L1_error) + __pyx_t_13 = (__pyx_t_11 + __pyx_t_12); + for (__pyx_t_14 = __pyx_t_11; __pyx_t_14 < __pyx_t_13; __pyx_t_14++) { + __pyx_t_10 = __pyx_t_14; + __pyx_v_c = (__pyx_t_10[0]); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + __pyx_v_i = __pyx_t_8; + + /* "View.MemoryView":519 + * bytesvalue = struct.pack(self.view.format, value) + * + * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< + * itemp[i] = c + * +*/ + __pyx_t_8 = (__pyx_t_8 + 1); + + /* "View.MemoryView":520 + * + * for i, c in enumerate(bytesvalue): + * itemp[i] = c # <<<<<<<<<<<<<< + * + * @cname('getbuffer') +*/ + (__pyx_v_itemp[__pyx_v_i]) = __pyx_v_c; + } + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + + /* "View.MemoryView":506 + * return result + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * """Only used if instantiated manually by the user, or if Cython doesn't + * know how to convert the type""" +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("View.MemoryView.memoryview.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_struct); + __Pyx_XDECREF(__pyx_v_bytesvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + +/* Python wrapper */ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags); /*proto*/ +CYTHON_UNUSED static int __pyx_memoryview_getbuffer(PyObject *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__getbuffer__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(((struct __pyx_memoryview_obj *)__pyx_v_self), ((Py_buffer *)__pyx_v_info), ((int)__pyx_v_flags)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbuffer__(struct __pyx_memoryview_obj *__pyx_v_self, Py_buffer *__pyx_v_info, int __pyx_v_flags) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t *__pyx_t_3; + char *__pyx_t_4; + void *__pyx_t_5; + int __pyx_t_6; + Py_ssize_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + if (unlikely(__pyx_v_info == NULL)) { + PyErr_SetString(PyExc_BufferError, "PyObject_GetBuffer: view==NULL argument is obsolete"); + return -1; + } + __Pyx_RefNannySetupContext("__getbuffer__", 0); + __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(__pyx_v_info->obj); + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + __pyx_t_2 = ((__pyx_v_flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L4_bool_binop_done; + } + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_L4_bool_binop_done:; + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":525 + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: + * raise ValueError, "Cannot create writable memory view from read-only memoryview" # <<<<<<<<<<<<<< + * + * if flags & PyBUF_ND: +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Cannot_create_writable_memory_vi, 0, 0); + __PYX_ERR(1, 525, __pyx_L1_error) + + /* "View.MemoryView":524 + * @cname('getbuffer') + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * +*/ + } + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":528 + * + * if flags & PyBUF_ND: + * info.shape = self.view.shape # <<<<<<<<<<<<<< + * else: + * info.shape = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.shape; + __pyx_v_info->shape = __pyx_t_3; + + /* "View.MemoryView":527 + * raise ValueError, "Cannot create writable memory view from read-only memoryview" + * + * if flags & PyBUF_ND: # <<<<<<<<<<<<<< + * info.shape = self.view.shape + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":530 + * info.shape = self.view.shape + * else: + * info.shape = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_STRIDES: +*/ + /*else*/ { + __pyx_v_info->shape = NULL; + } + __pyx_L6:; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":533 + * + * if flags & PyBUF_STRIDES: + * info.strides = self.view.strides # <<<<<<<<<<<<<< + * else: + * info.strides = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.strides; + __pyx_v_info->strides = __pyx_t_3; + + /* "View.MemoryView":532 + * info.shape = NULL + * + * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< + * info.strides = self.view.strides + * else: +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":535 + * info.strides = self.view.strides + * else: + * info.strides = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_INDIRECT: +*/ + /*else*/ { + __pyx_v_info->strides = NULL; + } + __pyx_L7:; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_INDIRECT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":538 + * + * if flags & PyBUF_INDIRECT: + * info.suboffsets = self.view.suboffsets # <<<<<<<<<<<<<< + * else: + * info.suboffsets = NULL +*/ + __pyx_t_3 = __pyx_v_self->view.suboffsets; + __pyx_v_info->suboffsets = __pyx_t_3; + + /* "View.MemoryView":537 + * info.strides = NULL + * + * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< + * info.suboffsets = self.view.suboffsets + * else: +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":540 + * info.suboffsets = self.view.suboffsets + * else: + * info.suboffsets = NULL # <<<<<<<<<<<<<< + * + * if flags & PyBUF_FORMAT: +*/ + /*else*/ { + __pyx_v_info->suboffsets = NULL; + } + __pyx_L8:; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":543 + * + * if flags & PyBUF_FORMAT: + * info.format = self.view.format # <<<<<<<<<<<<<< + * else: + * info.format = NULL +*/ + __pyx_t_4 = __pyx_v_self->view.format; + __pyx_v_info->format = __pyx_t_4; + + /* "View.MemoryView":542 + * info.suboffsets = NULL + * + * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< + * info.format = self.view.format + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":545 + * info.format = self.view.format + * else: + * info.format = NULL # <<<<<<<<<<<<<< + * + * info.buf = self.view.buf +*/ + /*else*/ { + __pyx_v_info->format = NULL; + } + __pyx_L9:; + + /* "View.MemoryView":547 + * info.format = NULL + * + * info.buf = self.view.buf # <<<<<<<<<<<<<< + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize +*/ + __pyx_t_5 = __pyx_v_self->view.buf; + __pyx_v_info->buf = __pyx_t_5; + + /* "View.MemoryView":548 + * + * info.buf = self.view.buf + * info.ndim = self.view.ndim # <<<<<<<<<<<<<< + * info.itemsize = self.view.itemsize + * info.len = self.view.len +*/ + __pyx_t_6 = __pyx_v_self->view.ndim; + __pyx_v_info->ndim = __pyx_t_6; + + /* "View.MemoryView":549 + * info.buf = self.view.buf + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize # <<<<<<<<<<<<<< + * info.len = self.view.len + * info.readonly = self.view.readonly +*/ + __pyx_t_7 = __pyx_v_self->view.itemsize; + __pyx_v_info->itemsize = __pyx_t_7; + + /* "View.MemoryView":550 + * info.ndim = self.view.ndim + * info.itemsize = self.view.itemsize + * info.len = self.view.len # <<<<<<<<<<<<<< + * info.readonly = self.view.readonly + * info.obj = self +*/ + __pyx_t_7 = __pyx_v_self->view.len; + __pyx_v_info->len = __pyx_t_7; + + /* "View.MemoryView":551 + * info.itemsize = self.view.itemsize + * info.len = self.view.len + * info.readonly = self.view.readonly # <<<<<<<<<<<<<< + * info.obj = self + * +*/ + __pyx_t_1 = __pyx_v_self->view.readonly; + __pyx_v_info->readonly = __pyx_t_1; + + /* "View.MemoryView":552 + * info.len = self.view.len + * info.readonly = self.view.readonly + * info.obj = self # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF((PyObject *)__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_v_self); + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); + __pyx_v_info->obj = ((PyObject *)__pyx_v_self); + + /* "View.MemoryView":522 + * itemp[i] = c + * + * @cname('getbuffer') # <<<<<<<<<<<<<< + * def __getbuffer__(self, Py_buffer *info, int flags): + * if flags & PyBUF_WRITABLE and self.view.readonly: +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__getbuffer__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + if (__pyx_v_info->obj != NULL) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + goto __pyx_L2; + __pyx_L0:; + if (__pyx_v_info->obj == Py_None) { + __Pyx_GOTREF(__pyx_v_info->obj); + __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = 0; + } + __pyx_L2:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":557 + * @property + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) # <<<<<<<<<<<<<< + * transpose_memslice(&result.from_slice) + * return result +*/ + __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 557, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 557, __pyx_L1_error) + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":558 + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 558, __pyx_L1_error) + + /* "View.MemoryView":559 + * cdef _memoryviewslice result = memoryview_copy(self) + * transpose_memslice(&result.from_slice) + * return result # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":555 + * + * + * @property # <<<<<<<<<<<<<< + * def T(self): + * cdef _memoryviewslice result = memoryview_copy(self) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.T.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":563 + * @property + * def base(self): + * return self._get_base() # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->_get_base(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 563, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":561 + * return result + * + * @property # <<<<<<<<<<<<<< + * def base(self): + * return self._get_base() +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.base.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + +static PyObject *__pyx_memoryview__get_base(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":566 + * + * cdef _get_base(self): + * return self.obj # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->obj); + __pyx_r = __pyx_v_self->obj; + goto __pyx_L0; + + /* "View.MemoryView":565 + * return self._get_base() + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.obj + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_7genexpr__pyx_v_length; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":570 + * @property + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_7genexpr__pyx_v_length = (__pyx_t_2[0]); + __pyx_t_5 = PyLong_FromSsize_t(__pyx_7genexpr__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } /* exit inner scope */ + __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 570, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "View.MemoryView":568 + * return self.obj + * + * @property # <<<<<<<<<<<<<< + * def shape(self): + * return tuple([length for length in self.view.shape[:self.view.ndim]]) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.shape.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr1__pyx_v_stride; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + __pyx_t_1 = (__pyx_v_self->view.strides == NULL); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":576 + * if self.view.strides == NULL: + * + * raise ValueError, "Buffer view does not expose strides" # <<<<<<<<<<<<<< + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Buffer_view_does_not_expose_stri, 0, 0); + __PYX_ERR(1, 576, __pyx_L1_error) + + /* "View.MemoryView":574 + * @property + * def strides(self): + * if self.view.strides == NULL: # <<<<<<<<<<<<<< + * + * raise ValueError, "Buffer view does not expose strides" +*/ + } + + /* "View.MemoryView":578 + * raise ValueError, "Buffer view does not expose strides" + * + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.strides + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.strides; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr1__pyx_v_stride = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr1__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 578, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":572 + * return tuple([length for length in self.view.shape[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def strides(self): + * if self.view.strides == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.strides.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_8genexpr2__pyx_v_suboffset; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + Py_ssize_t *__pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + __pyx_t_1 = (__pyx_v_self->view.suboffsets == NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PySequence_Multiply(__pyx_mstate_global->__pyx_tuple[1], __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":582 + * @property + * def suboffsets(self): + * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< + * return (-1,) * self.view.ndim + * +*/ + } + + /* "View.MemoryView":585 + * return (-1,) * self.view.ndim + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + { /* enter inner scope */ + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (__pyx_v_self->view.suboffsets + __pyx_v_self->view.ndim); + for (__pyx_t_5 = __pyx_v_self->view.suboffsets; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { + __pyx_t_3 = __pyx_t_5; + __pyx_8genexpr2__pyx_v_suboffset = (__pyx_t_3[0]); + __pyx_t_6 = PyLong_FromSsize_t(__pyx_8genexpr2__pyx_v_suboffset); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } /* exit inner scope */ + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_6; + __pyx_t_6 = 0; + goto __pyx_L0; + + /* "View.MemoryView":580 + * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def suboffsets(self): + * if self.view.suboffsets == NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("View.MemoryView.memoryview.suboffsets.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":589 + * @property + * def ndim(self): + * return self.view.ndim # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 589, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":587 + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) + * + * @property # <<<<<<<<<<<<<< + * def ndim(self): + * return self.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.ndim.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":593 + * @property + * def itemsize(self): + * return self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 593, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":591 + * return self.view.ndim + * + * @property # <<<<<<<<<<<<<< + * def itemsize(self): + * return self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview.itemsize.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":597 + * @property + * def nbytes(self): + * return self.size * self.view.itemsize # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":595 + * return self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def nbytes(self): + * return self.size * self.view.itemsize +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.nbytes.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_v_result = NULL; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__get__", 0); + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + __pyx_t_1 = (__pyx_v_self->_size == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":602 + * def size(self): + * if self._size is None: + * result = 1 # <<<<<<<<<<<<<< + * + * for length in self.view.shape[:self.view.ndim]: +*/ + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_1); + __pyx_v_result = __pyx_mstate_global->__pyx_int_1; + + /* "View.MemoryView":604 + * result = 1 + * + * for length in self.view.shape[:self.view.ndim]: # <<<<<<<<<<<<<< + * result *= length + * +*/ + __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); + for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_t_5 = PyLong_FromSsize_t((__pyx_t_2[0])); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 604, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_5); + __pyx_t_5 = 0; + + /* "View.MemoryView":605 + * + * for length in self.view.shape[:self.view.ndim]: + * result *= length # <<<<<<<<<<<<<< + * + * self._size = result +*/ + __pyx_t_5 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 605, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF_SET(__pyx_v_result, __pyx_t_5); + __pyx_t_5 = 0; + } + + /* "View.MemoryView":607 + * result *= length + * + * self._size = result # <<<<<<<<<<<<<< + * + * return self._size +*/ + __Pyx_INCREF(__pyx_v_result); + __Pyx_GIVEREF(__pyx_v_result); + __Pyx_GOTREF(__pyx_v_self->_size); + __Pyx_DECREF(__pyx_v_self->_size); + __pyx_v_self->_size = __pyx_v_result; + + /* "View.MemoryView":601 + * @property + * def size(self): + * if self._size is None: # <<<<<<<<<<<<<< + * result = 1 + * +*/ + } + + /* "View.MemoryView":609 + * self._size = result + * + * return self._size # <<<<<<<<<<<<<< + * + * def __len__(self): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->_size); + __pyx_r = __pyx_v_self->_size; + goto __pyx_L0; + + /* "View.MemoryView":599 + * return self.size * self.view.itemsize + * + * @property # <<<<<<<<<<<<<< + * def size(self): + * if self._size is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.memoryview.size.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + +/* Python wrapper */ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_memoryview___len__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_10__len__(struct __pyx_memoryview_obj *__pyx_v_self) { + Py_ssize_t __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + __pyx_t_1 = (__pyx_v_self->view.ndim >= 1); + if (__pyx_t_1) { + + /* "View.MemoryView":613 + * def __len__(self): + * if self.view.ndim >= 1: + * return self.view.shape[0] # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_r = (__pyx_v_self->view.shape[0]); + goto __pyx_L0; + + /* "View.MemoryView":612 + * + * def __len__(self): + * if self.view.ndim >= 1: # <<<<<<<<<<<<<< + * return self.view.shape[0] + * +*/ + } + + /* "View.MemoryView":615 + * return self.view.shape[0] + * + * return 0 # <<<<<<<<<<<<<< + * + * def __repr__(self): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":611 + * return self._size + * + * def __len__(self): # <<<<<<<<<<<<<< + * if self.view.ndim >= 1: + * return self.view.shape[0] +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___repr__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12__repr__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[5]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__repr__", 0); + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":619 + * def __repr__(self): + * return "" % (self.base.__class__.__name__, + * id(self)) # <<<<<<<<<<<<<< + * + * def __str__(self): +*/ + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_Format(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_x); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_4[1] = __pyx_t_2; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_at_0x; + __pyx_t_4[3] = __pyx_t_3; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u__3; + + /* "View.MemoryView":618 + * + * def __repr__(self): + * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< + * id(self)) + * +*/ + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_4, 5, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2) | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":617 + * return 0 + * + * def __repr__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__, + * id(self)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_memoryview___str__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14__str__(struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3[3]; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__str__", 0); + + /* "View.MemoryView":622 + * + * def __str__(self): + * return "" % (self.base.__class__.__name__,) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_mstate_global->__pyx_n_u_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_FormatSimpleAndDecref(PyObject_Repr(__pyx_t_1), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3[0] = __pyx_mstate_global->__pyx_kp_u_MemoryView_of; + __pyx_t_3[1] = __pyx_t_2; + __pyx_t_3[2] = __pyx_mstate_global->__pyx_kp_u_object; + __pyx_t_1 = __Pyx_PyUnicode_Join(__pyx_t_3, 3, 15 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 8, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_2)); + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 622, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":621 + * id(self)) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return "" % (self.base.__class__.__name__,) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_c_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_c_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_c_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_c_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16is_c_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_c_contig", 0); + + /* "View.MemoryView":628 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 628, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":629 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'C', self.view.ndim) # <<<<<<<<<<<<<< + * + * def is_f_contig(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 629, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":625 + * + * + * def is_c_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_c_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_is_f_contig(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("is_f_contig (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("is_f_contig", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("is_f_contig", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18is_f_contig(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice *__pyx_v_mslice; + __Pyx_memviewslice __pyx_v_tmp; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice *__pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_f_contig", 0); + + /* "View.MemoryView":634 + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * +*/ + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((void *)NULL))) __PYX_ERR(1, 634, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":635 + * cdef __Pyx_memviewslice tmp + * mslice = get_slice_from_memview(self, &tmp) + * return slice_is_contig(mslice[0], 'F', self.view.ndim) # <<<<<<<<<<<<<< + * + * def copy(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 635, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":631 + * return slice_is_contig(mslice[0], 'C', self.view.ndim) + * + * def is_f_contig(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *mslice + * cdef __Pyx_memviewslice tmp +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.is_f_contig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20copy(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_mslice; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy", 0); + + /* "View.MemoryView":639 + * def copy(self): + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &mslice) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_F_CONTIGUOUS)); + + /* "View.MemoryView":641 + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS + * + * slice_copy(self, &mslice) # <<<<<<<<<<<<<< + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_mslice)); + + /* "View.MemoryView":642 + * + * slice_copy(self, &mslice) + * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_C_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char const *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 642, __pyx_L1_error) + __pyx_v_mslice = __pyx_t_1; + + /* "View.MemoryView":647 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &mslice) # <<<<<<<<<<<<<< + * + * def copy_fortran(self): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 647, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":637 + * return slice_is_contig(mslice[0], 'F', self.view.ndim) + * + * def copy(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice mslice + * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + +/* Python wrapper */ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_memoryview_copy_fortran(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("copy_fortran (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("copy_fortran", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("copy_fortran", __pyx_kwds); return NULL;} + __pyx_r = __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22copy_fortran(struct __pyx_memoryview_obj *__pyx_v_self) { + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + int __pyx_v_flags; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_memviewslice __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("copy_fortran", 0); + + /* "View.MemoryView":651 + * def copy_fortran(self): + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS # <<<<<<<<<<<<<< + * + * slice_copy(self, &src) +*/ + __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_C_CONTIGUOUS)); + + /* "View.MemoryView":653 + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS + * + * slice_copy(self, &src) # <<<<<<<<<<<<<< + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, + * self.view.itemsize, +*/ + __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_src)); + + /* "View.MemoryView":654 + * + * slice_copy(self, &src) + * dst = slice_copy_contig(&src, "fortran", self.view.ndim, # <<<<<<<<<<<<<< + * self.view.itemsize, + * flags|PyBUF_F_CONTIGUOUS, +*/ + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char const *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 654, __pyx_L1_error) + __pyx_v_dst = __pyx_t_1; + + /* "View.MemoryView":659 + * self.dtype_is_object) + * + * return memoryview_copy_from_slice(self, &dst) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 659, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":649 + * return memoryview_copy_from_slice(self, &mslice) + * + * def copy_fortran(self): # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice src, dst + * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.memoryview.copy_fortran", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryview___reduce_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryview_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryview_2__setstate_cython__(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryview_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.memoryview.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + +static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, int __pyx_v_dtype_is_object, __Pyx_TypeInfo const *__pyx_v_typeinfo) { + struct __pyx_memoryview_obj *__pyx_v_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_cwrapper", 0); + + /* "View.MemoryView":664 + * @cname('__pyx_memoryview_new') + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) # <<<<<<<<<<<<<< + * result.typeinfo = typeinfo + * return result +*/ + __pyx_t_2 = NULL; + __pyx_t_3 = __Pyx_PyLong_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[4] = {__pyx_t_2, __pyx_v_o, __pyx_t_3, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_memoryview_type, __pyx_callargs+__pyx_t_5, (4-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 664, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_1); + } + __pyx_v_result = ((struct __pyx_memoryview_obj *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":665 + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo # <<<<<<<<<<<<<< + * return result + * +*/ + __pyx_v_result->typeinfo = __pyx_v_typeinfo; + + /* "View.MemoryView":666 + * cdef memoryview result = memoryview(o, flags, dtype_is_object) + * result.typeinfo = typeinfo + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_check') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":662 + * + * + * @cname('__pyx_memoryview_new') # <<<<<<<<<<<<<< + * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, const __Pyx_TypeInfo *typeinfo): + * cdef memoryview result = memoryview(o, flags, dtype_is_object) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_cwrapper", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + +static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { + int __pyx_r; + int __pyx_t_1; + + /* "View.MemoryView":670 + * @cname('__pyx_memoryview_check') + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) # <<<<<<<<<<<<<< + * + * cdef tuple _unellipsify(object index, int ndim): +*/ + __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_o, __pyx_mstate_global->__pyx_memoryview_type); + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":668 + * return result + * + * @cname('__pyx_memoryview_check') # <<<<<<<<<<<<<< + * cdef inline bint memoryview_check(object o) noexcept: + * return isinstance(o, memoryview) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + +static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_idx; + PyObject *__pyx_v_tup = NULL; + PyObject *__pyx_v_result = NULL; + int __pyx_v_have_slices; + int __pyx_v_seen_ellipsis; + PyObject *__pyx_v_item = NULL; + Py_ssize_t __pyx_v_nslices; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6[3]; + PyObject *__pyx_t_7 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_unellipsify", 0); + + /* "View.MemoryView":678 + * """ + * cdef Py_ssize_t idx + * tup = index if isinstance(index, tuple) else (index,) # <<<<<<<<<<<<<< + * + * result = [slice(None)] * ndim +*/ + __pyx_t_2 = PyTuple_Check(__pyx_v_index); + if (__pyx_t_2) { + __Pyx_INCREF(((PyObject*)__pyx_v_index)); + __pyx_t_1 = __pyx_v_index; + } else { + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 678, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_index); + __Pyx_GIVEREF(__pyx_v_index); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_index) != (0)) __PYX_ERR(1, 678, __pyx_L1_error); + __pyx_t_1 = __pyx_t_3; + __pyx_t_3 = 0; + } + __pyx_v_tup = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_t_1 = PyList_New(1 * ((__pyx_v_ndim<0) ? 0:__pyx_v_ndim)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + { Py_ssize_t __pyx_temp; + for (__pyx_temp=0; __pyx_temp < __pyx_v_ndim; __pyx_temp++) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + if (__Pyx_PyList_SET_ITEM(__pyx_t_1, __pyx_temp, __pyx_mstate_global->__pyx_slice[0]) != (0)) __PYX_ERR(1, 680, __pyx_L1_error); + } + } + __pyx_v_result = ((PyObject*)__pyx_t_1); + __pyx_t_1 = 0; + + /* "View.MemoryView":681 + * + * result = [slice(None)] * ndim + * have_slices = False # <<<<<<<<<<<<<< + * seen_ellipsis = False + * idx = 0 +*/ + __pyx_v_have_slices = 0; + + /* "View.MemoryView":682 + * result = [slice(None)] * ndim + * have_slices = False + * seen_ellipsis = False # <<<<<<<<<<<<<< + * idx = 0 + * for item in tup: +*/ + __pyx_v_seen_ellipsis = 0; + + /* "View.MemoryView":683 + * have_slices = False + * seen_ellipsis = False + * idx = 0 # <<<<<<<<<<<<<< + * for item in tup: + * if item is Ellipsis: +*/ + __pyx_v_idx = 0; + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + __PYX_ERR(1, 684, __pyx_L1_error) + } + __pyx_t_1 = __pyx_v_tup; __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + for (;;) { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 684, __pyx_L1_error) + #endif + if (__pyx_t_4 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4)); + #else + __pyx_t_3 = __Pyx_PySequence_ITEM(__pyx_t_1, __pyx_t_4); + #endif + ++__pyx_t_4; + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 684, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_3); + __pyx_t_3 = 0; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + __pyx_t_2 = (__pyx_v_item == __pyx_builtin_Ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + __pyx_t_2 = (!__pyx_v_seen_ellipsis); + if (__pyx_t_2) { + + /* "View.MemoryView":687 + * if item is Ellipsis: + * if not seen_ellipsis: + * idx += ndim - len(tup) # <<<<<<<<<<<<<< + * seen_ellipsis = True + * have_slices = True +*/ + if (unlikely(__pyx_v_tup == Py_None)) { + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); + __PYX_ERR(1, 687, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_PyTuple_GET_SIZE(__pyx_v_tup); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 687, __pyx_L1_error) + __pyx_v_idx = (__pyx_v_idx + (__pyx_v_ndim - __pyx_t_5)); + + /* "View.MemoryView":688 + * if not seen_ellipsis: + * idx += ndim - len(tup) + * seen_ellipsis = True # <<<<<<<<<<<<<< + * have_slices = True + * else: +*/ + __pyx_v_seen_ellipsis = 1; + + /* "View.MemoryView":686 + * for item in tup: + * if item is Ellipsis: + * if not seen_ellipsis: # <<<<<<<<<<<<<< + * idx += ndim - len(tup) + * seen_ellipsis = True +*/ + } + + /* "View.MemoryView":689 + * idx += ndim - len(tup) + * seen_ellipsis = True + * have_slices = True # <<<<<<<<<<<<<< + * else: + * if isinstance(item, slice): +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":685 + * idx = 0 + * for item in tup: + * if item is Ellipsis: # <<<<<<<<<<<<<< + * if not seen_ellipsis: + * idx += ndim - len(tup) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + /*else*/ { + __pyx_t_2 = PySlice_Check(__pyx_v_item); + if (__pyx_t_2) { + + /* "View.MemoryView":692 + * else: + * if isinstance(item, slice): + * have_slices = True # <<<<<<<<<<<<<< + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" +*/ + __pyx_v_have_slices = 1; + + /* "View.MemoryView":691 + * have_slices = True + * else: + * if isinstance(item, slice): # <<<<<<<<<<<<<< + * have_slices = True + * elif not PyIndex_Check(item): +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + __pyx_t_2 = (!(PyIndex_Check(__pyx_v_item) != 0)); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":694 + * have_slices = True + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" # <<<<<<<<<<<<<< + * result[idx] = item + * idx += 1 +*/ + __pyx_t_3 = __Pyx_PyObject_FormatSimple(((PyObject *)Py_TYPE(__pyx_v_item)), __pyx_mstate_global->__pyx_empty_unicode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6[0] = __pyx_mstate_global->__pyx_kp_u_Cannot_index_with_type; + __pyx_t_6[1] = __pyx_t_3; + __pyx_t_6[2] = __pyx_mstate_global->__pyx_kp_u__4; + __pyx_t_7 = __Pyx_PyUnicode_Join(__pyx_t_6, 3, 24 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_3)); + if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 694, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_t_7, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 694, __pyx_L1_error) + + /* "View.MemoryView":693 + * if isinstance(item, slice): + * have_slices = True + * elif not PyIndex_Check(item): # <<<<<<<<<<<<<< + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item +*/ + } + __pyx_L7:; + + /* "View.MemoryView":695 + * elif not PyIndex_Check(item): + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item # <<<<<<<<<<<<<< + * idx += 1 + * +*/ + if (unlikely((__Pyx_SetItemInt(__pyx_v_result, __pyx_v_idx, __pyx_v_item, Py_ssize_t, 1, PyLong_FromSsize_t, 1, 1, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(1, 695, __pyx_L1_error) + } + __pyx_L5:; + + /* "View.MemoryView":696 + * raise TypeError, f"Cannot index with type '{type(item)}'" + * result[idx] = item + * idx += 1 # <<<<<<<<<<<<<< + * + * nslices = ndim - idx +*/ + __pyx_v_idx = (__pyx_v_idx + 1); + + /* "View.MemoryView":684 + * seen_ellipsis = False + * idx = 0 + * for item in tup: # <<<<<<<<<<<<<< + * if item is Ellipsis: + * if not seen_ellipsis: +*/ + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "View.MemoryView":698 + * idx += 1 + * + * nslices = ndim - idx # <<<<<<<<<<<<<< + * return have_slices or nslices, tuple(result) + * +*/ + __pyx_v_nslices = (__pyx_v_ndim - __pyx_v_idx); + + /* "View.MemoryView":699 + * + * nslices = ndim - idx + * return have_slices or nslices, tuple(result) # <<<<<<<<<<<<<< + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: +*/ + __Pyx_XDECREF(__pyx_r); + if (!__pyx_v_have_slices) { + } else { + __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_7 = PyLong_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __pyx_t_7; + __pyx_t_7 = 0; + __pyx_L9_bool_binop_done:; + __pyx_t_7 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_7) != (0)) __PYX_ERR(1, 699, __pyx_L1_error); + __pyx_t_1 = 0; + __pyx_t_7 = 0; + __pyx_r = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "View.MemoryView":672 + * return isinstance(o, memoryview) + * + * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< + * """ + * Replace all ellipses with full slices and fill incomplete indices with +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("View.MemoryView._unellipsify", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_tup); + __Pyx_XDECREF(__pyx_v_result); + __Pyx_XDECREF(__pyx_v_item); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + +static int assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_suboffset; + int __pyx_r; + Py_ssize_t *__pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "View.MemoryView":702 + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" +*/ + __pyx_t_2 = (__pyx_v_suboffsets + __pyx_v_ndim); + for (__pyx_t_3 = __pyx_v_suboffsets; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_suboffset = (__pyx_t_1[0]); + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + __pyx_t_4 = (__pyx_v_suboffset >= 0); + if (unlikely(__pyx_t_4)) { + + /* "View.MemoryView":704 + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" # <<<<<<<<<<<<<< + * return 0 # return type just used as an error flag + * +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_mstate_global->__pyx_kp_u_Indirect_dimensions_not_supporte, 0, 0); + __PYX_ERR(1, 704, __pyx_L1_error) + + /* "View.MemoryView":703 + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag +*/ + } + } + + /* "View.MemoryView":705 + * if suboffset >= 0: + * raise ValueError, "Indirect dimensions not supported" + * return 0 # return type just used as an error flag # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":701 + * return have_slices or nslices, tuple(result) + * + * cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: # <<<<<<<<<<<<<< + * for suboffset in suboffsets[:ndim]: + * if suboffset >= 0: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView.assert_direct_dimensions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + +static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_obj *__pyx_v_memview, PyObject *__pyx_v_indices) { + int __pyx_v_new_ndim; + int __pyx_v_suboffset_dim; + int __pyx_v_dim; + __Pyx_memviewslice __pyx_v_src; + __Pyx_memviewslice __pyx_v_dst; + __Pyx_memviewslice *__pyx_v_p_src; + struct __pyx_memoryviewslice_obj *__pyx_v_memviewsliceobj = 0; + __Pyx_memviewslice *__pyx_v_p_dst; + int *__pyx_v_p_suboffset_dim; + Py_ssize_t __pyx_v_start; + Py_ssize_t __pyx_v_stop; + Py_ssize_t __pyx_v_step; + Py_ssize_t __pyx_v_cindex; + int __pyx_v_have_start; + int __pyx_v_have_stop; + int __pyx_v_have_step; + PyObject *__pyx_v_index = NULL; + struct __pyx_memoryview_obj *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + struct __pyx_memoryview_obj *__pyx_t_3; + char *__pyx_t_4; + int __pyx_t_5; + Py_ssize_t __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + Py_ssize_t __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memview_slice", 0); + + /* "View.MemoryView":713 + * @cname('__pyx_memview_slice') + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim # <<<<<<<<<<<<<< + * cdef bint negative_step + * cdef __Pyx_memviewslice src, dst +*/ + __pyx_v_new_ndim = 0; + __pyx_v_suboffset_dim = -1; + + /* "View.MemoryView":720 + * + * + * memset(&dst, 0, sizeof(dst)) # <<<<<<<<<<<<<< + * + * cdef _memoryviewslice memviewsliceobj +*/ + (void)(memset((&__pyx_v_dst), 0, (sizeof(__pyx_v_dst)))); + + /* "View.MemoryView":724 + * cdef _memoryviewslice memviewsliceobj + * + * assert memview.view.ndim > 0 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_1 = (__pyx_v_memview->view.ndim > 0); + if (unlikely(!__pyx_t_1)) { + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_AssertionError))), 0, 0, 0); + __PYX_ERR(1, 724, __pyx_L1_error) + } + } + #else + if ((1)); else __PYX_ERR(1, 724, __pyx_L1_error) + #endif + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":727 + * + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview # <<<<<<<<<<<<<< + * p_src = &memviewsliceobj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 727, __pyx_L1_error) + __pyx_v_memviewsliceobj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":728 + * if isinstance(memview, _memoryviewslice): + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, &src) +*/ + __pyx_v_p_src = (&__pyx_v_memviewsliceobj->from_slice); + + /* "View.MemoryView":726 + * assert memview.view.ndim > 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * memviewsliceobj = memview + * p_src = &memviewsliceobj.from_slice +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":730 + * p_src = &memviewsliceobj.from_slice + * else: + * slice_copy(memview, &src) # <<<<<<<<<<<<<< + * p_src = &src + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_src)); + + /* "View.MemoryView":731 + * else: + * slice_copy(memview, &src) + * p_src = &src # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_p_src = (&__pyx_v_src); + } + __pyx_L3:; + + /* "View.MemoryView":737 + * + * + * dst.memview = p_src.memview # <<<<<<<<<<<<<< + * dst.data = p_src.data + * +*/ + __pyx_t_3 = __pyx_v_p_src->memview; + __pyx_v_dst.memview = __pyx_t_3; + + /* "View.MemoryView":738 + * + * dst.memview = p_src.memview + * dst.data = p_src.data # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_v_p_src->data; + __pyx_v_dst.data = __pyx_t_4; + + /* "View.MemoryView":743 + * + * + * cdef __Pyx_memviewslice *p_dst = &dst # <<<<<<<<<<<<<< + * cdef int *p_suboffset_dim = &suboffset_dim + * cdef Py_ssize_t start, stop, step, cindex +*/ + __pyx_v_p_dst = (&__pyx_v_dst); + + /* "View.MemoryView":744 + * + * cdef __Pyx_memviewslice *p_dst = &dst + * cdef int *p_suboffset_dim = &suboffset_dim # <<<<<<<<<<<<<< + * cdef Py_ssize_t start, stop, step, cindex + * cdef bint have_start, have_stop, have_step +*/ + __pyx_v_p_suboffset_dim = (&__pyx_v_suboffset_dim); + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_indices)) || PyTuple_CheckExact(__pyx_v_indices)) { + __pyx_t_2 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_6 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 748, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 748, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRefFast(__pyx_t_2, __pyx_t_6, __Pyx_ReferenceSharing_OwnStrongReference); + ++__pyx_t_6; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 748, __pyx_L1_error) + #endif + if (__pyx_t_6 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_6)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_6); + #endif + ++__pyx_t_6; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 748, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_2); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 748, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_index, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_dim = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + __pyx_t_1 = (PyIndex_Check(__pyx_v_index) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":750 + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): + * cindex = index # <<<<<<<<<<<<<< + * slice_memviewslice( + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], +*/ + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 750, __pyx_L1_error) + __pyx_v_cindex = __pyx_t_9; + + /* "View.MemoryView":751 + * if PyIndex_Check(index): + * cindex = index + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_cindex, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 751, __pyx_L1_error) + + /* "View.MemoryView":749 + * + * for dim, index in enumerate(indices): + * if PyIndex_Check(index): # <<<<<<<<<<<<<< + * cindex = index + * slice_memviewslice( +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + __pyx_t_1 = (__pyx_v_index == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":758 + * False) + * elif index is None: + * p_dst.shape[new_ndim] = 1 # <<<<<<<<<<<<<< + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 +*/ + (__pyx_v_p_dst->shape[__pyx_v_new_ndim]) = 1; + + /* "View.MemoryView":759 + * elif index is None: + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 # <<<<<<<<<<<<<< + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 +*/ + (__pyx_v_p_dst->strides[__pyx_v_new_ndim]) = 0; + + /* "View.MemoryView":760 + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 # <<<<<<<<<<<<<< + * new_ndim += 1 + * else: +*/ + (__pyx_v_p_dst->suboffsets[__pyx_v_new_ndim]) = -1L; + + /* "View.MemoryView":761 + * p_dst.strides[new_ndim] = 0 + * p_dst.suboffsets[new_ndim] = -1 + * new_ndim += 1 # <<<<<<<<<<<<<< + * else: + * start = index.start or 0 +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + + /* "View.MemoryView":757 + * 0, 0, 0, # have_{start,stop,step} + * False) + * elif index is None: # <<<<<<<<<<<<<< + * p_dst.shape[new_ndim] = 1 + * p_dst.strides[new_ndim] = 0 +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":763 + * new_ndim += 1 + * else: + * start = index.start or 0 # <<<<<<<<<<<<<< + * stop = index.stop or 0 + * step = index.step or 0 +*/ + /*else*/ { + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 763, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 763, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 763, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L7_bool_binop_done:; + __pyx_v_start = __pyx_t_9; + + /* "View.MemoryView":764 + * else: + * start = index.start or 0 + * stop = index.stop or 0 # <<<<<<<<<<<<<< + * step = index.step or 0 + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 764, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 764, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 764, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L9_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L9_bool_binop_done:; + __pyx_v_stop = __pyx_t_9; + + /* "View.MemoryView":765 + * start = index.start or 0 + * stop = index.stop or 0 + * step = index.step or 0 # <<<<<<<<<<<<<< + * + * have_start = index.start is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 765, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(1, 765, __pyx_L1_error) + if (!__pyx_t_1) { + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } else { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 765, __pyx_L1_error) + __pyx_t_9 = __pyx_t_11; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_9 = 0; + __pyx_L11_bool_binop_done:; + __pyx_v_step = __pyx_t_9; + + /* "View.MemoryView":767 + * step = index.step or 0 + * + * have_start = index.start is not None # <<<<<<<<<<<<<< + * have_stop = index.stop is not None + * have_step = index.step is not None +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_start); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 767, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_start = __pyx_t_1; + + /* "View.MemoryView":768 + * + * have_start = index.start is not None + * have_stop = index.stop is not None # <<<<<<<<<<<<<< + * have_step = index.step is not None + * +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_stop); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 768, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_stop = __pyx_t_1; + + /* "View.MemoryView":769 + * have_start = index.start is not None + * have_stop = index.stop is not None + * have_step = index.step is not None # <<<<<<<<<<<<<< + * + * slice_memviewslice( +*/ + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_mstate_global->__pyx_n_u_step); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_1 = (__pyx_t_8 != Py_None); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_have_step = __pyx_t_1; + + /* "View.MemoryView":771 + * have_step = index.step is not None + * + * slice_memviewslice( # <<<<<<<<<<<<<< + * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], + * dim, new_ndim, p_suboffset_dim, +*/ + __pyx_t_10 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_10 == ((int)-1))) __PYX_ERR(1, 771, __pyx_L1_error) + + /* "View.MemoryView":777 + * have_start, have_stop, have_step, + * True) + * new_ndim += 1 # <<<<<<<<<<<<<< + * + * if isinstance(memview, _memoryviewslice): +*/ + __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); + } + __pyx_L6:; + + /* "View.MemoryView":748 + * cdef bint have_start, have_stop, have_step + * + * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< + * if PyIndex_Check(index): + * cindex = index +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":781 + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, # <<<<<<<<<<<<<< + * memviewsliceobj.to_dtype_func, + * memview.dtype_is_object) +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 781, __pyx_L1_error) } + + /* "View.MemoryView":782 + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * else: +*/ + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 782, __pyx_L1_error) } + + /* "View.MemoryView":780 + * + * if isinstance(memview, _memoryviewslice): + * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< + * memviewsliceobj.to_object_func, + * memviewsliceobj.to_dtype_func, +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 780, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":779 + * new_ndim += 1 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * return memoryview_fromslice(dst, new_ndim, + * memviewsliceobj.to_object_func, +*/ + } + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + /*else*/ { + __Pyx_XDECREF((PyObject *)__pyx_r); + + /* "View.MemoryView":786 + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 785, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + + /* "View.MemoryView":785 + * memview.dtype_is_object) + * else: + * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< + * memview.dtype_is_object) + * +*/ + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryview_type))))) __PYX_ERR(1, 785, __pyx_L1_error) + __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_2); + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":711 + * + * + * @cname('__pyx_memview_slice') # <<<<<<<<<<<<<< + * cdef memoryview memview_slice(memoryview memview, object indices): + * cdef int new_ndim = 0, suboffset_dim = -1, dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("View.MemoryView.memview_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_memviewsliceobj); + __Pyx_XDECREF(__pyx_v_index); + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + +static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, Py_ssize_t __pyx_v_shape, Py_ssize_t __pyx_v_stride, Py_ssize_t __pyx_v_suboffset, int __pyx_v_dim, int __pyx_v_new_ndim, int *__pyx_v_suboffset_dim, Py_ssize_t __pyx_v_start, Py_ssize_t __pyx_v_stop, Py_ssize_t __pyx_v_step, int __pyx_v_have_start, int __pyx_v_have_stop, int __pyx_v_have_step, int __pyx_v_is_slice) { + Py_ssize_t __pyx_v_new_shape; + int __pyx_v_negative_step; + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + __pyx_t_1 = (!__pyx_v_is_slice); + if (__pyx_t_1) { + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + __pyx_t_1 = (__pyx_v_start < 0); + if (__pyx_t_1) { + + /* "View.MemoryView":817 + * + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":816 + * if not is_slice: + * + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if not 0 <= start < shape: +*/ + } + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + __pyx_t_1 = (0 <= __pyx_v_start); + if (__pyx_t_1) { + __pyx_t_1 = (__pyx_v_start < __pyx_v_shape); + } + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "View.MemoryView":819 + * start += shape + * if not 0 <= start < shape: + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_Index_out_of_bounds_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 819, __pyx_L1_error) + + /* "View.MemoryView":818 + * if start < 0: + * start += shape + * if not 0 <= start < shape: # <<<<<<<<<<<<<< + * _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":814 + * cdef bint negative_step + * + * if not is_slice: # <<<<<<<<<<<<<< + * + * if start < 0: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + /*else*/ { + __pyx_t_2 = (__pyx_v_have_step != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":823 + * + * if have_step: + * negative_step = step < 0 # <<<<<<<<<<<<<< + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) +*/ + __pyx_v_negative_step = (__pyx_v_step < 0); + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + __pyx_t_2 = (__pyx_v_step == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":825 + * negative_step = step < 0 + * if step == 0: + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) # <<<<<<<<<<<<<< + * else: + * negative_step = False +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Step_may_not_be_zero_axis_d, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 825, __pyx_L1_error) + + /* "View.MemoryView":824 + * if have_step: + * negative_step = step < 0 + * if step == 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: +*/ + } + + /* "View.MemoryView":822 + * else: + * + * if have_step: # <<<<<<<<<<<<<< + * negative_step = step < 0 + * if step == 0: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":827 + * _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + * else: + * negative_step = False # <<<<<<<<<<<<<< + * step = 1 + * +*/ + /*else*/ { + __pyx_v_negative_step = 0; + + /* "View.MemoryView":828 + * else: + * negative_step = False + * step = 1 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_step = 1; + } + __pyx_L6:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + __pyx_t_2 = (__pyx_v_have_start != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":833 + * if have_start: + * if start < 0: + * start += shape # <<<<<<<<<<<<<< + * if start < 0: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_start + __pyx_v_shape); + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + __pyx_t_2 = (__pyx_v_start < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":835 + * start += shape + * if start < 0: + * start = 0 # <<<<<<<<<<<<<< + * elif start >= shape: + * if negative_step: +*/ + __pyx_v_start = 0; + + /* "View.MemoryView":834 + * if start < 0: + * start += shape + * if start < 0: # <<<<<<<<<<<<<< + * start = 0 + * elif start >= shape: +*/ + } + + /* "View.MemoryView":832 + * + * if have_start: + * if start < 0: # <<<<<<<<<<<<<< + * start += shape + * if start < 0: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + __pyx_t_2 = (__pyx_v_start >= __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + if (__pyx_v_negative_step) { + + /* "View.MemoryView":838 + * elif start >= shape: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = shape +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":837 + * start = 0 + * elif start >= shape: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L11; + } + + /* "View.MemoryView":840 + * start = shape - 1 + * else: + * start = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + /*else*/ { + __pyx_v_start = __pyx_v_shape; + } + __pyx_L11:; + + /* "View.MemoryView":836 + * if start < 0: + * start = 0 + * elif start >= shape: # <<<<<<<<<<<<<< + * if negative_step: + * start = shape - 1 +*/ + } + __pyx_L9:; + + /* "View.MemoryView":831 + * + * + * if have_start: # <<<<<<<<<<<<<< + * if start < 0: + * start += shape +*/ + goto __pyx_L8; + } + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":843 + * else: + * if negative_step: + * start = shape - 1 # <<<<<<<<<<<<<< + * else: + * start = 0 +*/ + __pyx_v_start = (__pyx_v_shape - 1); + + /* "View.MemoryView":842 + * start = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * start = shape - 1 + * else: +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":845 + * start = shape - 1 + * else: + * start = 0 # <<<<<<<<<<<<<< + * + * if have_stop: +*/ + /*else*/ { + __pyx_v_start = 0; + } + __pyx_L12:; + } + __pyx_L8:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + __pyx_t_2 = (__pyx_v_have_stop != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":849 + * if have_stop: + * if stop < 0: + * stop += shape # <<<<<<<<<<<<<< + * if stop < 0: + * stop = 0 +*/ + __pyx_v_stop = (__pyx_v_stop + __pyx_v_shape); + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + __pyx_t_2 = (__pyx_v_stop < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":851 + * stop += shape + * if stop < 0: + * stop = 0 # <<<<<<<<<<<<<< + * elif stop > shape: + * stop = shape +*/ + __pyx_v_stop = 0; + + /* "View.MemoryView":850 + * if stop < 0: + * stop += shape + * if stop < 0: # <<<<<<<<<<<<<< + * stop = 0 + * elif stop > shape: +*/ + } + + /* "View.MemoryView":848 + * + * if have_stop: + * if stop < 0: # <<<<<<<<<<<<<< + * stop += shape + * if stop < 0: +*/ + goto __pyx_L14; + } + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + __pyx_t_2 = (__pyx_v_stop > __pyx_v_shape); + if (__pyx_t_2) { + + /* "View.MemoryView":853 + * stop = 0 + * elif stop > shape: + * stop = shape # <<<<<<<<<<<<<< + * else: + * if negative_step: +*/ + __pyx_v_stop = __pyx_v_shape; + + /* "View.MemoryView":852 + * if stop < 0: + * stop = 0 + * elif stop > shape: # <<<<<<<<<<<<<< + * stop = shape + * else: +*/ + } + __pyx_L14:; + + /* "View.MemoryView":847 + * start = 0 + * + * if have_stop: # <<<<<<<<<<<<<< + * if stop < 0: + * stop += shape +*/ + goto __pyx_L13; + } + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + /*else*/ { + if (__pyx_v_negative_step) { + + /* "View.MemoryView":856 + * else: + * if negative_step: + * stop = -1 # <<<<<<<<<<<<<< + * else: + * stop = shape +*/ + __pyx_v_stop = -1L; + + /* "View.MemoryView":855 + * stop = shape + * else: + * if negative_step: # <<<<<<<<<<<<<< + * stop = -1 + * else: +*/ + goto __pyx_L16; + } + + /* "View.MemoryView":858 + * stop = -1 + * else: + * stop = shape # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __pyx_v_stop = __pyx_v_shape; + } + __pyx_L16:; + } + __pyx_L13:; + + /* "View.MemoryView":862 + * + * with cython.cdivision(True): + * new_shape = (stop - start) // step # <<<<<<<<<<<<<< + * + * if (stop - start) - step * new_shape: +*/ + __pyx_v_new_shape = ((__pyx_v_stop - __pyx_v_start) / __pyx_v_step); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + __pyx_t_2 = (((__pyx_v_stop - __pyx_v_start) - (__pyx_v_step * __pyx_v_new_shape)) != 0); + if (__pyx_t_2) { + + /* "View.MemoryView":865 + * + * if (stop - start) - step * new_shape: + * new_shape += 1 # <<<<<<<<<<<<<< + * + * if new_shape < 0: +*/ + __pyx_v_new_shape = (__pyx_v_new_shape + 1); + + /* "View.MemoryView":864 + * new_shape = (stop - start) // step + * + * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< + * new_shape += 1 + * +*/ + } + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + __pyx_t_2 = (__pyx_v_new_shape < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":868 + * + * if new_shape < 0: + * new_shape = 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_new_shape = 0; + + /* "View.MemoryView":867 + * new_shape += 1 + * + * if new_shape < 0: # <<<<<<<<<<<<<< + * new_shape = 0 + * +*/ + } + + /* "View.MemoryView":871 + * + * + * dst.strides[new_ndim] = stride * step # <<<<<<<<<<<<<< + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset +*/ + (__pyx_v_dst->strides[__pyx_v_new_ndim]) = (__pyx_v_stride * __pyx_v_step); + + /* "View.MemoryView":872 + * + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape # <<<<<<<<<<<<<< + * dst.suboffsets[new_ndim] = suboffset + * +*/ + (__pyx_v_dst->shape[__pyx_v_new_ndim]) = __pyx_v_new_shape; + + /* "View.MemoryView":873 + * dst.strides[new_ndim] = stride * step + * dst.shape[new_ndim] = new_shape + * dst.suboffsets[new_ndim] = suboffset # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_dst->suboffsets[__pyx_v_new_ndim]) = __pyx_v_suboffset; + } + __pyx_L3:; + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + __pyx_t_2 = ((__pyx_v_suboffset_dim[0]) < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":877 + * + * if suboffset_dim[0] < 0: + * dst.data += start * stride # <<<<<<<<<<<<<< + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride +*/ + __pyx_v_dst->data = (__pyx_v_dst->data + (__pyx_v_start * __pyx_v_stride)); + + /* "View.MemoryView":876 + * + * + * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< + * dst.data += start * stride + * else: +*/ + goto __pyx_L19; + } + + /* "View.MemoryView":879 + * dst.data += start * stride + * else: + * dst.suboffsets[suboffset_dim[0]] += start * stride # <<<<<<<<<<<<<< + * + * if suboffset >= 0: +*/ + /*else*/ { + __pyx_t_3 = (__pyx_v_suboffset_dim[0]); + (__pyx_v_dst->suboffsets[__pyx_t_3]) = ((__pyx_v_dst->suboffsets[__pyx_t_3]) + (__pyx_v_start * __pyx_v_stride)); + } + __pyx_L19:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + __pyx_t_2 = (!__pyx_v_is_slice); + if (__pyx_t_2) { + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + __pyx_t_2 = (__pyx_v_new_ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":884 + * if not is_slice: + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset # <<<<<<<<<<<<<< + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " +*/ + __pyx_v_dst->data = ((((char **)__pyx_v_dst->data)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":883 + * if suboffset >= 0: + * if not is_slice: + * if new_ndim == 0: # <<<<<<<<<<<<<< + * dst.data = ( dst.data)[0] + suboffset + * else: +*/ + goto __pyx_L22; + } + + /* "View.MemoryView":886 + * dst.data = ( dst.data)[0] + suboffset + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " # <<<<<<<<<<<<<< + * "must be indexed and not sliced", dim) + * else: +*/ + /*else*/ { + + /* "View.MemoryView":887 + * else: + * _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " + * "must be indexed and not sliced", dim) # <<<<<<<<<<<<<< + * else: + * suboffset_dim[0] = new_ndim +*/ + __pyx_t_3 = __pyx_memoryview_err_dim(PyExc_IndexError, __pyx_mstate_global->__pyx_kp_u_All_dimensions_preceding_dimensi, __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 886, __pyx_L1_error) + } + __pyx_L22:; + + /* "View.MemoryView":882 + * + * if suboffset >= 0: + * if not is_slice: # <<<<<<<<<<<<<< + * if new_ndim == 0: + * dst.data = ( dst.data)[0] + suboffset +*/ + goto __pyx_L21; + } + + /* "View.MemoryView":889 + * "must be indexed and not sliced", dim) + * else: + * suboffset_dim[0] = new_ndim # <<<<<<<<<<<<<< + * + * return 0 +*/ + /*else*/ { + (__pyx_v_suboffset_dim[0]) = __pyx_v_new_ndim; + } + __pyx_L21:; + + /* "View.MemoryView":881 + * dst.suboffsets[suboffset_dim[0]] += start * stride + * + * if suboffset >= 0: # <<<<<<<<<<<<<< + * if not is_slice: + * if new_ndim == 0: +*/ + } + + /* "View.MemoryView":891 + * suboffset_dim[0] = new_ndim + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":793 + * + * + * @cname('__pyx_memoryview_slice_memviewslice') # <<<<<<<<<<<<<< + * cdef int slice_memviewslice( + * __Pyx_memviewslice *dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.slice_memviewslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + +static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, Py_ssize_t __pyx_v_index, Py_ssize_t __pyx_v_dim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_suboffset; + Py_ssize_t __pyx_v_itemsize; + char *__pyx_v_resultp; + char *__pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[3]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("pybuffer_index", 0); + + /* "View.MemoryView":899 + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 # <<<<<<<<<<<<<< + * cdef Py_ssize_t itemsize = view.itemsize + * cdef char *resultp +*/ + __pyx_v_suboffset = -1L; + + /* "View.MemoryView":900 + * Py_ssize_t dim) except NULL: + * cdef Py_ssize_t shape, stride, suboffset = -1 + * cdef Py_ssize_t itemsize = view.itemsize # <<<<<<<<<<<<<< + * cdef char *resultp + * +*/ + __pyx_t_1 = __pyx_v_view->itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + __pyx_t_2 = (__pyx_v_view->ndim == 0); + if (__pyx_t_2) { + + /* "View.MemoryView":904 + * + * if view.ndim == 0: + * shape = view.len // itemsize # <<<<<<<<<<<<<< + * stride = itemsize + * else: +*/ + if (unlikely(__pyx_v_itemsize == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(__pyx_v_view->len))) { + PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); + __PYX_ERR(1, 904, __pyx_L1_error) + } + __pyx_v_shape = __Pyx_div_Py_ssize_t(__pyx_v_view->len, __pyx_v_itemsize, 0); + + /* "View.MemoryView":905 + * if view.ndim == 0: + * shape = view.len // itemsize + * stride = itemsize # <<<<<<<<<<<<<< + * else: + * shape = view.shape[dim] +*/ + __pyx_v_stride = __pyx_v_itemsize; + + /* "View.MemoryView":903 + * cdef char *resultp + * + * if view.ndim == 0: # <<<<<<<<<<<<<< + * shape = view.len // itemsize + * stride = itemsize +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":907 + * stride = itemsize + * else: + * shape = view.shape[dim] # <<<<<<<<<<<<<< + * stride = view.strides[dim] + * if view.suboffsets != NULL: +*/ + /*else*/ { + __pyx_v_shape = (__pyx_v_view->shape[__pyx_v_dim]); + + /* "View.MemoryView":908 + * else: + * shape = view.shape[dim] + * stride = view.strides[dim] # <<<<<<<<<<<<<< + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] +*/ + __pyx_v_stride = (__pyx_v_view->strides[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + __pyx_t_2 = (__pyx_v_view->suboffsets != NULL); + if (__pyx_t_2) { + + /* "View.MemoryView":910 + * stride = view.strides[dim] + * if view.suboffsets != NULL: + * suboffset = view.suboffsets[dim] # <<<<<<<<<<<<<< + * + * if index < 0: +*/ + __pyx_v_suboffset = (__pyx_v_view->suboffsets[__pyx_v_dim]); + + /* "View.MemoryView":909 + * shape = view.shape[dim] + * stride = view.strides[dim] + * if view.suboffsets != NULL: # <<<<<<<<<<<<<< + * suboffset = view.suboffsets[dim] + * +*/ + } + } + __pyx_L3:; + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (__pyx_t_2) { + + /* "View.MemoryView":913 + * + * if index < 0: + * index += view.shape[dim] # <<<<<<<<<<<<<< + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" +*/ + __pyx_v_index = (__pyx_v_index + (__pyx_v_view->shape[__pyx_v_dim])); + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index < 0); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":915 + * index += view.shape[dim] + * if index < 0: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * if index >= shape: +*/ + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_3; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 915, __pyx_L1_error) + + /* "View.MemoryView":914 + * if index < 0: + * index += view.shape[dim] + * if index < 0: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":912 + * suboffset = view.suboffsets[dim] + * + * if index < 0: # <<<<<<<<<<<<<< + * index += view.shape[dim] + * if index < 0: +*/ + } + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + __pyx_t_2 = (__pyx_v_index >= __pyx_v_shape); + if (unlikely(__pyx_t_2)) { + + /* "View.MemoryView":918 + * + * if index >= shape: + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" # <<<<<<<<<<<<<< + * + * resultp = bufp + index * stride +*/ + __pyx_t_5 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_dim, 0, ' ', 'd'); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_Out_of_bounds_on_buffer_access_a; + __pyx_t_4[1] = __pyx_t_5; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_3 = __Pyx_PyUnicode_Join(__pyx_t_4, 3, 37 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_5) + 1, 127); + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 918, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_IndexError))), __pyx_t_3, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(1, 918, __pyx_L1_error) + + /* "View.MemoryView":917 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * if index >= shape: # <<<<<<<<<<<<<< + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * +*/ + } + + /* "View.MemoryView":920 + * raise IndexError, f"Out of bounds on buffer access (axis {dim})" + * + * resultp = bufp + index * stride # <<<<<<<<<<<<<< + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset +*/ + __pyx_v_resultp = (__pyx_v_bufp + (__pyx_v_index * __pyx_v_stride)); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + __pyx_t_2 = (__pyx_v_suboffset >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":922 + * resultp = bufp + index * stride + * if suboffset >= 0: + * resultp = ( resultp)[0] + suboffset # <<<<<<<<<<<<<< + * + * return resultp +*/ + __pyx_v_resultp = ((((char **)__pyx_v_resultp)[0]) + __pyx_v_suboffset); + + /* "View.MemoryView":921 + * + * resultp = bufp + index * stride + * if suboffset >= 0: # <<<<<<<<<<<<<< + * resultp = ( resultp)[0] + suboffset + * +*/ + } + + /* "View.MemoryView":924 + * resultp = ( resultp)[0] + suboffset + * + * return resultp # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_resultp; + goto __pyx_L0; + + /* "View.MemoryView":896 + * + * + * @cname('__pyx_pybuffer_index') # <<<<<<<<<<<<<< + * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, + * Py_ssize_t dim) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView.pybuffer_index", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + +static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { + int __pyx_v_ndim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + int __pyx_v_i; + int __pyx_v_j; + int __pyx_r; + int __pyx_t_1; + Py_ssize_t *__pyx_t_2; + long __pyx_t_3; + long __pyx_t_4; + Py_ssize_t __pyx_t_5; + Py_ssize_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":931 + * @cname('__pyx_memslice_transpose') + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim # <<<<<<<<<<<<<< + * + * cdef Py_ssize_t *shape = memslice.shape +*/ + __pyx_t_1 = __pyx_v_memslice->memview->view.ndim; + __pyx_v_ndim = __pyx_t_1; + + /* "View.MemoryView":933 + * cdef int ndim = memslice.memview.view.ndim + * + * cdef Py_ssize_t *shape = memslice.shape # <<<<<<<<<<<<<< + * cdef Py_ssize_t *strides = memslice.strides + * +*/ + __pyx_t_2 = __pyx_v_memslice->shape; + __pyx_v_shape = __pyx_t_2; + + /* "View.MemoryView":934 + * + * cdef Py_ssize_t *shape = memslice.shape + * cdef Py_ssize_t *strides = memslice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = __pyx_v_memslice->strides; + __pyx_v_strides = __pyx_t_2; + + /* "View.MemoryView":938 + * + * cdef int i, j + * for i in range(ndim // 2): # <<<<<<<<<<<<<< + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] +*/ + __pyx_t_3 = __Pyx_div_long(__pyx_v_ndim, 2, 1); + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_4; __pyx_t_1+=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":939 + * cdef int i, j + * for i in range(ndim // 2): + * j = ndim - 1 - i # <<<<<<<<<<<<<< + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] +*/ + __pyx_v_j = ((__pyx_v_ndim - 1) - __pyx_v_i); + + /* "View.MemoryView":940 + * for i in range(ndim // 2): + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] # <<<<<<<<<<<<<< + * shape[i], shape[j] = shape[j], shape[i] + * +*/ + __pyx_t_5 = (__pyx_v_strides[__pyx_v_j]); + __pyx_t_6 = (__pyx_v_strides[__pyx_v_i]); + (__pyx_v_strides[__pyx_v_i]) = __pyx_t_5; + (__pyx_v_strides[__pyx_v_j]) = __pyx_t_6; + + /* "View.MemoryView":941 + * j = ndim - 1 - i + * strides[i], strides[j] = strides[j], strides[i] + * shape[i], shape[j] = shape[j], shape[i] # <<<<<<<<<<<<<< + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: +*/ + __pyx_t_6 = (__pyx_v_shape[__pyx_v_j]); + __pyx_t_5 = (__pyx_v_shape[__pyx_v_i]); + (__pyx_v_shape[__pyx_v_i]) = __pyx_t_6; + (__pyx_v_shape[__pyx_v_j]) = __pyx_t_5; + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_i]) >= 0); + if (!__pyx_t_8) { + } else { + __pyx_t_7 = __pyx_t_8; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_8 = ((__pyx_v_memslice->suboffsets[__pyx_v_j]) >= 0); + __pyx_t_7 = __pyx_t_8; + __pyx_L6_bool_binop_done:; + if (__pyx_t_7) { + + /* "View.MemoryView":944 + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") # <<<<<<<<<<<<<< + * + * return 0 +*/ + __pyx_t_9 = __pyx_memoryview_err(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Cannot_transpose_memoryview_with); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 944, __pyx_L1_error) + + /* "View.MemoryView":943 + * shape[i], shape[j] = shape[j], shape[i] + * + * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * +*/ + } + } + + /* "View.MemoryView":946 + * _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") + * + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":929 + * + * + * @cname('__pyx_memslice_transpose') # <<<<<<<<<<<<<< + * cdef int transpose_memslice(__Pyx_memviewslice *memslice) except -1 nogil: + * cdef int ndim = memslice.memview.view.ndim +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.transpose_memslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + +/* Python wrapper */ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_memoryviewslice___dealloc__(PyObject *__pyx_v_self) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewslice___dealloc__(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + + /* "View.MemoryView":965 + * + * def __dealloc__(self): + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) # <<<<<<<<<<<<<< + * + * cdef convert_item_to_object(self, char *itemp): +*/ + __PYX_XCLEAR_MEMVIEW((&__pyx_v_self->from_slice), 1); + + /* "View.MemoryView":964 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + +static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("convert_item_to_object", 0); + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_object_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":969 + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) # <<<<<<<<<<<<<< + * else: + * return memoryview.convert_item_to_object(self, itemp) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 969, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "View.MemoryView":968 + * + * cdef convert_item_to_object(self, char *itemp): + * if self.to_object_func != NULL: # <<<<<<<<<<<<<< + * return self.to_object_func(itemp) + * else: +*/ + } + + /* "View.MemoryView":971 + * return self.to_object_func(itemp) + * else: + * return memoryview.convert_item_to_object(self, itemp) # <<<<<<<<<<<<<< + * + * cdef assign_item_from_object(self, char *itemp, object value): +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 971, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + } + + /* "View.MemoryView":967 + * __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) + * + * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< + * if self.to_object_func != NULL: + * return self.to_object_func(itemp) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + +static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memoryviewslice_obj *__pyx_v_self, char *__pyx_v_itemp, PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("assign_item_from_object", 0); + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + __pyx_t_1 = (__pyx_v_self->to_dtype_func != NULL); + if (__pyx_t_1) { + + /* "View.MemoryView":975 + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) # <<<<<<<<<<<<<< + * else: + * memoryview.assign_item_from_object(self, itemp, value) +*/ + __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 975, __pyx_L1_error) + + /* "View.MemoryView":974 + * + * cdef assign_item_from_object(self, char *itemp, object value): + * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< + * self.to_dtype_func(itemp, value) + * else: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":977 + * self.to_dtype_func(itemp, value) + * else: + * memoryview.assign_item_from_object(self, itemp, value) # <<<<<<<<<<<<<< + * + * cdef _get_base(self): +*/ + /*else*/ { + __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 977, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __pyx_L3:; + + /* "View.MemoryView":973 + * return memoryview.convert_item_to_object(self, itemp) + * + * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< + * if self.to_dtype_func != NULL: + * self.to_dtype_func(itemp, value) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.assign_item_from_object", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + +static PyObject *__pyx_memoryviewslice__get_base(struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_get_base", 0); + + /* "View.MemoryView":980 + * + * cdef _get_base(self): + * return self.from_object # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->from_object); + __pyx_r = __pyx_v_self->from_object; + goto __pyx_L0; + + /* "View.MemoryView":979 + * memoryview.assign_item_from_object(self, itemp, value) + * + * cdef _get_base(self): # <<<<<<<<<<<<<< + * return self.from_object + * +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_1__reduce_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + if (unlikely(__pyx_nargs > 0)) { __Pyx_RaiseArgtupleInvalid("__reduce_cython__", 1, 0, 0, __pyx_nargs); return NULL; } + const Py_ssize_t __pyx_kwds_len = unlikely(__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len < 0)) return NULL; + if (unlikely(__pyx_kwds_len > 0)) {__Pyx_RejectKeywords("__reduce_cython__", __pyx_kwds); return NULL;} + __pyx_r = __pyx_pf___pyx_memoryviewslice___reduce_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice___reduce_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_pw___pyx_memoryviewslice_3__setstate_cython__(PyObject *__pyx_v_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + CYTHON_UNUSED PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 3, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__setstate_cython__", 0) < (0)) __PYX_ERR(1, 3, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, i); __PYX_ERR(1, 3, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 3, __pyx_L3_error) + } + __pyx_v___pyx_state = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__setstate_cython__", 1, 1, 1, __pyx_nargs); __PYX_ERR(1, 3, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf___pyx_memoryviewslice_2__setstate_cython__(((struct __pyx_memoryviewslice_obj *)__pyx_v_self), __pyx_v___pyx_state); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUSED struct __pyx_memoryviewslice_obj *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + + /* "(tree fragment)":4 + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" # <<<<<<<<<<<<<< +*/ + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_TypeError))), __pyx_mstate_global->__pyx_kp_u_no_default___reduce___due_to_non, 0, 0); + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError, "no default __reduce__ due to non-trivial __cinit__" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._memoryviewslice.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + +static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewslice, int __pyx_v_ndim, PyObject *(*__pyx_v_to_object_func)(char *), int (*__pyx_v_to_dtype_func)(char *, PyObject *), int __pyx_v_dtype_is_object) { + struct __pyx_memoryviewslice_obj *__pyx_v_result = 0; + Py_ssize_t __pyx_v_suboffset; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_TypeInfo const *__pyx_t_4; + Py_buffer __pyx_t_5; + Py_ssize_t *__pyx_t_6; + Py_ssize_t *__pyx_t_7; + Py_ssize_t *__pyx_t_8; + Py_ssize_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_fromslice", 0); + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + __pyx_t_1 = (((PyObject *)__pyx_v_memviewslice.memview) == Py_None); + if (__pyx_t_1) { + + /* "View.MemoryView":1009 + * + * if memviewslice.memview == Py_None: + * return None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "View.MemoryView":1008 + * cdef _memoryviewslice result + * + * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< + * return None + * +*/ + } + + /* "View.MemoryView":1014 + * + * + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) # <<<<<<<<<<<<<< + * + * result.from_slice = memviewslice +*/ + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(Py_None); + __Pyx_GIVEREF(Py_None); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 0, Py_None) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_0); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_0); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_mstate_global->__pyx_int_0) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2) != (0)) __PYX_ERR(1, 1014, __pyx_L1_error); + __pyx_t_2 = 0; + __pyx_t_2 = ((PyObject *)__pyx_tp_new__memoryviewslice(((PyTypeObject *)__pyx_mstate_global->__pyx_memoryviewslice_type), __pyx_t_3, NULL)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1014, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1016 + * result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) + * + * result.from_slice = memviewslice # <<<<<<<<<<<<<< + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * +*/ + __pyx_v_result->from_slice = __pyx_v_memviewslice; + + /* "View.MemoryView":1017 + * + * result.from_slice = memviewslice + * __PYX_INC_MEMVIEW(&memviewslice, 1) # <<<<<<<<<<<<<< + * + * result.from_object = ( memviewslice.memview)._get_base() +*/ + __PYX_INC_MEMVIEW((&__pyx_v_memviewslice), 1); + + /* "View.MemoryView":1019 + * __PYX_INC_MEMVIEW(&memviewslice, 1) + * + * result.from_object = ( memviewslice.memview)._get_base() # <<<<<<<<<<<<<< + * result.typeinfo = memviewslice.memview.typeinfo + * +*/ + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->__pyx_vtab)->_get_base(((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1019, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_v_result->from_object); + __Pyx_DECREF(__pyx_v_result->from_object); + __pyx_v_result->from_object = __pyx_t_2; + __pyx_t_2 = 0; + + /* "View.MemoryView":1020 + * + * result.from_object = ( memviewslice.memview)._get_base() + * result.typeinfo = memviewslice.memview.typeinfo # <<<<<<<<<<<<<< + * + * result.view = memviewslice.memview.view +*/ + __pyx_t_4 = __pyx_v_memviewslice.memview->typeinfo; + __pyx_v_result->__pyx_base.typeinfo = __pyx_t_4; + + /* "View.MemoryView":1022 + * result.typeinfo = memviewslice.memview.typeinfo + * + * result.view = memviewslice.memview.view # <<<<<<<<<<<<<< + * result.view.buf = memviewslice.data + * result.view.ndim = ndim +*/ + __pyx_t_5 = __pyx_v_memviewslice.memview->view; + __pyx_v_result->__pyx_base.view = __pyx_t_5; + + /* "View.MemoryView":1023 + * + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data # <<<<<<<<<<<<<< + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None +*/ + __pyx_v_result->__pyx_base.view.buf = ((void *)__pyx_v_memviewslice.data); + + /* "View.MemoryView":1024 + * result.view = memviewslice.memview.view + * result.view.buf = memviewslice.data + * result.view.ndim = ndim # <<<<<<<<<<<<<< + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) +*/ + __pyx_v_result->__pyx_base.view.ndim = __pyx_v_ndim; + + /* "View.MemoryView":1025 + * result.view.buf = memviewslice.data + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None # <<<<<<<<<<<<<< + * Py_INCREF(Py_None) + * +*/ + ((Py_buffer *)(&__pyx_v_result->__pyx_base.view))->obj = Py_None; + + /* "View.MemoryView":1026 + * result.view.ndim = ndim + * (<__pyx_buffer *> &result.view).obj = Py_None + * Py_INCREF(Py_None) # <<<<<<<<<<<<<< + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: +*/ + Py_INCREF(Py_None); + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + __pyx_t_1 = ((((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->flags & PyBUF_WRITABLE) != 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1029 + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: + * result.flags = PyBUF_RECORDS # <<<<<<<<<<<<<< + * else: + * result.flags = PyBUF_RECORDS_RO +*/ + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS; + + /* "View.MemoryView":1028 + * Py_INCREF(Py_None) + * + * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< + * result.flags = PyBUF_RECORDS + * else: +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1031 + * result.flags = PyBUF_RECORDS + * else: + * result.flags = PyBUF_RECORDS_RO # <<<<<<<<<<<<<< + * + * result.view.shape = result.from_slice.shape +*/ + /*else*/ { + __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS_RO; + } + __pyx_L4:; + + /* "View.MemoryView":1033 + * result.flags = PyBUF_RECORDS_RO + * + * result.view.shape = result.from_slice.shape # <<<<<<<<<<<<<< + * result.view.strides = result.from_slice.strides + * +*/ + __pyx_v_result->__pyx_base.view.shape = ((Py_ssize_t *)__pyx_v_result->from_slice.shape); + + /* "View.MemoryView":1034 + * + * result.view.shape = result.from_slice.shape + * result.view.strides = result.from_slice.strides # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_result->__pyx_base.view.strides = ((Py_ssize_t *)__pyx_v_result->from_slice.strides); + + /* "View.MemoryView":1037 + * + * + * result.view.suboffsets = NULL # <<<<<<<<<<<<<< + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: +*/ + __pyx_v_result->__pyx_base.view.suboffsets = NULL; + + /* "View.MemoryView":1038 + * + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: # <<<<<<<<<<<<<< + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets +*/ + __pyx_t_7 = (__pyx_v_result->from_slice.suboffsets + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->from_slice.suboffsets; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_v_suboffset = (__pyx_t_6[0]); + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + __pyx_t_1 = (__pyx_v_suboffset >= 0); + if (__pyx_t_1) { + + /* "View.MemoryView":1040 + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_result->__pyx_base.view.suboffsets = ((Py_ssize_t *)__pyx_v_result->from_slice.suboffsets); + + /* "View.MemoryView":1041 + * if suboffset >= 0: + * result.view.suboffsets = result.from_slice.suboffsets + * break # <<<<<<<<<<<<<< + * + * result.view.len = result.view.itemsize +*/ + goto __pyx_L6_break; + + /* "View.MemoryView":1039 + * result.view.suboffsets = NULL + * for suboffset in result.from_slice.suboffsets[:ndim]: + * if suboffset >= 0: # <<<<<<<<<<<<<< + * result.view.suboffsets = result.from_slice.suboffsets + * break +*/ + } + } + __pyx_L6_break:; + + /* "View.MemoryView":1043 + * break + * + * result.view.len = result.view.itemsize # <<<<<<<<<<<<<< + * for length in result.view.shape[:ndim]: + * result.view.len *= length +*/ + __pyx_t_9 = __pyx_v_result->__pyx_base.view.itemsize; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + + /* "View.MemoryView":1044 + * + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: # <<<<<<<<<<<<<< + * result.view.len *= length + * +*/ + __pyx_t_7 = (__pyx_v_result->__pyx_base.view.shape + __pyx_v_ndim); + for (__pyx_t_8 = __pyx_v_result->__pyx_base.view.shape; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { + __pyx_t_6 = __pyx_t_8; + __pyx_t_2 = PyLong_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1044, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1045 + * result.view.len = result.view.itemsize + * for length in result.view.shape[:ndim]: + * result.view.len *= length # <<<<<<<<<<<<<< + * + * result.to_object_func = to_object_func +*/ + __pyx_t_2 = PyLong_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1045, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_result->__pyx_base.view.len = __pyx_t_9; + } + + /* "View.MemoryView":1047 + * result.view.len *= length + * + * result.to_object_func = to_object_func # <<<<<<<<<<<<<< + * result.to_dtype_func = to_dtype_func + * +*/ + __pyx_v_result->to_object_func = __pyx_v_to_object_func; + + /* "View.MemoryView":1048 + * + * result.to_object_func = to_object_func + * result.to_dtype_func = to_dtype_func # <<<<<<<<<<<<<< + * + * return result +*/ + __pyx_v_result->to_dtype_func = __pyx_v_to_dtype_func; + + /* "View.MemoryView":1050 + * result.to_dtype_func = to_dtype_func + * + * return result # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF((PyObject *)__pyx_v_result); + __pyx_r = ((PyObject *)__pyx_v_result); + goto __pyx_L0; + + /* "View.MemoryView":999 + * pass # ignore failure, it's a minor issue + * + * @cname('__pyx_memoryview_fromslice') # <<<<<<<<<<<<<< + * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, + * int ndim, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.memoryview_fromslice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_result); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + +static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_mslice) { + struct __pyx_memoryviewslice_obj *__pyx_v_obj = 0; + __Pyx_memviewslice *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_slice_from_memview", 0); + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1057 + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): + * obj = memview # <<<<<<<<<<<<<< + * return &obj.from_slice + * else: +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_memview); + __Pyx_INCREF(__pyx_t_2); + if (!(likely(((__pyx_t_2) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_2, __pyx_mstate_global->__pyx_memoryviewslice_type))))) __PYX_ERR(1, 1057, __pyx_L1_error) + __pyx_v_obj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); + __pyx_t_2 = 0; + + /* "View.MemoryView":1058 + * if isinstance(memview, _memoryviewslice): + * obj = memview + * return &obj.from_slice # <<<<<<<<<<<<<< + * else: + * slice_copy(memview, mslice) +*/ + __pyx_r = (&__pyx_v_obj->from_slice); + goto __pyx_L0; + + /* "View.MemoryView":1056 + * __Pyx_memviewslice *mslice) except NULL: + * cdef _memoryviewslice obj + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * obj = memview + * return &obj.from_slice +*/ + } + + /* "View.MemoryView":1060 + * return &obj.from_slice + * else: + * slice_copy(memview, mslice) # <<<<<<<<<<<<<< + * return mslice + * +*/ + /*else*/ { + __pyx_memoryview_slice_copy(__pyx_v_memview, __pyx_v_mslice); + + /* "View.MemoryView":1061 + * else: + * slice_copy(memview, mslice) + * return mslice # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_slice_copy') +*/ + __pyx_r = __pyx_v_mslice; + goto __pyx_L0; + } + + /* "View.MemoryView":1052 + * return result + * + * @cname('__pyx_memoryview_get_slice_from_memoryview') # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, + * __Pyx_memviewslice *mslice) except NULL: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView.get_slice_from_memview", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_obj); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + +static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_dst) { + int __pyx_v_dim; + Py_ssize_t *__pyx_v_shape; + Py_ssize_t *__pyx_v_strides; + Py_ssize_t *__pyx_v_suboffsets; + Py_ssize_t *__pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + Py_ssize_t __pyx_t_5; + int __pyx_t_6; + + /* "View.MemoryView":1068 + * cdef (Py_ssize_t*) shape, strides, suboffsets + * + * shape = memview.view.shape # <<<<<<<<<<<<<< + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets +*/ + __pyx_t_1 = __pyx_v_memview->view.shape; + __pyx_v_shape = __pyx_t_1; + + /* "View.MemoryView":1069 + * + * shape = memview.view.shape + * strides = memview.view.strides # <<<<<<<<<<<<<< + * suboffsets = memview.view.suboffsets + * +*/ + __pyx_t_1 = __pyx_v_memview->view.strides; + __pyx_v_strides = __pyx_t_1; + + /* "View.MemoryView":1070 + * shape = memview.view.shape + * strides = memview.view.strides + * suboffsets = memview.view.suboffsets # <<<<<<<<<<<<<< + * + * dst.memview = <__pyx_memoryview *> memview +*/ + __pyx_t_1 = __pyx_v_memview->view.suboffsets; + __pyx_v_suboffsets = __pyx_t_1; + + /* "View.MemoryView":1072 + * suboffsets = memview.view.suboffsets + * + * dst.memview = <__pyx_memoryview *> memview # <<<<<<<<<<<<<< + * dst.data = memview.view.buf + * +*/ + __pyx_v_dst->memview = ((struct __pyx_memoryview_obj *)__pyx_v_memview); + + /* "View.MemoryView":1073 + * + * dst.memview = <__pyx_memoryview *> memview + * dst.data = memview.view.buf # <<<<<<<<<<<<<< + * + * for dim in range(memview.view.ndim): +*/ + __pyx_v_dst->data = ((char *)__pyx_v_memview->view.buf); + + /* "View.MemoryView":1075 + * dst.data = memview.view.buf + * + * for dim in range(memview.view.ndim): # <<<<<<<<<<<<<< + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] +*/ + __pyx_t_2 = __pyx_v_memview->view.ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_dim = __pyx_t_4; + + /* "View.MemoryView":1076 + * + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] # <<<<<<<<<<<<<< + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 +*/ + (__pyx_v_dst->shape[__pyx_v_dim]) = (__pyx_v_shape[__pyx_v_dim]); + + /* "View.MemoryView":1077 + * for dim in range(memview.view.ndim): + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] # <<<<<<<<<<<<<< + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * +*/ + (__pyx_v_dst->strides[__pyx_v_dim]) = (__pyx_v_strides[__pyx_v_dim]); + + /* "View.MemoryView":1078 + * dst.shape[dim] = shape[dim] + * dst.strides[dim] = strides[dim] + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object') +*/ + __pyx_t_6 = (__pyx_v_suboffsets != 0); + if (__pyx_t_6) { + __pyx_t_5 = (__pyx_v_suboffsets[__pyx_v_dim]); + } else { + __pyx_t_5 = -1L; + } + (__pyx_v_dst->suboffsets[__pyx_v_dim]) = __pyx_t_5; + } + + /* "View.MemoryView":1063 + * return mslice + * + * @cname('__pyx_memoryview_slice_copy') # <<<<<<<<<<<<<< + * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst) noexcept: + * cdef int dim +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + +static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx_v_memview) { + __Pyx_memviewslice __pyx_v_memviewslice; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy", 0); + + /* "View.MemoryView":1084 + * "Create a new memoryview object" + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) # <<<<<<<<<<<<<< + * return memoryview_copy_from_slice(memview, &memviewslice) + * +*/ + __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_memviewslice)); + + /* "View.MemoryView":1085 + * cdef __Pyx_memviewslice memviewslice + * slice_copy(memview, &memviewslice) + * return memoryview_copy_from_slice(memview, &memviewslice) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_object_from_slice') +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1085, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1080 + * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 + * + * @cname('__pyx_memoryview_copy_object') # <<<<<<<<<<<<<< + * cdef memoryview_copy(memoryview memview): + * "Create a new memoryview object" +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + +static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview_obj *__pyx_v_memview, __Pyx_memviewslice *__pyx_v_memviewslice) { + PyObject *(*__pyx_v_to_object_func)(char *); + int (*__pyx_v_to_dtype_func)(char *, PyObject *); + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *(*__pyx_t_2)(char *); + int (*__pyx_t_3)(char *, PyObject *); + PyObject *__pyx_t_4 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("memoryview_copy_from_slice", 0); + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + __pyx_t_1 = __Pyx_TypeCheck(((PyObject *)__pyx_v_memview), __pyx_mstate_global->__pyx_memoryviewslice_type); + if (__pyx_t_1) { + + /* "View.MemoryView":1096 + * + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func # <<<<<<<<<<<<<< + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: +*/ + __pyx_t_2 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_object_func; + __pyx_v_to_object_func = __pyx_t_2; + + /* "View.MemoryView":1097 + * if isinstance(memview, _memoryviewslice): + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func # <<<<<<<<<<<<<< + * else: + * to_object_func = NULL +*/ + __pyx_t_3 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_dtype_func; + __pyx_v_to_dtype_func = __pyx_t_3; + + /* "View.MemoryView":1095 + * cdef int (*to_dtype_func)(char *, object) except 0 + * + * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< + * to_object_func = (<_memoryviewslice> memview).to_object_func + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1099 + * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func + * else: + * to_object_func = NULL # <<<<<<<<<<<<<< + * to_dtype_func = NULL + * +*/ + /*else*/ { + __pyx_v_to_object_func = NULL; + + /* "View.MemoryView":1100 + * else: + * to_object_func = NULL + * to_dtype_func = NULL # <<<<<<<<<<<<<< + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, +*/ + __pyx_v_to_dtype_func = NULL; + } + __pyx_L3:; + + /* "View.MemoryView":1102 + * to_dtype_func = NULL + * + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, # <<<<<<<<<<<<<< + * to_object_func, to_dtype_func, + * memview.dtype_is_object) +*/ + __Pyx_XDECREF(__pyx_r); + + /* "View.MemoryView":1104 + * return memoryview_fromslice(memviewslice[0], memview.view.ndim, + * to_object_func, to_dtype_func, + * memview.dtype_is_object) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_4 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1102, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "View.MemoryView":1087 + * return memoryview_copy_from_slice(memview, &memviewslice) + * + * @cname('__pyx_memoryview_copy_object_from_slice') # <<<<<<<<<<<<<< + * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): + * """ +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_from_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + +static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + + /* "View.MemoryView":1111 + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: + * return -arg if arg < 0 else arg # <<<<<<<<<<<<<< + * + * @cname('__pyx_get_best_slice_order') +*/ + __pyx_t_2 = (__pyx_v_arg < 0); + if (__pyx_t_2) { + __pyx_t_1 = (-__pyx_v_arg); + } else { + __pyx_t_1 = __pyx_v_arg; + } + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "View.MemoryView":1110 + * + * + * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: # <<<<<<<<<<<<<< + * return -arg if arg < 0 else arg + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + +static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim) { + int __pyx_v_i; + Py_ssize_t __pyx_v_c_stride; + Py_ssize_t __pyx_v_f_stride; + char __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1119 + * """ + * cdef int i + * cdef Py_ssize_t c_stride = 0 # <<<<<<<<<<<<<< + * cdef Py_ssize_t f_stride = 0 + * +*/ + __pyx_v_c_stride = 0; + + /* "View.MemoryView":1120 + * cdef int i + * cdef Py_ssize_t c_stride = 0 + * cdef Py_ssize_t f_stride = 0 # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_f_stride = 0; + + /* "View.MemoryView":1122 + * cdef Py_ssize_t f_stride = 0 + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1124 + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_c_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1125 + * if mslice.shape[i] > 1: + * c_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + goto __pyx_L4_break; + + /* "View.MemoryView":1123 + * + * for i in range(ndim - 1, -1, -1): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * c_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L4_break:; + + /* "View.MemoryView":1127 + * break + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] +*/ + __pyx_t_1 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_1; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + __pyx_t_2 = ((__pyx_v_mslice->shape[__pyx_v_i]) > 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1129 + * for i in range(ndim): + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] # <<<<<<<<<<<<<< + * break + * +*/ + __pyx_v_f_stride = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1130 + * if mslice.shape[i] > 1: + * f_stride = mslice.strides[i] + * break # <<<<<<<<<<<<<< + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): +*/ + goto __pyx_L7_break; + + /* "View.MemoryView":1128 + * + * for i in range(ndim): + * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< + * f_stride = mslice.strides[i] + * break +*/ + } + } + __pyx_L7_break:; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + __pyx_t_2 = (abs_py_ssize_t(__pyx_v_c_stride) <= abs_py_ssize_t(__pyx_v_f_stride)); + if (__pyx_t_2) { + + /* "View.MemoryView":1133 + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): + * return 'C' # <<<<<<<<<<<<<< + * else: + * return 'F' +*/ + __pyx_r = 'C'; + goto __pyx_L0; + + /* "View.MemoryView":1132 + * break + * + * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< + * return 'C' + * else: +*/ + } + + /* "View.MemoryView":1135 + * return 'C' + * else: + * return 'F' # <<<<<<<<<<<<<< + * + * @cython.cdivision(True) +*/ + /*else*/ { + __pyx_r = 'F'; + goto __pyx_L0; + } + + /* "View.MemoryView":1113 + * return -arg if arg < 0 else arg + * + * @cname('__pyx_get_best_slice_order') # <<<<<<<<<<<<<< + * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) noexcept nogil: + * """ +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + +static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v_src_strides, char *__pyx_v_dst_data, Py_ssize_t *__pyx_v_dst_strides, Py_ssize_t *__pyx_v_src_shape, Py_ssize_t *__pyx_v_dst_shape, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + CYTHON_UNUSED Py_ssize_t __pyx_v_src_extent; + Py_ssize_t __pyx_v_dst_extent; + Py_ssize_t __pyx_v_src_stride; + Py_ssize_t __pyx_v_dst_stride; + int __pyx_t_1; + int __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + + /* "View.MemoryView":1145 + * + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] +*/ + __pyx_v_src_extent = (__pyx_v_src_shape[0]); + + /* "View.MemoryView":1146 + * cdef Py_ssize_t i + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] +*/ + __pyx_v_dst_extent = (__pyx_v_dst_shape[0]); + + /* "View.MemoryView":1147 + * cdef Py_ssize_t src_extent = src_shape[0] + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t dst_stride = dst_strides[0] + * +*/ + __pyx_v_src_stride = (__pyx_v_src_strides[0]); + + /* "View.MemoryView":1148 + * cdef Py_ssize_t dst_extent = dst_shape[0] + * cdef Py_ssize_t src_stride = src_strides[0] + * cdef Py_ssize_t dst_stride = dst_strides[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_dst_stride = (__pyx_v_dst_strides[0]); + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + __pyx_t_2 = (__pyx_v_src_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_2 = (__pyx_v_dst_stride > 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + + /* "View.MemoryView":1152 + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: +*/ + __pyx_t_2 = (((size_t)__pyx_v_src_stride) == __pyx_v_itemsize); + if (__pyx_t_2) { + __pyx_t_2 = (__pyx_v_itemsize == ((size_t)__pyx_v_dst_stride)); + } + __pyx_t_1 = __pyx_t_2; + __pyx_L5_bool_binop_done:; + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + if (__pyx_t_1) { + + /* "View.MemoryView":1153 + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, (__pyx_v_itemsize * __pyx_v_dst_extent))); + + /* "View.MemoryView":1151 + * + * if ndim == 1: + * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< + * src_stride == itemsize == dst_stride): + * memcpy(dst_data, src_data, itemsize * dst_extent) +*/ + goto __pyx_L4; + } + + /* "View.MemoryView":1155 + * memcpy(dst_data, src_data, itemsize * dst_extent) + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1156 + * else: + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) # <<<<<<<<<<<<<< + * src_data += src_stride + * dst_data += dst_stride +*/ + (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, __pyx_v_itemsize)); + + /* "View.MemoryView":1157 + * for i in range(dst_extent): + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * else: +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1158 + * memcpy(dst_data, src_data, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * else: + * for i in range(dst_extent): +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L4:; + + /* "View.MemoryView":1150 + * cdef Py_ssize_t dst_stride = dst_strides[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * if (src_stride > 0 and dst_stride > 0 and + * src_stride == itemsize == dst_stride): +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1160 + * dst_data += dst_stride + * else: + * for i in range(dst_extent): # <<<<<<<<<<<<<< + * _copy_strided_to_strided(src_data, src_strides + 1, + * dst_data, dst_strides + 1, +*/ + /*else*/ { + __pyx_t_3 = __pyx_v_dst_extent; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "View.MemoryView":1161 + * else: + * for i in range(dst_extent): + * _copy_strided_to_strided(src_data, src_strides + 1, # <<<<<<<<<<<<<< + * dst_data, dst_strides + 1, + * src_shape + 1, dst_shape + 1, +*/ + _copy_strided_to_strided(__pyx_v_src_data, (__pyx_v_src_strides + 1), __pyx_v_dst_data, (__pyx_v_dst_strides + 1), (__pyx_v_src_shape + 1), (__pyx_v_dst_shape + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize); + + /* "View.MemoryView":1165 + * src_shape + 1, dst_shape + 1, + * ndim - 1, itemsize) + * src_data += src_stride # <<<<<<<<<<<<<< + * dst_data += dst_stride + * +*/ + __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); + + /* "View.MemoryView":1166 + * ndim - 1, itemsize) + * src_data += src_stride + * dst_data += dst_stride # <<<<<<<<<<<<<< + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, +*/ + __pyx_v_dst_data = (__pyx_v_dst_data + __pyx_v_dst_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1137 + * return 'F' + * + * @cython.cdivision(True) # <<<<<<<<<<<<<< + * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, + * char *dst_data, Py_ssize_t *dst_strides, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + +static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize) { + + /* "View.MemoryView":1171 + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: + * _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, # <<<<<<<<<<<<<< + * src.shape, dst.shape, ndim, itemsize) + * +*/ + _copy_strided_to_strided(__pyx_v_src->data, __pyx_v_src->strides, __pyx_v_dst->data, __pyx_v_dst->strides, __pyx_v_src->shape, __pyx_v_dst->shape, __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1168 + * dst_data += dst_stride + * + * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< + * __Pyx_memviewslice *dst, + * int ndim, size_t itemsize) noexcept nogil: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + +static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_src, int __pyx_v_ndim) { + Py_ssize_t __pyx_v_shape; + Py_ssize_t __pyx_v_size; + Py_ssize_t __pyx_r; + Py_ssize_t __pyx_t_1; + Py_ssize_t *__pyx_t_2; + Py_ssize_t *__pyx_t_3; + Py_ssize_t *__pyx_t_4; + + /* "View.MemoryView":1177 + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" + * cdef Py_ssize_t shape, size = src.memview.view.itemsize # <<<<<<<<<<<<<< + * + * for shape in src.shape[:ndim]: +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_size = __pyx_t_1; + + /* "View.MemoryView":1179 + * cdef Py_ssize_t shape, size = src.memview.view.itemsize + * + * for shape in src.shape[:ndim]: # <<<<<<<<<<<<<< + * size *= shape + * +*/ + __pyx_t_3 = (__pyx_v_src->shape + __pyx_v_ndim); + for (__pyx_t_4 = __pyx_v_src->shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { + __pyx_t_2 = __pyx_t_4; + __pyx_v_shape = (__pyx_t_2[0]); + + /* "View.MemoryView":1180 + * + * for shape in src.shape[:ndim]: + * size *= shape # <<<<<<<<<<<<<< + * + * return size +*/ + __pyx_v_size = (__pyx_v_size * __pyx_v_shape); + } + + /* "View.MemoryView":1182 + * size *= shape + * + * return size # <<<<<<<<<<<<<< + * + * @cname('__pyx_fill_contig_strides_array') +*/ + __pyx_r = __pyx_v_size; + goto __pyx_L0; + + /* "View.MemoryView":1174 + * src.shape, dst.shape, ndim, itemsize) + * + * @cname('__pyx_memoryview_slice_get_size') # <<<<<<<<<<<<<< + * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) noexcept nogil: + * "Return the size of the memory occupied by the slice in number of bytes" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + +static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, Py_ssize_t __pyx_v_stride, int __pyx_v_ndim, char __pyx_v_order) { + int __pyx_v_idx; + Py_ssize_t __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + __pyx_t_1 = (__pyx_v_order == 'F'); + if (__pyx_t_1) { + + /* "View.MemoryView":1195 + * + * if order == 'F': + * for idx in range(ndim): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + __pyx_t_2 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_idx = __pyx_t_4; + + /* "View.MemoryView":1196 + * if order == 'F': + * for idx in range(ndim): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * else: +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1197 + * for idx in range(ndim): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * else: + * for idx in range(ndim - 1, -1, -1): +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + + /* "View.MemoryView":1194 + * cdef int idx + * + * if order == 'F': # <<<<<<<<<<<<<< + * for idx in range(ndim): + * strides[idx] = stride +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1199 + * stride *= shape[idx] + * else: + * for idx in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * strides[idx] = stride + * stride *= shape[idx] +*/ + /*else*/ { + for (__pyx_t_2 = (__pyx_v_ndim - 1); __pyx_t_2 > -1; __pyx_t_2-=1) { + __pyx_v_idx = __pyx_t_2; + + /* "View.MemoryView":1200 + * else: + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride # <<<<<<<<<<<<<< + * stride *= shape[idx] + * +*/ + (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; + + /* "View.MemoryView":1201 + * for idx in range(ndim - 1, -1, -1): + * strides[idx] = stride + * stride *= shape[idx] # <<<<<<<<<<<<<< + * + * return stride +*/ + __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); + } + } + __pyx_L3:; + + /* "View.MemoryView":1203 + * stride *= shape[idx] + * + * return stride # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_copy_data_to_temp') +*/ + __pyx_r = __pyx_v_stride; + goto __pyx_L0; + + /* "View.MemoryView":1184 + * return size + * + * @cname('__pyx_fill_contig_strides_array') # <<<<<<<<<<<<<< + * cdef Py_ssize_t fill_contig_strides_array( + * Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + +static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_tmpslice, char __pyx_v_order, int __pyx_v_ndim) { + int __pyx_v_i; + void *__pyx_v_result; + size_t __pyx_v_itemsize; + size_t __pyx_v_size; + void *__pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + struct __pyx_memoryview_obj *__pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1217 + * cdef void *result + * + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef size_t size = slice_get_size(src, ndim) + * +*/ + __pyx_t_1 = __pyx_v_src->memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1218 + * + * cdef size_t itemsize = src.memview.view.itemsize + * cdef size_t size = slice_get_size(src, ndim) # <<<<<<<<<<<<<< + * + * result = malloc(size) +*/ + __pyx_v_size = __pyx_memoryview_slice_get_size(__pyx_v_src, __pyx_v_ndim); + + /* "View.MemoryView":1220 + * cdef size_t size = slice_get_size(src, ndim) + * + * result = malloc(size) # <<<<<<<<<<<<<< + * if not result: + * _err_no_memory() +*/ + __pyx_v_result = malloc(__pyx_v_size); + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + __pyx_t_2 = (!(__pyx_v_result != 0)); + if (__pyx_t_2) { + + /* "View.MemoryView":1222 + * result = malloc(size) + * if not result: + * _err_no_memory() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __pyx_memoryview_err_no_memory(); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1222, __pyx_L1_error) + + /* "View.MemoryView":1221 + * + * result = malloc(size) + * if not result: # <<<<<<<<<<<<<< + * _err_no_memory() + * +*/ + } + + /* "View.MemoryView":1225 + * + * + * tmpslice.data = result # <<<<<<<<<<<<<< + * tmpslice.memview = src.memview + * for i in range(ndim): +*/ + __pyx_v_tmpslice->data = ((char *)__pyx_v_result); + + /* "View.MemoryView":1226 + * + * tmpslice.data = result + * tmpslice.memview = src.memview # <<<<<<<<<<<<<< + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] +*/ + __pyx_t_4 = __pyx_v_src->memview; + __pyx_v_tmpslice->memview = __pyx_t_4; + + /* "View.MemoryView":1227 + * tmpslice.data = result + * tmpslice.memview = src.memview + * for i in range(ndim): # <<<<<<<<<<<<<< + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1228 + * tmpslice.memview = src.memview + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] # <<<<<<<<<<<<<< + * tmpslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_tmpslice->shape[__pyx_v_i]) = (__pyx_v_src->shape[__pyx_v_i]); + + /* "View.MemoryView":1229 + * for i in range(ndim): + * tmpslice.shape[i] = src.shape[i] + * tmpslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) +*/ + (__pyx_v_tmpslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1231 + * tmpslice.suboffsets[i] = -1 + * + * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # <<<<<<<<<<<<<< + * + * +*/ + (void)(__pyx_fill_contig_strides_array((&(__pyx_v_tmpslice->shape[0])), (&(__pyx_v_tmpslice->strides[0])), __pyx_v_itemsize, __pyx_v_ndim, __pyx_v_order)); + + /* "View.MemoryView":1234 + * + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 +*/ + __pyx_t_3 = __pyx_v_ndim; + __pyx_t_5 = __pyx_t_3; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + __pyx_t_2 = ((__pyx_v_tmpslice->shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1236 + * for i in range(ndim): + * if tmpslice.shape[i] == 1: + * tmpslice.strides[i] = 0 # <<<<<<<<<<<<<< + * + * if slice_is_contig(src[0], order, ndim): +*/ + (__pyx_v_tmpslice->strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1235 + * + * for i in range(ndim): + * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< + * tmpslice.strides[i] = 0 + * +*/ + } + } + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig((__pyx_v_src[0]), __pyx_v_order, __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1239 + * + * if slice_is_contig(src[0], order, ndim): + * memcpy(result, src.data, size) # <<<<<<<<<<<<<< + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) +*/ + (void)(memcpy(__pyx_v_result, __pyx_v_src->data, __pyx_v_size)); + + /* "View.MemoryView":1238 + * tmpslice.strides[i] = 0 + * + * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< + * memcpy(result, src.data, size) + * else: +*/ + goto __pyx_L9; + } + + /* "View.MemoryView":1241 + * memcpy(result, src.data, size) + * else: + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) # <<<<<<<<<<<<<< + * + * return result +*/ + /*else*/ { + copy_strided_to_strided(__pyx_v_src, __pyx_v_tmpslice, __pyx_v_ndim, __pyx_v_itemsize); + } + __pyx_L9:; + + /* "View.MemoryView":1243 + * copy_strided_to_strided(src, tmpslice, ndim, itemsize) + * + * return result # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_result; + goto __pyx_L0; + + /* "View.MemoryView":1205 + * return stride + * + * @cname('__pyx_memoryview_copy_data_to_temp') # <<<<<<<<<<<<<< + * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, + * __Pyx_memviewslice *tmpslice, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.copy_data_to_temp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + +static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent1, Py_ssize_t __pyx_v_extent2) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4[7]; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_extents", 0); + + /* "View.MemoryView":1250 + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_dim') +*/ + __pyx_t_1 = __Pyx_PyUnicode_From_int(__pyx_v_i, 0, ' ', 'd'); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent1, 0, ' ', 'd'); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyUnicode_From_Py_ssize_t(__pyx_v_extent2, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4[0] = __pyx_mstate_global->__pyx_kp_u_got_differing_extents_in_dimensi; + __pyx_t_4[1] = __pyx_t_1; + __pyx_t_4[2] = __pyx_mstate_global->__pyx_kp_u_got; + __pyx_t_4[3] = __pyx_t_2; + __pyx_t_4[4] = __pyx_mstate_global->__pyx_kp_u_and; + __pyx_t_4[5] = __pyx_t_3; + __pyx_t_4[6] = __pyx_mstate_global->__pyx_kp_u__5; + __pyx_t_5 = __Pyx_PyUnicode_Join(__pyx_t_4, 7, 35 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_1) + 6 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_2) + 5 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_3) + 1, 127); + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1250, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(((PyObject *)(((PyTypeObject*)PyExc_ValueError))), __pyx_t_5, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 1250, __pyx_L1_error) + + /* "View.MemoryView":1247 + * + * + * @cname('__pyx_memoryview_err_extents') # <<<<<<<<<<<<<< + * cdef int _err_extents(int i, Py_ssize_t extent1, + * Py_ssize_t extent2) except -1 with gil: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("View.MemoryView._err_extents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + +static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, PyObject *__pyx_v_msg, int __pyx_v_dim) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err_dim", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1254 + * @cname('__pyx_memoryview_err_dim') + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err') +*/ + __pyx_t_1 = __Pyx_PyLong_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyUnicode_FormatSafe(__pyx_v_msg, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1254, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_t_2, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 1254, __pyx_L1_error) + + /* "View.MemoryView":1252 + * raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" + * + * @cname('__pyx_memoryview_err_dim') # <<<<<<<<<<<<<< + * cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + * raise error, msg % dim +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("View.MemoryView._err_dim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + +static int __pyx_memoryview_err(PyObject *__pyx_v_error, PyObject *__pyx_v_msg) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("_err", 0); + __Pyx_INCREF(__pyx_v_msg); + + /* "View.MemoryView":1258 + * @cname('__pyx_memoryview_err') + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_err_no_memory') +*/ + __Pyx_Raise(((PyObject *)__pyx_v_error), __pyx_v_msg, 0, 0); + __PYX_ERR(1, 1258, __pyx_L1_error) + + /* "View.MemoryView":1256 + * raise error, msg % dim + * + * @cname('__pyx_memoryview_err') # <<<<<<<<<<<<<< + * cdef int _err(PyObject *error, str msg) except -1 with gil: + * raise error, msg +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_XDECREF(__pyx_v_msg); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + +static int __pyx_memoryview_err_no_memory(void) { + int __pyx_r; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1262 + * @cname('__pyx_memoryview_err_no_memory') + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError # <<<<<<<<<<<<<< + * + * +*/ + PyErr_NoMemory(); __PYX_ERR(1, 1262, __pyx_L1_error) + + /* "View.MemoryView":1260 + * raise error, msg + * + * @cname('__pyx_memoryview_err_no_memory') # <<<<<<<<<<<<<< + * cdef int _err_no_memory() except -1 with gil: + * raise MemoryError +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("View.MemoryView._err_no_memory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + +static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_memviewslice __pyx_v_dst, int __pyx_v_src_ndim, int __pyx_v_dst_ndim, int __pyx_v_dtype_is_object) { + void *__pyx_v_tmpdata; + size_t __pyx_v_itemsize; + int __pyx_v_i; + char __pyx_v_order; + int __pyx_v_broadcasting; + int __pyx_v_direct_copy; + __Pyx_memviewslice __pyx_v_tmp; + int __pyx_v_ndim; + int __pyx_r; + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + int __pyx_t_6; + void *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "View.MemoryView":1274 + * Check for overlapping memory and verify the shapes. + * """ + * cdef void *tmpdata = NULL # <<<<<<<<<<<<<< + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i +*/ + __pyx_v_tmpdata = NULL; + + /* "View.MemoryView":1275 + * """ + * cdef void *tmpdata = NULL + * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) +*/ + __pyx_t_1 = __pyx_v_src.memview->view.itemsize; + __pyx_v_itemsize = __pyx_t_1; + + /* "View.MemoryView":1277 + * cdef size_t itemsize = src.memview.view.itemsize + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) # <<<<<<<<<<<<<< + * cdef bint broadcasting = False + * cdef bint direct_copy = False +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_src), __pyx_v_src_ndim); + + /* "View.MemoryView":1278 + * cdef int i + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False # <<<<<<<<<<<<<< + * cdef bint direct_copy = False + * cdef __Pyx_memviewslice tmp +*/ + __pyx_v_broadcasting = 0; + + /* "View.MemoryView":1279 + * cdef char order = get_best_order(&src, src_ndim) + * cdef bint broadcasting = False + * cdef bint direct_copy = False # <<<<<<<<<<<<<< + * cdef __Pyx_memviewslice tmp + * +*/ + __pyx_v_direct_copy = 0; + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + __pyx_t_2 = (__pyx_v_src_ndim < __pyx_v_dst_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1283 + * + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_src), __pyx_v_src_ndim, __pyx_v_dst_ndim); + + /* "View.MemoryView":1282 + * cdef __Pyx_memviewslice tmp + * + * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + __pyx_t_2 = (__pyx_v_dst_ndim < __pyx_v_src_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1285 + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: + * broadcast_leading(&dst, dst_ndim, src_ndim) # <<<<<<<<<<<<<< + * + * cdef int ndim = max(src_ndim, dst_ndim) +*/ + __pyx_memoryview_broadcast_leading((&__pyx_v_dst), __pyx_v_dst_ndim, __pyx_v_src_ndim); + + /* "View.MemoryView":1284 + * if src_ndim < dst_ndim: + * broadcast_leading(&src, src_ndim, dst_ndim) + * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< + * broadcast_leading(&dst, dst_ndim, src_ndim) + * +*/ + } + __pyx_L3:; + + /* "View.MemoryView":1287 + * broadcast_leading(&dst, dst_ndim, src_ndim) + * + * cdef int ndim = max(src_ndim, dst_ndim) # <<<<<<<<<<<<<< + * + * for i in range(ndim): +*/ + __pyx_t_3 = __pyx_v_dst_ndim; + __pyx_t_4 = __pyx_v_src_ndim; + __pyx_t_2 = (__pyx_t_3 > __pyx_t_4); + if (__pyx_t_2) { + __pyx_t_5 = __pyx_t_3; + } else { + __pyx_t_5 = __pyx_t_4; + } + __pyx_v_ndim = __pyx_t_5; + + /* "View.MemoryView":1289 + * cdef int ndim = max(src_ndim, dst_ndim) + * + * for i in range(ndim): # <<<<<<<<<<<<<< + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: +*/ + __pyx_t_5 = __pyx_v_ndim; + __pyx_t_3 = __pyx_t_5; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) != (__pyx_v_dst.shape[__pyx_v_i])); + if (__pyx_t_2) { + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + __pyx_t_2 = ((__pyx_v_src.shape[__pyx_v_i]) == 1); + if (__pyx_t_2) { + + /* "View.MemoryView":1292 + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: + * broadcasting = True # <<<<<<<<<<<<<< + * src.strides[i] = 0 + * else: +*/ + __pyx_v_broadcasting = 1; + + /* "View.MemoryView":1293 + * if src.shape[i] == 1: + * broadcasting = True + * src.strides[i] = 0 # <<<<<<<<<<<<<< + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) +*/ + (__pyx_v_src.strides[__pyx_v_i]) = 0; + + /* "View.MemoryView":1291 + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: + * if src.shape[i] == 1: # <<<<<<<<<<<<<< + * broadcasting = True + * src.strides[i] = 0 +*/ + goto __pyx_L7; + } + + /* "View.MemoryView":1295 + * src.strides[i] = 0 + * else: + * _err_extents(i, dst.shape[i], src.shape[i]) # <<<<<<<<<<<<<< + * + * if src.suboffsets[i] >= 0: +*/ + /*else*/ { + __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1295, __pyx_L1_error) + } + __pyx_L7:; + + /* "View.MemoryView":1290 + * + * for i in range(ndim): + * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< + * if src.shape[i] == 1: + * broadcasting = True +*/ + } + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + __pyx_t_2 = ((__pyx_v_src.suboffsets[__pyx_v_i]) >= 0); + if (__pyx_t_2) { + + /* "View.MemoryView":1298 + * + * if src.suboffsets[i] >= 0: + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) # <<<<<<<<<<<<<< + * + * if slices_overlap(&src, &dst, ndim, itemsize): +*/ + __pyx_t_6 = __pyx_memoryview_err_dim(PyExc_ValueError, __pyx_mstate_global->__pyx_kp_u_Dimension_d_is_not_direct, __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1298, __pyx_L1_error) + + /* "View.MemoryView":1297 + * _err_extents(i, dst.shape[i], src.shape[i]) + * + * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * +*/ + } + } + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + __pyx_t_2 = __pyx_slices_overlap((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + if (__pyx_t_2) { + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + __pyx_t_2 = (!__pyx_memviewslice_is_contig(__pyx_v_src, __pyx_v_order, __pyx_v_ndim)); + if (__pyx_t_2) { + + /* "View.MemoryView":1303 + * + * if not slice_is_contig(src, order, ndim): + * order = get_best_order(&dst, ndim) # <<<<<<<<<<<<<< + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) +*/ + __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim); + + /* "View.MemoryView":1302 + * if slices_overlap(&src, &dst, ndim, itemsize): + * + * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< + * order = get_best_order(&dst, ndim) + * +*/ + } + + /* "View.MemoryView":1305 + * order = get_best_order(&dst, ndim) + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) # <<<<<<<<<<<<<< + * src = tmp + * +*/ + __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1305, __pyx_L1_error) + __pyx_v_tmpdata = __pyx_t_7; + + /* "View.MemoryView":1306 + * + * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) + * src = tmp # <<<<<<<<<<<<<< + * + * if not broadcasting: +*/ + __pyx_v_src = __pyx_v_tmp; + + /* "View.MemoryView":1300 + * _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) + * + * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< + * + * if not slice_is_contig(src, order, ndim): +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (!__pyx_v_broadcasting); + if (__pyx_t_2) { + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'C', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1312 + * + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) # <<<<<<<<<<<<<< + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'C', __pyx_v_ndim); + + /* "View.MemoryView":1311 + * + * + * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): +*/ + goto __pyx_L12; + } + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + __pyx_t_2 = __pyx_memviewslice_is_contig(__pyx_v_src, 'F', __pyx_v_ndim); + if (__pyx_t_2) { + + /* "View.MemoryView":1314 + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): + * direct_copy = slice_is_contig(dst, 'F', ndim) # <<<<<<<<<<<<<< + * + * if direct_copy: +*/ + __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'F', __pyx_v_ndim); + + /* "View.MemoryView":1313 + * if slice_is_contig(src, 'C', ndim): + * direct_copy = slice_is_contig(dst, 'C', ndim) + * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< + * direct_copy = slice_is_contig(dst, 'F', ndim) + * +*/ + } + __pyx_L12:; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + if (__pyx_v_direct_copy) { + + /* "View.MemoryView":1318 + * if direct_copy: + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1319 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) +*/ + (void)(memcpy(__pyx_v_dst.data, __pyx_v_src.data, __pyx_memoryview_slice_get_size((&__pyx_v_src), __pyx_v_ndim))); + + /* "View.MemoryView":1320 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * free(tmpdata) + * return 0 +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1321 + * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1322 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * if order == 'F' == get_best_order(&dst, ndim): +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1316 + * direct_copy = slice_is_contig(dst, 'F', ndim) + * + * if direct_copy: # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + } + + /* "View.MemoryView":1308 + * src = tmp + * + * if not broadcasting: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = (__pyx_v_order == 'F'); + if (__pyx_t_2) { + __pyx_t_2 = ('F' == __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim)); + } + if (__pyx_t_2) { + + /* "View.MemoryView":1327 + * + * + * transpose_memslice(&src) # <<<<<<<<<<<<<< + * transpose_memslice(&dst) + * +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1327, __pyx_L1_error) + + /* "View.MemoryView":1328 + * + * transpose_memslice(&src) + * transpose_memslice(&dst) # <<<<<<<<<<<<<< + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) +*/ + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)-1))) __PYX_ERR(1, 1328, __pyx_L1_error) + + /* "View.MemoryView":1324 + * return 0 + * + * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":1330 + * transpose_memslice(&dst) + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1331 + * + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) # <<<<<<<<<<<<<< + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * +*/ + copy_strided_to_strided((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); + + /* "View.MemoryView":1332 + * refcount_copying(&dst, dtype_is_object, ndim, inc=False) + * copy_strided_to_strided(&src, &dst, ndim, itemsize) + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * free(tmpdata) +*/ + __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1334 + * refcount_copying(&dst, dtype_is_object, ndim, inc=True) + * + * free(tmpdata) # <<<<<<<<<<<<<< + * return 0 + * +*/ + free(__pyx_v_tmpdata); + + /* "View.MemoryView":1335 + * + * free(tmpdata) + * return 0 # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_broadcast_leading') +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "View.MemoryView":1265 + * + * + * @cname('__pyx_memoryview_copy_contents') # <<<<<<<<<<<<<< + * cdef int memoryview_copy_contents(__Pyx_memviewslice src, + * __Pyx_memviewslice dst, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("View.MemoryView.memoryview_copy_contents", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + +static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslice, int __pyx_v_ndim, int __pyx_v_ndim_other) { + int __pyx_v_i; + int __pyx_v_offset; + int __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + + /* "View.MemoryView":1342 + * int ndim_other) noexcept nogil: + * cdef int i + * cdef int offset = ndim_other - ndim # <<<<<<<<<<<<<< + * + * for i in range(ndim - 1, -1, -1): +*/ + __pyx_v_offset = (__pyx_v_ndim_other - __pyx_v_ndim); + + /* "View.MemoryView":1344 + * cdef int offset = ndim_other - ndim + * + * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] +*/ + for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { + __pyx_v_i = __pyx_t_1; + + /* "View.MemoryView":1345 + * + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] # <<<<<<<<<<<<<< + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] +*/ + (__pyx_v_mslice->shape[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->shape[__pyx_v_i]); + + /* "View.MemoryView":1346 + * for i in range(ndim - 1, -1, -1): + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] # <<<<<<<<<<<<<< + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * +*/ + (__pyx_v_mslice->strides[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->strides[__pyx_v_i]); + + /* "View.MemoryView":1347 + * mslice.shape[i + offset] = mslice.shape[i] + * mslice.strides[i + offset] = mslice.strides[i] + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] # <<<<<<<<<<<<<< + * + * for i in range(offset): +*/ + (__pyx_v_mslice->suboffsets[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->suboffsets[__pyx_v_i]); + } + + /* "View.MemoryView":1349 + * mslice.suboffsets[i + offset] = mslice.suboffsets[i] + * + * for i in range(offset): # <<<<<<<<<<<<<< + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] +*/ + __pyx_t_1 = __pyx_v_offset; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1350 + * + * for i in range(offset): + * mslice.shape[i] = 1 # <<<<<<<<<<<<<< + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 +*/ + (__pyx_v_mslice->shape[__pyx_v_i]) = 1; + + /* "View.MemoryView":1351 + * for i in range(offset): + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] # <<<<<<<<<<<<<< + * mslice.suboffsets[i] = -1 + * +*/ + (__pyx_v_mslice->strides[__pyx_v_i]) = (__pyx_v_mslice->strides[0]); + + /* "View.MemoryView":1352 + * mslice.shape[i] = 1 + * mslice.strides[i] = mslice.strides[0] + * mslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< + * + * +*/ + (__pyx_v_mslice->suboffsets[__pyx_v_i]) = -1L; + } + + /* "View.MemoryView":1337 + * return 0 + * + * @cname('__pyx_memoryview_broadcast_leading') # <<<<<<<<<<<<<< + * cdef void broadcast_leading(__Pyx_memviewslice *mslice, + * int ndim, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + +static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_dtype_is_object, int __pyx_v_ndim, int __pyx_v_inc) { + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + if (__pyx_v_dtype_is_object) { + + /* "View.MemoryView":1363 + * + * if dtype_is_object: + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') +*/ + __pyx_memoryview_refcount_objects_in_slice_with_gil(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1362 + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * + * if dtype_is_object: # <<<<<<<<<<<<<< + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * +*/ + } + + /* "View.MemoryView":1359 + * + * + * @cname('__pyx_memoryview_refcount_copying') # <<<<<<<<<<<<<< + * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + * +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + + /* "View.MemoryView":1369 + * Py_ssize_t *strides, int ndim, + * bint inc) noexcept with gil: + * refcount_objects_in_slice(data, shape, strides, ndim, inc) # <<<<<<<<<<<<<< + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') +*/ + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, __pyx_v_shape, __pyx_v_strides, __pyx_v_ndim, __pyx_v_inc); + + /* "View.MemoryView":1365 + * refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ + __Pyx_PyGILState_Release(__pyx_gilstate_save); +} + +/* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + +static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, int __pyx_v_inc) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + int __pyx_t_4; + + /* "View.MemoryView":1375 + * Py_ssize_t *strides, int ndim, bint inc) noexcept: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * + * for i in range(shape[0]): +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1377 + * cdef Py_ssize_t stride = strides[0] + * + * for i in range(shape[0]): # <<<<<<<<<<<<<< + * if ndim == 1: + * if inc: +*/ + __pyx_t_1 = (__pyx_v_shape[0]); + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + __pyx_t_4 = (__pyx_v_ndim == 1); + if (__pyx_t_4) { + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + if (__pyx_v_inc) { + + /* "View.MemoryView":1380 + * if ndim == 1: + * if inc: + * Py_INCREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * Py_DECREF(( data)[0]) +*/ + Py_INCREF((((PyObject **)__pyx_v_data)[0])); + + /* "View.MemoryView":1379 + * for i in range(shape[0]): + * if ndim == 1: + * if inc: # <<<<<<<<<<<<<< + * Py_INCREF(( data)[0]) + * else: +*/ + goto __pyx_L6; + } + + /* "View.MemoryView":1382 + * Py_INCREF(( data)[0]) + * else: + * Py_DECREF(( data)[0]) # <<<<<<<<<<<<<< + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) +*/ + /*else*/ { + Py_DECREF((((PyObject **)__pyx_v_data)[0])); + } + __pyx_L6:; + + /* "View.MemoryView":1378 + * + * for i in range(shape[0]): + * if ndim == 1: # <<<<<<<<<<<<<< + * if inc: + * Py_INCREF(( data)[0]) +*/ + goto __pyx_L5; + } + + /* "View.MemoryView":1384 + * Py_DECREF(( data)[0]) + * else: + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) # <<<<<<<<<<<<<< + * + * data += stride +*/ + /*else*/ { + __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_inc); + } + __pyx_L5:; + + /* "View.MemoryView":1386 + * refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) + * + * data += stride # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1371 + * refcount_objects_in_slice(data, shape, strides, ndim, inc) + * + * @cname('__pyx_memoryview_refcount_objects_in_slice') # <<<<<<<<<<<<<< + * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, bint inc) noexcept: +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + +static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item, int __pyx_v_dtype_is_object) { + + /* "View.MemoryView":1395 + * size_t itemsize, void *item, + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) # <<<<<<<<<<<<<< + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 0); + + /* "View.MemoryView":1396 + * bint dtype_is_object) noexcept nogil: + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) # <<<<<<<<<<<<<< + * refcount_copying(dst, dtype_is_object, ndim, inc=True) + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1397 + * refcount_copying(dst, dtype_is_object, ndim, inc=False) + * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + * refcount_copying(dst, dtype_is_object, ndim, inc=True) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 1); + + /* "View.MemoryView":1391 + * + * + * @cname('__pyx_memoryview_slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, + * size_t itemsize, void *item, +*/ + + /* function exit code */ +} + +/* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + +static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t *__pyx_v_shape, Py_ssize_t *__pyx_v_strides, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item) { + CYTHON_UNUSED Py_ssize_t __pyx_v_i; + Py_ssize_t __pyx_v_stride; + Py_ssize_t __pyx_v_extent; + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + Py_ssize_t __pyx_t_3; + Py_ssize_t __pyx_t_4; + + /* "View.MemoryView":1405 + * size_t itemsize, void *item) noexcept nogil: + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< + * cdef Py_ssize_t extent = shape[0] + * +*/ + __pyx_v_stride = (__pyx_v_strides[0]); + + /* "View.MemoryView":1406 + * cdef Py_ssize_t i + * cdef Py_ssize_t stride = strides[0] + * cdef Py_ssize_t extent = shape[0] # <<<<<<<<<<<<<< + * + * if ndim == 1: +*/ + __pyx_v_extent = (__pyx_v_shape[0]); + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + __pyx_t_1 = (__pyx_v_ndim == 1); + if (__pyx_t_1) { + + /* "View.MemoryView":1409 + * + * if ndim == 1: + * for i in range(extent): # <<<<<<<<<<<<<< + * memcpy(data, item, itemsize) + * data += stride +*/ + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1410 + * if ndim == 1: + * for i in range(extent): + * memcpy(data, item, itemsize) # <<<<<<<<<<<<<< + * data += stride + * else: +*/ + (void)(memcpy(__pyx_v_data, __pyx_v_item, __pyx_v_itemsize)); + + /* "View.MemoryView":1411 + * for i in range(extent): + * memcpy(data, item, itemsize) + * data += stride # <<<<<<<<<<<<<< + * else: + * for i in range(extent): +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + + /* "View.MemoryView":1408 + * cdef Py_ssize_t extent = shape[0] + * + * if ndim == 1: # <<<<<<<<<<<<<< + * for i in range(extent): + * memcpy(data, item, itemsize) +*/ + goto __pyx_L3; + } + + /* "View.MemoryView":1413 + * data += stride + * else: + * for i in range(extent): # <<<<<<<<<<<<<< + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride +*/ + /*else*/ { + __pyx_t_2 = __pyx_v_extent; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + + /* "View.MemoryView":1414 + * else: + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) # <<<<<<<<<<<<<< + * data += stride + * +*/ + __pyx_memoryview__slice_assign_scalar(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize, __pyx_v_item); + + /* "View.MemoryView":1415 + * for i in range(extent): + * _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) + * data += stride # <<<<<<<<<<<<<< + * +*/ + __pyx_v_data = (__pyx_v_data + __pyx_v_stride); + } + } + __pyx_L3:; + + /* "View.MemoryView":1400 + * + * + * @cname('__pyx_memoryview__slice_assign_scalar') # <<<<<<<<<<<<<< + * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, + * Py_ssize_t *strides, int ndim, +*/ + + /* function exit code */ +} + +/* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum = {"__pyx_unpickle_Enum", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_15View_dot_MemoryView_1__pyx_unpickle_Enum(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v___pyx_type = 0; + long __pyx_v___pyx_checksum; + PyObject *__pyx_v___pyx_state = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_pyx_type,&__pyx_mstate_global->__pyx_n_u_pyx_checksum,&__pyx_mstate_global->__pyx_n_u_pyx_state,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(1, 4, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__pyx_unpickle_Enum", 0) < (0)) __PYX_ERR(1, 4, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, i); __PYX_ERR(1, 4, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(1, 4, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(1, 4, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(1, 4, __pyx_L3_error) + } + __pyx_v___pyx_type = values[0]; + __pyx_v___pyx_checksum = __Pyx_PyLong_As_long(values[1]); if (unlikely((__pyx_v___pyx_checksum == (long)-1) && PyErr_Occurred())) __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_v___pyx_state = ((PyObject*)values[2]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__pyx_unpickle_Enum", 1, 3, 3, __pyx_nargs); __PYX_ERR(1, 4, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v___pyx_state), (&PyTuple_Type), 1, "__pyx_state", 1))) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_r = __pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(__pyx_self, __pyx_v___pyx_type, __pyx_v___pyx_checksum, __pyx_v___pyx_state); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_15View_dot_MemoryView___pyx_unpickle_Enum(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v___pyx_type, long __pyx_v___pyx_checksum, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_v___pyx_result = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + int __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum", 0); + + /* "(tree fragment)":6 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') # <<<<<<<<<<<<<< + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: +*/ + __pyx_t_1 = __Pyx_CheckUnpickleChecksum(__pyx_v___pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, ((char const *)"name")); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(1, 6, __pyx_L1_error) + + /* "(tree fragment)":7 + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) # <<<<<<<<<<<<<< + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) +*/ + __pyx_t_3 = ((PyObject *)__pyx_mstate_global->__pyx_MemviewEnum_type); + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_v___pyx_type}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_new, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v___pyx_result = __pyx_t_2; + __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + __pyx_t_5 = (__pyx_v___pyx_state != ((PyObject*)Py_None)); + if (__pyx_t_5) { + + /* "(tree fragment)":9 + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) # <<<<<<<<<<<<<< + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): +*/ + if (unlikely(__pyx_v___pyx_state == Py_None)) { + PyErr_SetString(PyExc_TypeError, "cannot pass None into a C function argument that is declared 'not None'"); + __PYX_ERR(1, 9, __pyx_L1_error) + } + __pyx_t_2 = __pyx_unpickle_Enum__set_state(((struct __pyx_MemviewEnum_obj *)__pyx_v___pyx_result), __pyx_v___pyx_state); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "(tree fragment)":8 + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') + * __pyx_result = Enum.__new__(__pyx_type) + * if __pyx_state is not None: # <<<<<<<<<<<<<< + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result +*/ + } + + /* "(tree fragment)":10 + * if __pyx_state is not None: + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result # <<<<<<<<<<<<<< + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v___pyx_result); + __pyx_r = __pyx_v___pyx_result; + goto __pyx_L0; + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v___pyx_result); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + +static PyObject *__pyx_unpickle_Enum__set_state(struct __pyx_MemviewEnum_obj *__pyx_v___pyx_result, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_unpickle_Enum__set_state", 0); + + /* "(tree fragment)":12 + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] # <<<<<<<<<<<<<< + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + __pyx_t_1 = __Pyx_GetItemInt_Tuple(__pyx_v___pyx_state, 0, long, 1, __Pyx_PyLong_From_long, 0, 0, 1, 1, __Pyx_ReferenceSharing_FunctionArgument); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_v___pyx_result->name); + __Pyx_DECREF(__pyx_v___pyx_result->name); + __pyx_v___pyx_result->name = __pyx_t_1; + __pyx_t_1 = 0; + + /* "(tree fragment)":13 + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) # <<<<<<<<<<<<<< +*/ + __pyx_t_2 = __Pyx_UpdateUnpickledDict(((PyObject *)__pyx_v___pyx_result), __pyx_v___pyx_state, 1); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(1, 13, __pyx_L1_error) + + /* "(tree fragment)":11 + * __pyx_unpickle_Enum__set_state( __pyx_result, __pyx_state) + * return __pyx_result + * cdef __pyx_unpickle_Enum__set_state(Enum __pyx_result, __pyx_state: tuple): # <<<<<<<<<<<<<< + * __pyx_result.name = __pyx_state[0] + * __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, 1) +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("View.MemoryView.__pyx_unpickle_Enum__set_state", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":242 + * cdef int type_num + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_8itemsize_itemsize(PyArray_Descr *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":244 + * @property + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyDataType_ELSIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":242 + * cdef int type_num + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp itemsize(self) noexcept nogil: + * return PyDataType_ELSIZE(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":246 + * return PyDataType_ELSIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_5dtype_9alignment_alignment(PyArray_Descr *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":248 + * @property + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) # <<<<<<<<<<<<<< + * + * # Use fields/names with care as they may be NULL. You must check +*/ + __pyx_r = PyDataType_ALIGNMENT(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":246 + * return PyDataType_ELSIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp alignment(self) noexcept nogil: + * return PyDataType_ALIGNMENT(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":252 + * # Use fields/names with care as they may be NULL. You must check + * # for this using PyDataType_HASFIELDS. + * @property # <<<<<<<<<<<<<< + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_6fields_fields(PyArray_Descr *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1; + __Pyx_RefNannySetupContext("fields", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":254 + * @property + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyDataType_FIELDS(__pyx_v_self); + __Pyx_INCREF(((PyObject *)__pyx_t_1)); + __pyx_r = ((PyObject *)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":252 + * # Use fields/names with care as they may be NULL. You must check + * # for this using PyDataType_HASFIELDS. + * @property # <<<<<<<<<<<<<< + * cdef inline object fields(self): + * return PyDataType_FIELDS(self) +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":256 + * return PyDataType_FIELDS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_5dtype_5names_names(PyArray_Descr *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1; + __Pyx_RefNannySetupContext("names", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":258 + * @property + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) # <<<<<<<<<<<<<< + * + * # Use PyDataType_HASSUBARRAY to test whether this field is +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyDataType_NAMES(__pyx_v_self); + __Pyx_INCREF(((PyObject*)__pyx_t_1)); + __pyx_r = ((PyObject*)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":256 + * return PyDataType_FIELDS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline tuple names(self): + * return PyDataType_NAMES(self) +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":263 + * # valid (the pointer can be NULL). Most users should access + * # this field via the inline helper method PyDataType_SHAPE. + * @property # <<<<<<<<<<<<<< + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) +*/ + +static CYTHON_INLINE PyArray_ArrayDescr *__pyx_f_5numpy_5dtype_8subarray_subarray(PyArray_Descr *__pyx_v_self) { + PyArray_ArrayDescr *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":265 + * @property + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyDataType_SUBARRAY(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":263 + * # valid (the pointer can be NULL). Most users should access + * # this field via the inline helper method PyDataType_SHAPE. + * @property # <<<<<<<<<<<<<< + * cdef inline PyArray_ArrayDescr* subarray(self) noexcept nogil: + * return PyDataType_SUBARRAY(self) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":267 + * return PyDataType_SUBARRAY(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" +*/ + +static CYTHON_INLINE npy_uint64 __pyx_f_5numpy_5dtype_5flags_flags(PyArray_Descr *__pyx_v_self) { + npy_uint64 __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":270 + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" + * return PyDataType_FLAGS(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyDataType_FLAGS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":267 + * return PyDataType_SUBARRAY(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_uint64 flags(self) noexcept nogil: + * """The data types flags.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":279 + * ctypedef class numpy.broadcast [object PyArrayMultiIterObject, check_size ignore]: + * + * @property # <<<<<<<<<<<<<< + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_7numiter_numiter(PyArrayMultiIterObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":282 + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" + * return PyArray_MultiIter_NUMITER(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_NUMITER(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":279 + * ctypedef class numpy.broadcast [object PyArrayMultiIterObject, check_size ignore]: + * + * @property # <<<<<<<<<<<<<< + * cdef inline int numiter(self) noexcept nogil: + * """The number of arrays that need to be broadcast to the same shape.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":284 + * return PyArray_MultiIter_NUMITER(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_4size_size(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":287 + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" + * return PyArray_MultiIter_SIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_SIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":284 + * return PyArray_MultiIter_NUMITER(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """The total broadcasted size.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":289 + * return PyArray_MultiIter_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_9broadcast_5index_index(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":292 + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" + * return PyArray_MultiIter_INDEX(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_INDEX(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":289 + * return PyArray_MultiIter_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp index(self) noexcept nogil: + * """The current (1-d) index into the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":294 + * return PyArray_MultiIter_INDEX(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_9broadcast_2nd_nd(PyArrayMultiIterObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":297 + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" + * return PyArray_MultiIter_NDIM(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_NDIM(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":294 + * return PyArray_MultiIter_INDEX(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int nd(self) noexcept nogil: + * """The number of dimensions in the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":299 + * return PyArray_MultiIter_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_9broadcast_10dimensions_dimensions(PyArrayMultiIterObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":302 + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" + * return PyArray_MultiIter_DIMS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_MultiIter_DIMS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":299 + * return PyArray_MultiIter_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp* dimensions(self) noexcept nogil: + * """The shape of the broadcasted result.""" +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":304 + * return PyArray_MultiIter_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline void** iters(self) noexcept nogil: + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. +*/ + +static CYTHON_INLINE void **__pyx_f_5numpy_9broadcast_5iters_iters(PyArrayMultiIterObject *__pyx_v_self) { + void **__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":308 + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. + * On return, the iterators are adjusted for broadcasting.""" + * return PyArray_MultiIter_ITERS(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyArray_MultiIter_ITERS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":304 + * return PyArray_MultiIter_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline void** iters(self) noexcept nogil: + * """An array of iterator objects that holds the iterators for the arrays to be broadcast together. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":322 + * # Instead, we use properties that map to the corresponding C-API functions. + * + * @property # <<<<<<<<<<<<<< + * cdef inline PyObject* base(self) noexcept nogil: + * """Returns a borrowed reference to the object owning the data/memory. +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_7ndarray_4base_base(PyArrayObject *__pyx_v_self) { + PyObject *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":326 + * """Returns a borrowed reference to the object owning the data/memory. + * """ + * return PyArray_BASE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_BASE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":322 + * # Instead, we use properties that map to the corresponding C-API functions. + * + * @property # <<<<<<<<<<<<<< + * cdef inline PyObject* base(self) noexcept nogil: + * """Returns a borrowed reference to the object owning the data/memory. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":328 + * return PyArray_BASE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline dtype descr(self): + * """Returns an owned reference to the dtype of the array. +*/ + +static CYTHON_INLINE PyArray_Descr *__pyx_f_5numpy_7ndarray_5descr_descr(PyArrayObject *__pyx_v_self) { + PyArray_Descr *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyArray_Descr *__pyx_t_1; + __Pyx_RefNannySetupContext("descr", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":332 + * """Returns an owned reference to the dtype of the array. + * """ + * return PyArray_DESCR(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __Pyx_XDECREF((PyObject *)__pyx_r); + __pyx_t_1 = PyArray_DESCR(__pyx_v_self); + __Pyx_INCREF((PyObject *)((PyArray_Descr *)__pyx_t_1)); + __pyx_r = ((PyArray_Descr *)__pyx_t_1); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":328 + * return PyArray_BASE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline dtype descr(self): + * """Returns an owned reference to the dtype of the array. +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF((PyObject *)__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":334 + * return PyArray_DESCR(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int ndim(self) noexcept nogil: + * """Returns the number of dimensions in the array. +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_7ndarray_4ndim_ndim(PyArrayObject *__pyx_v_self) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":338 + * """Returns the number of dimensions in the array. + * """ + * return PyArray_NDIM(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_NDIM(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":334 + * return PyArray_DESCR(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline int ndim(self) noexcept nogil: + * """Returns the number of dimensions in the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":340 + * return PyArray_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *shape(self) noexcept nogil: + * """Returns a pointer to the dimensions/shape of the array. +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_5shape_shape(PyArrayObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":346 + * Can return NULL for 0-dimensional arrays. + * """ + * return PyArray_DIMS(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_DIMS(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":340 + * return PyArray_NDIM(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *shape(self) noexcept nogil: + * """Returns a pointer to the dimensions/shape of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":348 + * return PyArray_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *strides(self) noexcept nogil: + * """Returns a pointer to the strides of the array. +*/ + +static CYTHON_INLINE npy_intp *__pyx_f_5numpy_7ndarray_7strides_strides(PyArrayObject *__pyx_v_self) { + npy_intp *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":353 + * The number of elements matches the number of dimensions of the array (ndim). + * """ + * return PyArray_STRIDES(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_STRIDES(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":348 + * return PyArray_DIMS(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp *strides(self) noexcept nogil: + * """Returns a pointer to the strides of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":355 + * return PyArray_STRIDES(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """Returns the total size (in number of elements) of the array. +*/ + +static CYTHON_INLINE npy_intp __pyx_f_5numpy_7ndarray_4size_size(PyArrayObject *__pyx_v_self) { + npy_intp __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":359 + * """Returns the total size (in number of elements) of the array. + * """ + * return PyArray_SIZE(self) # <<<<<<<<<<<<<< + * + * @property +*/ + __pyx_r = PyArray_SIZE(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":355 + * return PyArray_STRIDES(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline npy_intp size(self) noexcept nogil: + * """Returns the total size (in number of elements) of the array. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":361 + * return PyArray_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline char* data(self) noexcept nogil: + * """The pointer to the data buffer as a char*. +*/ + +static CYTHON_INLINE char *__pyx_f_5numpy_7ndarray_4data_data(PyArrayObject *__pyx_v_self) { + char *__pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":368 + * of `PyArray_DATA()` instead, which returns a 'void*'. + * """ + * return PyArray_BYTES(self) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyArray_BYTES(__pyx_v_self); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":361 + * return PyArray_SIZE(self) + * + * @property # <<<<<<<<<<<<<< + * cdef inline char* data(self) noexcept nogil: + * """The pointer to the data buffer as a char*. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":776 + * ctypedef long double complex clongdouble_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew1(PyObject *__pyx_v_a) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew1", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":777 + * + * cdef inline object PyArray_MultiIterNew1(a): + * return PyArray_MultiIterNew(1, a) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew2(a, b): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(1, ((void *)__pyx_v_a)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 777, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":776 + * ctypedef long double complex clongdouble_t + * + * cdef inline object PyArray_MultiIterNew1(a): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(1, a) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew1", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":779 + * return PyArray_MultiIterNew(1, a) + * + * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(2, a, b) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew2(PyObject *__pyx_v_a, PyObject *__pyx_v_b) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew2", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":780 + * + * cdef inline object PyArray_MultiIterNew2(a, b): + * return PyArray_MultiIterNew(2, a, b) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(2, ((void *)__pyx_v_a), ((void *)__pyx_v_b)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 780, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":779 + * return PyArray_MultiIterNew(1, a) + * + * cdef inline object PyArray_MultiIterNew2(a, b): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(2, a, b) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":782 + * return PyArray_MultiIterNew(2, a, b) + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(3, a, b, c) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew3(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew3", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":783 + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): + * return PyArray_MultiIterNew(3, a, b, c) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(3, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 783, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":782 + * return PyArray_MultiIterNew(2, a, b) + * + * cdef inline object PyArray_MultiIterNew3(a, b, c): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(3, a, b, c) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":785 + * return PyArray_MultiIterNew(3, a, b, c) + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(4, a, b, c, d) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew4(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew4", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":786 + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): + * return PyArray_MultiIterNew(4, a, b, c, d) # <<<<<<<<<<<<<< + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(4, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 786, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":785 + * return PyArray_MultiIterNew(3, a, b, c) + * + * cdef inline object PyArray_MultiIterNew4(a, b, c, d): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(4, a, b, c, d) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew4", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":788 + * return PyArray_MultiIterNew(4, a, b, c, d) + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyArray_MultiIterNew5(PyObject *__pyx_v_a, PyObject *__pyx_v_b, PyObject *__pyx_v_c, PyObject *__pyx_v_d, PyObject *__pyx_v_e) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("PyArray_MultiIterNew5", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":789 + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): + * return PyArray_MultiIterNew(5, a, b, c, d, e) # <<<<<<<<<<<<<< + * + * cdef inline tuple PyDataType_SHAPE(dtype d): +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyArray_MultiIterNew(5, ((void *)__pyx_v_a), ((void *)__pyx_v_b), ((void *)__pyx_v_c), ((void *)__pyx_v_d), ((void *)__pyx_v_e)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":788 + * return PyArray_MultiIterNew(4, a, b, c, d) + * + * cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): # <<<<<<<<<<<<<< + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("numpy.PyArray_MultiIterNew5", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":791 + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * + * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_PyDataType_SHAPE(PyArray_Descr *__pyx_v_d) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2; + __Pyx_RefNannySetupContext("PyDataType_SHAPE", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":792 + * + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< + * return d.subarray.shape + * else: +*/ + __pyx_t_1 = PyDataType_HASSUBARRAY(__pyx_v_d); + if (__pyx_t_1) { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":793 + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape # <<<<<<<<<<<<<< + * else: + * return () +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_f_5numpy_5dtype_8subarray_subarray(__pyx_v_d)->shape; + __Pyx_INCREF(((PyObject*)__pyx_t_2)); + __pyx_r = ((PyObject*)__pyx_t_2); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":792 + * + * cdef inline tuple PyDataType_SHAPE(dtype d): + * if PyDataType_HASSUBARRAY(d): # <<<<<<<<<<<<<< + * return d.subarray.shape + * else: +*/ + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":795 + * return d.subarray.shape + * else: + * return () # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_mstate_global->__pyx_empty_tuple); + __pyx_r = __pyx_mstate_global->__pyx_empty_tuple; + goto __pyx_L0; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":791 + * return PyArray_MultiIterNew(5, a, b, c, d, e) + * + * cdef inline tuple PyDataType_SHAPE(dtype d): # <<<<<<<<<<<<<< + * if PyDataType_HASSUBARRAY(d): + * return d.subarray.shape +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":994 + * int _import_umath() except -1 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: # <<<<<<<<<<<<<< + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) +*/ + +static CYTHON_INLINE void __pyx_f_5numpy_set_array_base(PyArrayObject *__pyx_v_arr, PyObject *__pyx_v_base) { + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":995 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: + * Py_INCREF(base) # important to do this before stealing the reference below! # <<<<<<<<<<<<<< + * PyArray_SetBaseObject(arr, base) + * +*/ + Py_INCREF(__pyx_v_base); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":996 + * cdef inline void set_array_base(ndarray arr, object base) except *: + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) # <<<<<<<<<<<<<< + * + * cdef inline object get_array_base(ndarray arr): +*/ + __pyx_t_1 = PyArray_SetBaseObject(__pyx_v_arr, __pyx_v_base); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 996, __pyx_L1_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":994 + * int _import_umath() except -1 + * + * cdef inline void set_array_base(ndarray arr, object base) except *: # <<<<<<<<<<<<<< + * Py_INCREF(base) # important to do this before stealing the reference below! + * PyArray_SetBaseObject(arr, base) +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("numpy.set_array_base", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":998 + * PyArray_SetBaseObject(arr, base) + * + * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< + * base = PyArray_BASE(arr) + * if base is NULL: +*/ + +static CYTHON_INLINE PyObject *__pyx_f_5numpy_get_array_base(PyArrayObject *__pyx_v_arr) { + PyObject *__pyx_v_base; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + __Pyx_RefNannySetupContext("get_array_base", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":999 + * + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) # <<<<<<<<<<<<<< + * if base is NULL: + * return None +*/ + __pyx_v_base = PyArray_BASE(__pyx_v_arr); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1000 + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) + * if base is NULL: # <<<<<<<<<<<<<< + * return None + * return base +*/ + __pyx_t_1 = (__pyx_v_base == NULL); + if (__pyx_t_1) { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1001 + * base = PyArray_BASE(arr) + * if base is NULL: + * return None # <<<<<<<<<<<<<< + * return base + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1000 + * cdef inline object get_array_base(ndarray arr): + * base = PyArray_BASE(arr) + * if base is NULL: # <<<<<<<<<<<<<< + * return None + * return base +*/ + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1002 + * if base is NULL: + * return None + * return base # <<<<<<<<<<<<<< + * + * # Versions of the import_* functions which are more suitable for +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_base)); + __pyx_r = ((PyObject *)__pyx_v_base); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":998 + * PyArray_SetBaseObject(arr, base) + * + * cdef inline object get_array_base(ndarray arr): # <<<<<<<<<<<<<< + * base = PyArray_BASE(arr) + * if base is NULL: +*/ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1006 + * # Versions of the import_* functions which are more suitable for + * # Cython code. + * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< + * try: + * __pyx_import_array() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_array(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_array", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1008 + * cdef inline int import_array() except -1: + * try: + * __pyx_import_array() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.multiarray failed to import") +*/ + __pyx_t_4 = _import_array(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1008, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1009 + * try: + * __pyx_import_array() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.multiarray failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_array", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1009, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1010 + * __pyx_import_array() + * except Exception: + * raise ImportError("numpy._core.multiarray failed to import") # <<<<<<<<<<<<<< + * + * cdef inline int import_umath() except -1: +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_multiarray_failed_to}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1010, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1010, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1007 + * # Cython code. + * cdef inline int import_array() except -1: + * try: # <<<<<<<<<<<<<< + * __pyx_import_array() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1006 + * # Versions of the import_* functions which are more suitable for + * # Cython code. + * cdef inline int import_array() except -1: # <<<<<<<<<<<<<< + * try: + * __pyx_import_array() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_array", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1012 + * raise ImportError("numpy._core.multiarray failed to import") + * + * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_umath(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_umath", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1014 + * cdef inline int import_umath() except -1: + * try: + * _import_umath() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.umath failed to import") +*/ + __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1014, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1015 + * try: + * _import_umath() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.umath failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_umath", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1015, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1016 + * _import_umath() + * except Exception: + * raise ImportError("numpy._core.umath failed to import") # <<<<<<<<<<<<<< + * + * cdef inline int import_ufunc() except -1: +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_umath_failed_to_impo}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1016, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1016, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1013 + * + * cdef inline int import_umath() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1012 + * raise ImportError("numpy._core.multiarray failed to import") + * + * cdef inline int import_umath() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_umath", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1018 + * raise ImportError("numpy._core.umath failed to import") + * + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_import_ufunc(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + size_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("import_ufunc", 0); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1020 + * cdef inline int import_ufunc() except -1: + * try: + * _import_umath() # <<<<<<<<<<<<<< + * except Exception: + * raise ImportError("numpy._core.umath failed to import") +*/ + __pyx_t_4 = _import_umath(); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(2, 1020, __pyx_L3_error) + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L8_try_end; + __pyx_L3_error:; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1021 + * try: + * _import_umath() + * except Exception: # <<<<<<<<<<<<<< + * raise ImportError("numpy._core.umath failed to import") + * +*/ + __pyx_t_4 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_4) { + __Pyx_AddTraceback("numpy.import_ufunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_6, &__pyx_t_7) < 0) __PYX_ERR(2, 1021, __pyx_L5_except_error) + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1022 + * _import_umath() + * except Exception: + * raise ImportError("numpy._core.umath failed to import") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_9 = NULL; + __pyx_t_10 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_9, __pyx_mstate_global->__pyx_kp_u_numpy__core_umath_failed_to_impo}; + __pyx_t_8 = __Pyx_PyObject_FastCall((PyObject*)(((PyTypeObject*)PyExc_ImportError)), __pyx_callargs+__pyx_t_10, (2-__pyx_t_10) | (__pyx_t_10*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 1022, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + } + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(2, 1022, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1019 + * + * cdef inline int import_ufunc() except -1: + * try: # <<<<<<<<<<<<<< + * _import_umath() + * except Exception: +*/ + __pyx_L5_except_error:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L8_try_end:; + } + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1018 + * raise ImportError("numpy._core.umath failed to import") + * + * cdef inline int import_ufunc() except -1: # <<<<<<<<<<<<<< + * try: + * _import_umath() +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("numpy.import_ufunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1025 + * + * + * cdef inline bint is_timedelta64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.timedelta64)` +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_is_timedelta64_object(PyObject *__pyx_v_obj) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1037 + * bool + * """ + * return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyObject_TypeCheck(__pyx_v_obj, (&PyTimedeltaArrType_Type)); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1025 + * + * + * cdef inline bint is_timedelta64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.timedelta64)` +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1040 + * + * + * cdef inline bint is_datetime64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.datetime64)` +*/ + +static CYTHON_INLINE int __pyx_f_5numpy_is_datetime64_object(PyObject *__pyx_v_obj) { + int __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1052 + * bool + * """ + * return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = PyObject_TypeCheck(__pyx_v_obj, (&PyDatetimeArrType_Type)); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1040 + * + * + * cdef inline bint is_datetime64_object(object obj) noexcept: # <<<<<<<<<<<<<< + * """ + * Cython equivalent of `isinstance(obj, np.datetime64)` +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1055 + * + * + * cdef inline npy_datetime get_datetime64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy datetime64 object +*/ + +static CYTHON_INLINE npy_datetime __pyx_f_5numpy_get_datetime64_value(PyObject *__pyx_v_obj) { + npy_datetime __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1062 + * also needed. That can be found using `get_datetime64_unit`. + * """ + * return (obj).obval # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((PyDatetimeScalarObject *)__pyx_v_obj)->obval; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1055 + * + * + * cdef inline npy_datetime get_datetime64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy datetime64 object +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1065 + * + * + * cdef inline npy_timedelta get_timedelta64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy timedelta64 object +*/ + +static CYTHON_INLINE npy_timedelta __pyx_f_5numpy_get_timedelta64_value(PyObject *__pyx_v_obj) { + npy_timedelta __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1069 + * returns the int64 value underlying scalar numpy timedelta64 object + * """ + * return (obj).obval # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((PyTimedeltaScalarObject *)__pyx_v_obj)->obval; + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1065 + * + * + * cdef inline npy_timedelta get_timedelta64_value(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the int64 value underlying scalar numpy timedelta64 object +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1072 + * + * + * cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the unit part of the dtype for a numpy datetime64 object. +*/ + +static CYTHON_INLINE NPY_DATETIMEUNIT __pyx_f_5numpy_get_datetime64_unit(PyObject *__pyx_v_obj) { + NPY_DATETIMEUNIT __pyx_r; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1076 + * returns the unit part of the dtype for a numpy datetime64 object. + * """ + * return (obj).obmeta.base # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((NPY_DATETIMEUNIT)((PyDatetimeScalarObject *)__pyx_v_obj)->obmeta.base); + goto __pyx_L0; + + /* "C:/Users/runneradmin/AppData/Local/Temp/build-env-kzdm41oa/Lib/site-packages/numpy/__init__.cython-30.pxd":1072 + * + * + * cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) noexcept nogil: # <<<<<<<<<<<<<< + * """ + * returns the unit part of the dtype for a numpy datetime64 object. +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":18 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_1SparseLinear(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_SparseLinear, "SparseLinear(nO: Optional[int] = None, length: int = 0x40000)"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_1SparseLinear = {"SparseLinear", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_1SparseLinear, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_SparseLinear}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_1SparseLinear(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_nO = 0; + PyObject *__pyx_v_length = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("SparseLinear (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_nO,&__pyx_mstate_global->__pyx_n_u_length,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 18, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 18, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 18, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "SparseLinear", 0) < (0)) __PYX_ERR(0, 18, __pyx_L3_error) + + /* "thinc/layers/sparselinear.pyx":20 + * @cython.binding(True) + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): # <<<<<<<<<<<<<< + * # NB: We can't have generic return type annotation if we want function to + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 +*/ + if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_262144))); + } else { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 18, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 18, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_262144))); + } + __pyx_v_nO = values[0]; + __pyx_v_length = values[1]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("SparseLinear", 0, 0, 2, __pyx_nargs); __PYX_ERR(0, 18, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear.SparseLinear", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_SparseLinear(__pyx_self, __pyx_v_nO, __pyx_v_length); + + /* "thinc/layers/sparselinear.pyx":18 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_SparseLinear(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nO, PyObject *__pyx_v_length) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("SparseLinear", 0); + + /* "thinc/layers/sparselinear.pyx":23 + * # NB: We can't have generic return type annotation if we want function to + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + * return Model( # <<<<<<<<<<<<<< + * "sparse_linear", + * forward, +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_Model); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + + /* "thinc/layers/sparselinear.pyx":25 + * return Model( + * "sparse_linear", + * forward, # <<<<<<<<<<<<<< + * init=init, + * params={"W": None, "b": None}, +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_forward); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + + /* "thinc/layers/sparselinear.pyx":26 + * "sparse_linear", + * forward, + * init=init, # <<<<<<<<<<<<<< + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, +*/ + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_init); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 26, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + + /* "thinc/layers/sparselinear.pyx":27 + * forward, + * init=init, + * params={"W": None, "b": None}, # <<<<<<<<<<<<<< + * dims={"nO": nO, "length": length}, + * attrs={"v1_indexing": True}, +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 27, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_W, Py_None) < (0)) __PYX_ERR(0, 27, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_b, Py_None) < (0)) __PYX_ERR(0, 27, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":28 + * init=init, + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, # <<<<<<<<<<<<<< + * attrs={"v1_indexing": True}, + * ) +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 28, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_nO, __pyx_v_nO) < (0)) __PYX_ERR(0, 28, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_length, __pyx_v_length) < (0)) __PYX_ERR(0, 28, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":29 + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, + * attrs={"v1_indexing": True}, # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_t_8 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (PyDict_SetItem(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_v1_indexing, Py_True) < (0)) __PYX_ERR(0, 29, __pyx_L1_error) + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 4 : 0)] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_sparse_linear, __pyx_t_4}; + __pyx_t_10 = __Pyx_MakeVectorcallBuilderKwds(4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_init, __pyx_t_5, __pyx_t_10, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 23, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_params, __pyx_t_6, __pyx_t_10, __pyx_callargs+3, 1) < (0)) __PYX_ERR(0, 23, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dims, __pyx_t_7, __pyx_t_10, __pyx_callargs+3, 2) < (0)) __PYX_ERR(0, 23, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_attrs, __pyx_t_8, __pyx_t_10, __pyx_callargs+3, 3) < (0)) __PYX_ERR(0, 23, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_10); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 23, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":18 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.layers.sparselinear.SparseLinear", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":33 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v2") + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_3SparseLinear_v2(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_2SparseLinear_v2, "SparseLinear_v2(nO: Optional[int] = None, length: int = 0x40000)"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_3SparseLinear_v2 = {"SparseLinear_v2", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_3SparseLinear_v2, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_2SparseLinear_v2}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_3SparseLinear_v2(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_nO = 0; + PyObject *__pyx_v_length = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("SparseLinear_v2 (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_nO,&__pyx_mstate_global->__pyx_n_u_length,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 33, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 33, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 33, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "SparseLinear_v2", 0) < (0)) __PYX_ERR(0, 33, __pyx_L3_error) + + /* "thinc/layers/sparselinear.pyx":35 + * @cython.binding(True) + * @registry.layers("SparseLinear.v2") + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): # <<<<<<<<<<<<<< + * # NB: We can't have generic return type annotation if we want function to + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 +*/ + if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_262144))); + } else { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 33, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 33, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[0]) values[0] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)((PyObject*)__pyx_mstate_global->__pyx_int_262144))); + } + __pyx_v_nO = values[0]; + __pyx_v_length = values[1]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("SparseLinear_v2", 0, 0, 2, __pyx_nargs); __PYX_ERR(0, 33, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear.SparseLinear_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_2SparseLinear_v2(__pyx_self, __pyx_v_nO, __pyx_v_length); + + /* "thinc/layers/sparselinear.pyx":33 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v2") + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_2SparseLinear_v2(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_nO, PyObject *__pyx_v_length) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("SparseLinear_v2", 0); + + /* "thinc/layers/sparselinear.pyx":38 + * # NB: We can't have generic return type annotation if we want function to + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + * return Model( # <<<<<<<<<<<<<< + * "sparse_linear", + * forward, +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_Model); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + + /* "thinc/layers/sparselinear.pyx":40 + * return Model( + * "sparse_linear", + * forward, # <<<<<<<<<<<<<< + * init=init, + * params={"W": None, "b": None}, +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_forward); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + + /* "thinc/layers/sparselinear.pyx":41 + * "sparse_linear", + * forward, + * init=init, # <<<<<<<<<<<<<< + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, +*/ + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_init); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + + /* "thinc/layers/sparselinear.pyx":42 + * forward, + * init=init, + * params={"W": None, "b": None}, # <<<<<<<<<<<<<< + * dims={"nO": nO, "length": length}, + * attrs={"v1_indexing": False}, +*/ + __pyx_t_6 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 42, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_W, Py_None) < (0)) __PYX_ERR(0, 42, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_b, Py_None) < (0)) __PYX_ERR(0, 42, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":43 + * init=init, + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, # <<<<<<<<<<<<<< + * attrs={"v1_indexing": False}, + * ) +*/ + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_nO, __pyx_v_nO) < (0)) __PYX_ERR(0, 43, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_mstate_global->__pyx_n_u_length, __pyx_v_length) < (0)) __PYX_ERR(0, 43, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":44 + * params={"W": None, "b": None}, + * dims={"nO": nO, "length": length}, + * attrs={"v1_indexing": False}, # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_t_8 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (PyDict_SetItem(__pyx_t_8, __pyx_mstate_global->__pyx_n_u_v1_indexing, Py_False) < (0)) __PYX_ERR(0, 44, __pyx_L1_error) + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[3 + ((CYTHON_VECTORCALL) ? 4 : 0)] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_sparse_linear, __pyx_t_4}; + __pyx_t_10 = __Pyx_MakeVectorcallBuilderKwds(4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_init, __pyx_t_5, __pyx_t_10, __pyx_callargs+3, 0) < (0)) __PYX_ERR(0, 38, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_params, __pyx_t_6, __pyx_t_10, __pyx_callargs+3, 1) < (0)) __PYX_ERR(0, 38, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dims, __pyx_t_7, __pyx_t_10, __pyx_callargs+3, 2) < (0)) __PYX_ERR(0, 38, __pyx_L1_error) + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_attrs, __pyx_t_8, __pyx_t_10, __pyx_callargs+3, 3) < (0)) __PYX_ERR(0, 38, __pyx_L1_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_9, (3-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_10); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":33 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v2") + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("thinc.layers.sparselinear.SparseLinear_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":48 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * def forward(model: Model, keys_values_lengths: InT, is_train: bool) -> Tuple[OutT, Callable]: + * # NB: We can't have generic Model annotation if we want function to +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_5forward(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_4forward, "forward(model: Model, keys_values_lengths: InT, is_train: bool) -> Tuple[OutT, Callable]"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_5forward = {"forward", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_5forward, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_4forward}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_5forward(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_model = 0; + PyObject *__pyx_v_keys_values_lengths = 0; + CYTHON_UNUSED PyObject *__pyx_v_is_train = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("forward (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_keys_values_lengths,&__pyx_mstate_global->__pyx_n_u_is_train,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 48, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 48, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 48, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 48, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "forward", 0) < (0)) __PYX_ERR(0, 48, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 3; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("forward", 1, 3, 3, i); __PYX_ERR(0, 48, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 3)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 48, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 48, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 48, __pyx_L3_error) + } + __pyx_v_model = values[0]; + __pyx_v_keys_values_lengths = values[1]; + __pyx_v_is_train = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("forward", 1, 3, 3, __pyx_nargs); __PYX_ERR(0, 48, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear.forward", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_4forward(__pyx_self, __pyx_v_model, __pyx_v_keys_values_lengths, __pyx_v_is_train); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_4forward(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys_values_lengths, CYTHON_UNUSED PyObject *__pyx_v_is_train) { + PyObject *__pyx_v_keys = NULL; + PyObject *__pyx_v_values = NULL; + PyObject *__pyx_v_lengths = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *(*__pyx_t_5)(PyObject *); + size_t __pyx_t_6; + int __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("forward", 0); + + /* "thinc/layers/sparselinear.pyx":52 + * # NB: We can't have generic Model annotation if we want function to + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + * keys, values, lengths = keys_values_lengths # <<<<<<<<<<<<<< + * if is_cupy_array(keys): + * # Currently we don't have a GPU-compatible implementation of this function :( +*/ + if ((likely(PyTuple_CheckExact(__pyx_v_keys_values_lengths))) || (PyList_CheckExact(__pyx_v_keys_values_lengths))) { + PyObject* sequence = __pyx_v_keys_values_lengths; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 3)) { + if (size > 3) __Pyx_RaiseTooManyValuesError(3); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 52, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_1); + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyTuple_GET_ITEM(sequence, 2); + __Pyx_INCREF(__pyx_t_3); + } else { + __pyx_t_1 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyList_GetItemRefFast(sequence, 2, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_3); + } + #else + __pyx_t_1 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } else { + Py_ssize_t index = -1; + __pyx_t_4 = PyObject_GetIter(__pyx_v_keys_values_lengths); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 52, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_4); + index = 0; __pyx_t_1 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_1)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_1); + index = 1; __pyx_t_2 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + index = 2; __pyx_t_3 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_3)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_5(__pyx_t_4), 3) < (0)) __PYX_ERR(0, 52, __pyx_L1_error) + __pyx_t_5 = NULL; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_5 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 52, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_keys = __pyx_t_1; + __pyx_t_1 = 0; + __pyx_v_values = __pyx_t_2; + __pyx_t_2 = 0; + __pyx_v_lengths = __pyx_t_3; + __pyx_t_3 = 0; + + /* "thinc/layers/sparselinear.pyx":53 + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + * keys, values, lengths = keys_values_lengths + * if is_cupy_array(keys): # <<<<<<<<<<<<<< + * # Currently we don't have a GPU-compatible implementation of this function :( + * # It sucks, but at least we can get the correct result by copying to CPU. +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_is_cupy_array); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_1); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_1, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_v_keys}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_1, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_t_7 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely((__pyx_t_7 < 0))) __PYX_ERR(0, 53, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_7) { + + /* "thinc/layers/sparselinear.pyx":56 + * # Currently we don't have a GPU-compatible implementation of this function :( + * # It sucks, but at least we can get the correct result by copying to CPU. + * return _begin_gpu_update(model, keys, values, lengths) # <<<<<<<<<<<<<< + * else: + * return _begin_cpu_update(model, keys, values, lengths) +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_begin_gpu_update); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_1); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_1, __pyx_v_model, __pyx_v_keys, __pyx_v_values, __pyx_v_lengths}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_6, (5-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":53 + * # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + * keys, values, lengths = keys_values_lengths + * if is_cupy_array(keys): # <<<<<<<<<<<<<< + * # Currently we don't have a GPU-compatible implementation of this function :( + * # It sucks, but at least we can get the correct result by copying to CPU. +*/ + } + + /* "thinc/layers/sparselinear.pyx":58 + * return _begin_gpu_update(model, keys, values, lengths) + * else: + * return _begin_cpu_update(model, keys, values, lengths) # <<<<<<<<<<<<<< + * + * +*/ + /*else*/ { + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_begin_cpu_update); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 58, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_1); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_1, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_2, __pyx_v_model, __pyx_v_keys, __pyx_v_values, __pyx_v_lengths}; + __pyx_t_3 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_1, __pyx_callargs+__pyx_t_6, (5-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 58, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + } + + /* "thinc/layers/sparselinear.pyx":48 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * def forward(model: Model, keys_values_lengths: InT, is_train: bool) -> Tuple[OutT, Callable]: + * # NB: We can't have generic Model annotation if we want function to +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("thinc.layers.sparselinear.forward", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_keys); + __Pyx_XDECREF(__pyx_v_values); + __Pyx_XDECREF(__pyx_v_lengths); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":61 + * + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: # <<<<<<<<<<<<<< + * if Y is not None: + * model.set_dim("nO", get_width(Y)) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_7init(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_6init, "init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_7init = {"init", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_7init, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_6init}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_7init(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_model = 0; + CYTHON_UNUSED PyObject *__pyx_v_X = 0; + PyObject *__pyx_v_Y = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[3] = {0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("init (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_X,&__pyx_mstate_global->__pyx_n_u_Y,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 61, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 61, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 61, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 61, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "init", 0) < (0)) __PYX_ERR(0, 61, __pyx_L3_error) + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("init", 0, 1, 3, i); __PYX_ERR(0, 61, __pyx_L3_error) } + } + } else { + switch (__pyx_nargs) { + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 61, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 61, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 61, __pyx_L3_error) + break; + default: goto __pyx_L5_argtuple_error; + } + if (!values[1]) values[1] = __Pyx_NewRef(((PyObject *)Py_None)); + if (!values[2]) values[2] = __Pyx_NewRef(((PyObject *)Py_None)); + } + __pyx_v_model = values[0]; + __pyx_v_X = values[1]; + __pyx_v_Y = values[2]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("init", 0, 1, 3, __pyx_nargs); __PYX_ERR(0, 61, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear.init", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_6init(__pyx_self, __pyx_v_model, __pyx_v_X, __pyx_v_Y); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_6init(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, CYTHON_UNUSED PyObject *__pyx_v_X, PyObject *__pyx_v_Y) { + PyObject *__pyx_v_nO = NULL; + PyObject *__pyx_v_length = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + size_t __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("init", 0); + + /* "thinc/layers/sparselinear.pyx":62 + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: + * if Y is not None: # <<<<<<<<<<<<<< + * model.set_dim("nO", get_width(Y)) + * nO = model.get_dim("nO") +*/ + __pyx_t_1 = (__pyx_v_Y != Py_None); + if (__pyx_t_1) { + + /* "thinc/layers/sparselinear.pyx":63 + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: + * if Y is not None: + * model.set_dim("nO", get_width(Y)) # <<<<<<<<<<<<<< + * nO = model.get_dim("nO") + * length = model.get_dim("length") +*/ + __pyx_t_3 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_5 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_get_width); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 63, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_6); + assert(__pyx_t_5); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_6, __pyx__function); + __pyx_t_7 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_v_Y}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_6, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (__pyx_t_7*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 63, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_mstate_global->__pyx_n_u_nO, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_set_dim, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 63, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/layers/sparselinear.pyx":62 + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: + * if Y is not None: # <<<<<<<<<<<<<< + * model.set_dim("nO", get_width(Y)) + * nO = model.get_dim("nO") +*/ + } + + /* "thinc/layers/sparselinear.pyx":64 + * if Y is not None: + * model.set_dim("nO", get_width(Y)) + * nO = model.get_dim("nO") # <<<<<<<<<<<<<< + * length = model.get_dim("length") + * model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) +*/ + __pyx_t_4 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_nO}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_nO = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/sparselinear.pyx":65 + * model.set_dim("nO", get_width(Y)) + * nO = model.get_dim("nO") + * length = model.get_dim("length") # <<<<<<<<<<<<<< + * model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) + * model.set_param("b", model.ops.alloc((nO,), dtype="f")) +*/ + __pyx_t_4 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_length}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_v_length = __pyx_t_2; + __pyx_t_2 = 0; + + /* "thinc/layers/sparselinear.pyx":66 + * nO = model.get_dim("nO") + * length = model.get_dim("length") + * model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) # <<<<<<<<<<<<<< + * model.set_param("b", model.ops.alloc((nO,), dtype="f")) + * return model +*/ + __pyx_t_4 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_8 = PyNumber_Multiply(__pyx_v_nO, __pyx_v_length); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_8) != (0)) __PYX_ERR(0, 66, __pyx_L1_error); + __pyx_t_8 = 0; + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_6, __pyx_t_9}; + __pyx_t_8 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_8, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_t_3 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_8); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_W, __pyx_t_3}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_set_param, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/layers/sparselinear.pyx":67 + * length = model.get_dim("length") + * model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) + * model.set_param("b", model.ops.alloc((nO,), dtype="f")) # <<<<<<<<<<<<<< + * return model + * +*/ + __pyx_t_3 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = __pyx_t_8; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_9 = PyTuple_New(1); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_INCREF(__pyx_v_nO); + __Pyx_GIVEREF(__pyx_v_nO); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_v_nO) != (0)) __PYX_ERR(0, 67, __pyx_L1_error); + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_5, __pyx_t_9}; + __pyx_t_6 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_dtype, __pyx_mstate_global->__pyx_n_u_f, __pyx_t_6, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 67, __pyx_L1_error) + __pyx_t_4 = __Pyx_Object_VectorcallMethod_CallFromBuilder((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_7, (2-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_6); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_7 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_3, __pyx_mstate_global->__pyx_n_u_b, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_set_param, __pyx_callargs+__pyx_t_7, (3-__pyx_t_7) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "thinc/layers/sparselinear.pyx":68 + * model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) + * model.set_param("b", model.ops.alloc((nO,), dtype="f")) + * return model # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_model); + __pyx_r = __pyx_v_model; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":61 + * + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: # <<<<<<<<<<<<<< + * if Y is not None: + * model.set_dim("nO", get_width(Y)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.layers.sparselinear.init", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_nO); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":71 + * + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: # <<<<<<<<<<<<<< + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_9_begin_gpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_8_begin_gpu_update, "_begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_9_begin_gpu_update = {"_begin_gpu_update", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_9_begin_gpu_update, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_8_begin_gpu_update}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_9_begin_gpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_model = 0; + PyObject *__pyx_v_keys = 0; + PyObject *__pyx_v_values = 0; + PyObject *__pyx_v_lengths = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_begin_gpu_update (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_keys,&__pyx_mstate_global->__pyx_n_u_values,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 71, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 71, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 71, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 71, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 71, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_begin_gpu_update", 0) < (0)) __PYX_ERR(0, 71, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_begin_gpu_update", 1, 4, 4, i); __PYX_ERR(0, 71, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 71, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 71, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 71, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 71, __pyx_L3_error) + } + __pyx_v_model = values[0]; + __pyx_v_keys = values[1]; + __pyx_v_values = values[2]; + __pyx_v_lengths = values[3]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_begin_gpu_update", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 71, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_gpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_8_begin_gpu_update(__pyx_self, __pyx_v_model, __pyx_v_keys, __pyx_v_values, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":75 + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) + * + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: # <<<<<<<<<<<<<< + * callback(d_scores.get()) + * return (keys, values, lengths) +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_17_begin_gpu_update_1backprop_gpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_17_begin_gpu_update_1backprop_gpu_update = {"backprop_gpu_update", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_17_begin_gpu_update_1backprop_gpu_update, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_17_begin_gpu_update_1backprop_gpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_d_scores = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("backprop_gpu_update (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_d_scores,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 75, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 75, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "backprop_gpu_update", 0) < (0)) __PYX_ERR(0, 75, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("backprop_gpu_update", 1, 1, 1, i); __PYX_ERR(0, 75, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 75, __pyx_L3_error) + } + __pyx_v_d_scores = values[0]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("backprop_gpu_update", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 75, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_gpu_update.backprop_gpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_17_begin_gpu_update_backprop_gpu_update(__pyx_self, __pyx_v_d_scores); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_17_begin_gpu_update_backprop_gpu_update(PyObject *__pyx_self, PyObject *__pyx_v_d_scores) { + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *__pyx_cur_scope; + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("backprop_gpu_update", 0); + __pyx_outer_scope = (struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + + /* "thinc/layers/sparselinear.pyx":76 + * + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: + * callback(d_scores.get()) # <<<<<<<<<<<<<< + * return (keys, values, lengths) + * +*/ + __pyx_t_2 = NULL; + if (unlikely(!__pyx_cur_scope->__pyx_v_callback)) { __Pyx_RaiseClosureNameError("callback"); __PYX_ERR(0, 76, __pyx_L1_error) } + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_callback); + __pyx_t_3 = __pyx_cur_scope->__pyx_v_callback; + __pyx_t_5 = __pyx_v_d_scores; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, NULL}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":77 + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: + * callback(d_scores.get()) + * return (keys, values, lengths) # <<<<<<<<<<<<<< + * + * return xp.asarray(scores_cpu), backprop_gpu_update +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(!__pyx_cur_scope->__pyx_v_keys)) { __Pyx_RaiseClosureNameError("keys"); __PYX_ERR(0, 77, __pyx_L1_error) } + if (unlikely(!__pyx_cur_scope->__pyx_v_values)) { __Pyx_RaiseClosureNameError("values"); __PYX_ERR(0, 77, __pyx_L1_error) } + if (unlikely(!__pyx_cur_scope->__pyx_v_lengths)) { __Pyx_RaiseClosureNameError("lengths"); __PYX_ERR(0, 77, __pyx_L1_error) } + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 77, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_keys); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_keys); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_cur_scope->__pyx_v_keys) != (0)) __PYX_ERR(0, 77, __pyx_L1_error); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_values); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_values); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_cur_scope->__pyx_v_values) != (0)) __PYX_ERR(0, 77, __pyx_L1_error); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_lengths); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_lengths); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_cur_scope->__pyx_v_lengths) != (0)) __PYX_ERR(0, 77, __pyx_L1_error); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":75 + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) + * + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: # <<<<<<<<<<<<<< + * callback(d_scores.get()) + * return (keys, values, lengths) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_gpu_update.backprop_gpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":71 + * + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: # <<<<<<<<<<<<<< + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) +*/ + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_8_begin_gpu_update(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys, PyObject *__pyx_v_values, PyObject *__pyx_v_lengths) { + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *__pyx_cur_scope; + PyObject *__pyx_v_xp = NULL; + PyObject *__pyx_v_scores_cpu = NULL; + PyObject *__pyx_v_backprop_gpu_update = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *(*__pyx_t_9)(PyObject *); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_begin_gpu_update", 0); + __pyx_cur_scope = (struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)__pyx_tp_new_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(__pyx_mstate_global->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 71, __pyx_L1_error) + } else { + __Pyx_GOTREF((PyObject *)__pyx_cur_scope); + } + __pyx_cur_scope->__pyx_v_keys = __pyx_v_keys; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_keys); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_keys); + __pyx_cur_scope->__pyx_v_values = __pyx_v_values; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_values); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_values); + __pyx_cur_scope->__pyx_v_lengths = __pyx_v_lengths; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_lengths); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_lengths); + + /* "thinc/layers/sparselinear.pyx":72 + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: + * xp = get_array_module(keys) # <<<<<<<<<<<<<< + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_get_array_module); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 72, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_cur_scope->__pyx_v_keys}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_3, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 72, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_xp = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":73 + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) # <<<<<<<<<<<<<< + * + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_begin_cpu_update); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = __pyx_cur_scope->__pyx_v_keys; + __Pyx_INCREF(__pyx_t_6); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_6, NULL}; + __pyx_t_5 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __pyx_t_7 = __pyx_cur_scope->__pyx_v_values; + __Pyx_INCREF(__pyx_t_7); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_7, NULL}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __pyx_t_8 = __pyx_cur_scope->__pyx_v_lengths; + __Pyx_INCREF(__pyx_t_8); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, NULL}; + __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get, __pyx_callargs+__pyx_t_4, (1-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + } + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_2, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_3, __pyx_v_model, __pyx_t_5, __pyx_t_6, __pyx_t_7}; + __pyx_t_1 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_2, __pyx_callargs+__pyx_t_4, (5-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { + PyObject* sequence = __pyx_t_1; + Py_ssize_t size = __Pyx_PySequence_SIZE(sequence); + if (unlikely(size != 2)) { + if (size > 2) __Pyx_RaiseTooManyValuesError(2); + else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); + __PYX_ERR(0, 73, __pyx_L1_error) + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_CheckExact(sequence))) { + __pyx_t_2 = PyTuple_GET_ITEM(sequence, 0); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); + __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_2 = __Pyx_PyList_GetItemRefFast(sequence, 0, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyList_GetItemRefFast(sequence, 1, __Pyx_ReferenceSharing_SharedReference); + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_XGOTREF(__pyx_t_7); + } + #else + __pyx_t_2 = __Pyx_PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + #endif + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + Py_ssize_t index = -1; + __pyx_t_6 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_9 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); + index = 0; __pyx_t_2 = __pyx_t_9(__pyx_t_6); if (unlikely(!__pyx_t_2)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_2); + index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_6); if (unlikely(!__pyx_t_7)) goto __pyx_L3_unpacking_failed; + __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_6), 2) < (0)) __PYX_ERR(0, 73, __pyx_L1_error) + __pyx_t_9 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L4_unpacking_done; + __pyx_L3_unpacking_failed:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = NULL; + if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + __PYX_ERR(0, 73, __pyx_L1_error) + __pyx_L4_unpacking_done:; + } + __pyx_v_scores_cpu = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_GIVEREF(__pyx_t_7); + __pyx_cur_scope->__pyx_v_callback = __pyx_t_7; + __pyx_t_7 = 0; + + /* "thinc/layers/sparselinear.pyx":75 + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) + * + * def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: # <<<<<<<<<<<<<< + * callback(d_scores.get()) + * return (keys, values, lengths) +*/ + __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_d_scores, __pyx_mstate_global->__pyx_n_u_ArrayXd) < (0)) __PYX_ERR(0, 75, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_kp_u_Tuple_ArrayXd_ArrayXd_ArrayXd) < (0)) __PYX_ERR(0, 75, __pyx_L1_error) + __pyx_t_7 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_17_begin_gpu_update_1backprop_gpu_update, 0, __pyx_mstate_global->__pyx_n_u_begin_gpu_update_locals_backpro, ((PyObject*)__pyx_cur_scope), __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 75, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_7, __pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_backprop_gpu_update = __pyx_t_7; + __pyx_t_7 = 0; + + /* "thinc/layers/sparselinear.pyx":79 + * return (keys, values, lengths) + * + * return xp.asarray(scores_cpu), backprop_gpu_update # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_v_xp; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_1, __pyx_v_scores_cpu}; + __pyx_t_7 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_asarray, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + } + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 79, __pyx_L1_error); + __Pyx_INCREF(__pyx_v_backprop_gpu_update); + __Pyx_GIVEREF(__pyx_v_backprop_gpu_update); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_backprop_gpu_update) != (0)) __PYX_ERR(0, 79, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":71 + * + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: # <<<<<<<<<<<<<< + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_gpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_xp); + __Pyx_XDECREF(__pyx_v_scores_cpu); + __Pyx_XDECREF(__pyx_v_backprop_gpu_update); + __Pyx_DECREF((PyObject *)__pyx_cur_scope); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":82 + * + * + * def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): # <<<<<<<<<<<<<< + * cdef int nO = model.get_dim("nO") + * cdef int length = model.get_dim("length") +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_11_begin_cpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_10_begin_cpu_update, "_begin_cpu_update(model, ndarray keys, ndarray values, ndarray lengths)"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_11_begin_cpu_update = {"_begin_cpu_update", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_11_begin_cpu_update, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_10_begin_cpu_update}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_11_begin_cpu_update(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_model = 0; + PyArrayObject *__pyx_v_keys = 0; + PyArrayObject *__pyx_v_values = 0; + PyArrayObject *__pyx_v_lengths = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[4] = {0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_begin_cpu_update (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_keys,&__pyx_mstate_global->__pyx_n_u_values,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 82, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 82, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_begin_cpu_update", 0) < (0)) __PYX_ERR(0, 82, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 4; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_begin_cpu_update", 1, 4, 4, i); __PYX_ERR(0, 82, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 4)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 82, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 82, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 82, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 82, __pyx_L3_error) + } + __pyx_v_model = values[0]; + __pyx_v_keys = ((PyArrayObject *)values[1]); + __pyx_v_values = ((PyArrayObject *)values[2]); + __pyx_v_lengths = ((PyArrayObject *)values[3]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_begin_cpu_update", 1, 4, 4, __pyx_nargs); __PYX_ERR(0, 82, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_cpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_keys), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "keys", 0))) __PYX_ERR(0, 82, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_values), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "values", 0))) __PYX_ERR(0, 82, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lengths), __pyx_mstate_global->__pyx_ptype_5numpy_ndarray, 1, "lengths", 0))) __PYX_ERR(0, 82, __pyx_L1_error) + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_10_begin_cpu_update(__pyx_self, __pyx_v_model, __pyx_v_keys, __pyx_v_values, __pyx_v_lengths); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_10_begin_cpu_update(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_model, PyArrayObject *__pyx_v_keys, PyArrayObject *__pyx_v_values, PyArrayObject *__pyx_v_lengths) { + int __pyx_v_nO; + int __pyx_v_length; + PyArrayObject *__pyx_v_W = 0; + PyArrayObject *__pyx_v_b = 0; + PyArrayObject *__pyx_v_scores = 0; + int __pyx_v_v1_indexing; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + size_t __pyx_t_3; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_begin_cpu_update", 0); + + /* "thinc/layers/sparselinear.pyx":83 + * + * def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): + * cdef int nO = model.get_dim("nO") # <<<<<<<<<<<<<< + * cdef int length = model.get_dim("length") + * cdef np.ndarray W = model.get_param("W") +*/ + __pyx_t_2 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_nO}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_1); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 83, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_nO = __pyx_t_4; + + /* "thinc/layers/sparselinear.pyx":84 + * def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): + * cdef int nO = model.get_dim("nO") + * cdef int length = model.get_dim("length") # <<<<<<<<<<<<<< + * cdef np.ndarray W = model.get_param("W") + * cdef np.ndarray b = model.get_param("b") +*/ + __pyx_t_2 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_length}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_4 = __Pyx_PyLong_As_int(__pyx_t_1); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_length = __pyx_t_4; + + /* "thinc/layers/sparselinear.pyx":85 + * cdef int nO = model.get_dim("nO") + * cdef int length = model.get_dim("length") + * cdef np.ndarray W = model.get_param("W") # <<<<<<<<<<<<<< + * cdef np.ndarray b = model.get_param("b") + * cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) +*/ + __pyx_t_2 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_W}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_param, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 85, __pyx_L1_error) + __pyx_v_W = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":86 + * cdef int length = model.get_dim("length") + * cdef np.ndarray W = model.get_param("W") + * cdef np.ndarray b = model.get_param("b") # <<<<<<<<<<<<<< + * cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) + * cdef bint v1_indexing = model.attrs["v1_indexing"] +*/ + __pyx_t_2 = __pyx_v_model; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_b}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_param, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 86, __pyx_L1_error) + __pyx_v_b = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":87 + * cdef np.ndarray W = model.get_param("W") + * cdef np.ndarray b = model.get_param("b") + * cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) # <<<<<<<<<<<<<< + * cdef bint v1_indexing = model.attrs["v1_indexing"] + * scores += b +*/ + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_6 = PyObject_Length(((PyObject *)__pyx_v_lengths)); if (unlikely(__pyx_t_6 == ((Py_ssize_t)-1))) __PYX_ERR(0, 87, __pyx_L1_error) + __pyx_t_7 = PyLong_FromSsize_t(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_PyLong_From_int(__pyx_v_nO); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_GIVEREF(__pyx_t_7); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_7) != (0)) __PYX_ERR(0, 87, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_8); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_8) != (0)) __PYX_ERR(0, 87, __pyx_L1_error); + __pyx_t_7 = 0; + __pyx_t_8 = 0; + __pyx_t_3 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_9}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_3, (2-__pyx_t_3) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 87, __pyx_L1_error) + __pyx_v_scores = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":88 + * cdef np.ndarray b = model.get_param("b") + * cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) + * cdef bint v1_indexing = model.attrs["v1_indexing"] # <<<<<<<<<<<<<< + * scores += b + * set_scoresC(scores.data, +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_model, __pyx_mstate_global->__pyx_n_u_attrs); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = __Pyx_PyObject_Dict_GetItem(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_v1_indexing); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 88, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_v_v1_indexing = __pyx_t_10; + + /* "thinc/layers/sparselinear.pyx":89 + * cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) + * cdef bint v1_indexing = model.attrs["v1_indexing"] + * scores += b # <<<<<<<<<<<<<< + * set_scoresC(scores.data, + * keys.data, values.data, lengths.data, +*/ + __pyx_t_5 = PyNumber_InPlaceAdd(((PyObject *)__pyx_v_scores), ((PyObject *)__pyx_v_b)); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 89, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (!(likely(((__pyx_t_5) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_5, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 89, __pyx_L1_error) + __Pyx_DECREF_SET(__pyx_v_scores, ((PyArrayObject *)__pyx_t_5)); + __pyx_t_5 = 0; + + /* "thinc/layers/sparselinear.pyx":90 + * cdef bint v1_indexing = model.attrs["v1_indexing"] + * scores += b + * set_scoresC(scores.data, # <<<<<<<<<<<<<< + * keys.data, values.data, lengths.data, + * lengths.shape[0], nO, +*/ + __pyx_f_5thinc_6layers_12sparselinear_set_scoresC(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_scores)), ((uint64_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_keys)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_values)), ((int32_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_lengths)), (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_lengths)[0]), __pyx_v_nO, ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_W)), __pyx_v_length, __pyx_v_v1_indexing); + + /* "thinc/layers/sparselinear.pyx":94 + * lengths.shape[0], nO, + * W.data, length, v1_indexing) + * return scores, _finish_linear_update(model, keys, values, lengths) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_9, __pyx_mstate_global->__pyx_n_u_finish_linear_update); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_3 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_9))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_9); + assert(__pyx_t_1); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_9); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_9, __pyx__function); + __pyx_t_3 = 0; + } + #endif + { + PyObject *__pyx_callargs[5] = {__pyx_t_1, __pyx_v_model, ((PyObject *)__pyx_v_keys), ((PyObject *)__pyx_v_values), ((PyObject *)__pyx_v_lengths)}; + __pyx_t_5 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_9, __pyx_callargs+__pyx_t_3, (5-__pyx_t_3) | (__pyx_t_3*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + } + __pyx_t_9 = PyTuple_New(2); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_INCREF((PyObject *)__pyx_v_scores); + __Pyx_GIVEREF((PyObject *)__pyx_v_scores); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 0, ((PyObject *)__pyx_v_scores)) != (0)) __PYX_ERR(0, 94, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_5); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_9, 1, __pyx_t_5) != (0)) __PYX_ERR(0, 94, __pyx_L1_error); + __pyx_t_5 = 0; + __pyx_r = __pyx_t_9; + __pyx_t_9 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":82 + * + * + * def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): # <<<<<<<<<<<<<< + * cdef int nO = model.get_dim("nO") + * cdef int length = model.get_dim("length") +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("thinc.layers.sparselinear._begin_cpu_update", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_W); + __Pyx_XDECREF((PyObject *)__pyx_v_b); + __Pyx_XDECREF((PyObject *)__pyx_v_scores); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":100 + * """Move this out of a closure, into its own callable object, to avoid + * pickling errors :(.""" + * def __init__(self, model, keys, values, lengths): # <<<<<<<<<<<<<< + * self.model = model + * self.keys = keys +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_1__init__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_21_finish_linear_update___init__, "_finish_linear_update.__init__(self, model, keys, values, lengths)"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_21_finish_linear_update_1__init__ = {"__init__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_1__init__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_21_finish_linear_update___init__}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_1__init__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + PyObject *__pyx_v_model = 0; + PyObject *__pyx_v_keys = 0; + PyObject *__pyx_v_values = 0; + PyObject *__pyx_v_lengths = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[5] = {0,0,0,0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_model,&__pyx_mstate_global->__pyx_n_u_keys,&__pyx_mstate_global->__pyx_n_u_values,&__pyx_mstate_global->__pyx_n_u_lengths,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 100, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 5: + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 100, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 4: + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 100, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 3: + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 100, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 100, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 100, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__init__", 0) < (0)) __PYX_ERR(0, 100, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 5; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__init__", 1, 5, 5, i); __PYX_ERR(0, 100, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 5)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 100, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 100, __pyx_L3_error) + values[2] = __Pyx_ArgRef_FASTCALL(__pyx_args, 2); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[2])) __PYX_ERR(0, 100, __pyx_L3_error) + values[3] = __Pyx_ArgRef_FASTCALL(__pyx_args, 3); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[3])) __PYX_ERR(0, 100, __pyx_L3_error) + values[4] = __Pyx_ArgRef_FASTCALL(__pyx_args, 4); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[4])) __PYX_ERR(0, 100, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_model = values[1]; + __pyx_v_keys = values[2]; + __pyx_v_values = values[3]; + __pyx_v_lengths = values[4]; + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 1, 5, 5, __pyx_nargs); __PYX_ERR(0, 100, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("thinc.layers.sparselinear._finish_linear_update.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update___init__(__pyx_self, __pyx_v_self, __pyx_v_model, __pyx_v_keys, __pyx_v_values, __pyx_v_lengths); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update___init__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, PyObject *__pyx_v_model, PyObject *__pyx_v_keys, PyObject *__pyx_v_values, PyObject *__pyx_v_lengths) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__init__", 0); + + /* "thinc/layers/sparselinear.pyx":101 + * pickling errors :(.""" + * def __init__(self, model, keys, values, lengths): + * self.model = model # <<<<<<<<<<<<<< + * self.keys = keys + * self.values = values +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model, __pyx_v_model) < (0)) __PYX_ERR(0, 101, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":102 + * def __init__(self, model, keys, values, lengths): + * self.model = model + * self.keys = keys # <<<<<<<<<<<<<< + * self.values = values + * self.lengths = lengths +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_keys, __pyx_v_keys) < (0)) __PYX_ERR(0, 102, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":103 + * self.model = model + * self.keys = keys + * self.values = values # <<<<<<<<<<<<<< + * self.lengths = lengths + * +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_values, __pyx_v_values) < (0)) __PYX_ERR(0, 103, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":104 + * self.keys = keys + * self.values = values + * self.lengths = lengths # <<<<<<<<<<<<<< + * + * def __call__(self, float[:, ::1] d_scores): +*/ + if (__Pyx_PyObject_SetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_lengths, __pyx_v_lengths) < (0)) __PYX_ERR(0, 104, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":100 + * """Move this out of a closure, into its own callable object, to avoid + * pickling errors :(.""" + * def __init__(self, model, keys, values, lengths): # <<<<<<<<<<<<<< + * self.model = model + * self.keys = keys +*/ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("thinc.layers.sparselinear._finish_linear_update.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":106 + * self.lengths = lengths + * + * def __call__(self, float[:, ::1] d_scores): # <<<<<<<<<<<<<< + * nO = self.model.get_dim("nO") + * length = self.model.get_dim("length") +*/ + +/* Python wrapper */ +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_3__call__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_5thinc_6layers_12sparselinear_21_finish_linear_update_2__call__, "_finish_linear_update.__call__(self, float[:, ::1] d_scores)"); +static PyMethodDef __pyx_mdef_5thinc_6layers_12sparselinear_21_finish_linear_update_3__call__ = {"__call__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_3__call__, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_5thinc_6layers_12sparselinear_21_finish_linear_update_2__call__}; +static PyObject *__pyx_pw_5thinc_6layers_12sparselinear_21_finish_linear_update_3__call__(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_self = 0; + __Pyx_memviewslice __pyx_v_d_scores = { 0, 0, { 0 }, { 0 }, { 0 } }; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[2] = {0,0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__call__ (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_self,&__pyx_mstate_global->__pyx_n_u_d_scores,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 106, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 2: + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 106, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 106, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "__call__", 0) < (0)) __PYX_ERR(0, 106, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 2; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("__call__", 1, 2, 2, i); __PYX_ERR(0, 106, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 2)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 106, __pyx_L3_error) + values[1] = __Pyx_ArgRef_FASTCALL(__pyx_args, 1); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[1])) __PYX_ERR(0, 106, __pyx_L3_error) + } + __pyx_v_self = values[0]; + __pyx_v_d_scores = __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(values[1], PyBUF_WRITABLE); if (unlikely(!__pyx_v_d_scores.memview)) __PYX_ERR(0, 106, __pyx_L3_error) + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__call__", 1, 2, 2, __pyx_nargs); __PYX_ERR(0, 106, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_scores, 1); + __Pyx_AddTraceback("thinc.layers.sparselinear._finish_linear_update.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update_2__call__(__pyx_self, __pyx_v_self, __pyx_v_d_scores); + + /* function exit code */ + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __PYX_XCLEAR_MEMVIEW(&__pyx_v_d_scores, 1); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5thinc_6layers_12sparselinear_21_finish_linear_update_2__call__(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_self, __Pyx_memviewslice __pyx_v_d_scores) { + PyObject *__pyx_v_nO = NULL; + PyObject *__pyx_v_length = NULL; + PyArrayObject *__pyx_v_d_weights = 0; + PyArrayObject *__pyx_v_d_bias = 0; + PyArrayObject *__pyx_v_keys = 0; + PyArrayObject *__pyx_v_values = 0; + PyArrayObject *__pyx_v_lengths = 0; + int __pyx_v_v1_indexing; + int __pyx_v_i; + int __pyx_v_j; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + Py_ssize_t __pyx_t_10; + int __pyx_t_11; + Py_ssize_t __pyx_t_12; + Py_ssize_t __pyx_t_13; + Py_ssize_t __pyx_t_14; + Py_ssize_t __pyx_t_15; + int __pyx_t_16; + int __pyx_t_17; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__call__", 0); + + /* "thinc/layers/sparselinear.pyx":107 + * + * def __call__(self, float[:, ::1] d_scores): + * nO = self.model.get_dim("nO") # <<<<<<<<<<<<<< + * length = self.model.get_dim("length") + * cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __pyx_t_3; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_nO}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_nO = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":108 + * def __call__(self, float[:, ::1] d_scores): + * nO = self.model.get_dim("nO") + * length = self.model.get_dim("length") # <<<<<<<<<<<<<< + * cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) + * cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_3); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_mstate_global->__pyx_n_u_length}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_get_dim, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_v_length = __pyx_t_1; + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":109 + * nO = self.model.get_dim("nO") + * length = self.model.get_dim("length") + * cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) # <<<<<<<<<<<<<< + * cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) + * cdef np.ndarray keys = self.keys +*/ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_2 = __pyx_t_5; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = PyNumber_Multiply(__pyx_v_nO, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3) != (0)) __PYX_ERR(0, 109, __pyx_L1_error); + __pyx_t_3 = 0; + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 109, __pyx_L1_error) + __pyx_v_d_weights = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":110 + * length = self.model.get_dim("length") + * cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) + * cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) # <<<<<<<<<<<<<< + * cdef np.ndarray keys = self.keys + * cdef np.ndarray values = self.values +*/ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_mstate_global->__pyx_n_u_ops); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_5 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_v_nO); + __Pyx_GIVEREF(__pyx_v_nO); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_nO) != (0)) __PYX_ERR(0, 110, __pyx_L1_error); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_alloc, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 110, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 110, __pyx_L1_error) + __pyx_v_d_bias = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":111 + * cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) + * cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) + * cdef np.ndarray keys = self.keys # <<<<<<<<<<<<<< + * cdef np.ndarray values = self.values + * cdef np.ndarray lengths = self.lengths +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_keys); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 111, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_keys = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":112 + * cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) + * cdef np.ndarray keys = self.keys + * cdef np.ndarray values = self.values # <<<<<<<<<<<<<< + * cdef np.ndarray lengths = self.lengths + * cdef bint v1_indexing = self.model.attrs["v1_indexing"] +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_values); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 112, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 112, __pyx_L1_error) + __pyx_v_values = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":113 + * cdef np.ndarray keys = self.keys + * cdef np.ndarray values = self.values + * cdef np.ndarray lengths = self.lengths # <<<<<<<<<<<<<< + * cdef bint v1_indexing = self.model.attrs["v1_indexing"] + * set_gradientC(d_weights.data, +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_lengths); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_mstate_global->__pyx_ptype_5numpy_ndarray))))) __PYX_ERR(0, 113, __pyx_L1_error) + __pyx_v_lengths = ((PyArrayObject *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "thinc/layers/sparselinear.pyx":114 + * cdef np.ndarray values = self.values + * cdef np.ndarray lengths = self.lengths + * cdef bint v1_indexing = self.model.attrs["v1_indexing"] # <<<<<<<<<<<<<< + * set_gradientC(d_weights.data, + * keys.data, values.data, lengths.data, +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_attrs); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_Dict_GetItem(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_v1_indexing); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_7 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_v1_indexing = __pyx_t_7; + + /* "thinc/layers/sparselinear.pyx":117 + * set_gradientC(d_weights.data, + * keys.data, values.data, lengths.data, + * lengths.shape[0], nO, &d_scores[0,0], length, v1_indexing) # <<<<<<<<<<<<<< + * cdef int i, j + * for i in range(d_scores.shape[0]): +*/ + __pyx_t_8 = __Pyx_PyLong_As_int(__pyx_v_nO); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 117, __pyx_L1_error) + __pyx_t_9 = 0; + __pyx_t_10 = 0; + __pyx_t_11 = -1; + if (__pyx_t_9 < 0) { + __pyx_t_11 = 0; + } else if (unlikely(__pyx_t_9 >= __pyx_v_d_scores.shape[0])) __pyx_t_11 = 0; + if (__pyx_t_10 < 0) { + __pyx_t_11 = 1; + } else if (unlikely(__pyx_t_10 >= __pyx_v_d_scores.shape[1])) __pyx_t_11 = 1; + if (unlikely(__pyx_t_11 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_11); + __PYX_ERR(0, 117, __pyx_L1_error) + } + __pyx_t_11 = __Pyx_PyLong_As_int(__pyx_v_length); if (unlikely((__pyx_t_11 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 117, __pyx_L1_error) + + /* "thinc/layers/sparselinear.pyx":115 + * cdef np.ndarray lengths = self.lengths + * cdef bint v1_indexing = self.model.attrs["v1_indexing"] + * set_gradientC(d_weights.data, # <<<<<<<<<<<<<< + * keys.data, values.data, lengths.data, + * lengths.shape[0], nO, &d_scores[0,0], length, v1_indexing) +*/ + __pyx_f_5thinc_6layers_12sparselinear_set_gradientC(((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_d_weights)), ((uint64_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_keys)), ((float *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_values)), ((int32_t *)__pyx_f_5numpy_7ndarray_4data_data(__pyx_v_lengths)), (__pyx_f_5numpy_7ndarray_5shape_shape(__pyx_v_lengths)[0]), __pyx_t_8, (&(*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_d_scores.data + __pyx_t_9 * __pyx_v_d_scores.strides[0]) )) + __pyx_t_10)) )))), __pyx_t_11, __pyx_v_v1_indexing); + + /* "thinc/layers/sparselinear.pyx":119 + * lengths.shape[0], nO, &d_scores[0,0], length, v1_indexing) + * cdef int i, j + * for i in range(d_scores.shape[0]): # <<<<<<<<<<<<<< + * for j in range(d_scores.shape[1]): + * d_bias[j] += d_scores[i, j] +*/ + __pyx_t_12 = (__pyx_v_d_scores.shape[0]); + __pyx_t_13 = __pyx_t_12; + for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_13; __pyx_t_11+=1) { + __pyx_v_i = __pyx_t_11; + + /* "thinc/layers/sparselinear.pyx":120 + * cdef int i, j + * for i in range(d_scores.shape[0]): + * for j in range(d_scores.shape[1]): # <<<<<<<<<<<<<< + * d_bias[j] += d_scores[i, j] + * self.model.inc_grad("W", d_weights) +*/ + __pyx_t_14 = (__pyx_v_d_scores.shape[1]); + __pyx_t_15 = __pyx_t_14; + for (__pyx_t_8 = 0; __pyx_t_8 < __pyx_t_15; __pyx_t_8+=1) { + __pyx_v_j = __pyx_t_8; + + /* "thinc/layers/sparselinear.pyx":121 + * for i in range(d_scores.shape[0]): + * for j in range(d_scores.shape[1]): + * d_bias[j] += d_scores[i, j] # <<<<<<<<<<<<<< + * self.model.inc_grad("W", d_weights) + * self.model.inc_grad("b", d_bias) +*/ + __pyx_t_16 = __pyx_v_j; + __pyx_t_1 = __Pyx_GetItemInt(((PyObject *)__pyx_v_d_bias), __pyx_t_16, int, 1, __Pyx_PyLong_From_int, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_10 = __pyx_v_i; + __pyx_t_9 = __pyx_v_j; + __pyx_t_17 = -1; + if (__pyx_t_10 < 0) { + __pyx_t_17 = 0; + } else if (unlikely(__pyx_t_10 >= __pyx_v_d_scores.shape[0])) __pyx_t_17 = 0; + if (__pyx_t_9 < 0) { + __pyx_t_17 = 1; + } else if (unlikely(__pyx_t_9 >= __pyx_v_d_scores.shape[1])) __pyx_t_17 = 1; + if (unlikely(__pyx_t_17 != -1)) { + __Pyx_RaiseBufferIndexError(__pyx_t_17); + __PYX_ERR(0, 121, __pyx_L1_error) + } + __pyx_t_2 = PyFloat_FromDouble((*((float *) ( /* dim=1 */ ((char *) (((float *) ( /* dim=0 */ (__pyx_v_d_scores.data + __pyx_t_10 * __pyx_v_d_scores.strides[0]) )) + __pyx_t_9)) )))); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyNumber_InPlaceAdd(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely((__Pyx_SetItemInt(((PyObject *)__pyx_v_d_bias), __pyx_t_16, __pyx_t_6, int, 1, __Pyx_PyLong_From_int, 0, 0, 1, 1, __Pyx_ReferenceSharing_OwnStrongReference) < 0))) __PYX_ERR(0, 121, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + + /* "thinc/layers/sparselinear.pyx":122 + * for j in range(d_scores.shape[1]): + * d_bias[j] += d_scores[i, j] + * self.model.inc_grad("W", d_weights) # <<<<<<<<<<<<<< + * self.model.inc_grad("b", d_bias) + * return (self.keys, self.values, self.lengths) +*/ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_t_1; + __Pyx_INCREF(__pyx_t_2); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_W, ((PyObject *)__pyx_v_d_weights)}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_inc_grad, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 122, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "thinc/layers/sparselinear.pyx":123 + * d_bias[j] += d_scores[i, j] + * self.model.inc_grad("W", d_weights) + * self.model.inc_grad("b", d_bias) # <<<<<<<<<<<<<< + * return (self.keys, self.values, self.lengths) + * +*/ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_model); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __pyx_t_2; + __Pyx_INCREF(__pyx_t_1); + __pyx_t_4 = 0; + { + PyObject *__pyx_callargs[3] = {__pyx_t_1, __pyx_mstate_global->__pyx_n_u_b, ((PyObject *)__pyx_v_d_bias)}; + __pyx_t_6 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_inc_grad, __pyx_callargs+__pyx_t_4, (3-__pyx_t_4) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "thinc/layers/sparselinear.pyx":124 + * self.model.inc_grad("W", d_weights) + * self.model.inc_grad("b", d_bias) + * return (self.keys, self.values, self.lengths) # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_keys); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_values); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_self, __pyx_mstate_global->__pyx_n_u_lengths); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_5 = PyTuple_New(3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 124, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_6); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_6) != (0)) __PYX_ERR(0, 124, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_2); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_2) != (0)) __PYX_ERR(0, 124, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_t_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_t_5, 2, __pyx_t_1) != (0)) __PYX_ERR(0, 124, __pyx_L1_error); + __pyx_t_6 = 0; + __pyx_t_2 = 0; + __pyx_t_1 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":106 + * self.lengths = lengths + * + * def __call__(self, float[:, ::1] d_scores): # <<<<<<<<<<<<<< + * nO = self.model.get_dim("nO") + * length = self.model.get_dim("length") +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("thinc.layers.sparselinear._finish_linear_update.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_nO); + __Pyx_XDECREF(__pyx_v_length); + __Pyx_XDECREF((PyObject *)__pyx_v_d_weights); + __Pyx_XDECREF((PyObject *)__pyx_v_d_bias); + __Pyx_XDECREF((PyObject *)__pyx_v_keys); + __Pyx_XDECREF((PyObject *)__pyx_v_values); + __Pyx_XDECREF((PyObject *)__pyx_v_lengths); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "thinc/layers/sparselinear.pyx":129 + * # v1_indexing is invalid and only uses a subset of the weight matrix, v1 + * # indexing is provided here for compatibility. See #752 for more information. + * cdef void set_scoresC(float* scores, # <<<<<<<<<<<<<< + * const uint64_t* keys, const float* values, const int32_t* lengths, + * int batch_size, int nr_out, const float* weights, int nr_weight, +*/ + +static void __pyx_f_5thinc_6layers_12sparselinear_set_scoresC(float *__pyx_v_scores, uint64_t const *__pyx_v_keys, float const *__pyx_v_values, int32_t const *__pyx_v_lengths, int __pyx_v_batch_size, int __pyx_v_nr_out, float const *__pyx_v_weights, int __pyx_v_nr_weight, int __pyx_v_v1_indexing) { + uint32_t __pyx_v_idx1; + uint32_t __pyx_v_idx2; + uint32_t __pyx_v_hash1; + uint32_t __pyx_v_hash2; + int32_t __pyx_v_length; + int32_t __pyx_v_i; + float __pyx_v_value; + int __pyx_v_clas; + int32_t const *__pyx_t_1; + int32_t const *__pyx_t_2; + int32_t const *__pyx_t_3; + int32_t __pyx_t_4; + int32_t __pyx_t_5; + int32_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + int __pyx_t_10; + + /* "thinc/layers/sparselinear.pyx":135 + * cdef uint32_t idx1, idx2 + * cdef uint32_t hash1, hash2 + * for length in lengths[:batch_size]: # <<<<<<<<<<<<<< + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) +*/ + __pyx_t_2 = (__pyx_v_lengths + __pyx_v_batch_size); + for (__pyx_t_3 = __pyx_v_lengths; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_length = (__pyx_t_1[0]); + + /* "thinc/layers/sparselinear.pyx":136 + * cdef uint32_t hash1, hash2 + * for length in lengths[:batch_size]: + * for i in range(length): # <<<<<<<<<<<<<< + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) +*/ + __pyx_t_4 = __pyx_v_length; + __pyx_t_5 = __pyx_t_4; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "thinc/layers/sparselinear.pyx":137 + * for length in lengths[:batch_size]: + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) # <<<<<<<<<<<<<< + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: +*/ + __pyx_v_hash1 = __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64((__pyx_v_keys[__pyx_v_i]), 0); + + /* "thinc/layers/sparselinear.pyx":138 + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) # <<<<<<<<<<<<<< + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) +*/ + __pyx_v_hash2 = __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64((__pyx_v_keys[__pyx_v_i]), 1); + + /* "thinc/layers/sparselinear.pyx":139 + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: # <<<<<<<<<<<<<< + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) +*/ + if (__pyx_v_v1_indexing) { + + /* "thinc/layers/sparselinear.pyx":140 + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) # <<<<<<<<<<<<<< + * idx2 = hash2 & (nr_weight-1) + * else: +*/ + __pyx_v_idx1 = (__pyx_v_hash1 & (__pyx_v_nr_weight - 1)); + + /* "thinc/layers/sparselinear.pyx":141 + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) # <<<<<<<<<<<<<< + * else: + * idx1 = hash1 % nr_weight +*/ + __pyx_v_idx2 = (__pyx_v_hash2 & (__pyx_v_nr_weight - 1)); + + /* "thinc/layers/sparselinear.pyx":139 + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: # <<<<<<<<<<<<<< + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) +*/ + goto __pyx_L7; + } + + /* "thinc/layers/sparselinear.pyx":143 + * idx2 = hash2 & (nr_weight-1) + * else: + * idx1 = hash1 % nr_weight # <<<<<<<<<<<<<< + * idx2 = hash2 % nr_weight + * value = values[i] +*/ + /*else*/ { + __pyx_v_idx1 = (__pyx_v_hash1 % __pyx_v_nr_weight); + + /* "thinc/layers/sparselinear.pyx":144 + * else: + * idx1 = hash1 % nr_weight + * idx2 = hash2 % nr_weight # <<<<<<<<<<<<<< + * value = values[i] + * for clas in range(nr_out): +*/ + __pyx_v_idx2 = (__pyx_v_hash2 % __pyx_v_nr_weight); + } + __pyx_L7:; + + /* "thinc/layers/sparselinear.pyx":145 + * idx1 = hash1 % nr_weight + * idx2 = hash2 % nr_weight + * value = values[i] # <<<<<<<<<<<<<< + * for clas in range(nr_out): + * if v1_indexing: +*/ + __pyx_v_value = (__pyx_v_values[__pyx_v_i]); + + /* "thinc/layers/sparselinear.pyx":146 + * idx2 = hash2 % nr_weight + * value = values[i] + * for clas in range(nr_out): # <<<<<<<<<<<<<< + * if v1_indexing: + * scores[clas] += weights[idx1 + clas] * value +*/ + __pyx_t_7 = __pyx_v_nr_out; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_clas = __pyx_t_9; + + /* "thinc/layers/sparselinear.pyx":147 + * value = values[i] + * for clas in range(nr_out): + * if v1_indexing: # <<<<<<<<<<<<<< + * scores[clas] += weights[idx1 + clas] * value + * scores[clas] += weights[idx2 + clas] * value +*/ + if (__pyx_v_v1_indexing) { + + /* "thinc/layers/sparselinear.pyx":148 + * for clas in range(nr_out): + * if v1_indexing: + * scores[clas] += weights[idx1 + clas] * value # <<<<<<<<<<<<<< + * scores[clas] += weights[idx2 + clas] * value + * else: +*/ + __pyx_t_10 = __pyx_v_clas; + (__pyx_v_scores[__pyx_t_10]) = ((__pyx_v_scores[__pyx_t_10]) + ((__pyx_v_weights[(__pyx_v_idx1 + __pyx_v_clas)]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":149 + * if v1_indexing: + * scores[clas] += weights[idx1 + clas] * value + * scores[clas] += weights[idx2 + clas] * value # <<<<<<<<<<<<<< + * else: + * scores[clas] += weights[(clas * nr_weight) + idx1] * value +*/ + __pyx_t_10 = __pyx_v_clas; + (__pyx_v_scores[__pyx_t_10]) = ((__pyx_v_scores[__pyx_t_10]) + ((__pyx_v_weights[(__pyx_v_idx2 + __pyx_v_clas)]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":147 + * value = values[i] + * for clas in range(nr_out): + * if v1_indexing: # <<<<<<<<<<<<<< + * scores[clas] += weights[idx1 + clas] * value + * scores[clas] += weights[idx2 + clas] * value +*/ + goto __pyx_L10; + } + + /* "thinc/layers/sparselinear.pyx":151 + * scores[clas] += weights[idx2 + clas] * value + * else: + * scores[clas] += weights[(clas * nr_weight) + idx1] * value # <<<<<<<<<<<<<< + * scores[clas] += weights[(clas * nr_weight) + idx2] * value + * scores += nr_out +*/ + /*else*/ { + __pyx_t_10 = __pyx_v_clas; + (__pyx_v_scores[__pyx_t_10]) = ((__pyx_v_scores[__pyx_t_10]) + ((__pyx_v_weights[((__pyx_v_clas * __pyx_v_nr_weight) + __pyx_v_idx1)]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":152 + * else: + * scores[clas] += weights[(clas * nr_weight) + idx1] * value + * scores[clas] += weights[(clas * nr_weight) + idx2] * value # <<<<<<<<<<<<<< + * scores += nr_out + * keys += length +*/ + __pyx_t_10 = __pyx_v_clas; + (__pyx_v_scores[__pyx_t_10]) = ((__pyx_v_scores[__pyx_t_10]) + ((__pyx_v_weights[((__pyx_v_clas * __pyx_v_nr_weight) + __pyx_v_idx2)]) * __pyx_v_value)); + } + __pyx_L10:; + } + } + + /* "thinc/layers/sparselinear.pyx":153 + * scores[clas] += weights[(clas * nr_weight) + idx1] * value + * scores[clas] += weights[(clas * nr_weight) + idx2] * value + * scores += nr_out # <<<<<<<<<<<<<< + * keys += length + * values += length +*/ + __pyx_v_scores = (__pyx_v_scores + __pyx_v_nr_out); + + /* "thinc/layers/sparselinear.pyx":154 + * scores[clas] += weights[(clas * nr_weight) + idx2] * value + * scores += nr_out + * keys += length # <<<<<<<<<<<<<< + * values += length + * +*/ + __pyx_v_keys = (__pyx_v_keys + __pyx_v_length); + + /* "thinc/layers/sparselinear.pyx":155 + * scores += nr_out + * keys += length + * values += length # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_values = (__pyx_v_values + __pyx_v_length); + } + + /* "thinc/layers/sparselinear.pyx":129 + * # v1_indexing is invalid and only uses a subset of the weight matrix, v1 + * # indexing is provided here for compatibility. See #752 for more information. + * cdef void set_scoresC(float* scores, # <<<<<<<<<<<<<< + * const uint64_t* keys, const float* values, const int32_t* lengths, + * int batch_size, int nr_out, const float* weights, int nr_weight, +*/ + + /* function exit code */ +} + +/* "thinc/layers/sparselinear.pyx":160 + * # v1_indexing is invalid and only uses a subset of the weight matrix, v1 + * # indexing is provided here for compatibility. See #752 for more information. + * cdef void set_gradientC(float* d_weights, # <<<<<<<<<<<<<< + * const uint64_t* keys, const float* values, const int32_t* lengths, + * int batch_size, int nr_out, const float* d_scores, int nr_weight, +*/ + +static void __pyx_f_5thinc_6layers_12sparselinear_set_gradientC(float *__pyx_v_d_weights, uint64_t const *__pyx_v_keys, float const *__pyx_v_values, int32_t const *__pyx_v_lengths, int __pyx_v_batch_size, int __pyx_v_nr_out, float const *__pyx_v_d_scores, int __pyx_v_nr_weight, int __pyx_v_v1_indexing) { + uint32_t __pyx_v_idx1; + uint32_t __pyx_v_idx2; + uint32_t __pyx_v_hash1; + uint32_t __pyx_v_hash2; + int32_t __pyx_v_length; + int32_t __pyx_v_i; + float __pyx_v_value; + int __pyx_v_clas; + int32_t const *__pyx_t_1; + int32_t const *__pyx_t_2; + int32_t const *__pyx_t_3; + int32_t __pyx_t_4; + int32_t __pyx_t_5; + int32_t __pyx_t_6; + int __pyx_t_7; + int __pyx_t_8; + int __pyx_t_9; + uint32_t __pyx_t_10; + + /* "thinc/layers/sparselinear.pyx":166 + * cdef uint32_t idx1, idx2 + * cdef uint32_t hash1, hash2 + * for length in lengths[:batch_size]: # <<<<<<<<<<<<<< + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) +*/ + __pyx_t_2 = (__pyx_v_lengths + __pyx_v_batch_size); + for (__pyx_t_3 = __pyx_v_lengths; __pyx_t_3 < __pyx_t_2; __pyx_t_3++) { + __pyx_t_1 = __pyx_t_3; + __pyx_v_length = (__pyx_t_1[0]); + + /* "thinc/layers/sparselinear.pyx":167 + * cdef uint32_t hash1, hash2 + * for length in lengths[:batch_size]: + * for i in range(length): # <<<<<<<<<<<<<< + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) +*/ + __pyx_t_4 = __pyx_v_length; + __pyx_t_5 = __pyx_t_4; + for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { + __pyx_v_i = __pyx_t_6; + + /* "thinc/layers/sparselinear.pyx":168 + * for length in lengths[:batch_size]: + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) # <<<<<<<<<<<<<< + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: +*/ + __pyx_v_hash1 = __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64((__pyx_v_keys[__pyx_v_i]), 0); + + /* "thinc/layers/sparselinear.pyx":169 + * for i in range(length): + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) # <<<<<<<<<<<<<< + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) +*/ + __pyx_v_hash2 = __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64((__pyx_v_keys[__pyx_v_i]), 1); + + /* "thinc/layers/sparselinear.pyx":170 + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: # <<<<<<<<<<<<<< + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) +*/ + if (__pyx_v_v1_indexing) { + + /* "thinc/layers/sparselinear.pyx":171 + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) # <<<<<<<<<<<<<< + * idx2 = hash2 & (nr_weight-1) + * else: +*/ + __pyx_v_idx1 = (__pyx_v_hash1 & (__pyx_v_nr_weight - 1)); + + /* "thinc/layers/sparselinear.pyx":172 + * if v1_indexing: + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) # <<<<<<<<<<<<<< + * else: + * idx1 = hash1 % nr_weight +*/ + __pyx_v_idx2 = (__pyx_v_hash2 & (__pyx_v_nr_weight - 1)); + + /* "thinc/layers/sparselinear.pyx":170 + * hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + * hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + * if v1_indexing: # <<<<<<<<<<<<<< + * idx1 = hash1 & (nr_weight-1) + * idx2 = hash2 & (nr_weight-1) +*/ + goto __pyx_L7; + } + + /* "thinc/layers/sparselinear.pyx":174 + * idx2 = hash2 & (nr_weight-1) + * else: + * idx1 = hash1 % nr_weight # <<<<<<<<<<<<<< + * idx2 = hash2 % nr_weight + * value = values[i] +*/ + /*else*/ { + __pyx_v_idx1 = (__pyx_v_hash1 % __pyx_v_nr_weight); + + /* "thinc/layers/sparselinear.pyx":175 + * else: + * idx1 = hash1 % nr_weight + * idx2 = hash2 % nr_weight # <<<<<<<<<<<<<< + * value = values[i] + * for clas in range(nr_out): +*/ + __pyx_v_idx2 = (__pyx_v_hash2 % __pyx_v_nr_weight); + } + __pyx_L7:; + + /* "thinc/layers/sparselinear.pyx":176 + * idx1 = hash1 % nr_weight + * idx2 = hash2 % nr_weight + * value = values[i] # <<<<<<<<<<<<<< + * for clas in range(nr_out): + * if v1_indexing: +*/ + __pyx_v_value = (__pyx_v_values[__pyx_v_i]); + + /* "thinc/layers/sparselinear.pyx":177 + * idx2 = hash2 % nr_weight + * value = values[i] + * for clas in range(nr_out): # <<<<<<<<<<<<<< + * if v1_indexing: + * d_weights[idx1 + clas] += d_scores[clas] * value +*/ + __pyx_t_7 = __pyx_v_nr_out; + __pyx_t_8 = __pyx_t_7; + for (__pyx_t_9 = 0; __pyx_t_9 < __pyx_t_8; __pyx_t_9+=1) { + __pyx_v_clas = __pyx_t_9; + + /* "thinc/layers/sparselinear.pyx":178 + * value = values[i] + * for clas in range(nr_out): + * if v1_indexing: # <<<<<<<<<<<<<< + * d_weights[idx1 + clas] += d_scores[clas] * value + * d_weights[idx2 + clas] += d_scores[clas] * value +*/ + if (__pyx_v_v1_indexing) { + + /* "thinc/layers/sparselinear.pyx":179 + * for clas in range(nr_out): + * if v1_indexing: + * d_weights[idx1 + clas] += d_scores[clas] * value # <<<<<<<<<<<<<< + * d_weights[idx2 + clas] += d_scores[clas] * value + * else: +*/ + __pyx_t_10 = (__pyx_v_idx1 + __pyx_v_clas); + (__pyx_v_d_weights[__pyx_t_10]) = ((__pyx_v_d_weights[__pyx_t_10]) + ((__pyx_v_d_scores[__pyx_v_clas]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":180 + * if v1_indexing: + * d_weights[idx1 + clas] += d_scores[clas] * value + * d_weights[idx2 + clas] += d_scores[clas] * value # <<<<<<<<<<<<<< + * else: + * d_weights[(clas * nr_weight) + idx1] += d_scores[clas] * value +*/ + __pyx_t_10 = (__pyx_v_idx2 + __pyx_v_clas); + (__pyx_v_d_weights[__pyx_t_10]) = ((__pyx_v_d_weights[__pyx_t_10]) + ((__pyx_v_d_scores[__pyx_v_clas]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":178 + * value = values[i] + * for clas in range(nr_out): + * if v1_indexing: # <<<<<<<<<<<<<< + * d_weights[idx1 + clas] += d_scores[clas] * value + * d_weights[idx2 + clas] += d_scores[clas] * value +*/ + goto __pyx_L10; + } + + /* "thinc/layers/sparselinear.pyx":182 + * d_weights[idx2 + clas] += d_scores[clas] * value + * else: + * d_weights[(clas * nr_weight) + idx1] += d_scores[clas] * value # <<<<<<<<<<<<<< + * d_weights[(clas * nr_weight) + idx2] += d_scores[clas] * value + * d_scores += nr_out +*/ + /*else*/ { + __pyx_t_10 = ((__pyx_v_clas * __pyx_v_nr_weight) + __pyx_v_idx1); + (__pyx_v_d_weights[__pyx_t_10]) = ((__pyx_v_d_weights[__pyx_t_10]) + ((__pyx_v_d_scores[__pyx_v_clas]) * __pyx_v_value)); + + /* "thinc/layers/sparselinear.pyx":183 + * else: + * d_weights[(clas * nr_weight) + idx1] += d_scores[clas] * value + * d_weights[(clas * nr_weight) + idx2] += d_scores[clas] * value # <<<<<<<<<<<<<< + * d_scores += nr_out + * keys += length +*/ + __pyx_t_10 = ((__pyx_v_clas * __pyx_v_nr_weight) + __pyx_v_idx2); + (__pyx_v_d_weights[__pyx_t_10]) = ((__pyx_v_d_weights[__pyx_t_10]) + ((__pyx_v_d_scores[__pyx_v_clas]) * __pyx_v_value)); + } + __pyx_L10:; + } + } + + /* "thinc/layers/sparselinear.pyx":184 + * d_weights[(clas * nr_weight) + idx1] += d_scores[clas] * value + * d_weights[(clas * nr_weight) + idx2] += d_scores[clas] * value + * d_scores += nr_out # <<<<<<<<<<<<<< + * keys += length + * values += length +*/ + __pyx_v_d_scores = (__pyx_v_d_scores + __pyx_v_nr_out); + + /* "thinc/layers/sparselinear.pyx":185 + * d_weights[(clas * nr_weight) + idx2] += d_scores[clas] * value + * d_scores += nr_out + * keys += length # <<<<<<<<<<<<<< + * values += length + * +*/ + __pyx_v_keys = (__pyx_v_keys + __pyx_v_length); + + /* "thinc/layers/sparselinear.pyx":186 + * d_scores += nr_out + * keys += length + * values += length # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_values = (__pyx_v_values + __pyx_v_length); + } + + /* "thinc/layers/sparselinear.pyx":160 + * # v1_indexing is invalid and only uses a subset of the weight matrix, v1 + * # indexing is provided here for compatibility. See #752 for more information. + * cdef void set_gradientC(float* d_weights, # <<<<<<<<<<<<<< + * const uint64_t* keys, const float* values, const int32_t* lengths, + * int batch_size, int nr_out, const float* d_scores, int nr_weight, +*/ + + /* function exit code */ +} + +/* "thinc/layers/sparselinear.pyx":189 + * + * + * cdef uint32_t MurmurHash3_x86_32_uint64(uint64_t key, uint32_t seed) noexcept nogil: # <<<<<<<<<<<<<< + * cdef uint32_t h1 = seed + * cdef uint32_t c1 = 0xcc9e2d51u +*/ + +static uint32_t __pyx_f_5thinc_6layers_12sparselinear_MurmurHash3_x86_32_uint64(uint64_t __pyx_v_key, uint32_t __pyx_v_seed) { + uint32_t __pyx_v_h1; + uint32_t __pyx_v_c1; + uint32_t __pyx_v_c2; + uint32_t __pyx_v_k1; + uint32_t __pyx_r; + + /* "thinc/layers/sparselinear.pyx":190 + * + * cdef uint32_t MurmurHash3_x86_32_uint64(uint64_t key, uint32_t seed) noexcept nogil: + * cdef uint32_t h1 = seed # <<<<<<<<<<<<<< + * cdef uint32_t c1 = 0xcc9e2d51u + * cdef uint32_t c2 = 0x1b873593u +*/ + __pyx_v_h1 = __pyx_v_seed; + + /* "thinc/layers/sparselinear.pyx":191 + * cdef uint32_t MurmurHash3_x86_32_uint64(uint64_t key, uint32_t seed) noexcept nogil: + * cdef uint32_t h1 = seed + * cdef uint32_t c1 = 0xcc9e2d51u # <<<<<<<<<<<<<< + * cdef uint32_t c2 = 0x1b873593u + * cdef uint32_t k1 +*/ + __pyx_v_c1 = 0xcc9e2d51U; + + /* "thinc/layers/sparselinear.pyx":192 + * cdef uint32_t h1 = seed + * cdef uint32_t c1 = 0xcc9e2d51u + * cdef uint32_t c2 = 0x1b873593u # <<<<<<<<<<<<<< + * cdef uint32_t k1 + * +*/ + __pyx_v_c2 = 0x1b873593U; + + /* "thinc/layers/sparselinear.pyx":195 + * cdef uint32_t k1 + * + * k1 = key & 0xffffffffu # <<<<<<<<<<<<<< + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) +*/ + __pyx_v_k1 = (__pyx_v_key & 0xffffffffU); + + /* "thinc/layers/sparselinear.pyx":196 + * + * k1 = key & 0xffffffffu + * k1 *= c1 # <<<<<<<<<<<<<< + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 +*/ + __pyx_v_k1 = (__pyx_v_k1 * __pyx_v_c1); + + /* "thinc/layers/sparselinear.pyx":197 + * k1 = key & 0xffffffffu + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) # <<<<<<<<<<<<<< + * k1 *= c2 + * h1 ^= k1 +*/ + __pyx_v_k1 = ((__pyx_v_k1 << 15) | (__pyx_v_k1 >> 17)); + + /* "thinc/layers/sparselinear.pyx":198 + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 # <<<<<<<<<<<<<< + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) +*/ + __pyx_v_k1 = (__pyx_v_k1 * __pyx_v_c2); + + /* "thinc/layers/sparselinear.pyx":199 + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 + * h1 ^= k1 # <<<<<<<<<<<<<< + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ __pyx_v_k1); + + /* "thinc/layers/sparselinear.pyx":200 + * k1 *= c2 + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) # <<<<<<<<<<<<<< + * h1 = h1*5+0xe6546b64u + * k1 = key >> 32 +*/ + __pyx_v_h1 = ((__pyx_v_h1 << 13) | (__pyx_v_h1 >> 19)); + + /* "thinc/layers/sparselinear.pyx":201 + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u # <<<<<<<<<<<<<< + * k1 = key >> 32 + * k1 *= c1 +*/ + __pyx_v_h1 = ((__pyx_v_h1 * 5) + 0xe6546b64U); + + /* "thinc/layers/sparselinear.pyx":202 + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u + * k1 = key >> 32 # <<<<<<<<<<<<<< + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) +*/ + __pyx_v_k1 = (__pyx_v_key >> 32); + + /* "thinc/layers/sparselinear.pyx":203 + * h1 = h1*5+0xe6546b64u + * k1 = key >> 32 + * k1 *= c1 # <<<<<<<<<<<<<< + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 +*/ + __pyx_v_k1 = (__pyx_v_k1 * __pyx_v_c1); + + /* "thinc/layers/sparselinear.pyx":204 + * k1 = key >> 32 + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) # <<<<<<<<<<<<<< + * k1 *= c2 + * h1 ^= k1 +*/ + __pyx_v_k1 = ((__pyx_v_k1 << 15) | (__pyx_v_k1 >> 17)); + + /* "thinc/layers/sparselinear.pyx":205 + * k1 *= c1 + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 # <<<<<<<<<<<<<< + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) +*/ + __pyx_v_k1 = (__pyx_v_k1 * __pyx_v_c2); + + /* "thinc/layers/sparselinear.pyx":206 + * k1 = (k1 << 15) | (k1 >> 17) + * k1 *= c2 + * h1 ^= k1 # <<<<<<<<<<<<<< + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ __pyx_v_k1); + + /* "thinc/layers/sparselinear.pyx":207 + * k1 *= c2 + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) # <<<<<<<<<<<<<< + * h1 = h1*5+0xe6546b64u + * h1 ^= 8 +*/ + __pyx_v_h1 = ((__pyx_v_h1 << 13) | (__pyx_v_h1 >> 19)); + + /* "thinc/layers/sparselinear.pyx":208 + * h1 ^= k1 + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u # <<<<<<<<<<<<<< + * h1 ^= 8 + * h1 ^= h1 >> 16 +*/ + __pyx_v_h1 = ((__pyx_v_h1 * 5) + 0xe6546b64U); + + /* "thinc/layers/sparselinear.pyx":209 + * h1 = (h1 << 13) | (h1 >> 19) + * h1 = h1*5+0xe6546b64u + * h1 ^= 8 # <<<<<<<<<<<<<< + * h1 ^= h1 >> 16 + * h1 *= 0x85ebca6bu +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ 8); + + /* "thinc/layers/sparselinear.pyx":210 + * h1 = h1*5+0xe6546b64u + * h1 ^= 8 + * h1 ^= h1 >> 16 # <<<<<<<<<<<<<< + * h1 *= 0x85ebca6bu + * h1 ^= h1 >> 13 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 16)); + + /* "thinc/layers/sparselinear.pyx":211 + * h1 ^= 8 + * h1 ^= h1 >> 16 + * h1 *= 0x85ebca6bu # <<<<<<<<<<<<<< + * h1 ^= h1 >> 13 + * h1 *= 0xc2b2ae35u +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0x85ebca6bU); + + /* "thinc/layers/sparselinear.pyx":212 + * h1 ^= h1 >> 16 + * h1 *= 0x85ebca6bu + * h1 ^= h1 >> 13 # <<<<<<<<<<<<<< + * h1 *= 0xc2b2ae35u + * h1 ^= h1 >> 16 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 13)); + + /* "thinc/layers/sparselinear.pyx":213 + * h1 *= 0x85ebca6bu + * h1 ^= h1 >> 13 + * h1 *= 0xc2b2ae35u # <<<<<<<<<<<<<< + * h1 ^= h1 >> 16 + * +*/ + __pyx_v_h1 = (__pyx_v_h1 * 0xc2b2ae35U); + + /* "thinc/layers/sparselinear.pyx":214 + * h1 ^= h1 >> 13 + * h1 *= 0xc2b2ae35u + * h1 ^= h1 >> 16 # <<<<<<<<<<<<<< + * + * return h1 +*/ + __pyx_v_h1 = (__pyx_v_h1 ^ (__pyx_v_h1 >> 16)); + + /* "thinc/layers/sparselinear.pyx":216 + * h1 ^= h1 >> 16 + * + * return h1 # <<<<<<<<<<<<<< +*/ + __pyx_r = __pyx_v_h1; + goto __pyx_L0; + + /* "thinc/layers/sparselinear.pyx":189 + * + * + * cdef uint32_t MurmurHash3_x86_32_uint64(uint64_t key, uint32_t seed) noexcept nogil: # <<<<<<<<<<<<<< + * cdef uint32_t h1 = seed + * cdef uint32_t c1 = 0xcc9e2d51u +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyObject *__pyx_tp_new_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update > 0) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(t, __pyx_mstate_global->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, sizeof(struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update)))) + { + o = (PyObject*)__pyx_mstate_global->__pyx_freelist_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update[--__pyx_mstate_global->__pyx_freecount_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update]; + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(Py_TYPE(o)); + #endif + memset(o, 0, sizeof(struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update)); + #if CYTHON_COMPILING_IN_LIMITED_API + (void) PyObject_Init(o, t); + #else + (void) PyObject_INIT(o, t); + #endif + PyObject_GC_Track(o); + } else + #endif + { + o = __Pyx_AllocateExtensionType(t, 1); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(PyObject *o) { + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *p = (struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_v_callback); + Py_CLEAR(p->__pyx_v_keys); + Py_CLEAR(p->__pyx_v_lengths); + Py_CLEAR(p->__pyx_v_values); + #if CYTHON_USE_FREELISTS + if (likely((int)(__pyx_mstate_global->__pyx_freecount_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update < 8) & __PYX_CHECK_FINAL_TYPE_FOR_FREELISTS(Py_TYPE(o), __pyx_mstate_global->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, sizeof(struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update)))) + { + __pyx_mstate_global->__pyx_freelist_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update[__pyx_mstate_global->__pyx_freecount_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update++] = ((struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)o); + } else + #endif + { + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif + } +} + +static int __pyx_tp_traverse_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *p = (struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->__pyx_v_callback) { + e = (*v)(p->__pyx_v_callback, a); if (e) return e; + } + if (p->__pyx_v_keys) { + e = (*v)(p->__pyx_v_keys, a); if (e) return e; + } + if (p->__pyx_v_lengths) { + e = (*v)(p->__pyx_v_lengths, a); if (e) return e; + } + if (p->__pyx_v_values) { + e = (*v)(p->__pyx_v_values, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *p = (struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update *)o; + tmp = ((PyObject*)p->__pyx_v_callback); + p->__pyx_v_callback = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_keys); + p->__pyx_v_keys = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_lengths); + p->__pyx_v_lengths = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_values); + p->__pyx_v_values = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update}, + {Py_tp_clear, (void *)__pyx_tp_clear_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update}, + {Py_tp_new, (void *)__pyx_tp_new_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update}, + {0, 0}, +}; +static PyType_Spec __pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update_spec = { + "thinc.layers.sparselinear.__pyx_scope_struct___begin_gpu_update", + sizeof(struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, + __pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update_slots, +}; +#else + +static PyTypeObject __pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.sparselinear.""__pyx_scope_struct___begin_gpu_update", /*tp_name*/ + sizeof(struct __pyx_obj_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, /*tp_traverse*/ + __pyx_tp_clear_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_array __pyx_vtable_array; + +static PyObject *__pyx_tp_new_array(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_array_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_array_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_array; + p->mode = ((PyObject*)Py_None); Py_INCREF(Py_None); + p->_format = ((PyObject*)Py_None); Py_INCREF(Py_None); + if (unlikely(__pyx_array___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_array(PyObject *o) { + struct __pyx_array_obj *p = (struct __pyx_array_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && (!PyType_IS_GC(Py_TYPE(o)) || !__Pyx_PyObject_GC_IsFinalized(o))) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_array) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_array___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->mode); + Py_CLEAR(p->_format); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static PyObject *__pyx_sq_item_array(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_array(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_array___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_tp_getattro_array(PyObject *o, PyObject *n) { + PyObject *v = PyObject_GenericGetAttr(o, n); + if (!v && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + v = __pyx_array___getattr__(o, n); + } + return v; +} + +static PyObject *__pyx_getprop___pyx_array_memview(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_5array_7memview_1__get__(o); +} + +static PyMethodDef __pyx_methods_array[] = { + {"__getattr__", (PyCFunction)__pyx_array___getattr__, METH_O|METH_COEXIST, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_array_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_array[] = { + {"memview", __pyx_getprop___pyx_array_memview, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_array_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_array}, + {Py_sq_length, (void *)__pyx_array___len__}, + {Py_sq_item, (void *)__pyx_sq_item_array}, + {Py_mp_length, (void *)__pyx_array___len__}, + {Py_mp_subscript, (void *)__pyx_array___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_array}, + {Py_tp_getattro, (void *)__pyx_tp_getattro_array}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_array_getbuffer}, + #endif + {Py_tp_methods, (void *)__pyx_methods_array}, + {Py_tp_getset, (void *)__pyx_getsets_array}, + {Py_tp_new, (void *)__pyx_tp_new_array}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_array_spec = { + "thinc.layers.sparselinear.array", + sizeof(struct __pyx_array_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_array_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_array = { + __pyx_array___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_array, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_array = { + __pyx_array___len__, /*mp_length*/ + __pyx_array___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_array, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_array = { + __pyx_array_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_array = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.sparselinear.""array", /*tp_name*/ + sizeof(struct __pyx_array_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_array, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_array, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_array, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + __pyx_tp_getattro_array, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_array, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_array, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_array, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_array, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyObject *__pyx_tp_new_Enum(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_MemviewEnum_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_MemviewEnum_obj *)o); + p->name = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_Enum(PyObject *o) { + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_Enum) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + Py_CLEAR(p->name); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_Enum(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->name) { + e = (*v)(p->name, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_Enum(PyObject *o) { + PyObject* tmp; + struct __pyx_MemviewEnum_obj *p = (struct __pyx_MemviewEnum_obj *)o; + tmp = ((PyObject*)p->name); + p->name = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_Enum[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_MemviewEnum_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_MemviewEnum_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_Enum}, + {Py_tp_repr, (void *)__pyx_MemviewEnum___repr__}, + {Py_tp_traverse, (void *)__pyx_tp_traverse_Enum}, + {Py_tp_clear, (void *)__pyx_tp_clear_Enum}, + {Py_tp_methods, (void *)__pyx_methods_Enum}, + {Py_tp_init, (void *)__pyx_MemviewEnum___init__}, + {Py_tp_new, (void *)__pyx_tp_new_Enum}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_MemviewEnum_spec = { + "thinc.layers.sparselinear.Enum", + sizeof(struct __pyx_MemviewEnum_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_MemviewEnum_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_MemviewEnum = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.sparselinear.""Enum", /*tp_name*/ + sizeof(struct __pyx_MemviewEnum_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_Enum, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_MemviewEnum___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_Enum, /*tp_traverse*/ + __pyx_tp_clear_Enum, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_Enum, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + __pyx_MemviewEnum___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_Enum, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct_memoryview __pyx_vtable_memoryview; + +static PyObject *__pyx_tp_new_memoryview(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryview_obj *p; + PyObject *o; + o = __Pyx_AllocateExtensionType(t, 0); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryview_obj *)o); + p->__pyx_vtab = __pyx_vtabptr_memoryview; + p->obj = Py_None; Py_INCREF(Py_None); + p->_size = Py_None; Py_INCREF(Py_None); + p->view.obj = NULL; + if (unlikely(__pyx_memoryview___cinit__(o, a, k) < 0)) goto bad; + return o; + bad: + Py_DECREF(o); o = 0; + return NULL; +} + +static void __pyx_tp_dealloc_memoryview(PyObject *o) { + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc_memoryview) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryview___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->obj); + Py_CLEAR(p->_size); + PyTypeObject *tp = Py_TYPE(o); + #if CYTHON_USE_TYPE_SLOTS + (*tp->tp_free)(o); + #else + { + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + if (tp_free) tp_free(o); + } + #endif + #if CYTHON_USE_TYPE_SPECS + Py_DECREF(tp); + #endif +} + +static int __pyx_tp_traverse_memoryview(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + { + e = __Pyx_call_type_traverse(o, 1, v, a); + if (e) return e; + } + if (p->obj) { + e = (*v)(p->obj, a); if (e) return e; + } + if (p->_size) { + e = (*v)(p->_size, a); if (e) return e; + } + if (p->view.obj) { + e = (*v)(p->view.obj, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_memoryview(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryview_obj *p = (struct __pyx_memoryview_obj *)o; + tmp = ((PyObject*)p->obj); + p->obj = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->_size); + p->_size = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + Py_CLEAR(p->view.obj); + return 0; +} + +static PyObject *__pyx_sq_item_memoryview(PyObject *o, Py_ssize_t i) { + PyObject *r; + PyObject *x = PyLong_FromSsize_t(i); if(!x) return 0; + #if CYTHON_USE_TYPE_SLOTS || (!CYTHON_USE_TYPE_SPECS && __PYX_LIMITED_VERSION_HEX < 0x030A0000) + r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x); + #else + r = ((binaryfunc)PyType_GetSlot(Py_TYPE(o), Py_mp_subscript))(o, x); + #endif + Py_DECREF(x); + return r; +} + +static int __pyx_mp_ass_subscript_memoryview(PyObject *o, PyObject *i, PyObject *v) { + if (v) { + return __pyx_memoryview___setitem__(o, i, v); + } + else { + __Pyx_TypeName o_type_name; + o_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(o)); + PyErr_Format(PyExc_NotImplementedError, + "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + return -1; + } +} + +static PyObject *__pyx_getprop___pyx_memoryview_T(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_1T_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_base(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4base_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_shape(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_5shape_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_strides(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_7strides_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_suboffsets(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_10suboffsets_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_ndim(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4ndim_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_itemsize(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_8itemsize_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_nbytes(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_6nbytes_1__get__(o); +} + +static PyObject *__pyx_getprop___pyx_memoryview_size(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_15View_dot_MemoryView_10memoryview_4size_1__get__(o); +} + +static PyMethodDef __pyx_methods_memoryview[] = { + {"is_c_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_c_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"is_f_contig", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_is_f_contig, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"copy_fortran", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_memoryview_copy_fortran, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryview_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_memoryview[] = { + {"T", __pyx_getprop___pyx_memoryview_T, 0, 0, 0}, + {"base", __pyx_getprop___pyx_memoryview_base, 0, 0, 0}, + {"shape", __pyx_getprop___pyx_memoryview_shape, 0, 0, 0}, + {"strides", __pyx_getprop___pyx_memoryview_strides, 0, 0, 0}, + {"suboffsets", __pyx_getprop___pyx_memoryview_suboffsets, 0, 0, 0}, + {"ndim", __pyx_getprop___pyx_memoryview_ndim, 0, 0, 0}, + {"itemsize", __pyx_getprop___pyx_memoryview_itemsize, 0, 0, 0}, + {"nbytes", __pyx_getprop___pyx_memoryview_nbytes, 0, 0, 0}, + {"size", __pyx_getprop___pyx_memoryview_size, 0, 0, 0}, + {0, 0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +#if !CYTHON_COMPILING_IN_LIMITED_API + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; +#endif +static PyType_Slot __pyx_type___pyx_memoryview_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc_memoryview}, + {Py_tp_repr, (void *)__pyx_memoryview___repr__}, + {Py_sq_length, (void *)__pyx_memoryview___len__}, + {Py_sq_item, (void *)__pyx_sq_item_memoryview}, + {Py_mp_length, (void *)__pyx_memoryview___len__}, + {Py_mp_subscript, (void *)__pyx_memoryview___getitem__}, + {Py_mp_ass_subscript, (void *)__pyx_mp_ass_subscript_memoryview}, + {Py_tp_str, (void *)__pyx_memoryview___str__}, + #if defined(Py_bf_getbuffer) + {Py_bf_getbuffer, (void *)__pyx_memoryview_getbuffer}, + #endif + {Py_tp_traverse, (void *)__pyx_tp_traverse_memoryview}, + {Py_tp_clear, (void *)__pyx_tp_clear_memoryview}, + {Py_tp_methods, (void *)__pyx_methods_memoryview}, + {Py_tp_getset, (void *)__pyx_getsets_memoryview}, + {Py_tp_new, (void *)__pyx_tp_new_memoryview}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryview_spec = { + "thinc.layers.sparselinear.memoryview", + sizeof(struct __pyx_memoryview_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, + __pyx_type___pyx_memoryview_slots, +}; +#else + +static PySequenceMethods __pyx_tp_as_sequence_memoryview = { + __pyx_memoryview___len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + __pyx_sq_item_memoryview, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_memoryview = { + __pyx_memoryview___len__, /*mp_length*/ + __pyx_memoryview___getitem__, /*mp_subscript*/ + __pyx_mp_ass_subscript_memoryview, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_memoryview = { + __pyx_memoryview_getbuffer, /*bf_getbuffer*/ + 0, /*bf_releasebuffer*/ +}; + +static PyTypeObject __pyx_type___pyx_memoryview = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.sparselinear.""memoryview", /*tp_name*/ + sizeof(struct __pyx_memoryview_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_memoryview, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + __pyx_memoryview___repr__, /*tp_repr*/ + 0, /*tp_as_number*/ + &__pyx_tp_as_sequence_memoryview, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_memoryview, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + __pyx_memoryview___str__, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_memoryview, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_memoryview, /*tp_traverse*/ + __pyx_tp_clear_memoryview, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_memoryview, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_memoryview, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_memoryview, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif +static struct __pyx_vtabstruct__memoryviewslice __pyx_vtable__memoryviewslice; + +static PyObject *__pyx_tp_new__memoryviewslice(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_memoryviewslice_obj *p; + PyObject *o = __pyx_tp_new_memoryview(t, a, k); + if (unlikely(!o)) return 0; + p = ((struct __pyx_memoryviewslice_obj *)o); + p->__pyx_base.__pyx_vtab = (struct __pyx_vtabstruct_memoryview*)__pyx_vtabptr__memoryviewslice; + __Pyx_default_placement_construct(&(p->from_slice)); + p->from_object = Py_None; Py_INCREF(Py_None); + p->from_slice.memview = NULL; + return o; +} + +static void __pyx_tp_dealloc__memoryviewslice(PyObject *o) { + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + #if CYTHON_USE_TP_FINALIZE + if (unlikely(__Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && !__Pyx_PyObject_GC_IsFinalized(o)) { + if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == __pyx_tp_dealloc__memoryviewslice) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1); + __pyx_memoryviewslice___dealloc__(o); + __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1); + PyErr_Restore(etype, eval, etb); + } + __Pyx_call_destructor(p->from_slice); + Py_CLEAR(p->from_object); + PyObject_GC_Track(o); + __pyx_tp_dealloc_memoryview(o); +} + +static int __pyx_tp_traverse__memoryviewslice(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + e = __pyx_tp_traverse_memoryview(o, v, a); if (e) return e; + if (p->from_object) { + e = (*v)(p->from_object, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear__memoryviewslice(PyObject *o) { + PyObject* tmp; + struct __pyx_memoryviewslice_obj *p = (struct __pyx_memoryviewslice_obj *)o; + __pyx_tp_clear_memoryview(o); + tmp = ((PyObject*)p->from_object); + p->from_object = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + __PYX_XCLEAR_MEMVIEW(&p->from_slice, 1); + return 0; +} + +static PyMethodDef __pyx_methods__memoryviewslice[] = { + {"__reduce_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_1__reduce_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {"__setstate_cython__", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw___pyx_memoryviewslice_3__setstate_cython__, __Pyx_METH_FASTCALL|METH_KEYWORDS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_type___pyx_memoryviewslice_slots[] = { + {Py_tp_dealloc, (void *)__pyx_tp_dealloc__memoryviewslice}, + {Py_tp_doc, (void *)PyDoc_STR("Internal class for passing memoryview slices to Python")}, + {Py_tp_traverse, (void *)__pyx_tp_traverse__memoryviewslice}, + {Py_tp_clear, (void *)__pyx_tp_clear__memoryviewslice}, + {Py_tp_methods, (void *)__pyx_methods__memoryviewslice}, + {Py_tp_new, (void *)__pyx_tp_new__memoryviewslice}, + {0, 0}, +}; +static PyType_Spec __pyx_type___pyx_memoryviewslice_spec = { + "thinc.layers.sparselinear._memoryviewslice", + sizeof(struct __pyx_memoryviewslice_obj), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, + __pyx_type___pyx_memoryviewslice_slots, +}; +#else + +static PyTypeObject __pyx_type___pyx_memoryviewslice = { + PyVarObject_HEAD_INIT(0, 0) + "thinc.layers.sparselinear.""_memoryviewslice", /*tp_name*/ + sizeof(struct __pyx_memoryviewslice_obj), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc__memoryviewslice, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___repr__, /*tp_repr*/ + #else + 0, /*tp_repr*/ + #endif + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY || 0 + __pyx_memoryview___str__, /*tp_str*/ + #else + 0, /*tp_str*/ + #endif + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC|Py_TPFLAGS_SEQUENCE, /*tp_flags*/ + PyDoc_STR("Internal class for passing memoryview slices to Python"), /*tp_doc*/ + __pyx_tp_traverse__memoryviewslice, /*tp_traverse*/ + __pyx_tp_clear__memoryviewslice, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods__memoryviewslice, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + #if !CYTHON_USE_TYPE_SPECS + 0, /*tp_dictoffset*/ + #endif + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new__memoryviewslice, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if CYTHON_USE_TP_FINALIZE + 0, /*tp_finalize*/ + #else + NULL, /*tp_finalize*/ + #endif + #if !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800 + 0, /*tp_vectorcall*/ + #endif + #if __PYX_NEED_TP_PRINT_SLOT == 1 + 0, /*tp_print*/ + #endif + #if PY_VERSION_HEX >= 0x030C0000 + 0, /*tp_watched*/ + #endif + #if PY_VERSION_HEX >= 0x030d00A4 + 0, /*tp_versions_used*/ + #endif + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000 + 0, /*tp_pypy_flags*/ + #endif +}; +#endif + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_collections_abc_Sequence = Py_None; Py_INCREF(Py_None); + generic = Py_None; Py_INCREF(Py_None); + strided = Py_None; Py_INCREF(Py_None); + indirect = Py_None; Py_INCREF(Py_None); + contiguous = Py_None; Py_INCREF(Py_None); + indirect_contiguous = Py_None; Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update)) __PYX_ERR(0, 71, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update_spec, __pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + #else + __pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update = &__pyx_type_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update->tp_dictoffset && __pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_ptype_5thinc_6layers_12sparselinear___pyx_scope_struct___begin_gpu_update->tp_getattro = PyObject_GenericGetAttr; + } + #endif + __pyx_vtabptr_array = &__pyx_vtable_array; + __pyx_vtable_array.get_memview = (PyObject *(*)(struct __pyx_array_obj *))__pyx_array_get_memview; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_array_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_array_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_array_type)) __PYX_ERR(1, 110, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_array_type->tp_as_buffer = &__pyx_tp_as_buffer_array; + if (!__pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_array_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_array_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_array_spec, __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #else + __pyx_mstate->__pyx_array_type = &__pyx_type___pyx_array; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_array_type); + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_array_type, __pyx_vtabptr_array) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_array_type) < (0)) __PYX_ERR(1, 110, __pyx_L1_error) + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_MemviewEnum_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_MemviewEnum_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_MemviewEnum_type)) __PYX_ERR(1, 299, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_MemviewEnum_spec, __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #else + __pyx_mstate->__pyx_MemviewEnum_type = &__pyx_type___pyx_MemviewEnum; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_MemviewEnum_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_MemviewEnum_type->tp_dictoffset && __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_MemviewEnum_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_MemviewEnum_type) < (0)) __PYX_ERR(1, 299, __pyx_L1_error) + __pyx_vtabptr_memoryview = &__pyx_vtable_memoryview; + __pyx_vtable_memoryview.get_item_pointer = (char *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_get_item_pointer; + __pyx_vtable_memoryview.is_slice = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_is_slice; + __pyx_vtable_memoryview.setitem_slice_assignment = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_slice_assignment; + __pyx_vtable_memoryview.setitem_slice_assign_scalar = (PyObject *(*)(struct __pyx_memoryview_obj *, struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_setitem_slice_assign_scalar; + __pyx_vtable_memoryview.setitem_indexed = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_indexed; + __pyx_vtable_memoryview.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryview_convert_item_to_object; + __pyx_vtable_memoryview.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryview_assign_item_from_object; + __pyx_vtable_memoryview._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryview__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_mstate->__pyx_memoryview_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryview_spec, NULL); if (unlikely(!__pyx_mstate->__pyx_memoryview_type)) __PYX_ERR(1, 334, __pyx_L1_error) + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer = &__pyx_tp_as_buffer_memoryview; + if (!__pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer && __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer) { + __pyx_mstate->__pyx_memoryview_type->tp_as_buffer->bf_releasebuffer = __pyx_mstate->__pyx_memoryview_type->tp_base->tp_as_buffer->bf_releasebuffer; + } + #elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer) + /* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */ + #elif defined(_MSC_VER) + #pragma message ("The buffer protocol is not supported in the Limited C-API < 3.11.") + #else + #warning "The buffer protocol is not supported in the Limited C-API < 3.11." + #endif + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryview_spec, __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryview_type = &__pyx_type___pyx_memoryview; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryview_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryview_type->tp_dictoffset && __pyx_mstate->__pyx_memoryview_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryview_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryview_type, __pyx_vtabptr_memoryview) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryview_type) < (0)) __PYX_ERR(1, 334, __pyx_L1_error) + __pyx_vtabptr__memoryviewslice = &__pyx_vtable__memoryviewslice; + __pyx_vtable__memoryviewslice.__pyx_base = *__pyx_vtabptr_memoryview; + __pyx_vtable__memoryviewslice.__pyx_base.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryviewslice_convert_item_to_object; + __pyx_vtable__memoryviewslice.__pyx_base.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryviewslice_assign_item_from_object; + __pyx_vtable__memoryviewslice.__pyx_base._get_base = (PyObject *(*)(struct __pyx_memoryview_obj *))__pyx_memoryviewslice__get_base; + #if CYTHON_USE_TYPE_SPECS + __pyx_t_1 = PyTuple_Pack(1, (PyObject *)__pyx_mstate_global->__pyx_memoryview_type); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_memoryviewslice_type = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(__pyx_m, &__pyx_type___pyx_memoryviewslice_spec, __pyx_t_1); + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_mstate->__pyx_memoryviewslice_type)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_fix_up_extension_type_from_spec(&__pyx_type___pyx_memoryviewslice_spec, __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #else + __pyx_mstate->__pyx_memoryviewslice_type = &__pyx_type___pyx_memoryviewslice; + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + __pyx_mstate_global->__pyx_memoryviewslice_type->tp_base = __pyx_mstate_global->__pyx_memoryview_type; + #endif + #if !CYTHON_USE_TYPE_SPECS + if (__Pyx_PyType_Ready(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount((PyObject*)__pyx_mstate->__pyx_memoryviewslice_type); + #endif + #if !CYTHON_COMPILING_IN_LIMITED_API + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_mstate->__pyx_memoryviewslice_type->tp_dictoffset && __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro == PyObject_GenericGetAttr)) { + __pyx_mstate->__pyx_memoryviewslice_type->tp_getattro = PyObject_GenericGetAttr; + } + #endif + if (__Pyx_SetVtable(__pyx_mstate->__pyx_memoryviewslice_type, __pyx_vtabptr__memoryviewslice) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_MergeVtables(__pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject *) __pyx_mstate->__pyx_memoryviewslice_type) < (0)) __PYX_ERR(1, 951, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7cpython_4type_type = __Pyx_ImportType_3_2_1(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyTypeObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + 0, 0, + #else + sizeof(PyHeapTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyHeapTypeObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("numpy"); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_5numpy_dtype = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "dtype", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #else + sizeof(PyArray_Descr), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArray_Descr), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_dtype) __PYX_ERR(2, 228, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_flatiter = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "flatiter", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #else + sizeof(PyArrayIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayIterObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_flatiter) __PYX_ERR(2, 273, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_broadcast = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "broadcast", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #else + sizeof(PyArrayMultiIterObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayMultiIterObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_broadcast) __PYX_ERR(2, 277, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_ndarray = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "ndarray", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #else + sizeof(PyArrayObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyArrayObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_ndarray) __PYX_ERR(2, 316, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_generic = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "generic", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_generic) __PYX_ERR(2, 825, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_number = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "number", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_number) __PYX_ERR(2, 827, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_integer = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "integer", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_integer) __PYX_ERR(2, 829, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_signedinteger = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "signedinteger", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_signedinteger) __PYX_ERR(2, 831, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_unsignedinteger = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "unsignedinteger", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_unsignedinteger) __PYX_ERR(2, 833, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_inexact = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "inexact", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_inexact) __PYX_ERR(2, 835, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_floating = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "floating", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_floating) __PYX_ERR(2, 837, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_complexfloating = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "complexfloating", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_complexfloating) __PYX_ERR(2, 839, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_flexible = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "flexible", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_flexible) __PYX_ERR(2, 841, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_character = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "character", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #else + sizeof(PyObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_character) __PYX_ERR(2, 843, __pyx_L1_error) + __pyx_mstate->__pyx_ptype_5numpy_ufunc = __Pyx_ImportType_3_2_1(__pyx_t_1, "numpy", "ufunc", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #else + sizeof(PyUFuncObject), __PYX_GET_STRUCT_ALIGNMENT_3_2_1(PyUFuncObject), + #endif + __Pyx_ImportType_CheckSize_Ignore_3_2_1); if (!__pyx_mstate->__pyx_ptype_5numpy_ufunc) __PYX_ERR(2, 907, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_sparselinear(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_sparselinear}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "sparselinear", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_sparselinear(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_sparselinear(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL && defined(GRAALPY_VERSION_NUM) && GRAALPY_VERSION_NUM > 0x19000000 + PY_INT64_T current_id = GraalPyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && (__PYX_LIMITED_VERSION_HEX < 0x03090000\ + || ((defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS)) && __PYX_LIMITED_VERSION_HEX < 0x030A0000)) + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_sparselinear(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + static PyThread_type_lock __pyx_t_7[8]; + int __pyx_t_8; + Py_ssize_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'sparselinear' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "sparselinear" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + +__Pyx_RefNannySetupContext("PyInit_sparselinear", 0); + __Pyx_init_runtime_version(); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Library function declarations ---*/ + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__pyx_module_is_main_thinc__layers__sparselinear) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "thinc.layers.sparselinear")) { + if (unlikely((PyDict_SetItemString(modules, "thinc.layers.sparselinear", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + (void)__Pyx_modinit_function_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_init_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":101 + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence # <<<<<<<<<<<<<< + * except: + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_collections_abc}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_builtin___import__, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + } + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_abc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_Sequence); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L2_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":100 + * + * cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" + * try: # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L7_try_end; + __pyx_L2_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":102 + * try: + * __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + * except: # <<<<<<<<<<<<<< + * + * __pyx_collections_abc_Sequence = None +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + + /* "View.MemoryView":104 + * except: + * + * __pyx_collections_abc_Sequence = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_collections_abc_Sequence); + __Pyx_DECREF_SET(__pyx_collections_abc_Sequence, Py_None); + __Pyx_GIVEREF(Py_None); + goto __pyx_L3_exception_handled; + } + __pyx_L3_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L7_try_end:; + } + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":240 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 240, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":241 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_array_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 241, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":239 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L15_try_end; + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":242 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L11_exception_handled; + } + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L15_try_end:; + } + + /* "View.MemoryView":307 + * return self.name + * + * cdef generic = Enum("") # <<<<<<<<<<<<<< + * cdef strided = Enum("") # default + * cdef indirect = Enum("") +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct_or_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 307, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(generic); + __Pyx_DECREF_SET(generic, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":308 + * + * cdef generic = Enum("") + * cdef strided = Enum("") # default # <<<<<<<<<<<<<< + * cdef indirect = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 308, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(strided); + __Pyx_DECREF_SET(strided, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":309 + * cdef generic = Enum("") + * cdef strided = Enum("") # default + * cdef indirect = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_strided_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 309, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect); + __Pyx_DECREF_SET(indirect, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":312 + * + * + * cdef contiguous = Enum("") # <<<<<<<<<<<<<< + * cdef indirect_contiguous = Enum("") + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_direct}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 312, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(contiguous); + __Pyx_DECREF_SET(contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":313 + * + * cdef contiguous = Enum("") + * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_5 = NULL; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, __pyx_mstate_global->__pyx_kp_u_contiguous_and_indirect}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_mstate_global->__pyx_MemviewEnum_type, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 313, __pyx_L1_error) + __Pyx_GOTREF((PyObject *)__pyx_t_4); + } + __Pyx_XGOTREF(indirect_contiguous); + __Pyx_DECREF_SET(indirect_contiguous, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF((PyObject *)__pyx_t_4); + __pyx_t_4 = 0; + + /* "View.MemoryView":321 + * + * + * cdef int __pyx_memoryview_thread_locks_used = 0 # <<<<<<<<<<<<<< + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ + * PyThread_allocate_lock(), +*/ + __pyx_memoryview_thread_locks_used = 0; + + /* "View.MemoryView":322 + * + * cdef int __pyx_memoryview_thread_locks_used = 0 + * cdef PyThread_type_lock[8] __pyx_memoryview_thread_locks = [ # <<<<<<<<<<<<<< + * PyThread_allocate_lock(), + * PyThread_allocate_lock(), +*/ + __pyx_t_7[0] = PyThread_allocate_lock(); + __pyx_t_7[1] = PyThread_allocate_lock(); + __pyx_t_7[2] = PyThread_allocate_lock(); + __pyx_t_7[3] = PyThread_allocate_lock(); + __pyx_t_7[4] = PyThread_allocate_lock(); + __pyx_t_7[5] = PyThread_allocate_lock(); + __pyx_t_7[6] = PyThread_allocate_lock(); + __pyx_t_7[7] = PyThread_allocate_lock(); + memcpy(&(__pyx_memoryview_thread_locks[0]), __pyx_t_7, sizeof(__pyx_memoryview_thread_locks[0]) * (8)); + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "View.MemoryView":984 + * + * try: + * count = __pyx_collections_abc_Sequence.count # <<<<<<<<<<<<<< + * index = __pyx_collections_abc_Sequence.index + * except: +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_count, __pyx_t_4) < (0)) __PYX_ERR(1, 984, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":985 + * try: + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index # <<<<<<<<<<<<<< + * except: + * pass +*/ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_collections_abc_Sequence, __pyx_mstate_global->__pyx_n_u_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_SetItemOnTypeDict(__pyx_mstate_global->__pyx_memoryviewslice_type, __pyx_mstate_global->__pyx_n_u_index, __pyx_t_4) < (0)) __PYX_ERR(1, 985, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":983 + * + * + * try: # <<<<<<<<<<<<<< + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index +*/ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L23_try_end; + __pyx_L18_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":986 + * count = __pyx_collections_abc_Sequence.count + * index = __pyx_collections_abc_Sequence.index + * except: # <<<<<<<<<<<<<< + * pass + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L19_exception_handled; + } + __pyx_L19_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __pyx_L23_try_end:; + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_2, &__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_1); + /*try:*/ { + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_8 = __Pyx_PyObject_IsTrue(__pyx_collections_abc_Sequence); if (unlikely((__pyx_t_8 < 0))) __PYX_ERR(1, 990, __pyx_L26_error) + if (__pyx_t_8) { + + /* "View.MemoryView":994 + * + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) # <<<<<<<<<<<<<< + * __pyx_collections_abc_Sequence.register(array) + * except: +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_memoryviewslice_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 994, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":995 + * + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) # <<<<<<<<<<<<<< + * except: + * pass # ignore failure, it's a minor issue +*/ + __pyx_t_5 = __pyx_collections_abc_Sequence; + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = 0; + { + PyObject *__pyx_callargs[2] = {__pyx_t_5, ((PyObject *)__pyx_mstate_global->__pyx_array_type)}; + __pyx_t_4 = __Pyx_PyObject_FastCallMethod((PyObject*)__pyx_mstate_global->__pyx_n_u_register, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (1*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 995, __pyx_L26_error) + __Pyx_GOTREF(__pyx_t_4); + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "View.MemoryView":990 + * + * try: + * if __pyx_collections_abc_Sequence: # <<<<<<<<<<<<<< + * + * +*/ + } + + /* "View.MemoryView":989 + * pass + * + * try: # <<<<<<<<<<<<<< + * if __pyx_collections_abc_Sequence: + * +*/ + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L31_try_end; + __pyx_L26_error:; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "View.MemoryView":996 + * __pyx_collections_abc_Sequence.register(_memoryviewslice) + * __pyx_collections_abc_Sequence.register(array) + * except: # <<<<<<<<<<<<<< + * pass # ignore failure, it's a minor issue + * +*/ + /*except:*/ { + __Pyx_ErrRestore(0,0,0); + goto __pyx_L27_exception_handled; + } + __pyx_L27_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_2, __pyx_t_1); + __pyx_L31_try_end:; + } + + /* "(tree fragment)":4 + * int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1 + * int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1 + * def __pyx_unpickle_Enum(__pyx_type, long __pyx_checksum, tuple __pyx_state): # <<<<<<<<<<<<<< + * cdef object __pyx_result + * __Pyx_CheckUnpickleChecksum(__pyx_checksum, 0x82a3537, 0x6ae9995, 0xb068931, b'name') +*/ + __pyx_t_4 = PyCFunction_NewEx(&__pyx_mdef_15View_dot_MemoryView_1__pyx_unpickle_Enum, NULL, __pyx_mstate_global->__pyx_n_u_View_MemoryView); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_pyx_unpickle_Enum, __pyx_t_4) < (0)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":6 + * from libc.stdint cimport int32_t, uint32_t, uint64_t + * + * from typing import Callable, Optional, Tuple # <<<<<<<<<<<<<< + * + * from ..backends import CupyOps, NumpyOps +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Callable,__pyx_mstate_global->__pyx_n_u_Optional,__pyx_mstate_global->__pyx_n_u_Tuple}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_typing, __pyx_imported_names, 3, __pyx_mstate_global->__pyx_kp_u_thinc_layers_typing, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 6, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Callable,__pyx_mstate_global->__pyx_n_u_Optional,__pyx_mstate_global->__pyx_n_u_Tuple}; + for (__pyx_t_9=0; __pyx_t_9 < 3; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":8 + * from typing import Callable, Optional, Tuple + * + * from ..backends import CupyOps, NumpyOps # <<<<<<<<<<<<<< + * from ..config import registry + * from ..model import Model +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_CupyOps,__pyx_mstate_global->__pyx_n_u_NumpyOps}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_backends, __pyx_imported_names, 2, __pyx_mstate_global->__pyx_kp_u_thinc_backends, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 8, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_CupyOps,__pyx_mstate_global->__pyx_n_u_NumpyOps}; + for (__pyx_t_9=0; __pyx_t_9 < 2; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":9 + * + * from ..backends import CupyOps, NumpyOps + * from ..config import registry # <<<<<<<<<<<<<< + * from ..model import Model + * from ..types import ArrayXd +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_config, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_config, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 9, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_registry}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":10 + * from ..backends import CupyOps, NumpyOps + * from ..config import registry + * from ..model import Model # <<<<<<<<<<<<<< + * from ..types import ArrayXd + * from ..util import get_array_module, get_width, is_cupy_array, is_numpy_array +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Model}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_model, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_model, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 10, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_Model}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":11 + * from ..config import registry + * from ..model import Model + * from ..types import ArrayXd # <<<<<<<<<<<<<< + * from ..util import get_array_module, get_width, is_cupy_array, is_numpy_array + * +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_ArrayXd}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_types, __pyx_imported_names, 1, __pyx_mstate_global->__pyx_kp_u_thinc_types, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 11, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_ArrayXd}; + __pyx_t_9 = 0; { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 11, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":12 + * from ..model import Model + * from ..types import ArrayXd + * from ..util import get_array_module, get_width, is_cupy_array, is_numpy_array # <<<<<<<<<<<<<< + * + * InT = Tuple[ArrayXd, ArrayXd, ArrayXd] +*/ + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_array_module,__pyx_mstate_global->__pyx_n_u_get_width,__pyx_mstate_global->__pyx_n_u_is_cupy_array,__pyx_mstate_global->__pyx_n_u_is_numpy_array}; + __pyx_t_1 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_util, __pyx_imported_names, 4, __pyx_mstate_global->__pyx_kp_u_thinc_util, 2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 12, __pyx_L1_error) + } + __pyx_t_4 = __pyx_t_1; + __Pyx_GOTREF(__pyx_t_4); + { + PyObject* const __pyx_imported_names[] = {__pyx_mstate_global->__pyx_n_u_get_array_module,__pyx_mstate_global->__pyx_n_u_get_width,__pyx_mstate_global->__pyx_n_u_is_cupy_array,__pyx_mstate_global->__pyx_n_u_is_numpy_array}; + for (__pyx_t_9=0; __pyx_t_9 < 4; __pyx_t_9++) { + __pyx_t_5 = __Pyx_ImportFrom(__pyx_t_4, __pyx_imported_names[__pyx_t_9]); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_imported_names[__pyx_t_9], __pyx_t_5) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "thinc/layers/sparselinear.pyx":14 + * from ..util import get_array_module, get_width, is_cupy_array, is_numpy_array + * + * InT = Tuple[ArrayXd, ArrayXd, ArrayXd] # <<<<<<<<<<<<<< + * OutT = ArrayXd + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Tuple); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_ArrayXd); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_ArrayXd); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_ArrayXd); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = PyTuple_Pack(3, __pyx_t_5, __pyx_t_10, __pyx_t_11); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_11 = __Pyx_PyObject_GetItem(__pyx_t_4, __pyx_t_12); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_InT, __pyx_t_11) < (0)) __PYX_ERR(0, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":15 + * + * InT = Tuple[ArrayXd, ArrayXd, ArrayXd] + * OutT = ArrayXd # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_GetModuleGlobalName(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_ArrayXd); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_OutT, __pyx_t_11) < (0)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":19 + * + * @cython.binding(True) + * @registry.layers("SparseLinear.v1") # <<<<<<<<<<<<<< + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): + * # NB: We can't have generic return type annotation if we want function to +*/ + __pyx_t_12 = NULL; + __pyx_t_10 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_registry); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_13 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_layers); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_10, __pyx_mstate_global->__pyx_kp_u_SparseLinear_v1}; + __pyx_t_4 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_13, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + } + + /* "thinc/layers/sparselinear.pyx":18 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + __pyx_t_13 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_nO, __pyx_mstate_global->__pyx_kp_u_Optional_int) < (0)) __PYX_ERR(0, 18, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_length, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 18, __pyx_L1_error) + __pyx_t_10 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_1SparseLinear, 0, __pyx_mstate_global->__pyx_n_u_SparseLinear, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_10); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_10, __pyx_mstate_global->__pyx_tuple[2]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_10, __pyx_t_13); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_12, __pyx_t_10}; + __pyx_t_11 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_4, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_SparseLinear, __pyx_t_11) < (0)) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":34 + * + * @cython.binding(True) + * @registry.layers("SparseLinear.v2") # <<<<<<<<<<<<<< + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): + * # NB: We can't have generic return type annotation if we want function to +*/ + __pyx_t_4 = NULL; + __pyx_t_12 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_registry); if (unlikely(!__pyx_t_13)) __PYX_ERR(0, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_13); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_13, __pyx_mstate_global->__pyx_n_u_layers); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_13); __pyx_t_13 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_12, __pyx_mstate_global->__pyx_kp_u_SparseLinear_v2}; + __pyx_t_10 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + } + + /* "thinc/layers/sparselinear.pyx":33 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v2") + * def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + __pyx_t_5 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_nO, __pyx_mstate_global->__pyx_kp_u_Optional_int) < (0)) __PYX_ERR(0, 33, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_length, __pyx_mstate_global->__pyx_n_u_int) < (0)) __PYX_ERR(0, 33, __pyx_L1_error) + __pyx_t_12 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_3SparseLinear_v2, 0, __pyx_mstate_global->__pyx_n_u_SparseLinear_v2_2, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[2])); if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_12); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_12); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_12, __pyx_mstate_global->__pyx_tuple[2]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_12, __pyx_t_5); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_t_12}; + __pyx_t_11 = __Pyx_PyObject_FastCall((PyObject*)__pyx_t_10, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 34, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + } + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_SparseLinear_v2_2, __pyx_t_11) < (0)) __PYX_ERR(0, 33, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":48 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * def forward(model: Model, keys_values_lengths: InT, is_train: bool) -> Tuple[OutT, Callable]: + * # NB: We can't have generic Model annotation if we want function to +*/ + __pyx_t_11 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_model, __pyx_mstate_global->__pyx_n_u_Model) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_keys_values_lengths, __pyx_mstate_global->__pyx_n_u_InT) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_is_train, __pyx_mstate_global->__pyx_n_u_bool) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_kp_u_Tuple_OutT_Callable) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + __pyx_t_10 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_5forward, 0, __pyx_mstate_global->__pyx_n_u_forward, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[3])); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_10); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_10, __pyx_t_11); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_forward, __pyx_t_10) < (0)) __PYX_ERR(0, 48, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/sparselinear.pyx":61 + * + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: # <<<<<<<<<<<<<< + * if Y is not None: + * model.set_dim("nO", get_width(Y)) +*/ + __pyx_t_10 = __Pyx_PyDict_NewPresized(4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_model, __pyx_mstate_global->__pyx_kp_u_Model_InT_OutT) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_X, __pyx_mstate_global->__pyx_kp_u_Optional_InT) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_Y, __pyx_mstate_global->__pyx_kp_u_Optional_OutT) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_kp_u_Model_InT_OutT) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + __pyx_t_11 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_7init, 0, __pyx_mstate_global->__pyx_n_u_init, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[4])); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_11); + #endif + __Pyx_CyFunction_SetDefaultsTuple(__pyx_t_11, __pyx_mstate_global->__pyx_tuple[3]); + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_11, __pyx_t_10); + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_init, __pyx_t_11) < (0)) __PYX_ERR(0, 61, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":71 + * + * + * def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: # <<<<<<<<<<<<<< + * xp = get_array_module(keys) + * scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) +*/ + __pyx_t_11 = __Pyx_PyDict_NewPresized(5); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_model, __pyx_mstate_global->__pyx_kp_u_Model_InT_OutT) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_keys, __pyx_mstate_global->__pyx_n_u_ArrayXd) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_values, __pyx_mstate_global->__pyx_n_u_ArrayXd) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_lengths, __pyx_mstate_global->__pyx_n_u_ArrayXd) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_11, __pyx_mstate_global->__pyx_n_u_return, __pyx_mstate_global->__pyx_kp_u_Tuple_ArrayXd_Callable) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + __pyx_t_10 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_9_begin_gpu_update, 0, __pyx_mstate_global->__pyx_n_u_begin_gpu_update, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[5])); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_10); + #endif + __Pyx_CyFunction_SetAnnotationsDict(__pyx_t_10, __pyx_t_11); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_begin_gpu_update, __pyx_t_10) < (0)) __PYX_ERR(0, 71, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/sparselinear.pyx":82 + * + * + * def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): # <<<<<<<<<<<<<< + * cdef int nO = model.get_dim("nO") + * cdef int length = model.get_dim("length") +*/ + __pyx_t_10 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_11_begin_cpu_update, 0, __pyx_mstate_global->__pyx_n_u_begin_cpu_update, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[6])); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_10); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_begin_cpu_update, __pyx_t_10) < (0)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/sparselinear.pyx":97 + * + * + * class _finish_linear_update: # <<<<<<<<<<<<<< + * """Move this out of a closure, into its own callable object, to avoid + * pickling errors :(.""" +*/ + __pyx_t_10 = __Pyx_Py3MetaclassPrepare((PyObject *) NULL, __pyx_mstate_global->__pyx_empty_tuple, __pyx_mstate_global->__pyx_n_u_finish_linear_update, __pyx_mstate_global->__pyx_n_u_finish_linear_update, (PyObject *) NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_kp_u_Move_this_out_of_a_closure_into); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + + /* "thinc/layers/sparselinear.pyx":100 + * """Move this out of a closure, into its own callable object, to avoid + * pickling errors :(.""" + * def __init__(self, model, keys, values, lengths): # <<<<<<<<<<<<<< + * self.model = model + * self.keys = keys +*/ + __pyx_t_11 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_21_finish_linear_update_1__init__, 0, __pyx_mstate_global->__pyx_n_u_finish_linear_update___init, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[7])); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 100, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_11); + #endif + if (__Pyx_SetNameInClass(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_init_2, __pyx_t_11) < (0)) __PYX_ERR(0, 100, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":106 + * self.lengths = lengths + * + * def __call__(self, float[:, ::1] d_scores): # <<<<<<<<<<<<<< + * nO = self.model.get_dim("nO") + * length = self.model.get_dim("length") +*/ + __pyx_t_11 = __Pyx_CyFunction_New(&__pyx_mdef_5thinc_6layers_12sparselinear_21_finish_linear_update_3__call__, 0, __pyx_mstate_global->__pyx_n_u_finish_linear_update___call, NULL, __pyx_mstate_global->__pyx_n_u_thinc_layers_sparselinear, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[8])); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_11); + #endif + if (__Pyx_SetNameInClass(__pyx_t_10, __pyx_mstate_global->__pyx_n_u_call, __pyx_t_11) < (0)) __PYX_ERR(0, 106, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "thinc/layers/sparselinear.pyx":97 + * + * + * class _finish_linear_update: # <<<<<<<<<<<<<< + * """Move this out of a closure, into its own callable object, to avoid + * pickling errors :(.""" +*/ + __pyx_t_11 = __Pyx_Py3ClassCreate(((PyObject*)&__Pyx_DefaultClassType), __pyx_mstate_global->__pyx_n_u_finish_linear_update, __pyx_mstate_global->__pyx_empty_tuple, __pyx_t_10, NULL, 0, 1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_11); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030E0000 + PyUnstable_Object_EnableDeferredRefcount(__pyx_t_11); + #endif + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_finish_linear_update, __pyx_t_11) < (0)) __PYX_ERR(0, 97, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /* "thinc/layers/sparselinear.pyx":1 + * # cython: infer_types=True, cdivision=True, bounds_check=False, wraparound=False, profile=False # <<<<<<<<<<<<<< + * cimport cython + * cimport numpy as np +*/ + __pyx_t_10 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_10); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_10) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init thinc.layers.sparselinear", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init thinc.layers.sparselinear"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin___import__ = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_import); if (!__pyx_builtin___import__) __PYX_ERR(1, 101, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 409, __pyx_L1_error) + __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_id); if (!__pyx_builtin_id) __PYX_ERR(1, 619, __pyx_L1_error) + + /* Cached unbound methods */ + __pyx_mstate->__pyx_umethod_PyDict_Type_items.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_items.method_name = &__pyx_mstate->__pyx_n_u_items; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_values.method_name = &__pyx_mstate->__pyx_n_u_values; + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "(tree fragment)":11 + * use_setstate = True + * else: + * use_setstate = ('self.name is not None',) # <<<<<<<<<<<<<< + * if use_setstate: + * return __pyx_unpickle_Enum, (type(self), 0x82a3537, None), state +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_mstate_global->__pyx_kp_u_self_name_is_not_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + + /* "View.MemoryView":583 + * def suboffsets(self): + * if self.view.suboffsets == NULL: + * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< + * + * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) +*/ + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_New(1); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(1, 583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_INCREF(__pyx_mstate_global->__pyx_int_neg_1); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_int_neg_1); + if (__Pyx_PyTuple_SET_ITEM(__pyx_mstate_global->__pyx_tuple[1], 0, __pyx_mstate_global->__pyx_int_neg_1) != (0)) __PYX_ERR(1, 583, __pyx_L1_error); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "View.MemoryView":680 + * tup = index if isinstance(index, tuple) else (index,) + * + * result = [slice(None)] * ndim # <<<<<<<<<<<<<< + * have_slices = False + * seen_ellipsis = False +*/ + __pyx_mstate_global->__pyx_slice[0] = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_slice[0])) __PYX_ERR(1, 680, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_slice[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_slice[0]); + + /* "thinc/layers/sparselinear.pyx":18 + * + * + * @cython.binding(True) # <<<<<<<<<<<<<< + * @registry.layers("SparseLinear.v1") + * def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): +*/ + __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(2, Py_None, ((PyObject*)__pyx_mstate_global->__pyx_int_262144)); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]); + + /* "thinc/layers/sparselinear.pyx":61 + * + * + * def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: # <<<<<<<<<<<<<< + * if Y is not None: + * model.set_dim("nO", get_width(Y)) +*/ + __pyx_mstate_global->__pyx_tuple[3] = PyTuple_Pack(2, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[3])) __PYX_ERR(0, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[3]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[3]); + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_tuple; + for (Py_ssize_t i=0; i<4; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_slice; + for (Py_ssize_t i=0; i<1; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + { + const struct { const unsigned int length: 8; } index[] = {{2},{68},{35},{54},{37},{60},{24},{52},{26},{34},{29},{33},{45},{22},{15},{16},{90},{179},{13},{14},{13},{37},{15},{15},{30},{32},{24},{21},{32},{1},{1},{1},{1},{1},{8},{5},{6},{15},{23},{25},{7},{6},{2},{6},{35},{9},{30},{50},{39},{34},{8},{21},{20},{32},{22},{14},{12},{29},{19},{11},{11},{10},{30},{37},{5},{7},{8},{7},{8},{3},{5},{8},{8},{4},{20},{8},{12},{15},{5},{15},{1},{1},{1},{3},{5},{15},{7},{18},{5},{1},{8},{19},{4},{17},{17},{46},{4},{1},{8},{8},{9},{17},{18},{6},{5},{6},{8},{9},{8},{4},{7},{5},{15},{6},{9},{5},{1},{21},{30},{30},{5},{6},{7},{7},{8},{3},{16},{7},{9},{9},{12},{1},{2},{10},{8},{5},{4},{8},{3},{13},{13},{14},{8},{5},{8},{1},{4},{19},{6},{6},{7},{8},{7},{13},{4},{5},{10},{2},{4},{8},{4},{7},{3},{3},{4},{6},{3},{11},{14},{11},{10},{19},{14},{12},{10},{17},{13},{8},{8},{6},{6},{10},{4},{7},{12},{9},{10},{12},{19},{5},{4},{13},{5},{4},{4},{6},{8},{25},{5},{6},{6},{6},{4},{11},{6},{1},{2},{120},{84},{61},{30},{241},{61},{65},{27},{151},{1}}; + #if (CYTHON_COMPRESS_STRINGS) == 2 /* compression: bz2 (2202 bytes) */ +const char* const cstring = "BZh91AY&SY\221\037\022O\000\001&\177\377\367\377\377\376\377\377\377\337\277\377\377\372\277\377\377\362\300@@@@@@@@@@@@\000@\000`\007\375\367[[\256\032\347m\240k,\215V$\221\234\016\032\236\251\252{T\324\361\032\231\033\325=O'\232\2325&\232z\217P\374\224\323M\017(\362\230\203j\0312h7\252=F\214i\250\3104z\2156)\372\247\2504\322\023F\231\002zF\201Si<\215\r1 \03242\006\200\000\000\r\000\000\001\243@\004\030\000&\000\002`\230\000\000\000\000&\0010\020\300\000\021\200\000\000\224\320\204@&\224\360\010\364\232\236A\242d\000\032\003A\2404\000\000\000\000h\003\006\220A\200\002`\000&\t\200\000\000\000\002`\023\001\014\000\001\030\000\000\tD\t\244\310\324\236\217H\324z\237\251=\021\243\322z'\250h\000\001\240\000\000\000\000\006\200i\240\002\226M\302{\273\210o\354\352\035^\263\277Q\334\226v\341\226\223\261\331`?\325\206\353\253\\\201-\374?\300\306='?\330\203!0N\250\210\252\210\221\214\263\304,\207\241\345\247\022E\032\224\004\r$\207\205P\032\305Me\033\246\025\266\002\333\r\341i\205o\256\251B\240\377\371\356qv\354\316\216$fs\213\030\223\304~\256\314\243\336*\331\242\332Uk\344i\036\301\272M\321]\016w\331\262/K\240\023o)\372jq[\372\233\353\212\3621\331$z0[5pAx\3757w\276\2345\302;\032\337\311\353#$=_\353\205\367\271\177H\337\346\322\367\330\035`b\323l\203\335m\rVS\373\271\246\"\247^\270e\022\2517bM%\331\2428UR\353\337k\337\005\311\342\304\344\204\361\001\2214\334\"\004\213=\360\364\300\333\2645\257\242\213\256\014\021uz\356A\221\252r\354\371\352\331D\205{,\307TM\014ou\270;\224,G\204c\215g\247\317R\216\271lk\366\360\201\227eh\317\230Y|\325\261\320\3114\255\334t\222KI\334IV\274f\014\030N2l\307Yk\033\263\244\307\211\261\253\216\341\315f,\205\213\005\200\201\205\177L\367M\014\323\013,\200W\232VRE\034\324\026\261\260r\027\242\365\005E\345\312\024\"\250*i\324@\267E\005\353\311\203\036K\264\230\346\215W*y\206\3242\253\035\204\220\225\264mj>F\330\3304Ac*\033n\251\323Mu\332\234p\025\005W\013\307\362\205\t\024\206\255\363Nu\211R\235\222\360Y\026\336\221@A5\304 \250\n\000\261\324\341\314R\3727\023\000@\275\221c\252\320\034\352\032\233{\275\200H\033\300w\n\242O5\nJ8\3505%\216\350\375\356\246\215\214# G\002\342Sp\206\213l\354\344h\312d\211\005\203\007\221Vl}\007b{\344\327\301\210\275d\201\033M<\263\003\223Qc\327\215\233\330LM\006\227y\232\352Q\235\232\215\222\300\366\226U\264r0\024\253\355\273\212)\305\210\226\303\3333\312\250,\306""\372#B{\261n\3305%\224K4\222\256\036\013\366\206\3303\363Y\254\362\035\251Dn\017j\tB7P<\226\221\350\204\010F\260\3710\201\210v\253\364\255\013z>ac|B?\327\263\203cj\277\235\243\343v\332-W\340\303\277\022O\320\210\263\026~6p\323\272-\253d}\3573@99\353\006\320\271\027\274V\254\306\025\332\003\365\224W\277\225\004^<\346\325'\274\031q\317\346\343\2353.\323\326]\315w\034m\326\361\224{~\360\002\330jB\315\343\245\246?\231\324Y1\331\361l\341\003\226!\314<\270J2\245BY\031!\215\357 \364\003Z\013\"\032\005\200\001\2570\t\227\360\232\360\250}\244\314\025\265_WXk:\010\271n}\354N\337wm '\004i\236h\003\022\316/JGB\215+\354\003\\B\244\024\256\207\211os\2645\\\260a\010)\207V\004\223\016\225\310Zx\037rQ\253+\270\306A<\341\331@\331\007w\016\226^?\250\220\324\020\031@\206\272p \276\036\315z\204Wi\025\272F\326\251!\326(\303\217)\255Q\022\277\266k\332\257\352\262\232\204.\206\226\312\277[\360:d\241\003[\221\007\241A\252\360Gu\215(\2205\202\261\001k\010\036_Px\246\205C\341\250\272\266\226\nC\242B\300\025\2463\321\227Mk!s\364\207\3378\036=\025f|Tt\224\201\237\306\035,t\333\037\255 <<\206\003(\237B?\275\342\035\211\377)|e#.\251\313\261\261\244i4\263\310U\2246\3404\034\315\320\323\003\021 \001T\rAq\000\266J4'L\315T\267\335\016\006g\246\007\323\327\322/\0077\376\321\377\333`\375`p\360bX\370\364M\330\235\217'\206\305\013\335\203x/\256&;\351^\312\206\205s\335\331n=fqsX\270\320\335\217\227\222\205\244\204\332?\305\245x+\231M\234\364f\366I\366e\3668{\335\237\3537?\334\272\331\233\354\301=\347\273\245\301\231\253\203\253\367\006\367\266\007\333\340\323\0318\034\254\007\347\376\230\314$\013\303\302t|~\360\347\345\254\234=\357m\366\013\375\315\267\223o\235w\327\337\325\006\373/\006/~x_\370\"\371>u{\013\277\024\317\234\235\352\336\325A6\322\322\233\211\3673\263\203\331\277f\253\275\322/\223\270u\017\2238\247\377.t\367\206\305\351\370l\374,\271\227\202\366\363n5\336L>K\027\206\305\213\335\237\023\366\246<,Nuw\342=|\375;\326\253\357c\206\257\377\304M\334\275\320}\032\317\305\355\244\211\307g\342\313p\323jZJa\347z\272\231MfN\357f\377\223\376\265~\271\017w,\244\227\323g\331J\326\352=\3565\207\305\277$*-\031-l^K\257\003\234E\222\026\322m\300d\24273,\316B\016\365D\244\315\341\324|\254\222\277\247K\260\363i\357\365\333\371w\023\303\251K\3617\200\314u\200q!]\315J\331\267\340m\251\277\360\036\242\333\212g\241$\355\244\245k4\205)\356CaV\323o\262\231la8=\007\273\255\344\000v\355l\006\323\331\216o%\245d+=\275\230\216\013\361w\311J\322\311\nY\031\260\274|ep\305\312\352=\366\373\261\234\030^\272\223\375\334\027o\233x\366|w)\376*)#\257\376\027\227\343=\320\235\233\036\\$\351D:\017\270\201\207\367\305\213'\326\203\302M\254/:\231\210\257$\227\023\355o\022\000\302\273\013\177\210o\344u(\314\305<\257B\341Jr6\371!mf\023'\304\253\311>\240]\005\304\354\336L\357\253>\030~\001\371\316\231\335\317\273@\307\213\335f\\L&\023(\311\245x%>L\352`\377]\357~\377\371\3332f\324\202\3228I\036\315|RD\277\347\273\253\203KK\331D6\017\224\256\367\331\356\377\001\357\217\237\320"; + PyObject *data = __Pyx_DecompressString(cstring, 2031, 1); + if (unlikely(!data)) __PYX_ERR(0, 1, __pyx_L1_error) + const char* const bytes = __Pyx_PyBytes_AsString(data); + #if !CYTHON_ASSUME_SAFE_MACROS + if (likely(bytes)); else { Py_DECREF(data); __PYX_ERR(0, 1, __pyx_L1_error) } + #endif + #else /* compression: none (3723 bytes) */ +const char* const bytes = ": All dimensions preceding dimension %d must be indexed and not slicedBuffer view does not expose stridesCan only create a buffer that is contiguous in memory.Cannot assign to read-only memoryviewCannot create writable memory view from read-only memoryviewCannot index with type 'Cannot transpose memoryview with indirect dimensionsDimension %d is not directEmpty shape tuple for cython.arrayIndex out of bounds (axis %d)Indirect dimensions not supportedInvalid mode, expected 'c' or 'fortran', got Invalid shape in axis ')?add_note and at 0xcollections.abcdisableenablegc (got got differing extents in dimension isenableditemsize <= 0 for cython.arrayno default __reduce__ due to non-trivial __cinit__numpy._core.multiarray failed to importnumpy._core.umath failed to import object>self.name is not Nonethinc.backendsthinc.configthinc/layers/sparselinear.pyxthinc.layers.typingthinc.modelthinc.typesthinc.utilunable to allocate array data.unable to allocate shape and strides.ASCIIArrayXdCallableCupyOpsEllipsisInTModelNumpyOpsOptionalOutT__Pyx_PyDict_NextRefSequenceSparseLinearSparseLinear_v2TupleView.MemoryViewWXYabcallocallocate_bufferasarrayasyncio.coroutinesattrsbbackendsbackprop_gpu_updatebase_begin_cpu_update_begin_gpu_update_begin_gpu_update..backprop_gpu_updateboolc__call__callback__class____class_getitem""__cline_in_tracebackconfigcountd_biasd_scoresd_weights__dict__dims__doc__dtypedtype_is_objectencodeenumerateerrorf_finish_linear_update_finish_linear_update.__call___finish_linear_update.__init__flagsformatfortranforward__func__getget_array_moduleget_dimget_paramget_width__getstate__iid__import__inc_gradindexinit__init__int_is_coroutineis_cupy_arrayis_numpy_arrayis_trainitemsitemsizejkeyskeys_values_lengthslayerslengthlengths__main__memview__metaclass__modemodel__module__nOname__name__ndim__new__objopspackparamspop__prepare____pyx_checksum__pyx_state__pyx_type__pyx_unpickle_Enum__pyx_vtable____qualname____reduce____reduce_cython____reduce_ex__registerregistryreturnscoresscores_cpuselfset_dim__set_name__set_paramsetdefault__setstate____setstate_cython__shapesizesparse_linearstartstepstopstruct__test__thinc.layers.sparselineartypestypingunpackupdateutilv1_indexingvaluesxxp\320\000\020\320\020\"\240#\320%;\2703\320>X\320XY\330\004\007\200r\210\027\220\001\330\010\r\210X\220Q\220f\230I\240Q\240a\330\004\t\210\025\210h\220a\220q\330\004\r\210U\220(\230!\2301\330\004\t\210\032\2201\220E\230\025\230d\240&\250\002\250#\250R\250z\270\026\270q\330\004\t\210\032\2201\220E\230\025\230d\240&\250\002\250&\260\006\260a\330\004\013\2101\320\000\035\320\0355\3205F\320FX\320Xd\320de\330\004\t\320\t\031\230\021\230!\330\004\020\220\013\320\033,\250A\250W\260D\270\004\270D\300\006\300d\310$\310g\320UY\320YZ\340\004\034\230J\240l\260!\360\010\000\005\014\2102\210X\220Q\220m\2401\200\001\340\021\025\320\025+\2508\2601\360\006\000\005\014\2105\220\001\330\010\t\330\010\t\330\010\r\210Q\330\010\020\220\005\220V\2305\240\001\330\010\016\210f\220D\230\n\240!\330\010\017\210\177\230a\200A\330\010\014\210I\220Q\330\010\014\210H\220A\330\010\014\210J\220a\330\010\014\210K\220q\200A\330\010\r\210T\220\026\220x\230q\240\001\330\010\021\220\024\220V\2308\2401\240A\330\010$\240D\250\006\250d\260&\270\002\270\"\270A\270Q\330\010!\240\024\240V\2504\250v\260R\260q\330\010\037\230t\2401\330\010!\240""\024\240Q\330\010\"\240$\240a\330\010 \240\004\240F\250&\260\001\260\021\330\010\025\220Q\220h\230i\240q\330\014\027\220t\2307\240(\250&\260\007\260z\300\027\310\001\330\014\023\2206\230\021\230$\230d\240!\2408\2501\250B\250d\260(\270!\340\010\014\210E\220\025\220a\220x\230v\240Q\240a\330\014\020\220\005\220U\230!\2308\2406\250\021\250!\330\020\026\220a\220v\230X\240Q\240c\250\021\330\010\014\210F\220)\2301\230E\240\021\330\010\014\210F\220)\2301\230E\240\021\330\010\020\220\004\220G\2304\230y\250\004\250A\200\001\340\024\030\320\030.\250h\260a\360\006\000\005\014\2105\220\001\330\010\t\330\010\t\330\010\r\210Q\330\010\020\220\005\220V\2305\240\001\330\010\016\210f\220D\230\n\240!\330\010\017\210\177\230a\200\001\330\023/\250\177\270i\300q\360\006\000\005\013\210(\220*\230A\330\004\007\200}\220A\220Q\360\006\000\t\020\320\017 \240\001\240\027\250\006\250h\260a\340\010\017\320\017 \240\001\240\027\250\006\250h\260a\320\004&\240l\260!\330\010\020\220\001\220\030\230\024\230Q\330\010\020\220\006\220h\230a\200\001\330\004\022\220%\220x\230q\240\001\330\004\026\220e\2308\2401\240A\330\004\030\230\005\230Z\240q\250\001\330\004\030\230\005\230Z\240q\250\001\330\004\035\230U\240$\240f\250B\250c\260\021\260*\270A\330\004\034\230E\240\026\240q\250\001\330\004\016\210a\330\004\017\210q\220\010\230\006\230a\330\010\023\2204\220w\230h\240f\250G\260:\270W\300A\330\010\017\210v\220Q\220d\230!\330\010\020\220\001\220\027\230\010\240\001\330\004\013\2108\320\023(\250\001\250\027\260\006\260h\270aO"; + PyObject *data = NULL; + CYTHON_UNUSED_VAR(__Pyx_DecompressString); + #endif + PyObject **stringtab = __pyx_mstate->__pyx_string_tab; + Py_ssize_t pos = 0; + for (int i = 0; i < 207; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyUnicode_DecodeUTF8(bytes + pos, bytes_length, NULL); + if (likely(string) && i >= 64) PyUnicode_InternInPlace(&string); + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + stringtab[i] = string; + pos += bytes_length; + } + for (int i = 207; i < 217; i++) { + Py_ssize_t bytes_length = index[i].length; + PyObject *string = PyBytes_FromStringAndSize(bytes + pos, bytes_length); + stringtab[i] = string; + pos += bytes_length; + if (unlikely(!string)) { + Py_XDECREF(data); + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + Py_XDECREF(data); + for (Py_ssize_t i = 0; i < 217; i++) { + if (unlikely(PyObject_Hash(stringtab[i]) == -1)) { + __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = stringtab + 207; + for (Py_ssize_t i=0; i<10; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + } + { + PyObject **numbertab = __pyx_mstate->__pyx_number_tab + 0; + int8_t const cint_constants_1[] = {0,-1,1}; + int32_t const cint_constants_4[] = {262144L,136983863L}; + for (int i = 0; i < 5; i++) { + numbertab[i] = PyLong_FromLong((i < 3 ? cint_constants_1[i - 0] : cint_constants_4[i - 3])); + if (unlikely(!numbertab[i])) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #if CYTHON_IMMORTAL_CONSTANTS + { + PyObject **table = __pyx_mstate->__pyx_number_tab; + for (Py_ssize_t i=0; i<5; ++i) { + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL); + #else + Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT); + #endif + } + } + #endif + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +typedef struct { + unsigned int argcount : 3; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 4; + unsigned int flags : 10; + unsigned int first_line : 7; +} __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 75}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_d_scores}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_backprop_gpu_update, __pyx_mstate->__pyx_kp_b_iso88591_l_Q_ha, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 18}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_length}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_SparseLinear, __pyx_mstate->__pyx_kp_b_iso88591_81_5_Q_V5_fD_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 2, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 33}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_length}; + __pyx_mstate_global->__pyx_codeobj_tab[2] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_SparseLinear_v2_2, __pyx_mstate->__pyx_kp_b_iso88591_ha_5_Q_V5_fD_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[2])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 6, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 48}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_keys_values_lengths, __pyx_mstate->__pyx_n_u_is_train, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_values, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[3] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_forward, __pyx_mstate->__pyx_kp_b_iso88591_iq_A_AQ_ha_ha, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[3])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {3, 0, 0, 5, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 61}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_X, __pyx_mstate->__pyx_n_u_Y, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_length}; + __pyx_mstate_global->__pyx_codeobj_tab[4] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_init, __pyx_mstate->__pyx_kp_b_iso88591_3_XXY_r_XQfIQa_haq_U_1_1E_d_Rz, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[4])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 9, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 71}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_values, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_xp, __pyx_mstate->__pyx_n_u_scores_cpu, __pyx_mstate->__pyx_n_u_callback, __pyx_mstate->__pyx_n_u_backprop_gpu_update, __pyx_mstate->__pyx_n_u_backprop_gpu_update}; + __pyx_mstate_global->__pyx_codeobj_tab[5] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_begin_gpu_update, __pyx_mstate->__pyx_kp_b_iso88591_55FFXXdde_AWD_D_d_gUYYZ_Jl_2XQm, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[5])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {4, 0, 0, 10, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 82}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_values, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_W, __pyx_mstate->__pyx_n_u_b, __pyx_mstate->__pyx_n_u_scores, __pyx_mstate->__pyx_n_u_v1_indexing}; + __pyx_mstate_global->__pyx_codeobj_tab[6] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_begin_cpu_update, __pyx_mstate->__pyx_kp_b_iso88591_xq_e81A_Zq_Zq_U_fBc_A_E_q_a_q_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[6])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {5, 0, 0, 5, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 100}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_model, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_values, __pyx_mstate->__pyx_n_u_lengths}; + __pyx_mstate_global->__pyx_codeobj_tab[7] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_init_2, __pyx_mstate->__pyx_kp_b_iso88591_A_IQ_HA_Ja_Kq, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[7])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {2, 0, 0, 12, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 106}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_self, __pyx_mstate->__pyx_n_u_d_scores, __pyx_mstate->__pyx_n_u_nO, __pyx_mstate->__pyx_n_u_length, __pyx_mstate->__pyx_n_u_d_weights, __pyx_mstate->__pyx_n_u_d_bias, __pyx_mstate->__pyx_n_u_keys, __pyx_mstate->__pyx_n_u_values, __pyx_mstate->__pyx_n_u_lengths, __pyx_mstate->__pyx_n_u_v1_indexing, __pyx_mstate->__pyx_n_u_i, __pyx_mstate->__pyx_n_u_j}; + __pyx_mstate_global->__pyx_codeobj_tab[8] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_thinc_layers_sparselinear_pyx, __pyx_mstate->__pyx_n_u_call, __pyx_mstate->__pyx_kp_b_iso88591_A_T_xq_V81A_D_d_AQ_V4vRq_t1_Q_a, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[8])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CommonTypesMetaclass.init */ + if (likely(__pyx_CommonTypesMetaclass_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API + { + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } + } // error handling follows + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CythonFunctionShared.init */ + if (likely(__pyx_CyFunction_init(__pyx_m) == 0)); else + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* NumpyImportArray.init */ + /* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * numpy._import_array + */ + #ifdef NPY_FEATURE_VERSION + #ifndef NO_IMPORT_ARRAY + if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use 'numpy._import_array' to disable if you are certain you don't need it)."); + } + #endif + #endif + + if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* TupleAndListFromArray (used by fastcall) */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals (used by UnicodeEquals) */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals (used by fastcall) */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO (used by PyObjectFastCall) */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall (used by PyObjectCallOneArg) */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL + #if CYTHON_COMPILING_IN_LIMITED_API + return PyObject_Vectorcall(func, args, _nargs, NULL); + #else + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #endif + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectCallOneArg (used by CallUnboundCMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetAttrStr (used by UnpackUnboundCMethod) */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* UnpackUnboundCMethod (used by CallUnboundCMethod0) */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + PyObject *result; + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} +#elif CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { + return _PyObject_Vectorcall + (method, args ? args+1 : NULL, nargs ? nargs-1 : 0, kwnames); +} +#else +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + return +#if PY_VERSION_HEX < 0x03090000 + _PyObject_Vectorcall +#else + PyObject_Vectorcall +#endif + (method, args ? args+1 : NULL, nargs ? (size_t) nargs-1 : 0, kwnames); +} +#endif +static PyMethodDef __Pyx_UnboundCMethod_Def = { + "CythonUnboundCMethod", + __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030C0000 + METH_VARARGS | METH_KEYWORDS, +#else + METH_FASTCALL | METH_KEYWORDS, +#endif + NULL +}; +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method, *result=NULL; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod0 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + if (likely(cfunc->flag == METH_NOARGS)) + return __Pyx_CallCFunction(cfunc, self, NULL); + if (likely(cfunc->flag == METH_FASTCALL)) + return __Pyx_CallCFunctionFast(cfunc, self, NULL, 0); + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, NULL, 0, NULL); + if (likely(cfunc->flag == (METH_VARARGS | METH_KEYWORDS))) + return __Pyx_CallCFunctionWithKeywords(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple, NULL); + if (cfunc->flag == METH_VARARGS) + return __Pyx_CallCFunction(cfunc, self, __pyx_mstate_global->__pyx_empty_tuple); + return __Pyx__CallUnboundCMethod0(cfunc, self); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod0(&tmp_cfunc, self); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod0(cfunc, self); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self) { + PyObject *result; + if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + result = __Pyx_PyObject_CallOneArg(cfunc->method, self); + return result; +} + +/* py_dict_items (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Items(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_items, d); +} + +/* py_dict_values (used by OwnedDictNext) */ +static CYTHON_INLINE PyObject* __Pyx_PyDict_Values(PyObject* d) { + return __Pyx_CallUnboundCMethod0(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_values, d); +} + +/* OwnedDictNext (used by ParseKeywordsImpl) */ +#if CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, PyObject **ppos, PyObject **pkey, PyObject **pvalue) { + PyObject *next = NULL; + if (!*ppos) { + if (pvalue) { + PyObject *dictview = pkey ? __Pyx_PyDict_Items(p) : __Pyx_PyDict_Values(p); + if (unlikely(!dictview)) goto bad; + *ppos = PyObject_GetIter(dictview); + Py_DECREF(dictview); + } else { + *ppos = PyObject_GetIter(p); + } + if (unlikely(!*ppos)) goto bad; + } + next = PyIter_Next(*ppos); + if (!next) { + if (PyErr_Occurred()) goto bad; + return 0; + } + if (pkey && pvalue) { + *pkey = __Pyx_PySequence_ITEM(next, 0); + if (unlikely(*pkey)) goto bad; + *pvalue = __Pyx_PySequence_ITEM(next, 1); + if (unlikely(*pvalue)) goto bad; + Py_DECREF(next); + } else if (pkey) { + *pkey = next; + } else { + assert(pvalue); + *pvalue = next; + } + return 1; + bad: + Py_XDECREF(next); +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + PyErr_FormatUnraisable("Exception ignored in __Pyx_PyDict_NextRef"); +#else + PyErr_WriteUnraisable(__pyx_mstate_global->__pyx_n_u_Pyx_PyDict_NextRef); +#endif + if (pkey) *pkey = NULL; + if (pvalue) *pvalue = NULL; + return 0; +} +#else // !CYTHON_AVOID_BORROWED_REFS +static int __Pyx_PyDict_NextRef(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { + int result = PyDict_Next(p, ppos, pkey, pvalue); + if (likely(result == 1)) { + if (pkey) Py_INCREF(*pkey); + if (pvalue) Py_INCREF(*pvalue); + } + return result; +} +#endif + +/* RaiseDoubleKeywords (used by ParseKeywordsImpl) */ +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywordsImpl (used by ParseKeywords) */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; + #else + Py_ssize_t pos = 0; + #endif + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while ( + #if CYTHON_AVOID_BORROWED_REFS + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL) + #else + PyDict_Next(kwds, &pos, &key, NULL) + #endif + ) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + __Pyx_END_CRITICAL_SECTION(); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); + #endif + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} + +/* ParseKeywords */ +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTestFunc (used by ArgTypeTest) */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* PyErrExceptionMatches (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore (used by PyObjectGetAttrStrNoError) */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStrNoError (used by GetBuiltinName) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* PyObjectFastCallMethod */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_PyObject_FastCallMethod(PyObject *name, PyObject *const *args, size_t nargsf) { + PyObject *result; + PyObject *attr = PyObject_GetAttr(args[0], name); + if (unlikely(!attr)) + return NULL; + result = __Pyx_PyObject_FastCall(attr, args+1, nargsf - 1); + Py_DECREF(attr); + return result; +} +#endif + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* CIntToDigits (used by CIntToPyUnicode) */ +static const char DIGIT_PAIRS_10[2*10*10+1] = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; +static const char DIGIT_PAIRS_8[2*8*8+1] = { + "0001020304050607" + "1011121314151617" + "2021222324252627" + "3031323334353637" + "4041424344454647" + "5051525354555657" + "6061626364656667" + "7071727374757677" +}; +static const char DIGITS_HEX[2*16+1] = { + "0123456789abcdef" + "0123456789ABCDEF" +}; + +/* BuildPyUnicode (used by COrdinalToPyUnicode) */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char) { + PyObject *uval; + Py_ssize_t uoffset = ulength - clength; +#if CYTHON_USE_UNICODE_INTERNALS + Py_ssize_t i; + void *udata; + uval = PyUnicode_New(ulength, 127); + if (unlikely(!uval)) return NULL; + udata = PyUnicode_DATA(uval); + if (uoffset > 0) { + i = 0; + if (prepend_sign) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); + i++; + } + for (; i < uoffset; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); + } + } + for (i=0; i < clength; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); + } +#else + { + PyObject *sign = NULL, *padding = NULL; + uval = NULL; + if (uoffset > 0) { + prepend_sign = !!prepend_sign; + if (uoffset > prepend_sign) { + padding = PyUnicode_FromOrdinal(padding_char); + if (likely(padding) && uoffset > prepend_sign + 1) { + PyObject *tmp = PySequence_Repeat(padding, uoffset - prepend_sign); + Py_DECREF(padding); + padding = tmp; + } + if (unlikely(!padding)) goto done_or_error; + } + if (prepend_sign) { + sign = PyUnicode_FromOrdinal('-'); + if (unlikely(!sign)) goto done_or_error; + } + } + uval = PyUnicode_DecodeASCII(chars, clength, NULL); + if (likely(uval) && padding) { + PyObject *tmp = PyUnicode_Concat(padding, uval); + Py_DECREF(uval); + uval = tmp; + } + if (likely(uval) && sign) { + PyObject *tmp = PyUnicode_Concat(sign, uval); + Py_DECREF(uval); + uval = tmp; + } +done_or_error: + Py_XDECREF(padding); + Py_XDECREF(sign); + } +#endif + return uval; +} + +/* COrdinalToPyUnicode (used by CIntToPyUnicode) */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value) { + return value <= 1114111; +} +static PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t ulength, char padding_char) { + Py_ssize_t padding_length = ulength - 1; + if (likely((padding_length <= 250) && (value < 0xD800 || value > 0xDFFF))) { + char chars[256]; + if (value <= 255) { + memset(chars, padding_char, (size_t) padding_length); + chars[ulength-1] = (char) value; + return PyUnicode_DecodeLatin1(chars, ulength, NULL); + } + char *cpos = chars + sizeof(chars); + if (value < 0x800) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xc0 | (value & 0x1f)); + } else if (value < 0x10000) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xe0 | (value & 0x0f)); + } else { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xf0 | (value & 0x07)); + } + cpos -= padding_length; + memset(cpos, padding_char, (size_t) padding_length); + return PyUnicode_DecodeUTF8(cpos, chars + sizeof(chars) - cpos, NULL); + } + if (value <= 127 && CYTHON_USE_UNICODE_INTERNALS) { + const char chars[1] = {(char) value}; + return __Pyx_PyUnicode_BuildFromAscii(ulength, chars, 1, 0, padding_char); + } + { + PyObject *uchar, *padding_uchar, *padding, *result; + padding_uchar = PyUnicode_FromOrdinal(padding_char); + if (unlikely(!padding_uchar)) return NULL; + padding = PySequence_Repeat(padding_uchar, padding_length); + Py_DECREF(padding_uchar); + if (unlikely(!padding)) return NULL; + uchar = PyUnicode_FromOrdinal(value); + if (unlikely(!uchar)) { + Py_DECREF(padding); + return NULL; + } + result = PyUnicode_Concat(padding, uchar); + Py_DECREF(padding); + Py_DECREF(uchar); + return result; + } +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (int) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(int)*3+2]; + char *dpos, *end = digits + sizeof(int)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + int remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (int) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (int) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (int) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_uchar___Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (Py_ssize_t) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); +} +static CYTHON_INLINE PyObject* __Pyx____Pyx_PyUnicode_From_Py_ssize_t(Py_ssize_t value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(Py_ssize_t)*3+2]; + char *dpos, *end = digits + sizeof(Py_ssize_t)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + Py_ssize_t remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const Py_ssize_t neg_one = (Py_ssize_t) -1, const_zero = (Py_ssize_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (Py_ssize_t) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (Py_ssize_t) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (Py_ssize_t) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* GetAttr */ +static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) + return __Pyx_PyObject_GetAttrStr(o, n); +#endif + return PyObject_GetAttr(o, n); +} + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS)) { + return __Pyx_PyList_GetItemRefFast(o, wrapped_i, unsafe_shared); + } else + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, wrapped_i)); + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + (void)wraparound; + (void)boundscheck; + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS)) { + return __Pyx_PyList_GetItemRefFast(o, n, unsafe_shared); + } else if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_NewRef(PyList_GET_ITEM(o, n)); + } + } else + #if !CYTHON_AVOID_BORROWED_REFS + if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + return __Pyx_NewRef(PyTuple_GET_ITEM(o, n)); + } + } else + #endif +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* ObjectGetItem */ +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + PyObject *runerr = NULL; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1, 1); + } + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, __pyx_mstate_global->__pyx_n_u_class_getitem); + if (!meth) { + PyErr_Clear(); + } else { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + +/* RejectKeywords */ +static void __Pyx_RejectKeywords(const char* function_name, PyObject *kwds) { + PyObject *key = NULL; + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) { + key = __Pyx_PySequence_ITEM(kwds, 0); + } else { +#if CYTHON_AVOID_BORROWED_REFS + PyObject *pos = NULL; +#else + Py_ssize_t pos = 0; +#endif +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return; +#endif + __Pyx_PyDict_NextRef(kwds, &pos, &key, NULL); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(pos); +#endif + } + if (likely(key)) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + Py_DECREF(key); + } +} + +/* DivInt[Py_ssize_t] */ +static CYTHON_INLINE Py_ssize_t __Pyx_div_Py_ssize_t(Py_ssize_t a, Py_ssize_t b, int b_is_constant) { + Py_ssize_t q = a / b; + Py_ssize_t r = a - q*b; + Py_ssize_t adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* GetAttr3 */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} +#endif +static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { + PyObject *r; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int res = PyObject_GetOptionalAttr(o, n, &r); + return (res != 0) ? r : __Pyx_NewRef(d); +#else + #if CYTHON_USE_TYPE_SLOTS + if (likely(PyUnicode_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } + #endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +#endif +} + +/* PyDictVersioning (used by GetModuleGlobalName) */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* RaiseTooManyValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %" CYTHON_FORMAT_SSIZE_T "d)", expected); +} + +/* RaiseNeedMoreValuesToUnpack */ +static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + "need more than %" CYTHON_FORMAT_SSIZE_T "d value%.1s to unpack", + index, (index == 1) ? "" : "s"); +} + +/* RaiseNoneIterError */ +static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); +} + +/* ExtTypeTest */ +static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (likely(__Pyx_TypeCheck(obj, type))) + return 1; + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); + return 0; +} + +/* GetTopmostException (used by SaveResetException) */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* SwapException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* HasAttr (used by ImportImpl) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!PyUnicode_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (!r) { + return (unlikely(PyErr_Occurred())) ? -1 : 0; + } else { + Py_DECREF(r); + return 1; + } +} +#endif + +/* ImportImpl (used by Import) */ +static int __Pyx__Import_GetModule(PyObject *qualname, PyObject **module) { + PyObject *imported_module = PyImport_GetModule(qualname); + if (unlikely(!imported_module)) { + *module = NULL; + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + *module = imported_module; + return 1; +} +static int __Pyx__Import_Lookup(PyObject *qualname, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject **module) { + PyObject *imported_module; + PyObject *top_level_package_name; + Py_ssize_t i; + int status, module_found; + Py_ssize_t dot_index; + module_found = __Pyx__Import_GetModule(qualname, &imported_module); + if (unlikely(!module_found || module_found == -1)) { + *module = NULL; + return module_found; + } + if (imported_names) { + for (i = 0; i < len_imported_names; i++) { + PyObject *imported_name = imported_names[i]; +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 + int has_imported_attribute = PyObject_HasAttr(imported_module, imported_name); +#else + int has_imported_attribute = PyObject_HasAttrWithError(imported_module, imported_name); + if (unlikely(has_imported_attribute == -1)) goto error; +#endif + if (!has_imported_attribute) { + goto not_found; + } + } + *module = imported_module; + return 1; + } + dot_index = PyUnicode_FindChar(qualname, '.', 0, PY_SSIZE_T_MAX, 1); + if (dot_index == -1) { + *module = imported_module; + return 1; + } + if (unlikely(dot_index == -2)) goto error; + top_level_package_name = PyUnicode_Substring(qualname, 0, dot_index); + if (unlikely(!top_level_package_name)) goto error; + Py_DECREF(imported_module); + status = __Pyx__Import_GetModule(top_level_package_name, module); + Py_DECREF(top_level_package_name); + return status; +error: + Py_DECREF(imported_module); + *module = NULL; + return -1; +not_found: + Py_DECREF(imported_module); + *module = NULL; + return 0; +} +static PyObject *__Pyx__Import(PyObject *name, PyObject *const *imported_names, Py_ssize_t len_imported_names, PyObject *qualname, PyObject *moddict, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *from_list = 0; + int module_found; + if (!qualname) { + qualname = name; + } + module_found = __Pyx__Import_Lookup(qualname, imported_names, len_imported_names, &module); + if (likely(module_found == 1)) { + return module; + } else if (unlikely(module_found == -1)) { + return NULL; + } + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (imported_names) { +#if CYTHON_COMPILING_IN_CPYTHON + from_list = __Pyx_PyList_FromArray(imported_names, len_imported_names); + if (unlikely(!from_list)) + goto bad; +#else + from_list = PyList_New(len_imported_names); + if (unlikely(!from_list)) goto bad; + for (Py_ssize_t i=0; i__pyx_d, level); +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; itp_as_sequence && type->tp_as_sequence->sq_repeat)) { + return type->tp_as_sequence->sq_repeat(seq, mul); + } else +#endif + { + return __Pyx_PySequence_Multiply_Generic(seq, mul); + } +} + +/* PyObjectFormatAndDecref */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatSimpleAndDecref(PyObject* s, PyObject* f) { + if (unlikely(!s)) return NULL; + if (likely(PyUnicode_CheckExact(s))) return s; + return __Pyx_PyObject_FormatAndDecref(s, f); +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_FormatAndDecref(PyObject* s, PyObject* f) { + PyObject *result; + if (unlikely(!s)) return NULL; + result = PyObject_Format(s, f); + Py_DECREF(s); + return result; +} + +/* PyObjectFormat */ +#if CYTHON_USE_UNICODE_WRITER +static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { + int ret; + _PyUnicodeWriter writer; + if (likely(PyFloat_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else if (likely(PyLong_CheckExact(obj))) { + _PyUnicodeWriter_Init(&writer); + ret = _PyLong_FormatAdvancedWriter( + &writer, + obj, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + } else { + return PyObject_Format(obj, format_spec); + } + if (unlikely(ret == -1)) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); +} +#endif + +/* SetItemInt */ +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { + int r; + if (unlikely(!j)) return -1; + r = PyObject_SetItem(o, j, v); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, + int wraparound, int boundscheck, int unsafe_shared) { + CYTHON_MAYBE_UNUSED_VAR(unsafe_shared); +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); + if ((CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS && !__Pyx_IS_UNIQUELY_REFERENCED(o, unsafe_shared))) { + Py_INCREF(v); + return PyList_SetItem(o, n, v); + } else if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { + PyObject* old; + Py_INCREF(v); + old = PyList_GET_ITEM(o, n); + PyList_SET_ITEM(o, n, v); + Py_DECREF(old); + return 0; + } + } else +#endif +#if CYTHON_USE_TYPE_SLOTS && !CYTHON_COMPILING_IN_PYPY + { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (!is_list && mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (is_list || likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return -1; + PyErr_Clear(); + } + } + return sm->sq_ass_item(o, i, v); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_SetItem(o, i, v); + } +#endif + (void)wraparound; + (void)boundscheck; + return __Pyx_SetItemInt_Generic(o, PyLong_FromSsize_t(i), v); +} + +/* RaiseUnboundLocalError */ +static void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + +/* DivInt[long] */ +static CYTHON_INLINE long __Pyx_div_long(long a, long b, int b_is_constant) { + long q = a / b; + long r = a - q*b; + long adapt_python = (b_is_constant ? + ((r != 0) & ((r < 0) ^ (b < 0))) : + ((r != 0) & ((r ^ b) < 0)) + ); + return q - adapt_python; +} + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* IterFinish */ +static CYTHON_INLINE int __Pyx_IterFinish(void) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_CurrentExceptionType(); + if (unlikely(exc_type)) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return -1; + __Pyx_PyErr_Clear(); + return 0; + } + return 0; +} + +/* UnpackItemEndCheck */ +static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { + if (unlikely(retval)) { + Py_DECREF(retval); + __Pyx_RaiseTooManyValuesError(expected); + return -1; + } + return __Pyx_IterFinish(); +} + +/* PyObjectVectorCallMethodKwBuilder */ +#if !CYTHON_VECTORCALL || PY_VERSION_HEX < 0x03090000 +static PyObject *__Pyx_Object_VectorcallMethod_CallFromBuilder(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyObject *result; + PyObject *obj = PyObject_GetAttr(args[0], name); + if (unlikely(!obj)) + return NULL; + result = __Pyx_Object_Vectorcall_CallFromBuilder(obj, args+1, nargsf-1, kwnames); + Py_DECREF(obj); + return result; +} +#endif + +/* RaiseClosureNameError */ +static void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + +/* dict_setdefault (used by FetchCommonType) */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) { + PyObject* value; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + PyObject *args[] = {d, key, default_value}; + value = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_n_u_setdefault, args, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethodObjArgs(d, __pyx_mstate_global->__pyx_n_u_setdefault, key, default_value, NULL); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* LimitedApiGetTypeDict (used by SetItemOnTypeDict) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict (used by FixUpExtensionType) */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType (used by FetchCommonType) */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_vectorcall_offset = memb->offset; + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* AddModuleRef (used by FetchSharedCythonModule) */ +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + static PyObject *__Pyx_PyImport_AddModuleObjectRef(PyObject *name) { + PyObject *module_dict = PyImport_GetModuleDict(); + PyObject *m; + if (PyMapping_GetOptionalItem(module_dict, name, &m) < 0) { + return NULL; + } + if (m != NULL && PyModule_Check(m)) { + return m; + } + Py_XDECREF(m); + m = PyModule_NewObject(name); + if (m == NULL) + return NULL; + if (PyDict_CheckExact(module_dict)) { + PyObject *new_m; + (void)PyDict_SetDefaultRef(module_dict, name, m, &new_m); + Py_DECREF(m); + return new_m; + } else { + if (PyObject_SetItem(module_dict, name, m) != 0) { + Py_DECREF(m); + return NULL; + } + return m; + } + } + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *py_name = PyUnicode_FromString(name); + if (!py_name) return NULL; + PyObject *module = __Pyx_PyImport_AddModuleObjectRef(py_name); + Py_DECREF(py_name); + return module; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif + +/* FetchSharedCythonModule (used by FetchCommonType) */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* FetchCommonType (used by CommonTypesMetaclass) */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + cached_type = __Pyx_PyType_FromMetaclass( + metaclass, + CYTHON_USE_MODULE_STATE ? module : abi_module, + spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass (used by CythonFunctionShared) */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +#if __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject* __pyx_CommonTypesMetaclass_call(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwds) { + PyErr_SetString(PyExc_TypeError, "Cannot instantiate Cython internal types"); + return NULL; +} +static int __pyx_CommonTypesMetaclass_setattr(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED PyObject *attr, CYTHON_UNUSED PyObject *value) { + PyErr_SetString(PyExc_TypeError, "Cython internal types are immutable"); + return -1; +} +#endif +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {Py_tp_call, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_new, (void*)__pyx_CommonTypesMetaclass_call}, + {Py_tp_setattro, (void*)__pyx_CommonTypesMetaclass_setattr}, + #endif + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse (used by CythonFunctionShared) */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew (used by CythonFunctionShared) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict (used by CythonFunctionShared) */ +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i; + #if CYTHON_AVOID_BORROWED_REFS + PyObject *pos; + #else + Py_ssize_t pos; + #endif + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = 0; + i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (__Pyx_PyDict_NextRef(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(pos); + #endif + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared (used by CythonFunction) */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +#endif +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)PyObject_GenericSetDict, 0, 0}, +#else + {"func_dict", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, + {"__dict__", (getter)PyObject_GenericGetDict, (setter)PyObject_GenericSetDict, 0, 0}, +#endif + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#endif +#if CYTHON_METH_FASTCALL +#if CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + op->func_dict = NULL; +#endif + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func_dict); +#elif PY_VERSION_HEX < 0x030d0000 + _PyObject_ClearManagedDict((PyObject*)m); +#else + PyObject_ClearManagedDict((PyObject*)m); +#endif + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif +#if PY_VERSION_HEX < 0x030C0000 || CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func_dict); +#else + { + int e = +#if PY_VERSION_HEX < 0x030d0000 + _PyObject_VisitManagedDict +#else + PyObject_VisitManagedDict +#endif + ((PyObject*)m, visit, arg); + if (e != 0) return e; + } +#endif + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && CYTHON_VECTORCALL +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + #if PY_VERSION_HEX < 0x030e00A6 + size_t nargs_value = (size_t) nargs; + #else + Py_ssize_t nargs_value = nargs; + #endif + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, nargs_value, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030C0000 && !CYTHON_COMPILING_IN_LIMITED_API + Py_TPFLAGS_MANAGED_DICT | +#endif + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION | + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* DictGetItem */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { + PyObject *value; + if (unlikely(__Pyx_PyDict_GetItemRef(d, key, &value) == 0)) { // no value, no error + if (unlikely(PyTuple_Check(key))) { + PyObject* args = PyTuple_Pack(1, key); + if (likely(args)) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + } else { + PyErr_SetObject(PyExc_KeyError, key); + } + } + return value; +} +#endif + +/* PyObjectSetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_setattro)) + return tp->tp_setattro(obj, attr_name, value); + return PyObject_SetAttr(obj, attr_name, value); +} +#endif + +/* BufferIndexError */ +static void __Pyx_RaiseBufferIndexError(int axis) { + PyErr_Format(PyExc_IndexError, + "Out of bounds on buffer access (axis %d)", axis); +} + +/* AllocateExtensionType */ +static PyObject *__Pyx_AllocateExtensionType(PyTypeObject *t, int is_final) { + if (is_final || likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) { + allocfunc alloc_func = __Pyx_PyType_GetSlot(t, tp_alloc, allocfunc); + return alloc_func(t, 0); + } else { + newfunc tp_new = __Pyx_PyType_TryGetSlot(&PyBaseObject_Type, tp_new, newfunc); + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (!tp_new) { + PyObject *new_str = PyUnicode_FromString("__new__"); + if (likely(new_str)) { + PyObject *o = PyObject_CallMethodObjArgs((PyObject *)&PyBaseObject_Type, new_str, t, NULL); + Py_DECREF(new_str); + return o; + } else + return NULL; + } else + #endif + return tp_new(t, __pyx_mstate_global->__pyx_empty_tuple, 0); + } +} + +/* PyObjectCallNoArg (used by PyObjectCallMethod0) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { + PyObject *arg[2] = {NULL, NULL}; + return __Pyx_PyObject_FastCall(func, arg + 1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectGetMethod (used by PyObjectCallMethod0) */ +#if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + assert (*method == NULL); + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#else + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + if (meth_found) { + *method = descr; + return 1; + } + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + type_name = __Pyx_PyType_GetFullyQualifiedName(tp); + PyErr_Format(PyExc_AttributeError, + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); + __Pyx_DECREF_TypeName(type_name); + return 0; +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif +try_unpack: +#if CYTHON_UNPACK_METHODS + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } +#endif + *method = attr; + return 0; +} +#endif + +/* PyObjectCallMethod0 (used by PyType_Ready) */ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { +#if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[1] = {obj}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_CallNoArg; + return PyObject_VectorcallMethod(method_name, args, 1 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); +#else + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +#endif +} + +/* ValidateBasesTuple (used by PyType_Ready) */ +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + Py_ssize_t i, n; +#if CYTHON_ASSUME_SAFE_SIZE + n = PyTuple_GET_SIZE(bases); +#else + n = PyTuple_Size(bases); + if (unlikely(n < 0)) return -1; +#endif + for (i = 1; i < n; i++) + { + PyTypeObject *b; +#if CYTHON_AVOID_BORROWED_REFS + PyObject *b0 = PySequence_GetItem(bases, i); + if (!b0) return -1; +#elif CYTHON_ASSUME_SAFE_MACROS + PyObject *b0 = PyTuple_GET_ITEM(bases, i); +#else + PyObject *b0 = PyTuple_GetItem(bases, i); + if (!b0) return -1; +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + if (dictoffset == 0) + { + Py_ssize_t b_dictoffset = 0; +#if CYTHON_USE_TYPE_SLOTS + b_dictoffset = b->tp_dictoffset; +#else + PyObject *py_b_dictoffset = PyObject_GetAttrString((PyObject*)b, "__dictoffset__"); + if (!py_b_dictoffset) goto dictoffset_return; + b_dictoffset = PyLong_AsSsize_t(py_b_dictoffset); + Py_DECREF(py_b_dictoffset); + if (b_dictoffset == -1 && PyErr_Occurred()) goto dictoffset_return; +#endif + if (b_dictoffset) { + { + __Pyx_TypeName b_name = __Pyx_PyType_GetFullyQualifiedName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + } +#if !CYTHON_USE_TYPE_SLOTS + dictoffset_return: +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + return -1; + } + } +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(b0); +#endif + } + return 0; +} +#endif + +/* PyType_Ready */ +CYTHON_UNUSED static int __Pyx_PyType_HasMultipleInheritance(PyTypeObject *t) { + while (t) { + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases) { + return 1; + } + t = __Pyx_PyType_GetSlot(t, tp_base, PyTypeObject*); + } + return 0; +} +static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS || !CYTHON_COMPILING_IN_CPYTHON || defined(PYSTON_MAJOR_VERSION) + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + return PyType_Ready(t); +#else + int r; + if (!__Pyx_PyType_HasMultipleInheritance(t)) { + return PyType_Ready(t); + } + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; +#if !defined(PYSTON_MAJOR_VERSION) + { + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + #else + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) &&\ + !CYTHON_COMPILING_IN_GRAAL + gc = PyImport_GetModule(__pyx_mstate_global->__pyx_kp_u_gc); + #endif + if (unlikely(!gc)) gc = PyImport_Import(__pyx_mstate_global->__pyx_kp_u_gc); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_isenabled); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_disable); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + (void)__Pyx_PyObject_CallMethod0; +#endif + r = PyType_Ready(t); +#if !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, __pyx_mstate_global->__pyx_kp_u_enable); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + PyErr_Restore(tp, v, tb); + } else { + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + return r; +#endif +} + +/* SetVTable */ +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { + PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#else + if (unlikely(PyDict_SetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable, ob) < 0)) +#endif + goto bad; + Py_DECREF(ob); + return 0; +bad: + Py_XDECREF(ob); + return -1; +} + +/* GetVTable (used by MergeVTables) */ +static void* __Pyx_GetVtable(PyTypeObject *type) { + void* ptr; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, __pyx_mstate_global->__pyx_n_u_pyx_vtable); +#endif + if (!ob) + goto bad; + ptr = PyCapsule_GetPointer(ob, 0); + if (!ptr && !PyErr_Occurred()) + PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); + Py_DECREF(ob); + return ptr; +bad: + Py_XDECREF(ob); + return NULL; +} + +/* MergeVTables */ +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i=0; + Py_ssize_t size; + void** base_vtables; + __Pyx_TypeName tp_base_name = NULL; + __Pyx_TypeName base_name = NULL; + void* unknown = (void*)-1; + PyObject* bases = __Pyx_PyType_GetSlot(type, tp_bases, PyObject*); + int base_depth = 0; + { + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (base) { + base_depth += 1; + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + base_vtables = (void**) PyMem_Malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; +#if CYTHON_COMPILING_IN_LIMITED_API + size = PyTuple_Size(bases); + if (size < 0) goto other_failure; +#else + size = PyTuple_GET_SIZE(bases); +#endif + for (i = 1; i < size; i++) { + PyObject *basei; + void* base_vtable; +#if CYTHON_AVOID_BORROWED_REFS + basei = PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto other_failure; +#else + basei = PyTuple_GET_ITEM(bases, i); +#endif + base_vtable = __Pyx_GetVtable((PyTypeObject*)basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + if (base_vtable != NULL) { + int j; + PyTypeObject* base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + goto bad; + } + base = __Pyx_PyType_GetSlot(base, tp_base, PyTypeObject*); + } + } + } + PyErr_Clear(); + PyMem_Free(base_vtables); + return 0; +bad: + { + PyTypeObject* basei = NULL; + PyTypeObject* tp_base = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + tp_base_name = __Pyx_PyType_GetFullyQualifiedName(tp_base); +#if CYTHON_AVOID_BORROWED_REFS + basei = (PyTypeObject*)PySequence_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#elif !CYTHON_ASSUME_SAFE_MACROS + basei = (PyTypeObject*)PyTuple_GetItem(bases, i); + if (unlikely(!basei)) goto really_bad; +#else + basei = (PyTypeObject*)PyTuple_GET_ITEM(bases, i); +#endif + base_name = __Pyx_PyType_GetFullyQualifiedName(basei); +#if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(basei); +#endif + } + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); +#if CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +really_bad: // bad has failed! +#endif + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_AVOID_BORROWED_REFS || !CYTHON_ASSUME_SAFE_MACROS +other_failure: +#endif + PyMem_Free(base_vtables); + return -1; +} + +/* DelItemOnTypeDict (used by SetupReduce) */ +static int __Pyx__DelItemOnTypeDict(PyTypeObject *tp, PyObject *k) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_DelItem(tp_dict, k); + if (likely(!result)) PyType_Modified(tp); + return result; +} + +/* SetupReduce */ +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, __pyx_mstate_global->__pyx_n_u_name_2); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_getstate = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + PyObject *getstate = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + getstate = _PyType_Lookup((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_getstate); +#else + getstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_getstate); + if (!getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (getstate) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_getstate = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); +#else + object_getstate = __Pyx_PyObject_GetAttrStrNoError((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_getstate); + if (!object_getstate && PyErr_Occurred()) { + goto __PYX_BAD; + } +#endif + if (object_getstate != getstate) { + goto __PYX_GOOD; + } + } +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (!object_reduce_ex) goto __PYX_BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_ex); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_mstate_global->__pyx_n_u_reduce); if (!object_reduce) goto __PYX_BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_mstate_global->__pyx_n_u_reduce); if (unlikely(!reduce)) goto __PYX_BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_mstate_global->__pyx_n_u_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); + if (likely(reduce_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce, reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + goto __PYX_BAD; + } + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_mstate_global->__pyx_n_u_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); + if (likely(setstate_cython)) { + ret = __Pyx_SetItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate, setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = __Pyx_DelItemOnTypeDict((PyTypeObject*)type_obj, __pyx_mstate_global->__pyx_n_u_setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; +__PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetFullyQualifiedName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; +__PYX_GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); + Py_XDECREF(object_getstate); + Py_XDECREF(getstate); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + +/* TypeImport */ +#ifndef __PYX_HAVE_RT_ImportType_3_2_1 +#define __PYX_HAVE_RT_ImportType_3_2_1 +static PyTypeObject *__Pyx_ImportType_3_2_1(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_2_1 check_size) +{ + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; +#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; +#endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; +#else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_2_1 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_2_1 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return NULL; +} +#endif + +/* ImportFrom */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u__2); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* CalculateMetaclass (used by Py3ClassCreate) */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { + Py_ssize_t i, nbases; +#if CYTHON_ASSUME_SAFE_SIZE + nbases = PyTuple_GET_SIZE(bases); +#else + nbases = PyTuple_Size(bases); + if (nbases < 0) return NULL; +#endif + for (i=0; i < nbases; i++) { + PyTypeObject *tmptype; +#if CYTHON_ASSUME_SAFE_MACROS + PyObject *tmp = PyTuple_GET_ITEM(bases, i); +#else + PyObject *tmp = PyTuple_GetItem(bases, i); + if (!tmp) return NULL; +#endif + tmptype = Py_TYPE(tmp); + if (!metaclass) { + metaclass = tmptype; + continue; + } + if (PyType_IsSubtype(metaclass, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, metaclass)) { + metaclass = tmptype; + continue; + } + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + if (!metaclass) { + metaclass = &PyType_Type; + } + Py_INCREF((PyObject*) metaclass); + return (PyObject*) metaclass; +} + +/* PyObjectCall2Args (used by Py3ClassCreate) */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectLookupSpecial (used by Py3ClassCreate) */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* Py3ClassCreate */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, + PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { + PyObject *ns; + if (metaclass) { + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, __pyx_mstate_global->__pyx_n_u_prepare); + if (prep) { + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_DECREF(prep); + } else { + if (unlikely(PyErr_Occurred())) + return NULL; + ns = PyDict_New(); + } + } else { + ns = PyDict_New(); + } + if (unlikely(!ns)) + return NULL; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_module, modname) < 0)) goto bad; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_qualname, qualname) < 0)) goto bad; + if (unlikely(doc && PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_doc, doc) < 0)) goto bad; + return ns; +bad: + Py_DECREF(ns); + return NULL; +} +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, + PyObject *dict, PyObject *mkw, + int calculate_metaclass, int allow_py2_metaclass) { + PyObject *result; + PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; + if (allow_py2_metaclass) { + owned_metaclass = PyObject_GetItem(dict, __pyx_mstate_global->__pyx_n_u_metaclass); + if (owned_metaclass) { + metaclass = owned_metaclass; + } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { + PyErr_Clear(); + } else { + return NULL; + } + } + if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { + metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); + Py_XDECREF(owned_metaclass); + if (unlikely(!metaclass)) + return NULL; + owned_metaclass = metaclass; + } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_XDECREF(owned_metaclass); + return result; +} + +/* CLineInTraceback (used by AddTraceback) */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 +#define __Pyx_PyProbablyModule_GetDict(o) __Pyx_XNewRef(PyModule_GetDict(o)) +#elif !CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyProbablyModule_GetDict(o) PyObject_GenericGetDict(o, NULL); +#else +PyObject* __Pyx_PyProbablyModule_GetDict(PyObject *o) { + PyObject **dict_ptr = _PyObject_GetDictPtr(o); + return dict_ptr ? __Pyx_XNewRef(*dict_ptr) : NULL; +} +#endif +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline = NULL; + PyObject *ptype, *pvalue, *ptraceback; + PyObject *cython_runtime_dict; + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + cython_runtime_dict = __Pyx_PyProbablyModule_GetDict(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, cython_runtime_dict, + __Pyx_PyDict_SetDefault(cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False)) + } + if (use_cline == NULL || use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + Py_XDECREF(cython_runtime_dict); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache (used by AddTraceback) */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + } else { + py_funcname = PyUnicode_FromString(funcname); + } + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* MemviewRefcount */ +#include +#ifndef _Py_NO_RETURN +#define _Py_NO_RETURN +#endif +_Py_NO_RETURN +static void __pyx_fatalerror(const char *fmt, ...) { + va_list vargs; + char msg[200]; +#if PY_VERSION_HEX >= 0x030A0000 || defined(HAVE_STDARG_PROTOTYPES) + va_start(vargs, fmt); +#else + va_start(vargs); +#endif + vsnprintf(msg, 200, fmt, vargs); + va_end(vargs); + Py_FatalError(msg); +} +static CYTHON_INLINE int +__pyx_add_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)++; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE int +__pyx_sub_acquisition_count_locked(__pyx_atomic_int_type *acquisition_count, + PyThread_type_lock lock) +{ + int result; + PyThread_acquire_lock(lock, 1); + result = (*acquisition_count)--; + PyThread_release_lock(lock); + return result; +} +static CYTHON_INLINE void +__Pyx_INC_MEMVIEW(__Pyx_memviewslice *memslice, int have_gil, int lineno) +{ + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + return; + } + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count+1, lineno); + } + } +} +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW(__Pyx_memviewslice *memslice, + int have_gil, int lineno) { + __pyx_nonatomic_int_type old_acquisition_count; + struct __pyx_memoryview_obj *memview = memslice->memview; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + memslice->memview = NULL; + return; + } + old_acquisition_count = __pyx_sub_acquisition_count(memview); + memslice->data = NULL; + if (likely(old_acquisition_count > 1)) { + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + if (have_gil) { + Py_CLEAR(memslice->memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_CLEAR(memslice->memview); + PyGILState_Release(_gilstate); + } + } else { + __pyx_fatalerror("Acquisition count is %d (line %d)", + old_acquisition_count-1, lineno); + } +} + +/* MemviewSliceIsContig */ +static int +__pyx_memviewslice_is_contig(const __Pyx_memviewslice mvs, char order, int ndim) +{ + int i, index, step, start; + Py_ssize_t itemsize = mvs.memview->view.itemsize; + if (order == 'F') { + step = 1; + start = 0; + } else { + step = -1; + start = ndim - 1; + } + for (i = 0; i < ndim; i++) { + index = start + step * i; + if (mvs.suboffsets[index] >= 0 || mvs.strides[index] != itemsize) + return 0; + itemsize *= mvs.shape[index]; + } + return 1; +} + +/* OverlappingSlices */ +static void +__pyx_get_array_memory_extents(__Pyx_memviewslice *slice, + void **out_start, void **out_end, + int ndim, size_t itemsize) +{ + char *start, *end; + int i; + start = end = slice->data; + for (i = 0; i < ndim; i++) { + Py_ssize_t stride = slice->strides[i]; + Py_ssize_t extent = slice->shape[i]; + if (extent == 0) { + *out_start = *out_end = start; + return; + } else { + if (stride > 0) + end += stride * (extent - 1); + else + start += stride * (extent - 1); + } + } + *out_start = start; + *out_end = end + itemsize; +} +static int +__pyx_slices_overlap(__Pyx_memviewslice *slice1, + __Pyx_memviewslice *slice2, + int ndim, size_t itemsize) +{ + void *start1, *end1, *start2, *end2; + __pyx_get_array_memory_extents(slice1, &start1, &end1, ndim, itemsize); + __pyx_get_array_memory_extents(slice2, &start2, &end2, ndim, itemsize); + return (start1 < end2) && (start2 < end1); +} + +/* MemviewSliceInit */ +static int +__Pyx_init_memviewslice(struct __pyx_memoryview_obj *memview, + int ndim, + __Pyx_memviewslice *memviewslice, + int memview_is_new_reference) +{ + __Pyx_RefNannyDeclarations + int i, retval=-1; + Py_buffer *buf = &memview->view; + __Pyx_RefNannySetupContext("init_memviewslice", 0); + if (unlikely(memviewslice->memview || memviewslice->data)) { + PyErr_SetString(PyExc_ValueError, + "memviewslice is already initialized!"); + goto fail; + } + if (buf->strides) { + for (i = 0; i < ndim; i++) { + memviewslice->strides[i] = buf->strides[i]; + } + } else { + Py_ssize_t stride = buf->itemsize; + for (i = ndim - 1; i >= 0; i--) { + memviewslice->strides[i] = stride; + stride *= buf->shape[i]; + } + } + for (i = 0; i < ndim; i++) { + memviewslice->shape[i] = buf->shape[i]; + if (buf->suboffsets) { + memviewslice->suboffsets[i] = buf->suboffsets[i]; + } else { + memviewslice->suboffsets[i] = -1; + } + } + memviewslice->memview = memview; + memviewslice->data = (char *)buf->buf; + if (__pyx_add_acquisition_count(memview) == 0 && !memview_is_new_reference) { + Py_INCREF((PyObject*)memview); + } + retval = 0; + goto no_fail; +fail: + memviewslice->memview = 0; + memviewslice->data = 0; + retval = -1; +no_fail: + __Pyx_RefNannyFinishContext(); + return retval; +} + +/* IsLittleEndian (used by BufferFormatCheck) */ +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } S; + S.u32 = 0x01020304; + return S.u8[0] == 4; +} + +/* BufferFormatCheck (used by MemviewSliceValidateAndInit) */ +static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, + __Pyx_BufFmt_StackElem* stack, + const __Pyx_TypeInfo* type) { + stack[0].field = &ctx->root; + stack[0].parent_offset = 0; + ctx->root.type = type; + ctx->root.name = "buffer dtype"; + ctx->root.offset = 0; + ctx->head = stack; + ctx->head->field = &ctx->root; + ctx->fmt_offset = 0; + ctx->head->parent_offset = 0; + ctx->new_packmode = '@'; + ctx->enc_packmode = '@'; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->is_complex = 0; + ctx->is_valid_array = 0; + ctx->struct_alignment = 0; + while (type->typegroup == 'S') { + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = 0; + type = type->fields->type; + } +} +static int __Pyx_BufFmt_ParseNumber(const char** ts) { + int count; + const char* t = *ts; + if (*t < '0' || *t > '9') { + return -1; + } else { + count = *t++ - '0'; + while (*t >= '0' && *t <= '9') { + count *= 10; + count += *t++ - '0'; + } + } + *ts = t; + return count; +} +static int __Pyx_BufFmt_ExpectNumber(const char **ts) { + int number = __Pyx_BufFmt_ParseNumber(ts); + if (number == -1) + PyErr_Format(PyExc_ValueError,\ + "Does not understand character buffer dtype format string ('%c')", **ts); + return number; +} +static void __Pyx_BufFmt_RaiseUnexpectedChar(char ch) { + PyErr_Format(PyExc_ValueError, + "Unexpected format string character: '%c'", ch); +} +static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { + switch (ch) { + case '?': return "'bool'"; + case 'c': return "'char'"; + case 'b': return "'signed char'"; + case 'B': return "'unsigned char'"; + case 'h': return "'short'"; + case 'H': return "'unsigned short'"; + case 'i': return "'int'"; + case 'I': return "'unsigned int'"; + case 'l': return "'long'"; + case 'L': return "'unsigned long'"; + case 'q': return "'long long'"; + case 'Q': return "'unsigned long long'"; + case 'f': return (is_complex ? "'complex float'" : "'float'"); + case 'd': return (is_complex ? "'complex double'" : "'double'"); + case 'g': return (is_complex ? "'complex long double'" : "'long double'"); + case 'T': return "a struct"; + case 'O': return "Python object"; + case 'P': return "a pointer"; + case 's': case 'p': return "a string"; + case 0: return "end"; + default: return "unparsable format string"; + } +} +static size_t __Pyx_BufFmt_TypeCharToStandardSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return 2; + case 'i': case 'I': case 'l': case 'L': return 4; + case 'q': case 'Q': return 8; + case 'f': return (is_complex ? 8 : 4); + case 'd': return (is_complex ? 16 : 8); + case 'g': { + PyErr_SetString(PyExc_ValueError, "Python does not define a standard format string size for long double ('g').."); + return 0; + } + case 'O': case 'P': return sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(short); + case 'i': case 'I': return sizeof(int); + case 'l': case 'L': return sizeof(long); + case 'q': case 'Q': return sizeof(PY_LONG_LONG); + case 'f': return sizeof(float) * (is_complex ? 2 : 1); + case 'd': return sizeof(double) * (is_complex ? 2 : 1); + case 'g': return sizeof(long double) * (is_complex ? 2 : 1); + case 'O': case 'P': return sizeof(void*); + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +typedef struct { char c; short x; } __Pyx_st_short; +typedef struct { char c; int x; } __Pyx_st_int; +typedef struct { char c; long x; } __Pyx_st_long; +typedef struct { char c; float x; } __Pyx_st_float; +typedef struct { char c; double x; } __Pyx_st_double; +typedef struct { char c; long double x; } __Pyx_st_longdouble; +typedef struct { char c; void *x; } __Pyx_st_void_p; +typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_st_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_st_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_st_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_st_float) - sizeof(float); + case 'd': return sizeof(__Pyx_st_double) - sizeof(double); + case 'g': return sizeof(__Pyx_st_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_st_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +/* These are for computing the padding at the end of the struct to align + on the first member of the struct. This will probably the same as above, + but we don't have any guarantees. + */ +typedef struct { short x; char c; } __Pyx_pad_short; +typedef struct { int x; char c; } __Pyx_pad_int; +typedef struct { long x; char c; } __Pyx_pad_long; +typedef struct { float x; char c; } __Pyx_pad_float; +typedef struct { double x; char c; } __Pyx_pad_double; +typedef struct { long double x; char c; } __Pyx_pad_longdouble; +typedef struct { void *x; char c; } __Pyx_pad_void_p; +typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); + switch (ch) { + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); + case 'i': case 'I': return sizeof(__Pyx_pad_int) - sizeof(int); + case 'l': case 'L': return sizeof(__Pyx_pad_long) - sizeof(long); + case 'q': case 'Q': return sizeof(__Pyx_pad_longlong) - sizeof(PY_LONG_LONG); + case 'f': return sizeof(__Pyx_pad_float) - sizeof(float); + case 'd': return sizeof(__Pyx_pad_double) - sizeof(double); + case 'g': return sizeof(__Pyx_pad_longdouble) - sizeof(long double); + case 'P': case 'O': return sizeof(__Pyx_pad_void_p) - sizeof(void*); + default: + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } +} +static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) { + switch (ch) { + case 'c': + return 'H'; + case 'b': case 'h': case 'i': + case 'l': case 'q': case 's': case 'p': + return 'I'; + case '?': case 'B': case 'H': case 'I': case 'L': case 'Q': + return 'U'; + case 'f': case 'd': case 'g': + return (is_complex ? 'C' : 'R'); + case 'O': + return 'O'; + case 'P': + return 'P'; + default: { + __Pyx_BufFmt_RaiseUnexpectedChar(ch); + return 0; + } + } +} +static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) { + if (ctx->head == NULL || ctx->head->field == &ctx->root) { + const char* expected; + const char* quote; + if (ctx->head == NULL) { + expected = "end"; + quote = ""; + } else { + expected = ctx->head->field->type->name; + quote = "'"; + } + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected %s%s%s but got %s", + quote, expected, quote, + __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex)); + } else { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_StructField* parent = (ctx->head - 1)->field; + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch, expected '%s' but got %s in '%s.%s'", + field->type->name, __Pyx_BufFmt_DescribeTypeChar(ctx->enc_type, ctx->is_complex), + parent->type->name, field->name); + } +} +static int __Pyx_BufFmt_ProcessTypeChunk(__Pyx_BufFmt_Context* ctx) { + char group; + size_t size, offset, arraysize = 1; + if (ctx->enc_type == 0) return 0; + if (ctx->head->field->type->arraysize[0]) { + int i, ndim = 0; + if (ctx->enc_type == 's' || ctx->enc_type == 'p') { + ctx->is_valid_array = ctx->head->field->type->ndim == 1; + ndim = 1; + if (ctx->enc_count != ctx->head->field->type->arraysize[0]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %zu", + ctx->head->field->type->arraysize[0], ctx->enc_count); + return -1; + } + } + if (!ctx->is_valid_array) { + PyErr_Format(PyExc_ValueError, "Expected %d dimensions, got %d", + ctx->head->field->type->ndim, ndim); + return -1; + } + for (i = 0; i < ctx->head->field->type->ndim; i++) { + arraysize *= ctx->head->field->type->arraysize[i]; + } + ctx->is_valid_array = 0; + ctx->enc_count = 1; + } + group = __Pyx_BufFmt_TypeCharToGroup(ctx->enc_type, ctx->is_complex); + do { + const __Pyx_StructField* field = ctx->head->field; + const __Pyx_TypeInfo* type = field->type; + if (ctx->enc_packmode == '@' || ctx->enc_packmode == '^') { + size = __Pyx_BufFmt_TypeCharToNativeSize(ctx->enc_type, ctx->is_complex); + } else { + size = __Pyx_BufFmt_TypeCharToStandardSize(ctx->enc_type, ctx->is_complex); + } + if (ctx->enc_packmode == '@') { + size_t align_at = __Pyx_BufFmt_TypeCharToAlignment(ctx->enc_type, ctx->is_complex); + size_t align_mod_offset; + if (align_at == 0) return -1; + align_mod_offset = ctx->fmt_offset % align_at; + if (align_mod_offset > 0) ctx->fmt_offset += align_at - align_mod_offset; + if (ctx->struct_alignment == 0) + ctx->struct_alignment = __Pyx_BufFmt_TypeCharToPadding(ctx->enc_type, + ctx->is_complex); + } + if (type->size != size || type->typegroup != group) { + if (type->typegroup == 'C' && type->fields != NULL) { + size_t parent_offset = ctx->head->parent_offset + field->offset; + ++ctx->head; + ctx->head->field = type->fields; + ctx->head->parent_offset = parent_offset; + continue; + } + if ((type->typegroup == 'H' || group == 'H') && type->size == size) { + } else { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + } + offset = ctx->head->parent_offset + field->offset; + if (ctx->fmt_offset != offset) { + PyErr_Format(PyExc_ValueError, + "Buffer dtype mismatch; next field is at offset %" CYTHON_FORMAT_SSIZE_T "d but %" CYTHON_FORMAT_SSIZE_T "d expected", + (Py_ssize_t)ctx->fmt_offset, (Py_ssize_t)offset); + return -1; + } + ctx->fmt_offset += size; + if (arraysize) + ctx->fmt_offset += (arraysize - 1) * size; + --ctx->enc_count; + while (1) { + if (field == &ctx->root) { + ctx->head = NULL; + if (ctx->enc_count != 0) { + __Pyx_BufFmt_RaiseExpected(ctx); + return -1; + } + break; + } + ctx->head->field = ++field; + if (field->type == NULL) { + --ctx->head; + field = ctx->head->field; + continue; + } else if (field->type->typegroup == 'S') { + size_t parent_offset = ctx->head->parent_offset + field->offset; + if (field->type->fields->type == NULL) continue; + field = field->type->fields; + ++ctx->head; + ctx->head->field = field; + ctx->head->parent_offset = parent_offset; + break; + } else { + break; + } + } + } while (ctx->enc_count); + ctx->enc_type = 0; + ctx->is_complex = 0; + return 0; +} +static int +__pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp) +{ + const char *ts = *tsp; + int i = 0, number, ndim; + ++ts; + if (ctx->new_count != 1) { + PyErr_SetString(PyExc_ValueError, + "Cannot handle repeated arrays in format string"); + return -1; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return -1; + ndim = ctx->head->field->type->ndim; + while (*ts && *ts != ')') { + switch (*ts) { + case ' ': case '\f': case '\r': case '\n': case '\t': case '\v': continue; + default: break; + } + number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return -1; + if (i < ndim && (size_t) number != ctx->head->field->type->arraysize[i]) { + PyErr_Format(PyExc_ValueError, + "Expected a dimension of size %zu, got %d", + ctx->head->field->type->arraysize[i], number); + return -1; + } + if (*ts != ',' && *ts != ')') { + PyErr_Format(PyExc_ValueError, + "Expected a comma in format string, got '%c'", *ts); + return -1; + } + if (*ts == ',') ts++; + i++; + } + if (i != ndim) { + PyErr_Format(PyExc_ValueError, "Expected %d dimension(s), got %d", + ctx->head->field->type->ndim, i); + return -1; + } + if (!*ts) { + PyErr_SetString(PyExc_ValueError, + "Unexpected end of format string, expected ')'"); + return -1; + } + ctx->is_valid_array = 1; + ctx->new_count = 1; + *tsp = ++ts; + return 0; +} +static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts) { + int got_Z = 0; + while (1) { + switch(*ts) { + case 0: + if (ctx->enc_type != 0 && ctx->head == NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + if (ctx->head != NULL) { + __Pyx_BufFmt_RaiseExpected(ctx); + return NULL; + } + return ts; + case ' ': + case '\r': + case '\n': + ++ts; + break; + case '<': + if (!__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '>': + case '!': + if (__Pyx_Is_Little_Endian()) { + PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); + return NULL; + } + ctx->new_packmode = '='; + ++ts; + break; + case '=': + case '@': + case '^': + ctx->new_packmode = *ts++; + break; + case 'T': + { + const char* ts_after_sub; + size_t i, struct_count = ctx->new_count; + size_t struct_alignment = ctx->struct_alignment; + ctx->new_count = 1; + ++ts; + if (*ts != '{') { + PyErr_SetString(PyExc_ValueError, "Buffer acquisition: Expected '{' after 'T'"); + return NULL; + } + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + ctx->enc_count = 0; + ctx->struct_alignment = 0; + ++ts; + ts_after_sub = ts; + for (i = 0; i != struct_count; ++i) { + ts_after_sub = __Pyx_BufFmt_CheckString(ctx, ts); + if (!ts_after_sub) return NULL; + } + ts = ts_after_sub; + if (struct_alignment) ctx->struct_alignment = struct_alignment; + } + break; + case '}': + { + size_t alignment = ctx->struct_alignment; + ++ts; + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_type = 0; + if (alignment && ctx->fmt_offset % alignment) { + ctx->fmt_offset += alignment - (ctx->fmt_offset % alignment); + } + } + return ts; + case 'x': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->fmt_offset += ctx->new_count; + ctx->new_count = 1; + ctx->enc_count = 0; + ctx->enc_type = 0; + ctx->enc_packmode = ctx->new_packmode; + ++ts; + break; + case 'Z': + got_Z = 1; + ++ts; + if (*ts != 'f' && *ts != 'd' && *ts != 'g') { + __Pyx_BufFmt_RaiseUnexpectedChar('Z'); + return NULL; + } + CYTHON_FALLTHROUGH; + case '?': case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': + case 'l': case 'L': case 'q': case 'Q': + case 'f': case 'd': case 'g': + case 'O': case 'p': + if ((ctx->enc_type == *ts) && (got_Z == ctx->is_complex) && + (ctx->enc_packmode == ctx->new_packmode) && (!ctx->is_valid_array)) { + ctx->enc_count += ctx->new_count; + ctx->new_count = 1; + got_Z = 0; + ++ts; + break; + } + CYTHON_FALLTHROUGH; + case 's': + if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + ctx->enc_count = ctx->new_count; + ctx->enc_packmode = ctx->new_packmode; + ctx->enc_type = *ts; + ctx->is_complex = got_Z; + ++ts; + ctx->new_count = 1; + got_Z = 0; + break; + case ':': + ++ts; + while(*ts != ':') ++ts; + ++ts; + break; + case '(': + if (__pyx_buffmt_parse_array(ctx, &ts) < 0) return NULL; + break; + default: + { + int number = __Pyx_BufFmt_ExpectNumber(&ts); + if (number == -1) return NULL; + ctx->new_count = (size_t)number; + } + } + } +} + +/* TypeInfoCompare (used by MemviewSliceValidateAndInit) */ + static int + __pyx_typeinfo_cmp(const __Pyx_TypeInfo *a, const __Pyx_TypeInfo *b) + { + int i; + if (!a || !b) + return 0; + if (a == b) + return 1; + if (a->size != b->size || a->typegroup != b->typegroup || + a->is_unsigned != b->is_unsigned || a->ndim != b->ndim) { + if (a->typegroup == 'H' || b->typegroup == 'H') { + return a->size == b->size; + } else { + return 0; + } + } + if (a->ndim) { + for (i = 0; i < a->ndim; i++) + if (a->arraysize[i] != b->arraysize[i]) + return 0; + } + if (a->typegroup == 'S') { + if (a->flags != b->flags) + return 0; + if (a->fields || b->fields) { + if (!(a->fields && b->fields)) + return 0; + for (i = 0; a->fields[i].type && b->fields[i].type; i++) { + const __Pyx_StructField *field_a = a->fields + i; + const __Pyx_StructField *field_b = b->fields + i; + if (field_a->offset != field_b->offset || + !__pyx_typeinfo_cmp(field_a->type, field_b->type)) + return 0; + } + return !a->fields[i].type && !b->fields[i].type; + } + } + return 1; + } + +/* MemviewSliceValidateAndInit (used by ObjectToMemviewSlice) */ + static int + __pyx_check_strides(Py_buffer *buf, int dim, int ndim, int spec) + { + if (buf->shape[dim] <= 1) + return 1; + if (buf->strides) { + if (spec & __Pyx_MEMVIEW_CONTIG) { + if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL)) { + if (unlikely(buf->strides[dim] != sizeof(void *))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly contiguous " + "in dimension %d.", dim); + goto fail; + } + } else if (unlikely(buf->strides[dim] != buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_FOLLOW) { + Py_ssize_t stride = buf->strides[dim]; + if (stride < 0) + stride = -stride; + if (unlikely(stride < buf->itemsize)) { + PyErr_SetString(PyExc_ValueError, + "Buffer and memoryview are not contiguous " + "in the same dimension."); + goto fail; + } + } + } else { + if (unlikely(spec & __Pyx_MEMVIEW_CONTIG && dim != ndim - 1)) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not contiguous in " + "dimension %d", dim); + goto fail; + } else if (unlikely(spec & (__Pyx_MEMVIEW_PTR))) { + PyErr_Format(PyExc_ValueError, + "C-contiguous buffer is not indirect in " + "dimension %d", dim); + goto fail; + } else if (unlikely(buf->suboffsets)) { + PyErr_SetString(PyExc_ValueError, + "Buffer exposes suboffsets but no strides"); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) + { + CYTHON_UNUSED_VAR(ndim); + if (spec & __Pyx_MEMVIEW_DIRECT) { + if (unlikely(buf->suboffsets && buf->suboffsets[dim] >= 0)) { + PyErr_Format(PyExc_ValueError, + "Buffer not compatible with direct access " + "in dimension %d.", dim); + goto fail; + } + } + if (spec & __Pyx_MEMVIEW_PTR) { + if (unlikely(!buf->suboffsets || (buf->suboffsets[dim] < 0))) { + PyErr_Format(PyExc_ValueError, + "Buffer is not indirectly accessible " + "in dimension %d.", dim); + goto fail; + } + } + return 1; + fail: + return 0; + } + static int + __pyx_verify_contig(Py_buffer *buf, int ndim, int c_or_f_flag) + { + int i; + if (c_or_f_flag & __Pyx_IS_F_CONTIG) { + Py_ssize_t stride = 1; + for (i = 0; i < ndim; i++) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not fortran contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } else if (c_or_f_flag & __Pyx_IS_C_CONTIG) { + Py_ssize_t stride = 1; + for (i = ndim - 1; i >- 1; i--) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { + PyErr_SetString(PyExc_ValueError, + "Buffer not C contiguous."); + goto fail; + } + stride = stride * buf->shape[i]; + } + } + return 1; + fail: + return 0; + } + static int __Pyx_ValidateAndInit_memviewslice( + int *axes_specs, + int c_or_f_flag, + int buf_flags, + int ndim, + const __Pyx_TypeInfo *dtype, + __Pyx_BufFmt_StackElem stack[], + __Pyx_memviewslice *memviewslice, + PyObject *original_obj) + { + struct __pyx_memoryview_obj *memview, *new_memview; + __Pyx_RefNannyDeclarations + Py_buffer *buf; + int i, spec = 0, retval = -1; + __Pyx_BufFmt_Context ctx; + int from_memoryview = __pyx_memoryview_check(original_obj); + __Pyx_RefNannySetupContext("ValidateAndInit_memviewslice", 0); + if (from_memoryview && __pyx_typeinfo_cmp(dtype, ((struct __pyx_memoryview_obj *) + original_obj)->typeinfo)) { + memview = (struct __pyx_memoryview_obj *) original_obj; + new_memview = NULL; + } else { + memview = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + original_obj, buf_flags, 0, dtype); + new_memview = memview; + if (unlikely(!memview)) + goto fail; + } + buf = &memview->view; + if (unlikely(buf->ndim != ndim)) { + PyErr_Format(PyExc_ValueError, + "Buffer has wrong number of dimensions (expected %d, got %d)", + ndim, buf->ndim); + goto fail; + } + if (new_memview) { + __Pyx_BufFmt_Init(&ctx, stack, dtype); + if (unlikely(!__Pyx_BufFmt_CheckString(&ctx, buf->format))) goto fail; + } + if (unlikely((unsigned) buf->itemsize != dtype->size)) { + PyErr_Format(PyExc_ValueError, + "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "u byte%s) " + "does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "u byte%s)", + buf->itemsize, + (buf->itemsize > 1) ? "s" : "", + dtype->name, + dtype->size, + (dtype->size > 1) ? "s" : ""); + goto fail; + } + if (buf->len > 0) { + for (i = 0; i < ndim; i++) { + spec = axes_specs[i]; + if (unlikely(!__pyx_check_strides(buf, i, ndim, spec))) + goto fail; + if (unlikely(!__pyx_check_suboffsets(buf, i, ndim, spec))) + goto fail; + } + if (unlikely(buf->strides && !__pyx_verify_contig(buf, ndim, c_or_f_flag))) + goto fail; + } + if (unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice, + new_memview != NULL) == -1)) { + goto fail; + } + retval = 0; + goto no_fail; + fail: + Py_XDECREF((PyObject*)new_memview); + retval = -1; + no_fail: + __Pyx_RefNannyFinishContext(); + return retval; + } + +/* ObjectToMemviewSlice */ + static CYTHON_INLINE __Pyx_memviewslice __Pyx_PyObject_to_MemoryviewSlice_d_dc_float(PyObject *obj, int writable_flag) { + __Pyx_memviewslice result = __Pyx_MEMSLICE_INIT; + __Pyx_BufFmt_StackElem stack[1]; + int axes_specs[] = { (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_FOLLOW), (__Pyx_MEMVIEW_DIRECT | __Pyx_MEMVIEW_CONTIG) }; + int retcode; + if (obj == Py_None) { + result.memview = (struct __pyx_memoryview_obj *) Py_None; + return result; + } + retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, __Pyx_IS_C_CONTIG, + (PyBUF_C_CONTIGUOUS | PyBUF_FORMAT) | writable_flag, 2, + &__Pyx_TypeInfo_float, stack, + &result, obj); + if (unlikely(retcode == -1)) + goto __pyx_fail; + return result; + __pyx_fail: + result.memview = NULL; + result.data = NULL; + return result; + } + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) + #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) + #define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + return ::std::complex< float >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + return x + y*(__pyx_t_float_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_float_complex __pyx_t_float_complex_from_parts(float x, float y) { + __pyx_t_float_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_sum_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_diff_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_prod_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + if (b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabsf(b.real) >= fabsf(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + float r = b.imag / b.real; + float s = (float)(1.0) / (b.real + b.imag * r); + return __pyx_t_float_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + float r = b.real / b.imag; + float s = (float)(1.0) / (b.imag + b.real * r); + return __pyx_t_float_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_quot_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + if (b.imag == 0) { + return __pyx_t_float_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + float denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_float_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_neg_float(__pyx_t_float_complex a) { + __pyx_t_float_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_float(__pyx_t_float_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_conj_float(__pyx_t_float_complex a) { + __pyx_t_float_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE float __Pyx_c_abs_float(__pyx_t_float_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrtf(z.real*z.real + z.imag*z.imag); + #else + return hypotf(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_float_complex __Pyx_c_pow_float(__pyx_t_float_complex a, __pyx_t_float_complex b) { + __pyx_t_float_complex z; + float r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + float denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_float(a, a); + case 3: + z = __Pyx_c_prod_float(a, a); + return __Pyx_c_prod_float(z, a); + case 4: + z = __Pyx_c_prod_float(a, a); + return __Pyx_c_prod_float(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = powf(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2f(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_float(a); + theta = atan2f(a.imag, a.real); + } + lnr = logf(r); + z_r = expf(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cosf(z_theta); + z.imag = z_r * sinf(z_theta); + return z; + } + #endif + #endif + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return ::std::complex< double >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + return x + y*(__pyx_t_double_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_double_complex __pyx_t_double_complex_from_parts(double x, double y) { + __pyx_t_double_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_sum_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_diff_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_prod_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabs(b.real) >= fabs(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + double r = b.imag / b.real; + double s = (double)(1.0) / (b.real + b.imag * r); + return __pyx_t_double_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + double r = b.real / b.imag; + double s = (double)(1.0) / (b.imag + b.real * r); + return __pyx_t_double_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_quot_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + if (b.imag == 0) { + return __pyx_t_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + double denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_double_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_neg_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_double(__pyx_t_double_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_conj_double(__pyx_t_double_complex a) { + __pyx_t_double_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE double __Pyx_c_abs_double(__pyx_t_double_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrt(z.real*z.real + z.imag*z.imag); + #else + return hypot(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_double_complex __Pyx_c_pow_double(__pyx_t_double_complex a, __pyx_t_double_complex b) { + __pyx_t_double_complex z; + double r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + double denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_double(a, a); + case 3: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, a); + case 4: + z = __Pyx_c_prod_double(a, a); + return __Pyx_c_prod_double(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = pow(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_double(a); + theta = atan2(a.imag, a.real); + } + lnr = log(r); + z_r = exp(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cos(z_theta); + z.imag = z_r * sin(z_theta); + return z; + } + #endif + #endif + +/* Declarations */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #ifdef __cplusplus + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + return ::std::complex< long double >(x, y); + } + #else + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + return x + y*(__pyx_t_long_double_complex)_Complex_I; + } + #endif + #else + static CYTHON_INLINE __pyx_t_long_double_complex __pyx_t_long_double_complex_from_parts(long double x, long double y) { + __pyx_t_long_double_complex z; + z.real = x; + z.imag = y; + return z; + } + #endif + +/* Arithmetic */ + #if CYTHON_CCOMPLEX && (1) && (!0 || __cplusplus) + #else + static CYTHON_INLINE int __Pyx_c_eq_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + return (a.real == b.real) && (a.imag == b.imag); + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_sum_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real + b.real; + z.imag = a.imag + b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_diff_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real - b.real; + z.imag = a.imag - b.imag; + return z; + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_prod_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + z.real = a.real * b.real - a.imag * b.imag; + z.imag = a.real * b.imag + a.imag * b.real; + return z; + } + #if 1 + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + if (b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else if (fabsl(b.real) >= fabsl(b.imag)) { + if (b.real == 0 && b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.imag); + } else { + long double r = b.imag / b.real; + long double s = (long double)(1.0) / (b.real + b.imag * r); + return __pyx_t_long_double_complex_from_parts( + (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); + } + } else { + long double r = b.real / b.imag; + long double s = (long double)(1.0) / (b.imag + b.real * r); + return __pyx_t_long_double_complex_from_parts( + (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); + } + } + #else + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_quot_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + if (b.imag == 0) { + return __pyx_t_long_double_complex_from_parts(a.real / b.real, a.imag / b.real); + } else { + long double denom = b.real * b.real + b.imag * b.imag; + return __pyx_t_long_double_complex_from_parts( + (a.real * b.real + a.imag * b.imag) / denom, + (a.imag * b.real - a.real * b.imag) / denom); + } + } + #endif + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_neg_long__double(__pyx_t_long_double_complex a) { + __pyx_t_long_double_complex z; + z.real = -a.real; + z.imag = -a.imag; + return z; + } + static CYTHON_INLINE int __Pyx_c_is_zero_long__double(__pyx_t_long_double_complex a) { + return (a.real == 0) && (a.imag == 0); + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_conj_long__double(__pyx_t_long_double_complex a) { + __pyx_t_long_double_complex z; + z.real = a.real; + z.imag = -a.imag; + return z; + } + #if 1 + static CYTHON_INLINE long double __Pyx_c_abs_long__double(__pyx_t_long_double_complex z) { + #if !defined(HAVE_HYPOT) || defined(_MSC_VER) + return sqrtl(z.real*z.real + z.imag*z.imag); + #else + return hypotl(z.real, z.imag); + #endif + } + static CYTHON_INLINE __pyx_t_long_double_complex __Pyx_c_pow_long__double(__pyx_t_long_double_complex a, __pyx_t_long_double_complex b) { + __pyx_t_long_double_complex z; + long double r, lnr, theta, z_r, z_theta; + if (b.imag == 0 && b.real == (int)b.real) { + if (b.real < 0) { + long double denom = a.real * a.real + a.imag * a.imag; + a.real = a.real / denom; + a.imag = -a.imag / denom; + b.real = -b.real; + } + switch ((int)b.real) { + case 0: + z.real = 1; + z.imag = 0; + return z; + case 1: + return a; + case 2: + return __Pyx_c_prod_long__double(a, a); + case 3: + z = __Pyx_c_prod_long__double(a, a); + return __Pyx_c_prod_long__double(z, a); + case 4: + z = __Pyx_c_prod_long__double(a, a); + return __Pyx_c_prod_long__double(z, z); + } + } + if (a.imag == 0) { + if (a.real == 0) { + return a; + } else if ((b.imag == 0) && (a.real >= 0)) { + z.real = powl(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2l(0.0, -1.0); + } + } else { + r = __Pyx_c_abs_long__double(a); + theta = atan2l(a.imag, a.real); + } + lnr = logl(r); + z_r = expl(lnr * b.real - theta * b.imag); + z_theta = theta * b.real + lnr * b.imag; + z.real = z_r * cosl(z_theta); + z.imag = z_r * sinl(z_theta); + return z; + } + #endif + #endif + +/* MemviewSliceCopy */ + static __Pyx_memviewslice + __pyx_memoryview_copy_new_contig(const __Pyx_memviewslice *from_mvs, + const char *mode, int ndim, + size_t sizeof_dtype, int contig_flag, + int dtype_is_object) + { + __Pyx_RefNannyDeclarations + int i; + __Pyx_memviewslice new_mvs = __Pyx_MEMSLICE_INIT; + struct __pyx_memoryview_obj *from_memview = from_mvs->memview; + Py_buffer *buf = &from_memview->view; + PyObject *shape_tuple = NULL; + PyObject *temp_int = NULL; + struct __pyx_array_obj *array_obj = NULL; + struct __pyx_memoryview_obj *memview_obj = NULL; + __Pyx_RefNannySetupContext("__pyx_memoryview_copy_new_contig", 0); + for (i = 0; i < ndim; i++) { + if (unlikely(from_mvs->suboffsets[i] >= 0)) { + PyErr_Format(PyExc_ValueError, "Cannot copy memoryview slice with " + "indirect dimensions (axis %d)", i); + goto fail; + } + } + shape_tuple = PyTuple_New(ndim); + if (unlikely(!shape_tuple)) { + goto fail; + } + __Pyx_GOTREF(shape_tuple); + for(i = 0; i < ndim; i++) { + temp_int = PyLong_FromSsize_t(from_mvs->shape[i]); + if(unlikely(!temp_int)) { + goto fail; + } else { + #if CYTHON_ASSUME_SAFE_MACROS + PyTuple_SET_ITEM(shape_tuple, i, temp_int); + #else + if (PyTuple_SetItem(shape_tuple, i, temp_int) < 0) { + goto fail; + } + #endif + temp_int = NULL; + } + } + array_obj = __pyx_array_new(shape_tuple, sizeof_dtype, buf->format, mode, NULL); + if (unlikely(!array_obj)) { + goto fail; + } + __Pyx_GOTREF(array_obj); + memview_obj = (struct __pyx_memoryview_obj *) __pyx_memoryview_new( + (PyObject *) array_obj, contig_flag, + dtype_is_object, + from_mvs->memview->typeinfo); + if (unlikely(!memview_obj)) + goto fail; + if (unlikely(__Pyx_init_memviewslice(memview_obj, ndim, &new_mvs, 1) < 0)) + goto fail; + if (unlikely(__pyx_memoryview_copy_contents(*from_mvs, new_mvs, ndim, ndim, + dtype_is_object) < 0)) + goto fail; + goto no_fail; + fail: + __Pyx_XDECREF((PyObject *) new_mvs.memview); + new_mvs.memview = NULL; + new_mvs.data = NULL; + no_fail: + __Pyx_XDECREF(shape_tuple); + __Pyx_XDECREF(temp_int); + __Pyx_XDECREF((PyObject *) array_obj); + __Pyx_RefNannyFinishContext(); + return new_mvs; + } + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int(int value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int neg_one = (int) -1, const_zero = (int) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_int32_t(int32_t value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int32_t) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(int32_t) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(int32_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(int32_t), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(int32_t)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int32_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int32_t) -1; + val = __Pyx_PyLong_As_int32_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 2 * PyLong_SHIFT)) { + return (int32_t) (((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 3 * PyLong_SHIFT)) { + return (int32_t) (((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 4 * PyLong_SHIFT)) { + return (int32_t) (((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int32_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(int32_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int32_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) ((((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) ((((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) ((((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(int32_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, long, PyLong_AsLong(x)) + } else if ((sizeof(int32_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + int32_t val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int32_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int32_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int32_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int32_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int32_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int32_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int32_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int32_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int32_t) 1) << (sizeof(int32_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (int32_t) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int32_t"); + return (int32_t) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int32_t"); + return (int32_t) -1; + } + +/* PyObjectCallMethod1 (used by UpdateUnpickledDict) */ + #if !(CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000))) + static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); + Py_DECREF(method); + return result; + } + #endif + static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + #if CYTHON_VECTORCALL && (__PYX_LIMITED_VERSION_HEX >= 0x030C0000 || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x03090000)) + PyObject *args[2] = {obj, arg}; + (void) __Pyx_PyObject_CallOneArg; + (void) __Pyx_PyObject_Call2Args; + return PyObject_VectorcallMethod(method_name, args, 2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + #else + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); + #endif + } + +/* UpdateUnpickledDict */ + static int __Pyx__UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + PyObject *state_dict = __Pyx_PySequence_ITEM(state, index); + if (unlikely(!state_dict)) { + return -1; + } + int non_empty = PyObject_IsTrue(state_dict); + if (non_empty == 0) { + Py_DECREF(state_dict); + return 0; + } else if (unlikely(non_empty == -1)) { + return -1; + } + PyObject *dict; + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = PyObject_GetAttrString(obj, "__dict__"); + #else + dict = PyObject_GenericGetDict(obj, NULL); + #endif + if (unlikely(!dict)) { + Py_DECREF(state_dict); + return -1; + } + int result; + if (likely(PyDict_CheckExact(dict))) { + result = PyDict_Update(dict, state_dict); + } else { + PyObject *obj_result = __Pyx_PyObject_CallMethod1(dict, __pyx_mstate_global->__pyx_n_u_update, state_dict); + if (likely(obj_result)) { + Py_DECREF(obj_result); + result = 0; + } else { + result = -1; + } + } + Py_DECREF(state_dict); + Py_DECREF(dict); + return result; + } + static int __Pyx_UpdateUnpickledDict(PyObject *obj, PyObject *state, Py_ssize_t index) { + Py_ssize_t state_size = __Pyx_PyTuple_GET_SIZE(state); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(state_size == -1)) return -1; + #endif + if (state_size <= index) { + return 0; + } + return __Pyx__UpdateUnpickledDict(obj, state, index); + } + +/* CheckUnpickleChecksum */ + static void __Pyx_RaiseUnpickleChecksumError(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + PyObject *pickle_module = PyImport_ImportModule("pickle"); + if (unlikely(!pickle_module)) return; + PyObject *pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (unlikely(!pickle_error)) return; + if (checksum2 == checksum1) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x) = (%s))", + checksum, checksum1, members); + } else if (checksum3 == checksum2) { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, members); + } else { + PyErr_Format(pickle_error, "Incompatible checksums (0x%x vs (0x%x, 0x%x, 0x%x) = (%s))", + checksum, checksum1, checksum2, checksum3, members); + } + Py_DECREF(pickle_error); + } + static int __Pyx_CheckUnpickleChecksum(long checksum, long checksum1, long checksum2, long checksum3, const char *members) { + int found = 0; + found |= checksum1 == checksum; + found |= checksum2 == checksum; + found |= checksum3 == checksum; + if (likely(found)) + return 0; + __Pyx_RaiseUnpickleChecksumError(checksum, checksum1, checksum2, checksum3, members); + return -1; + } + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + long val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (long) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; + } + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const long neg_one = (long) -1, const_zero = (long) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + #if !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + #endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + unsigned char *bytes = (unsigned char *)&value; + #if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + #else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; + #endif + } + } + +/* CIntFromPy */ + static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *x) { + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wconversion" + #endif + const char neg_one = (char) -1, const_zero = (char) 0; + #ifdef __Pyx_HAS_GCC_DIAGNOSTIC + #pragma GCC diagnostic pop + #endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + char val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (char) -1; + val = __Pyx_PyLong_As_char(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { + #if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 2 * PyLong_SHIFT)) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 3 * PyLong_SHIFT)) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 4 * PyLong_SHIFT)) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } + } + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } + #else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } + #endif + if ((sizeof(char) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) + } else if ((sizeof(char) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { + #if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(char) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } + } + #endif + if ((sizeof(char) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) + } else if ((sizeof(char) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { + char val; + int ret = -1; + #if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } + #elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); + #else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (char) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (char) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (char) -1; + } else { + stepval = v; + } + v = NULL; + val = (char) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(char) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((char) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(char) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((char) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((char) 1) << (sizeof(char) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); + #endif + if (unlikely(ret)) + return (char) -1; + return val; + } + raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; + raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; + } + +/* FormatTypeName */ + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 + static __Pyx_TypeName + __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) + { + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__6); + } + goto done; + } + #endif + +/* GetRuntimeVersion */ + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + void __Pyx_init_runtime_version(void) { + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + } + #endif + static unsigned long __Pyx_get_runtime_version(void) { + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + return Py_Version & ~0xFFUL; + #else + return __Pyx_cached_runtime_version; + #endif + } + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } + } + +/* NewCodeObj */ + #if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } + #elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } + #elif !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #endif + static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + PyObject *line_table, + PyObject *tuple_dedup_map + ) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL && !CYTHON_COMPILING_IN_GRAAL) { + Py_ssize_t line_table_length = __Pyx_PyBytes_GET_SIZE(line_table); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(line_table_length == -1)) goto done; + #endif + Py_ssize_t code_len = (line_table_length * 2 + 4) & ~3LL; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table) ? line_table : __pyx_mstate_global->__pyx_empty_bytes + ); + done: + Py_XDECREF(code_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; + } + +/* DecompressString */ + static PyObject *__Pyx_DecompressString(const char *s, Py_ssize_t length, int algo) { + PyObject *module, *decompress, *compressed_bytes, *decompressed; + const char* module_name = algo == 3 ? "compression.zstd" : algo == 2 ? "bz2" : "zlib"; + PyObject *methodname = PyUnicode_FromString("decompress"); + if (unlikely(!methodname)) return NULL; + #if __PYX_LIMITED_VERSION_HEX >= 0x030e0000 + if (algo == 3) { + PyObject *fromlist = Py_BuildValue("[O]", methodname); + if (unlikely(!fromlist)) return NULL; + module = PyImport_ImportModuleLevel("compression.zstd", NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + } else + #endif + module = PyImport_ImportModule(module_name); + if (unlikely(!module)) goto import_failed; + decompress = PyObject_GetAttr(module, methodname); + if (unlikely(!decompress)) goto import_failed; + { + #ifdef __cplusplus + char *memview_bytes = const_cast(s); + #else + #if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wcast-qual" + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wcast-qual" + #endif + char *memview_bytes = (char*) s; + #if defined(__clang__) + #pragma clang diagnostic pop + #elif !defined(__INTEL_COMPILER) && defined(__GNUC__) + #pragma GCC diagnostic pop + #endif + #endif + #if CYTHON_COMPILING_IN_LIMITED_API && !defined(PyBUF_READ) + int memview_flags = 0x100; + #else + int memview_flags = PyBUF_READ; + #endif + compressed_bytes = PyMemoryView_FromMemory(memview_bytes, length, memview_flags); + } + if (unlikely(!compressed_bytes)) { + Py_DECREF(decompress); + goto bad; + } + decompressed = PyObject_CallFunctionObjArgs(decompress, compressed_bytes, NULL); + Py_DECREF(compressed_bytes); + Py_DECREF(decompress); + Py_DECREF(module); + Py_DECREF(methodname); + return decompressed; + import_failed: + PyErr_Format(PyExc_ImportError, + "Failed to import '%.20s.decompress' - cannot initialise module strings. " + "String compression was configured with the C macro 'CYTHON_COMPRESS_STRINGS=%d'.", + module_name, algo); + bad: + Py_XDECREF(module); + Py_DECREF(methodname); + return NULL; + } + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return __Pyx_NewRef(b ? Py_True: Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} + + + /* MultiPhaseInitModuleState */ + #if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE + #ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 + #else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 + #endif + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS + #error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" + #endif + #if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + #define __Pyx_ModuleStateLookup_Lock() + #define __Pyx_ModuleStateLookup_Unlock() + #elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 + static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; + #define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(__cplusplus) && __cplusplus >= 201103L + #include + static std::mutex __Pyx_ModuleStateLookup_mutex; + #define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() + #define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() + #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) + #include + static mtx_t __Pyx_ModuleStateLookup_mutex; + static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; + static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); + } + #define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(HAVE_PTHREAD_H) + #include + static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; + #define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) + #elif defined(_WIN32) + #include // synchapi.h on its own doesn't work + static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; + #define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) + #else + #error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." + #endif + typedef struct { + int64_t id; + PyObject *module; + } __Pyx_InterpreterIdAndModule; + typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; + } __Pyx_ModuleStateLookupData; + #define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; + #endif + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; + #else + static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; + #endif + static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; + } + static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; + #else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + #endif + return result; + } + } + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); + } + #else + #define __Pyx_ModuleStateLookup_wait_until_no_readers() + #endif + static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; + } + static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; + } + static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; + #endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); + #else + __Pyx_ModuleStateLookup_data = new_data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return result; + } + static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); + #else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; + #endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: + #if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); + #else + __Pyx_ModuleStateLookup_data = data; + #endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; + } + #endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.pyx b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.pyx new file mode 100644 index 0000000000000000000000000000000000000000..464d90e043a8de1dee49e9e250c64220c60521db --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/sparselinear.pyx @@ -0,0 +1,216 @@ +# cython: infer_types=True, cdivision=True, bounds_check=False, wraparound=False, profile=False +cimport cython +cimport numpy as np +from libc.stdint cimport int32_t, uint32_t, uint64_t + +from typing import Callable, Optional, Tuple + +from ..backends import CupyOps, NumpyOps +from ..config import registry +from ..model import Model +from ..types import ArrayXd +from ..util import get_array_module, get_width, is_cupy_array, is_numpy_array + +InT = Tuple[ArrayXd, ArrayXd, ArrayXd] +OutT = ArrayXd + + +@cython.binding(True) +@registry.layers("SparseLinear.v1") +def SparseLinear(nO: Optional[int] = None, length: int = 2 ** 18): + # NB: We can't have generic return type annotation if we want function to + # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + return Model( + "sparse_linear", + forward, + init=init, + params={"W": None, "b": None}, + dims={"nO": nO, "length": length}, + attrs={"v1_indexing": True}, + ) + + +@cython.binding(True) +@registry.layers("SparseLinear.v2") +def SparseLinear_v2(nO: Optional[int] = None, length: int = 2 ** 18): + # NB: We can't have generic return type annotation if we want function to + # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + return Model( + "sparse_linear", + forward, + init=init, + params={"W": None, "b": None}, + dims={"nO": nO, "length": length}, + attrs={"v1_indexing": False}, + ) + + +@cython.binding(True) +def forward(model: Model, keys_values_lengths: InT, is_train: bool) -> Tuple[OutT, Callable]: + # NB: We can't have generic Model annotation if we want function to + # be bound (and inspectable): https://github.com/cython/cython/issues/2753 + keys, values, lengths = keys_values_lengths + if is_cupy_array(keys): + # Currently we don't have a GPU-compatible implementation of this function :( + # It sucks, but at least we can get the correct result by copying to CPU. + return _begin_gpu_update(model, keys, values, lengths) + else: + return _begin_cpu_update(model, keys, values, lengths) + + +def init(model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None) -> Model[InT, OutT]: + if Y is not None: + model.set_dim("nO", get_width(Y)) + nO = model.get_dim("nO") + length = model.get_dim("length") + model.set_param("W", model.ops.alloc((nO * length,), dtype="f")) + model.set_param("b", model.ops.alloc((nO,), dtype="f")) + return model + + +def _begin_gpu_update(model: Model[InT, OutT], keys: ArrayXd, values: ArrayXd, lengths: ArrayXd) -> Tuple[ArrayXd, Callable]: + xp = get_array_module(keys) + scores_cpu, callback = _begin_cpu_update(model, keys.get(), values.get(), lengths.get()) + + def backprop_gpu_update(d_scores: ArrayXd) -> Tuple[ArrayXd, ArrayXd, ArrayXd]: + callback(d_scores.get()) + return (keys, values, lengths) + + return xp.asarray(scores_cpu), backprop_gpu_update + + +def _begin_cpu_update(model, np.ndarray keys, np.ndarray values, np.ndarray lengths): + cdef int nO = model.get_dim("nO") + cdef int length = model.get_dim("length") + cdef np.ndarray W = model.get_param("W") + cdef np.ndarray b = model.get_param("b") + cdef np.ndarray scores = model.ops.alloc((len(lengths), nO)) + cdef bint v1_indexing = model.attrs["v1_indexing"] + scores += b + set_scoresC(scores.data, + keys.data, values.data, lengths.data, + lengths.shape[0], nO, + W.data, length, v1_indexing) + return scores, _finish_linear_update(model, keys, values, lengths) + + +class _finish_linear_update: + """Move this out of a closure, into its own callable object, to avoid + pickling errors :(.""" + def __init__(self, model, keys, values, lengths): + self.model = model + self.keys = keys + self.values = values + self.lengths = lengths + + def __call__(self, float[:, ::1] d_scores): + nO = self.model.get_dim("nO") + length = self.model.get_dim("length") + cdef np.ndarray d_weights = self.model.ops.alloc((nO*length,)) + cdef np.ndarray d_bias = self.model.ops.alloc((nO,)) + cdef np.ndarray keys = self.keys + cdef np.ndarray values = self.values + cdef np.ndarray lengths = self.lengths + cdef bint v1_indexing = self.model.attrs["v1_indexing"] + set_gradientC(d_weights.data, + keys.data, values.data, lengths.data, + lengths.shape[0], nO, &d_scores[0,0], length, v1_indexing) + cdef int i, j + for i in range(d_scores.shape[0]): + for j in range(d_scores.shape[1]): + d_bias[j] += d_scores[i, j] + self.model.inc_grad("W", d_weights) + self.model.inc_grad("b", d_bias) + return (self.keys, self.values, self.lengths) + + +# v1_indexing is invalid and only uses a subset of the weight matrix, v1 +# indexing is provided here for compatibility. See #752 for more information. +cdef void set_scoresC(float* scores, + const uint64_t* keys, const float* values, const int32_t* lengths, + int batch_size, int nr_out, const float* weights, int nr_weight, + bint v1_indexing) noexcept nogil: + cdef uint32_t idx1, idx2 + cdef uint32_t hash1, hash2 + for length in lengths[:batch_size]: + for i in range(length): + hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + if v1_indexing: + idx1 = hash1 & (nr_weight-1) + idx2 = hash2 & (nr_weight-1) + else: + idx1 = hash1 % nr_weight + idx2 = hash2 % nr_weight + value = values[i] + for clas in range(nr_out): + if v1_indexing: + scores[clas] += weights[idx1 + clas] * value + scores[clas] += weights[idx2 + clas] * value + else: + scores[clas] += weights[(clas * nr_weight) + idx1] * value + scores[clas] += weights[(clas * nr_weight) + idx2] * value + scores += nr_out + keys += length + values += length + + +# v1_indexing is invalid and only uses a subset of the weight matrix, v1 +# indexing is provided here for compatibility. See #752 for more information. +cdef void set_gradientC(float* d_weights, + const uint64_t* keys, const float* values, const int32_t* lengths, + int batch_size, int nr_out, const float* d_scores, int nr_weight, + bint v1_indexing) noexcept nogil: + cdef uint32_t idx1, idx2 + cdef uint32_t hash1, hash2 + for length in lengths[:batch_size]: + for i in range(length): + hash1 = MurmurHash3_x86_32_uint64(keys[i], 0) + hash2 = MurmurHash3_x86_32_uint64(keys[i], 1) + if v1_indexing: + idx1 = hash1 & (nr_weight-1) + idx2 = hash2 & (nr_weight-1) + else: + idx1 = hash1 % nr_weight + idx2 = hash2 % nr_weight + value = values[i] + for clas in range(nr_out): + if v1_indexing: + d_weights[idx1 + clas] += d_scores[clas] * value + d_weights[idx2 + clas] += d_scores[clas] * value + else: + d_weights[(clas * nr_weight) + idx1] += d_scores[clas] * value + d_weights[(clas * nr_weight) + idx2] += d_scores[clas] * value + d_scores += nr_out + keys += length + values += length + + +cdef uint32_t MurmurHash3_x86_32_uint64(uint64_t key, uint32_t seed) noexcept nogil: + cdef uint32_t h1 = seed + cdef uint32_t c1 = 0xcc9e2d51u + cdef uint32_t c2 = 0x1b873593u + cdef uint32_t k1 + + k1 = key & 0xffffffffu + k1 *= c1 + k1 = (k1 << 15) | (k1 >> 17) + k1 *= c2 + h1 ^= k1 + h1 = (h1 << 13) | (h1 >> 19) + h1 = h1*5+0xe6546b64u + k1 = key >> 32 + k1 *= c1 + k1 = (k1 << 15) | (k1 >> 17) + k1 *= c2 + h1 ^= k1 + h1 = (h1 << 13) | (h1 >> 19) + h1 = h1*5+0xe6546b64u + h1 ^= 8 + h1 ^= h1 >> 16 + h1 *= 0x85ebca6bu + h1 ^= h1 >> 13 + h1 *= 0xc2b2ae35u + h1 ^= h1 >> 16 + + return h1 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/strings2arrays.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/strings2arrays.py new file mode 100644 index 0000000000000000000000000000000000000000..475e398b25ea72822faf556b5d6ef1f8ace11509 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/strings2arrays.py @@ -0,0 +1,30 @@ +from ctypes import c_uint64 +from typing import Callable, List, Sequence, Tuple + +from murmurhash import hash_unicode + +from ..config import registry +from ..model import Model +from ..types import Ints2d + +InT = Sequence[Sequence[str]] +OutT = List[Ints2d] + + +@registry.layers("strings2arrays.v1") +def strings2arrays() -> Model[InT, OutT]: + """Transform a sequence of string sequences to a list of arrays.""" + return Model("strings2arrays", forward) + + +def forward(model: Model[InT, OutT], Xs: InT, is_train: bool) -> Tuple[OutT, Callable]: + # Cast 32-bit (signed) integer to 64-bit unsigned, since such casting + # is deprecated in NumPy. + hashes = [[c_uint64(hash_unicode(word)).value for word in X] for X in Xs] + hash_arrays = [model.ops.asarray1i(h, dtype="uint64") for h in hashes] + arrays = [model.ops.reshape2i(array, -1, 1) for array in hash_arrays] + + def backprop(dX: OutT) -> InT: + return [] + + return arrays, backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/swish.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/swish.py new file mode 100644 index 0000000000000000000000000000000000000000..b0bc9a73cf038550366856741a8d0ed0132be39d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/swish.py @@ -0,0 +1,70 @@ +from typing import Callable, Optional, Tuple, cast + +from ..config import registry +from ..initializers import he_normal_init, zero_init +from ..model import Model +from ..types import Floats1d, Floats2d +from ..util import get_width, partial +from .chain import chain +from .dropout import Dropout +from .layernorm import LayerNorm + + +@registry.layers("Swish.v1") +def Swish( + nO: Optional[int] = None, + nI: Optional[int] = None, + *, + init_W: Optional[Callable] = None, + init_b: Optional[Callable] = None, + dropout: Optional[float] = None, + normalize: bool = False, +) -> Model[Floats2d, Floats2d]: + if init_W is None: + init_W = he_normal_init + if init_b is None: + init_b = zero_init + model: Model[Floats2d, Floats2d] = Model( + "swish", + forward, + init=partial(init, init_W, init_b), + dims={"nO": nO, "nI": nI}, + params={"W": None, "b": None}, + ) + if normalize: + model = chain(model, LayerNorm(nI=nO)) + if dropout is not None: + model = chain(model, cast(Model[Floats2d, Floats2d], Dropout(dropout))) + return model + + +def forward( + model: Model[Floats2d, Floats2d], X: Floats2d, is_train: bool +) -> Tuple[Floats2d, Callable]: + W = cast(Floats2d, model.get_param("W")) + b = cast(Floats1d, model.get_param("b")) + Y_preact = model.ops.affine(X, W, b) + Y = model.ops.swish(Y_preact) + + def backprop(dY: Floats2d) -> Floats2d: + dY = model.ops.backprop_swish(dY, Y_preact, Y, inplace=False) + model.inc_grad("b", dY.sum(axis=0)) + model.inc_grad("W", model.ops.gemm(dY, X, trans1=True)) + return model.ops.gemm(dY, W) + + return Y, backprop + + +def init( + init_W: Callable, + init_b: Callable, + model: Model[Floats2d, Floats2d], + X: Optional[Floats2d] = None, + Y: Optional[Floats2d] = None, +) -> None: + if X is not None: + model.set_dim("nI", get_width(X)) + if Y is not None: + model.set_dim("nO", get_width(Y)) + model.set_param("W", init_W(model.ops, (model.get_dim("nO"), model.get_dim("nI")))) + model.set_param("b", init_b(model.ops, (model.get_dim("nO"),))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tensorflowwrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tensorflowwrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..77969a957cfcfe0e36c8f6db01d6261e35f90bf7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tensorflowwrapper.py @@ -0,0 +1,183 @@ +# mypy: ignore-errors +from typing import Any, Callable, Dict, Optional, Tuple, Type, TypeVar + +import srsly + +from ..compat import tensorflow as tf +from ..model import Model +from ..shims import TensorFlowShim, keras_model_fns, maybe_handshake_model +from ..types import ArgsKwargs, ArrayXd +from ..util import ( + assert_tensorflow_installed, + convert_recursive, + is_tensorflow_array, + is_xp_array, + tensorflow2xp, + xp2tensorflow, +) + +InT = TypeVar("InT") +OutT = TypeVar("OutT") +InFunc = TypeVar("InFunc") +XType = TypeVar("XType", bound=ArrayXd) +YType = TypeVar("YType", bound=ArrayXd) + + +def keras_subclass( + name: str, + X: XType, + Y: YType, + input_shape: Tuple[int, ...], + compile_args: Optional[Dict[str, Any]] = None, +) -> Callable[[InFunc], InFunc]: + """Decorate a custom keras subclassed model with enough information to + serialize and deserialize it reliably in the face of the many restrictions + on keras subclassed models. + + name (str): The unique namespace string to use to represent this model class. + X (Any): A sample X input for performing a forward pass on the network. + Y (Any): A sample Y input for performing a backward pass on the network. + input_shape (Tuple[int, ...]): A set of input shapes for building the network. + compile: Arguments to pass directly to the keras `model.compile` call. + + RETURNS (Callable): The decorated class. + """ + + compile_defaults = {"optimizer": "adam", "loss": "mse"} + if compile_args is None: + compile_args = compile_defaults + else: + compile_args = {**compile_defaults, **compile_args} + + def call_fn(clazz): + + clazz.catalogue_name = property(lambda inst: name) + clazz.eg_shape = property(lambda inst: input_shape) + clazz.eg_compile = property(lambda inst: compile_args) + clazz.eg_x = property(lambda inst: X) + clazz.eg_y = property(lambda inst: Y) + + @keras_model_fns(name) + def create_component(*call_args, **call_kwargs): + return clazz(*call_args, **call_kwargs) + + # Capture construction args and store them on the instance + wrapped_init = clazz.__init__ + + def __init__(self, *args, **kwargs): + wrapped_init(self, *args, **kwargs) + try: + srsly.json_dumps(args) + srsly.json_dumps(kwargs) + except BaseException as _err: + raise ValueError( + "In order to serialize Keras Subclass models, the constructor " + "arguments must be serializable. This allows thinc to recreate " + "the code-based model with the same configuration.\n" + f"The encountered error is: {_err}" + ) + self.eg_args = ArgsKwargs(args, kwargs) + + clazz.__init__ = __init__ + + return clazz + + return call_fn + + +def TensorFlowWrapper( + tensorflow_model: Any, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, + optimizer: Optional[Any] = None, + model_class: Type[Model] = Model, + model_name: str = "tensorflow", +) -> Model[InT, OutT]: + """Wrap a TensorFlow model, so that it has the same API as Thinc models. + To optimize the model, you'll need to create a TensorFlow optimizer and call + optimizer.apply_gradients after each batch. + """ + assert_tensorflow_installed() + if not isinstance(tensorflow_model, tf.keras.models.Model): + err = f"Expected tf.keras.models.Model, got: {type(tensorflow_model)}" + raise ValueError(err) + tensorflow_model = maybe_handshake_model(tensorflow_model) + if convert_inputs is None: + convert_inputs = _convert_inputs + if convert_outputs is None: + convert_outputs = _convert_outputs + return model_class( + model_name, + forward, + shims=[TensorFlowShim(tensorflow_model, optimizer=optimizer)], + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + """Return the output of the wrapped TensorFlow model for the given input, + along with a callback to handle the backward pass. + """ + convert_inputs = model.attrs["convert_inputs"] + convert_outputs = model.attrs["convert_outputs"] + tensorflow_model = model.shims[0] + X_tensorflow, get_dX = convert_inputs(model, X, is_train) + if is_train: + Y_tensorflow, tensorflow_backprop = tensorflow_model(X_tensorflow, is_train) + else: + Y_tensorflow = tensorflow_model(X_tensorflow, is_train) + Y, get_dY_tensorflow = convert_outputs(model, Y_tensorflow, is_train) + + def backprop(dY: OutT) -> InT: + dY_tensorflow = get_dY_tensorflow(dY) + dX_tensorflow = tensorflow_backprop(dY_tensorflow) + return get_dX(dX_tensorflow) + + return Y, backprop + + +# Default conversion functions +# These are pretty much the same as the PyTorch one, but I think we should +# leave the duplication -- I think the abstraction could get pretty messy, +# and then may need to be undone, as there can always be different specifics. + + +def _convert_inputs(model, X, is_train): + xp2tensorflow_ = lambda x: xp2tensorflow(x, requires_grad=is_train) + converted = convert_recursive(is_xp_array, xp2tensorflow_, X) + if isinstance(converted, ArgsKwargs): + + def reverse_conversion(dXtf): + return convert_recursive(is_tensorflow_array, tensorflow2xp, dXtf) + + return converted, reverse_conversion + elif isinstance(converted, dict): + + def reverse_conversion(dXtf): + dX = convert_recursive(is_tensorflow_array, tensorflow2xp, dXtf) + return dX.kwargs + + return ArgsKwargs(args=tuple(), kwargs=converted), reverse_conversion + elif isinstance(converted, (tuple, list)): + + def reverse_conversion(dXtf): + dX = convert_recursive(is_tensorflow_array, tensorflow2xp, dXtf) + return dX.args + + return ArgsKwargs(args=converted, kwargs={}), reverse_conversion + else: + + def reverse_conversion(dXtf): + dX = convert_recursive(is_tensorflow_array, tensorflow2xp, dXtf) + return dX.args[0] + + return ArgsKwargs(args=(converted,), kwargs={}), reverse_conversion + + +def _convert_outputs(model, Ytf, is_train): + Y = convert_recursive(is_tensorflow_array, tensorflow2xp, Ytf) + + def reverse_conversion(dY): + return convert_recursive(is_xp_array, xp2tensorflow, dY) + + return Y, reverse_conversion diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/torchscriptwrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/torchscriptwrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..42527f6312f1def3c28c8919c5909ca578d5d833 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/torchscriptwrapper.py @@ -0,0 +1,93 @@ +from typing import Any, Callable, Optional + +from ..compat import torch +from ..model import Model +from ..shims import PyTorchGradScaler, PyTorchShim, TorchScriptShim +from .pytorchwrapper import ( + convert_pytorch_default_inputs, + convert_pytorch_default_outputs, + forward, +) + + +def TorchScriptWrapper_v1( + torchscript_model: Optional["torch.jit.ScriptModule"] = None, + convert_inputs: Optional[Callable] = None, + convert_outputs: Optional[Callable] = None, + mixed_precision: bool = False, + grad_scaler: Optional[PyTorchGradScaler] = None, + device: Optional["torch.device"] = None, +) -> Model[Any, Any]: + """Wrap a TorchScript model, so that it has the same API as Thinc models. + + torchscript_model: + The TorchScript module. A value of `None` is also possible to + construct a shim to deserialize into. + convert_inputs: + Function that converts inputs and gradients that should be passed + to the model to Torch tensors. + convert_outputs: + Function that converts model outputs and gradients from Torch tensors + Thinc arrays. + mixed_precision: + Enable mixed-precision. This changes whitelisted ops to run + in half precision for better performance and lower memory use. + grad_scaler: + The gradient scaler to use for mixed-precision training. If this + argument is set to "None" and mixed precision is enabled, a gradient + scaler with the default configuration is used. + device: + The PyTorch device to run the model on. When this argument is + set to "None", the default device for the currently active Thinc + ops is used. + """ + + if convert_inputs is None: + convert_inputs = convert_pytorch_default_inputs + if convert_outputs is None: + convert_outputs = convert_pytorch_default_outputs + + return Model( + "pytorch_script", + forward, + attrs={"convert_inputs": convert_inputs, "convert_outputs": convert_outputs}, + shims=[ + TorchScriptShim( + model=torchscript_model, + mixed_precision=mixed_precision, + grad_scaler=grad_scaler, + device=device, + ) + ], + dims={"nI": None, "nO": None}, + ) + + +def pytorch_to_torchscript_wrapper(model: Model): + """Convert a PyTorch wrapper to a TorchScript wrapper. The embedded PyTorch + `Module` is converted to `ScriptModule`. + """ + shim = model.shims[0] + if not isinstance(shim, PyTorchShim): + raise ValueError("Expected PyTorchShim when converting a PyTorch wrapper") + + convert_inputs = model.attrs["convert_inputs"] + convert_outputs = model.attrs["convert_outputs"] + + pytorch_model = shim._model + if not isinstance(pytorch_model, torch.nn.Module): + raise ValueError("PyTorchShim does not wrap a PyTorch module") + + torchscript_model = torch.jit.script(pytorch_model) + grad_scaler = shim._grad_scaler + mixed_precision = shim._mixed_precision + device = shim.device + + return TorchScriptWrapper_v1( + torchscript_model, + convert_inputs=convert_inputs, + convert_outputs=convert_outputs, + mixed_precision=mixed_precision, + grad_scaler=grad_scaler, + device=device, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tuplify.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tuplify.py new file mode 100644 index 0000000000000000000000000000000000000000..70f5e6785636cc2b221124bd447eb3522e3bb4c5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/tuplify.py @@ -0,0 +1,70 @@ +from typing import Any, Optional, Tuple, TypeVar + +from ..config import registry +from ..model import Model + +InT = TypeVar("InT") +OutT = Tuple + + +@registry.layers("tuplify.v1") +def tuplify( + layer1: Model[InT, Any], layer2: Model[InT, Any], *layers +) -> Model[InT, Tuple]: + """Send a separate copy of the input to each child layer, and join the + outputs of the children into a tuple on the way out. + + Typically used to provide both modified data and the original input to a + downstream layer. + """ + + layers = (layer1, layer2) + layers + names = [layer.name for layer in layers] + return Model( + "tuple(" + ", ".join(names) + ")", + tuplify_forward, + init=init, + layers=layers, + dims={"nI": None}, + ) + + +def tuplify_forward(model, X, is_train): + Ys = [] + backprops = [] + for layer in model.layers: + Y, backprop = layer(X, is_train) + Ys.append(Y) + backprops.append(backprop) + + def backprop_tuplify(dYs): + dXs = [bp(dY) for bp, dY in zip(backprops, dYs)] + dX = dXs[0] + for dx in dXs[1:]: + dX += dx + return dX + + return tuple(Ys), backprop_tuplify + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + if X is None and Y is None: + for layer in model.layers: + layer.initialize() + if model.layers[0].has_dim("nI"): + model.set_dim("nI", model.layers[0].get_dim("nI")) + + # Try to set nO on each layer, where available. + # All layers have the same input, and the output should map directly from the + # given Y, if provided. + for ii, layer in enumerate(model.layers): + if Y is not None and layer.has_dim("nO") is None: + layer.initialize(X=X, Y=Y[ii]) + else: + layer.initialize(X=X) + + if model.layers[0].has_dim("nI"): + model.set_dim("nI", model.layers[0].get_dim("nI")) + # this model can have an input dimension, but can't have an output dimension diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/uniqued.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/uniqued.py new file mode 100644 index 0000000000000000000000000000000000000000..ee457bbb022237130a2782e17ce4d167b1105f88 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/uniqued.py @@ -0,0 +1,65 @@ +from typing import Callable, Optional, Tuple + +import numpy + +from ..config import registry +from ..model import Model +from ..types import Floats2d, Ints2d + +InT = Ints2d +OutT = Floats2d + + +@registry.layers("uniqued.v1") +def uniqued(layer: Model, *, column: int = 0) -> Model[InT, OutT]: + """Group inputs to a layer, so that the layer only has to compute for the + unique values. The data is transformed back before output, and the same + transformation is applied for the gradient. Effectively, this is a cache + local to each minibatch. + """ + return Model( + f"uniqued({layer.name})", + forward, + init=init, + layers=[layer], + dims={"nO": None, "nI": None}, + attrs={"column": column}, + ) + + +def forward(model: Model[InT, OutT], X: InT, is_train: bool) -> Tuple[OutT, Callable]: + column: int = model.attrs["column"] + layer = model.layers[0] + if X.size < 2: + return layer(X, is_train) + keys = X[:, column] + if not isinstance(keys, numpy.ndarray): + keys = keys.get() # pragma: no cover + uniq_keys, ind, inv, counts = layer.ops.xp.unique( + keys, return_index=True, return_inverse=True, return_counts=True + ) + counts = model.ops.reshape2i(counts, -1, 1) + X_uniq = X[ind] + Y_uniq, bp_Y_uniq = layer(X_uniq, is_train) + Y = Y_uniq[inv].reshape((X.shape[0],) + Y_uniq.shape[1:]) + uniq_shape = tuple(Y_uniq.shape) + + def backprop(dY: OutT) -> InT: + dY_uniq = layer.ops.alloc2f(*uniq_shape) + layer.ops.scatter_add(dY_uniq, layer.ops.asarray_i(inv), dY) + d_uniques = bp_Y_uniq(dY_uniq) + # This confusing bit of indexing "ununiques" + return (d_uniques / counts)[inv] + + return Y, backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + layer = model.layers[0] + layer.initialize(X=X, Y=Y) + if layer.has_dim("nI"): + model.set_dim("nI", layer.get_dim("nI")) # pragma: no cover + if layer.has_dim("nO"): + model.set_dim("nO", layer.get_dim("nO")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array.py new file mode 100644 index 0000000000000000000000000000000000000000..30511889b782497be4ed81d0c5cd559cf47f2132 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array.py @@ -0,0 +1,110 @@ +from typing import Callable, Optional, Tuple, TypeVar, Union, cast + +from ..backends import NumpyOps +from ..config import registry +from ..model import Model +from ..types import Array3d, ArrayXd, ListXd, Padded, Ragged + +NUMPY_OPS = NumpyOps() + + +ArrayTXd = TypeVar("ArrayTXd", bound=ArrayXd) +SeqT = TypeVar("SeqT", bound=Union[Padded, Ragged, ListXd, ArrayXd]) + + +@registry.layers("with_array.v1") +def with_array(layer: Model[ArrayTXd, ArrayTXd], pad: int = 0) -> Model[SeqT, SeqT]: + """Transform sequence data into a contiguous array on the way into and + out of a model. Handles a variety of sequence types: lists, padded and ragged. + If the input is an array, it is passed through unchanged. + """ + model: Model[SeqT, SeqT] = Model( + f"with_array({layer.name})", + forward, + init=init, + layers=[layer], + attrs={"pad": pad}, + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + return model + + +def forward( + model: Model[SeqT, SeqT], Xseq: SeqT, is_train: bool +) -> Tuple[SeqT, Callable]: + if isinstance(Xseq, Ragged): + return cast(Tuple[SeqT, Callable], _ragged_forward(model, Xseq, is_train)) + elif isinstance(Xseq, Padded): + return cast(Tuple[SeqT, Callable], _padded_forward(model, Xseq, is_train)) + elif not isinstance(Xseq, (list, tuple)): + return model.layers[0](Xseq, is_train) + else: + return cast(Tuple[SeqT, Callable], _list_forward(model, Xseq, is_train)) + + +def init( + model: Model[SeqT, SeqT], X: Optional[SeqT] = None, Y: Optional[SeqT] = None +) -> None: + layer: Model[ArrayXd, ArrayXd] = model.layers[0] + layer.initialize( + X=_get_array(model, X) if X is not None else X, + Y=_get_array(model, Y) if Y is not None else Y, + ) + for dim_name in layer.dim_names: + value = layer.maybe_get_dim(dim_name) + if value is not None: + model.set_dim(dim_name, value) + + +def _get_array(model, X: SeqT) -> ArrayXd: + if isinstance(X, Ragged): + return X.dataXd + elif isinstance(X, Padded): + return X.data + elif not isinstance(X, (list, tuple)): + return cast(ArrayXd, X) + else: + return model.ops.flatten(X) + + +def _list_forward( + model: Model[SeqT, SeqT], Xs: ListXd, is_train: bool +) -> Tuple[ListXd, Callable]: + layer: Model[ArrayXd, ArrayXd] = model.layers[0] + pad = model.attrs["pad"] + lengths = NUMPY_OPS.asarray1i([len(seq) for seq in Xs]) + Xf = layer.ops.flatten(Xs, pad=pad) + Yf, get_dXf = layer(Xf, is_train) + + def backprop(dYs: ListXd) -> ListXd: + dYf = layer.ops.flatten(dYs, pad=pad) + dXf = get_dXf(dYf) + return layer.ops.unflatten(dXf, lengths, pad=pad) + + return layer.ops.unflatten(Yf, lengths, pad=pad), backprop + + +def _ragged_forward( + model: Model[SeqT, SeqT], Xr: Ragged, is_train: bool +) -> Tuple[Ragged, Callable]: + layer: Model[ArrayXd, ArrayXd] = model.layers[0] + Y, get_dX = layer(Xr.dataXd, is_train) + + def backprop(dYr: Ragged) -> Ragged: + return Ragged(get_dX(dYr.dataXd), dYr.lengths) + + return Ragged(Y, Xr.lengths), backprop + + +def _padded_forward( + model: Model[SeqT, SeqT], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + layer: Model[Array3d, Array3d] = model.layers[0] + Y, get_dX = layer(Xp.data, is_train) + + def backprop(dYp: Padded) -> Padded: + assert isinstance(dYp, Padded) + dX = get_dX(dYp.data) + return Padded(dX, dYp.size_at_t, dYp.lengths, dYp.indices) + + return Padded(Y, Xp.size_at_t, Xp.lengths, Xp.indices), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array2d.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array2d.py new file mode 100644 index 0000000000000000000000000000000000000000..f0e63b3fa9f37860616aa7b48c1426274144d10f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_array2d.py @@ -0,0 +1,125 @@ +from typing import Callable, List, Optional, Tuple, TypeVar, Union, cast + +from ..backends import NumpyOps +from ..config import registry +from ..model import Model +from ..types import Array2d, Floats2d, List2d, Padded, Ragged + +NUMPY_OPS = NumpyOps() + + +ValT = TypeVar("ValT", bound=Array2d) +SeqT = TypeVar("SeqT", bound=Union[Padded, Ragged, List2d, Array2d]) + + +@registry.layers("with_array2d.v1") +def with_array2d(layer: Model[ValT, ValT], pad: int = 0) -> Model[SeqT, SeqT]: + """Transform sequence data into a contiguous 2d array on the way into and + out of a model. Handles a variety of sequence types: lists, padded and ragged. + If the input is a 2d array, it is passed through unchanged. + """ + return Model( + f"with_array({layer.name})", + forward, + init=init, + layers=[layer], + attrs={"pad": pad}, + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + + +def forward( + model: Model[SeqT, SeqT], Xseq: SeqT, is_train: bool +) -> Tuple[SeqT, Callable]: + if isinstance(Xseq, Ragged): + return cast(Tuple[SeqT, Callable], _ragged_forward(model, Xseq, is_train)) + elif isinstance(Xseq, Padded): + return cast(Tuple[SeqT, Callable], _padded_forward(model, Xseq, is_train)) + elif not isinstance(Xseq, (list, tuple)): + return model.layers[0](Xseq, is_train) + else: + return cast(Tuple[SeqT, Callable], _list_forward(model, Xseq, is_train)) + return + + +def init( + model: Model[SeqT, SeqT], X: Optional[SeqT] = None, Y: Optional[SeqT] = None +) -> None: + layer: Model[Array2d, Array2d] = model.layers[0] + layer.initialize( + X=_get_array(model, X) if X is not None else X, + Y=_get_array(model, Y) if Y is not None else Y, + ) + for dim_name in layer.dim_names: + value = layer.maybe_get_dim(dim_name) + if value is not None: + model.set_dim(dim_name, value) + + +def _get_array(model, X: SeqT) -> Array2d: + if isinstance(X, Ragged): + return X.data + elif isinstance(X, Padded): + return model.ops.reshape2f( + X.data, X.data.shape[0] * X.data.shape[1], X.data.shape[2] + ) + elif not isinstance(X, (list, tuple)): + return cast(Array2d, X) + else: + return model.ops.flatten(X) + + +def _list_forward( + model: Model[SeqT, SeqT], Xs: List2d, is_train: bool +) -> Tuple[List2d, Callable]: + layer: Model[Array2d, Array2d] = model.layers[0] + pad = model.attrs["pad"] + lengths = NUMPY_OPS.asarray1i([len(seq) for seq in Xs]) + Xf = layer.ops.flatten(Xs, pad=pad) + Yf, get_dXf = layer(Xf, is_train) + + def backprop(dYs: List2d) -> List2d: + dYf = layer.ops.flatten(dYs, pad=pad) + dXf = get_dXf(dYf) + return layer.ops.unflatten(dXf, lengths, pad=pad) + + return layer.ops.unflatten(Yf, lengths, pad=pad), backprop + + +def _ragged_forward( + model: Model[SeqT, SeqT], Xr: Ragged, is_train: bool +) -> Tuple[Ragged, Callable]: + layer: Model[Array2d, Array2d] = model.layers[0] + Y, get_dX = layer(Xr.data, is_train) + x_shape = Xr.dataXd.shape + + def backprop(dYr: Ragged) -> Ragged: + return Ragged(get_dX(dYr.dataXd).reshape(x_shape), dYr.lengths) + + return Ragged(Y, Xr.lengths), backprop + + +def _padded_forward( + model: Model[SeqT, SeqT], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + layer: Model[Array2d, Array2d] = model.layers[0] + X = model.ops.reshape2( + Xp.data, Xp.data.shape[0] * Xp.data.shape[1], Xp.data.shape[2] + ) + Y2d, get_dX = layer(X, is_train) + Y = model.ops.reshape3f( + cast(Floats2d, Y2d), Xp.data.shape[0], Xp.data.shape[1], Y2d.shape[1] + ) + + def backprop(dYp: Padded) -> Padded: + assert isinstance(dYp, Padded) + dY = model.ops.reshape2( + dYp.data, dYp.data.shape[0] * dYp.data.shape[1], dYp.data.shape[2] + ) + dX2d = get_dX(dY) + dX = model.ops.reshape3f( + dX2d, dYp.data.shape[0], dYp.data.shape[1], dX2d.shape[1] + ) + return Padded(dX, dYp.size_at_t, dYp.lengths, dYp.indices) + + return Padded(Y, Xp.size_at_t, Xp.lengths, Xp.indices), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_cpu.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_cpu.py new file mode 100644 index 0000000000000000000000000000000000000000..c5b9f61ccb8cc97a5d7254efcfadb9c7ba7c47de --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_cpu.py @@ -0,0 +1,58 @@ +from typing import Any, Callable, Tuple + +import numpy + +from thinc.backends import Ops + +from ..config import registry +from ..model import Model + + +@registry.layers("with_cpu.v1") +def with_cpu(layer: Model, ops: Ops) -> Model: + layer.to_cpu() + return Model( + f"with_cpu({layer.name})", + forward, + layers=[layer], + ops=ops, + init=init, + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + + +def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + cpu_outputs, backprop = model.layers[0].begin_update(_to_cpu(X)) + gpu_outputs = _to_device(model.ops, cpu_outputs) + + def with_cpu_backprop(d_outputs): + cpu_d_outputs = _to_cpu(d_outputs) + return backprop(cpu_d_outputs) + + return gpu_outputs, with_cpu_backprop + + +def init(model: Model, X: Any, Y: Any) -> None: + model.layers[0].initialize(X, Y) + + +def _to_cpu(X): + if isinstance(X, numpy.ndarray): + return X + elif isinstance(X, tuple): + return tuple([_to_cpu(x) for x in X]) + elif isinstance(X, list): + return [_to_cpu(x) for x in X] + elif hasattr(X, "get"): + return X.get() + else: + return X + + +def _to_device(ops, X): + if isinstance(X, tuple): + return tuple([_to_device(ops, x) for x in X]) + elif isinstance(X, list): + return [_to_device(ops, x) for x in X] + else: + return ops.asarray(X) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_debug.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_debug.py new file mode 100644 index 0000000000000000000000000000000000000000..94072cad5de6d6cec9612063244ac972bf524ff6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_debug.py @@ -0,0 +1,44 @@ +from typing import Any, Callable, Optional, Tuple, TypeVar + +from ..model import Model + +_ModelT = TypeVar("_ModelT", bound=Model) + +do_nothing = lambda *args, **kwargs: None + + +def with_debug( + layer: _ModelT, + name: Optional[str] = None, + *, + on_init: Callable[[Model, Any, Any], None] = do_nothing, + on_forward: Callable[[Model, Any, bool], None] = do_nothing, + on_backprop: Callable[[Any], None] = do_nothing, +) -> _ModelT: + """Debugging layer that wraps any layer and allows executing callbacks + during the forward pass, backward pass and initialization. The callbacks + will receive the same arguments as the functions they're called in. + """ + name = layer.name if name is None else name + + orig_forward = layer._func + orig_init = layer.init + + def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + on_forward(model, X, is_train) + layer_Y, layer_callback = orig_forward(layer, X, is_train=is_train) + + def backprop(dY: Any) -> Any: + on_backprop(dY) + return layer_callback(dY) + + return layer_Y, backprop + + def init(model: Model, X: Any, Y: Any) -> None: + on_init(model, X, Y) + if orig_init is not None: + orig_init(layer, X, Y) + + layer.replace_callbacks(forward, init=init) + + return layer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten.py new file mode 100644 index 0000000000000000000000000000000000000000..dd042f9962704cf72fbd3858994f974c77bfd1f3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten.py @@ -0,0 +1,53 @@ +from typing import Any, Callable, List, Optional, Sequence, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import ArrayXd, ListXd + +ItemT = TypeVar("ItemT") +InT = Sequence[Sequence[ItemT]] +OutT = ListXd +InnerInT = Sequence[ItemT] +InnerOutT = ArrayXd + + +@registry.layers("with_flatten.v1") +def with_flatten(layer: Model[InnerInT[ItemT], InnerOutT]) -> Model[InT[ItemT], OutT]: + return Model(f"with_flatten({layer.name})", forward, layers=[layer], init=init) + + +def forward( + model: Model[InT, OutT], Xnest: InT, is_train: bool +) -> Tuple[OutT, Callable]: + layer: Model[InnerInT, InnerOutT] = model.layers[0] + Xflat = _flatten(Xnest) + Yflat, backprop_layer = layer(Xflat, is_train) + # Get the split points. We want n-1 splits for n items. + arr = layer.ops.asarray1i([len(x) for x in Xnest[:-1]]) + splits = arr.cumsum() + Ynest = layer.ops.xp.split(Yflat, splits, axis=0) + + def backprop(dYnest: OutT) -> InT: + dYflat = model.ops.flatten(dYnest) # type: ignore[arg-type, var-annotated] + # type ignore necessary for older versions of Mypy/Pydantic + dXflat = backprop_layer(dYflat) + dXnest = layer.ops.xp.split(dXflat, splits, axis=-1) + return dXnest + + return Ynest, backprop + + +def _flatten(nested: InT) -> InnerInT: + flat: List = [] + for item in nested: + flat.extend(item) + return cast(InT, flat) + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + model.layers[0].initialize( + _flatten(X) if X is not None else None, + model.layers[0].ops.xp.hstack(Y) if Y is not None else None, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten_v2.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..e63b4a92fb5d34fcc6bc79644942e339b50f3fc7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_flatten_v2.py @@ -0,0 +1,66 @@ +from typing import Any, Callable, List, Optional, Sequence, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model + +InItemT = TypeVar("InItemT") +OutItemT = TypeVar("OutItemT") +ItemT = TypeVar("ItemT") + +NestedT = List[List[ItemT]] +FlatT = List[ItemT] + + +@registry.layers("with_flatten.v2") +def with_flatten_v2( + layer: Model[FlatT[InItemT], FlatT[OutItemT]] +) -> Model[NestedT[InItemT], NestedT[OutItemT]]: + return Model(f"with_flatten({layer.name})", forward, layers=[layer], init=init) + + +def forward( + model: Model[NestedT[InItemT], NestedT[OutItemT]], + Xnest: NestedT[InItemT], + is_train: bool, +) -> Tuple[NestedT[OutItemT], Callable]: + layer: Model[FlatT[InItemT], FlatT[OutItemT]] = model.layers[0] + Xflat, lens = _flatten(Xnest) + Yflat, backprop_layer = layer(Xflat, is_train) + Ynest = _unflatten(Yflat, lens) + + def backprop(dYnest: NestedT[InItemT]) -> NestedT[OutItemT]: + dYflat, _ = _flatten(dYnest) # type: ignore[arg-type, var-annotated] + # type ignore necessary for older versions of Mypy/Pydantic + dXflat = backprop_layer(dYflat) + dXnest = _unflatten(dXflat, lens) + return dXnest + + return Ynest, backprop + + +def _flatten(nested: NestedT[ItemT]) -> Tuple[FlatT[ItemT], List[int]]: + flat: List = [] + lens: List[int] = [] + for item in nested: + flat.extend(item) + lens.append(len(item)) + return cast(FlatT[ItemT], flat), lens + + +def _unflatten(flat: FlatT[ItemT], lens: List[int]) -> NestedT[ItemT]: + nested = [] + for l in lens: + nested.append(flat[:l]) + flat = flat[l:] + return nested + + +def init( + model: Model[NestedT[InItemT], NestedT[OutItemT]], + X: Optional[NestedT[InItemT]] = None, + Y: Optional[NestedT[OutItemT]] = None, +) -> None: + model.layers[0].initialize( + _flatten(X)[0] if X is not None else None, + model.layers[0].ops.xp.hstack(Y) if Y is not None else None, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_getitem.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_getitem.py new file mode 100644 index 0000000000000000000000000000000000000000..89c67dffad2245c350dded6d3ae0169154ffed00 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_getitem.py @@ -0,0 +1,43 @@ +from typing import Any, Callable, Optional, Tuple + +from ..config import registry +from ..model import Model + +InT = Tuple[Any, ...] +OutT = Tuple[Any, ...] + + +@registry.layers("with_getitem.v1") +def with_getitem(idx: int, layer: Model) -> Model[InT, OutT]: + """Transform data on the way into and out of a layer, by plucking an item + from a tuple. + """ + return Model( + f"with_getitem({layer.name})", + forward, + init=init, + layers=[layer], + attrs={"idx": idx}, + ) + + +def forward( + model: Model[InT, OutT], items: InT, is_train: bool +) -> Tuple[OutT, Callable]: + idx = model.attrs["idx"] + Y_i, backprop_item = model.layers[0](items[idx], is_train) + + def backprop(d_output: OutT) -> InT: + dY_i = backprop_item(d_output[idx]) + return d_output[:idx] + (dY_i,) + d_output[idx + 1 :] + + return items[:idx] + (Y_i,) + items[idx + 1 :], backprop + + +def init( + model: Model[InT, OutT], X: Optional[InT] = None, Y: Optional[OutT] = None +) -> None: + idx = model.attrs["idx"] + X_i = X[idx] if X is not None else X + Y_i = Y[idx] if Y is not None else Y + model.layers[0].initialize(X=X_i, Y=Y_i) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_list.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_list.py new file mode 100644 index 0000000000000000000000000000000000000000..58c09e2c1a6a39945c653c1d35e2566495375676 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_list.py @@ -0,0 +1,87 @@ +from typing import Callable, List, Optional, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..model import Model +from ..types import Array2d, Floats2d, Ints2d, List2d, Padded, Ragged + +SeqT = TypeVar("SeqT", Padded, Ragged, List2d, List[Floats2d], List[Ints2d]) + + +@registry.layers("with_list.v1") +def with_list(layer: Model[List2d, List2d]) -> Model[SeqT, SeqT]: + return Model( + f"with_list({layer.name})", + forward, + init=init, + layers=[layer], + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + + +def forward( + model: Model[SeqT, SeqT], Xseq: SeqT, is_train: bool +) -> Tuple[SeqT, Callable]: + layer: Model[List2d, List2d] = model.layers[0] + if isinstance(Xseq, Padded): + return _padded_forward(layer, Xseq, is_train) + elif isinstance(Xseq, Ragged): + return _ragged_forward(layer, Xseq, is_train) + else: + return cast(Tuple[SeqT, Callable], layer(cast(List2d, Xseq), is_train)) + + +def init( + model: Model[SeqT, SeqT], X: Optional[SeqT] = None, Y: Optional[SeqT] = None +) -> None: + model.layers[0].initialize( + X=_get_list(model, X) if X is not None else None, + Y=_get_list(model, Y) if Y is not None else None, + ) + + +def _get_list(model, seq): + if isinstance(seq, Padded): + return model.ops.padded2list(seq) + elif isinstance(seq, Ragged): + return model.ops.unflatten(seq.data, seq.lengths) + else: + return seq + + +def _ragged_forward( + layer: Model[List2d, List2d], Xr: Ragged, is_train: bool +) -> Tuple[Ragged, Callable]: + # Assign these to locals, to keep code a bit shorter. + unflatten = layer.ops.unflatten + flatten = layer.ops.flatten + # It's worth being a bit careful about memory here, as the activations + # are potentially large on GPU. So we make nested function calls instead + # of assigning to temporaries where possible, so memory can be reclaimed + # sooner. + Ys, get_dXs = layer(unflatten(Xr.data, Xr.lengths), is_train) + + def backprop(dYr: Ragged): + return Ragged( + flatten(get_dXs(unflatten(dYr.data, dYr.lengths))), + dYr.lengths, + ) + + return Ragged(flatten(Ys), Xr.lengths), backprop + + +def _padded_forward( + layer: Model[List2d, List2d], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + # Assign these to locals, to keep code a bit shorter. + padded2list = layer.ops.padded2list + list2padded = layer.ops.list2padded + # It's worth being a bit careful about memory here, as the activations + # are potentially large on GPU. So we make nested function calls instead + # of assigning to temporaries where possible, so memory can be reclaimed + # sooner. + Ys, get_dXs = layer(padded2list(Xp), is_train) + + def backprop(dYp): + return list2padded(get_dXs(padded2list(dYp))) + + return list2padded(Ys), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_nvtx_range.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_nvtx_range.py new file mode 100644 index 0000000000000000000000000000000000000000..da0fab76e47bed5280463a7e735129fc31ea3c46 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_nvtx_range.py @@ -0,0 +1,45 @@ +from typing import Any, Callable, Optional, Tuple, TypeVar + +from ..model import Model +from ..util import use_nvtx_range + +_ModelT = TypeVar("_ModelT", bound=Model) + + +def with_nvtx_range( + layer: _ModelT, + name: Optional[str] = None, + *, + forward_color: int = -1, + backprop_color: int = -1, +) -> _ModelT: + """Wraps any layer and marks the forward and backprop phases as + NVTX ranges for CUDA profiling. + + By default, the name of the layer is used as the name of the range, + followed by the name of the pass. + """ + name = layer.name if name is None else name + + orig_forward = layer._func + orig_init = layer.init + + def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + with use_nvtx_range(f"{name} forward", forward_color): + layer_Y, layer_callback = orig_forward(model, X, is_train=is_train) + + def backprop(dY: Any) -> Any: + with use_nvtx_range(f"{name} backprop", backprop_color): + return layer_callback(dY) + + return layer_Y, backprop + + def init(_model: Model, X: Any, Y: Any) -> Model: + if orig_init is not None: + return orig_init(layer, X, Y) + else: + return layer + + layer.replace_callbacks(forward, init=init) + + return layer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_padded.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_padded.py new file mode 100644 index 0000000000000000000000000000000000000000..d89093392dfad48dfa9e607626e5ce0f3b063e9a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_padded.py @@ -0,0 +1,143 @@ +from typing import Callable, List, Optional, Tuple, TypeVar, Union, cast + +from ..config import registry +from ..model import Model +from ..types import Array2d, Floats3d, Ints1d, List2d, Padded, Ragged +from ..util import is_xp_array + +PaddedData = Tuple[Floats3d, Ints1d, Ints1d, Ints1d] +SeqT = TypeVar("SeqT", bound=Union[Padded, Ragged, List2d, Floats3d, PaddedData]) + + +@registry.layers("with_padded.v1") +def with_padded(layer: Model[Padded, Padded]) -> Model[SeqT, SeqT]: + return Model( + f"with_padded({layer.name})", + forward, + init=init, + layers=[layer], + dims={name: layer.maybe_get_dim(name) for name in layer.dim_names}, + ) + + +def forward( + model: Model[SeqT, SeqT], Xseq: SeqT, is_train: bool +) -> Tuple[SeqT, Callable]: + layer: Model[Padded, Padded] = model.layers[0] + if isinstance(Xseq, Padded): + return cast(Tuple[SeqT, Callable], layer(Xseq, is_train)) + elif isinstance(Xseq, Ragged): + return cast(Tuple[SeqT, Callable], _ragged_forward(layer, Xseq, is_train)) + elif _is_padded_data(Xseq): + return cast( + Tuple[SeqT, Callable], + _tuple_forward(layer, cast(PaddedData, Xseq), is_train), + ) + elif is_xp_array(Xseq): + return cast( + Tuple[SeqT, Callable], _array_forward(layer, cast(Floats3d, Xseq), is_train) + ) + else: + return cast( + Tuple[SeqT, Callable], _list_forward(layer, cast(List2d, Xseq), is_train) + ) + + +def init( + model: Model[SeqT, SeqT], X: Optional[SeqT] = None, Y: Optional[SeqT] = None +) -> None: + model.layers[0].initialize( + X=_get_padded(model, X) if X is not None else None, + Y=_get_padded(model, Y) if Y is not None else None, + ) + + +def _is_padded_data(seq: SeqT) -> bool: + return isinstance(seq, tuple) and len(seq) == 4 and all(map(is_xp_array, seq)) + + +def _get_padded(model: Model, seq: SeqT) -> Padded: + if isinstance(seq, Padded): + return seq + elif isinstance(seq, Ragged): + return model.ops.list2padded(model.ops.unflatten(seq.data, seq.lengths)) + elif _is_padded_data(seq): + return Padded(*seq) # type: ignore[misc] + elif is_xp_array(seq): + floats3d_seq = cast(Floats3d, seq) + size_at_t = model.ops.asarray1i([floats3d_seq.shape[1]] * floats3d_seq.shape[0]) + lengths = model.ops.asarray1i([floats3d_seq.shape[0]] * floats3d_seq.shape[1]) + indices = model.ops.xp.arange(floats3d_seq.shape[1]) + return Padded(floats3d_seq, size_at_t, lengths, indices) + else: + assert isinstance(seq, list), seq + return model.ops.list2padded(seq) + + +def _array_forward( + layer: Model[Padded, Padded], X: Floats3d, is_train +) -> Tuple[Floats3d, Callable]: + # Create bogus metadata for Padded. + Xp = _get_padded(layer, X) + Yp, get_dXp = layer(Xp, is_train) + size_at_t = Xp.size_at_t + lengths = Xp.lengths + indices = Xp.indices + + def backprop(dY: Floats3d) -> Floats3d: + dYp = Padded(dY, size_at_t, lengths, indices) + dXp = get_dXp(dYp) + return dXp.data + + return cast(Floats3d, Yp.data), backprop + + +def _tuple_forward( + layer: Model[Padded, Padded], X: PaddedData, is_train: bool +) -> Tuple[PaddedData, Callable]: + Yp, get_dXp = layer(Padded(*X), is_train) + + def backprop(dY): + dXp = get_dXp(Padded(*dY)) + return (dXp.data, dXp.size_at_t, dXp.lengths, dXp.indices) + + return (cast(Floats3d, Yp.data), Yp.size_at_t, Yp.lengths, Yp.indices), backprop + + +def _ragged_forward( + layer: Model[Padded, Padded], Xr: Ragged, is_train: bool +) -> Tuple[Ragged, Callable]: + # Assign these to locals, to keep code a bit shorter. + list2padded = layer.ops.list2padded + padded2list = layer.ops.padded2list + unflatten = layer.ops.unflatten + flatten = layer.ops.flatten + # It's worth being a bit careful about memory here, as the activations + # are potentially large on GPU. So we make nested function calls instead + # of assigning to temporaries where possible, so memory can be reclaimed + # sooner. + Yp, get_dXp = layer(list2padded(unflatten(Xr.data, Xr.lengths)), is_train) + + def backprop(dYr: Ragged): + flattened = flatten( + padded2list(get_dXp(list2padded(unflatten(dYr.data, dYr.lengths)))), + ) + return Ragged(flattened, dYr.lengths) + + flattened = flatten(padded2list(Yp)) + return Ragged(flattened, Xr.lengths), backprop + + +def _list_forward( + layer: Model[Padded, Padded], Xs: List2d, is_train: bool +) -> Tuple[List2d, Callable]: + # Assign these to locals, to keep code a bit shorter. + list2padded = layer.ops.list2padded + padded2list = layer.ops.padded2list + + Yp, get_dXp = layer(list2padded(Xs), is_train) + + def backprop(dYs): + return padded2list(get_dXp(list2padded(dYs))) + + return padded2list(Yp), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_ragged.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_ragged.py new file mode 100644 index 0000000000000000000000000000000000000000..916b8e96fac382b814e3310e43394fe1ef632ba4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_ragged.py @@ -0,0 +1,128 @@ +from typing import Callable, List, Optional, Tuple, TypeVar, Union, cast + +from ..backends import NumpyOps +from ..config import registry +from ..model import Model +from ..types import Array2d, Ints1d, List2d, ListXd, Padded, Ragged + +NUMPY_OPS = NumpyOps() + + +RaggedData = Tuple[Array2d, Ints1d] +SeqT = TypeVar("SeqT", bound=Union[Padded, Ragged, ListXd, RaggedData]) + + +@registry.layers("with_ragged.v1") +def with_ragged(layer: Model[Ragged, Ragged]) -> Model[SeqT, SeqT]: + return Model(f"with_ragged({layer.name})", forward, init=init, layers=[layer]) + + +def forward( + model: Model[SeqT, SeqT], Xseq: SeqT, is_train: bool +) -> Tuple[SeqT, Callable]: + layer: Model[Ragged, Ragged] = model.layers[0] + if isinstance(Xseq, Ragged): + return cast(Tuple[SeqT, Callable], layer(Xseq, is_train)) + elif isinstance(Xseq, Padded): + return cast(Tuple[SeqT, Callable], _padded_forward(layer, Xseq, is_train)) + elif _is_ragged_data(Xseq): + return cast( + Tuple[SeqT, Callable], + _tuple_forward(layer, cast(RaggedData, Xseq), is_train), + ) + else: + return cast( + Tuple[SeqT, Callable], _list_forward(layer, cast(List, Xseq), is_train) + ) + + +def init( + model: Model[SeqT, SeqT], + X: Optional[SeqT] = None, + Y: Optional[SeqT] = None, +) -> None: + model.layers[0].initialize( + X=_get_ragged(model, X) if X is not None else None, + Y=_get_ragged(model, Y) if Y is not None else None, + ) + + +def _is_ragged_data(seq): + return isinstance(seq, tuple) and len(seq) == 2 + + +def _get_ragged(model: Model[SeqT, SeqT], seq: SeqT) -> Ragged: + if isinstance(seq, Ragged): + return seq + elif isinstance(seq, Padded): + lists = model.ops.padded2list(seq) + lengths = model.ops.asarray1i([len(x) for x in lists]) + k = model.ops.flatten(lists) + return Ragged(model.ops.flatten(lists), lengths) + elif _is_ragged_data(seq): + return Ragged(*seq) # type: ignore[misc] + else: + list2d_seq = cast(List2d, seq) + lengths = model.ops.asarray1i([len(x) for x in list2d_seq]) + return Ragged(model.ops.flatten(list2d_seq), lengths) + + +def _tuple_forward( + layer: Model[Ragged, Ragged], X: RaggedData, is_train: bool +) -> Tuple[RaggedData, Callable]: + Yr, get_dXr = layer(Ragged(*X), is_train) + + def backprop(dY: RaggedData) -> RaggedData: + dXr = get_dXr(Ragged(*dY)) + return (dXr.data, dXr.lengths) + + return (Yr.data, Yr.lengths), backprop + + +def _padded_forward( + layer: Model[Ragged, Ragged], Xp: Padded, is_train: bool +) -> Tuple[Padded, Callable]: + # Assign these to locals, to keep code a bit shorter. + list2padded = layer.ops.list2padded + padded2list = layer.ops.padded2list + unflatten = layer.ops.unflatten + flatten = layer.ops.flatten + # It's worth being a bit careful about memory here, as the activations + # are potentially large on GPU. So we make nested function calls instead + # of assigning to temporaries where possible, so memory can be reclaimed + # sooner. + Xs = padded2list(Xp) + # Bit annoying here: padded is in a different order, so we need to make new + # lengths. The lengths are unconditionally allocated in CPU memory, because + # otherwire unflatten would move GPU allocations to the CPU again. For the + # ragged arrays we let the layer's ops determine how lengths should be + # stored to ensure that the array and lengths use the same type of memory. + lengths = NUMPY_OPS.asarray1i([len(x) for x in Xs]) + Yr, get_dXr = layer(Ragged(flatten(Xs), layer.ops.asarray1i(lengths)), is_train) + + def backprop(dYp: Padded): + flattened = flatten(padded2list(dYp)) + dXr = get_dXr(Ragged(flattened, lengths)) + return list2padded(unflatten(dXr.data, lengths)) + + return ( + list2padded(unflatten(Yr.data, Yr.lengths)), + backprop, + ) + + +def _list_forward( + layer: Model[Ragged, Ragged], Xs: List, is_train: bool +) -> Tuple[List, Callable]: + # Assign these to locals, to keep code a bit shorter. + flatten = layer.ops.flatten + unflatten = layer.ops.unflatten + + lengths = [len(x) for x in Xs] + Yr, get_dXr = layer(Ragged(flatten(Xs), layer.ops.asarray1i(lengths)), is_train) + + def backprop(dYs): + flattened = flatten(dYs) + return unflatten(get_dXr(Ragged(flattened, lengths)).data, lengths) + + return unflatten(Yr.data, Yr.lengths), backprop diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_reshape.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_reshape.py new file mode 100644 index 0000000000000000000000000000000000000000..435ab9fa3de5a18368c67f1ac18fd5a1e2faee49 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_reshape.py @@ -0,0 +1,56 @@ +from typing import Callable, List, Optional, Tuple, TypeVar, cast + +from ..config import registry +from ..model import Model +from ..types import Array2d, Array3d + +InT = TypeVar("InT", bound=Array3d) +OutT = TypeVar("OutT", bound=Array2d) + + +@registry.layers("with_reshape.v1") +def with_reshape(layer: Model[OutT, OutT]) -> Model[InT, InT]: + """Reshape data on the way into and out from a layer.""" + return Model( + f"with_reshape({layer.name})", + forward, + init=init, + layers=[layer], + dims={"nO": None, "nI": None}, + ) + + +def forward(model: Model[InT, InT], X: InT, is_train: bool) -> Tuple[InT, Callable]: + layer = model.layers[0] + initial_shape = X.shape + final_shape = list(initial_shape[:-1]) + [layer.get_dim("nO")] + nB = X.shape[0] + nT = X.shape[1] + X2d = model.ops.reshape(X, (-1, X.shape[2])) + Y2d, Y2d_backprop = layer(X2d, is_train=is_train) + Y = model.ops.reshape3(Y2d, *final_shape) + + def backprop(dY: InT) -> InT: + reshaped = model.ops.reshape2(dY, nB * nT, -1) + return Y2d_backprop(model.ops.reshape3(reshaped, *initial_shape)) + + return cast(InT, Y), backprop + + +def init( + model: Model[InT, InT], X: Optional[Array3d] = None, Y: Optional[Array3d] = None +) -> None: + layer = model.layers[0] + if X is None and Y is None: + layer.initialize() + X2d: Optional[Array2d] = None + Y2d: Optional[Array2d] = None + if X is not None: + X2d = cast(Array2d, model.ops.reshape(X, (-1, X.shape[-1]))) + if Y is not None: + Y2d = cast(Array2d, model.ops.reshape(Y, (-1, Y.shape[-1]))) + layer.initialize(X=X2d, Y=Y2d) + if layer.has_dim("nI"): + model.set_dim("nI", layer.get_dim("nI")) + if layer.has_dim("nO"): + model.set_dim("nO", layer.get_dim("nO")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_signpost_interval.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_signpost_interval.py new file mode 100644 index 0000000000000000000000000000000000000000..97501a23a69e47bd1ed08233072d1c8bbbea2a38 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/layers/with_signpost_interval.py @@ -0,0 +1,49 @@ +from typing import Any, Callable, Optional, Tuple, TypeVar + +from ..compat import has_os_signpost, os_signpost +from ..model import Model + +_ModelT = TypeVar("_ModelT", bound=Model) + + +def with_signpost_interval( + layer: _ModelT, + signposter: "os_signpost.Signposter", + name: Optional[str] = None, +) -> _ModelT: + """Wraps any layer and marks the init, forward and backprop phases using + signpost intervals for macOS Instruments profiling + + By default, the name of the layer is used as the name of the range, + followed by the name of the pass. + """ + if not has_os_signpost: + raise ValueError( + "with_signpost_interval layer requires the 'os_signpost' package" + ) + + name = layer.name if name is None else name + + orig_forward = layer._func + orig_init = layer.init + + def forward(model: Model, X: Any, is_train: bool) -> Tuple[Any, Callable]: + with signposter.use_interval(f"{name} forward"): + layer_Y, layer_callback = orig_forward(model, X, is_train=is_train) + + def backprop(dY: Any) -> Any: + with signposter.use_interval(f"{name} backprop"): + return layer_callback(dY) + + return layer_Y, backprop + + def init(_model: Model, X: Any, Y: Any) -> Model: + if orig_init is not None: + with signposter.use_interval(f"{name} init"): + return orig_init(layer, X, Y) + else: + return layer + + layer.replace_callbacks(forward, init=init) + + return layer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15f46fd7387d699e374d4c5db0e15d9ebdfe6164 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__init__.py @@ -0,0 +1,19 @@ +from .mxnet import MXNetShim +from .pytorch import PyTorchShim +from .pytorch_grad_scaler import PyTorchGradScaler +from .shim import Shim +from .tensorflow import TensorFlowShim, keras_model_fns, maybe_handshake_model +from .torchscript import TorchScriptShim + +# fmt: off +__all__ = [ + "MXNetShim", + "PyTorchShim", + "PyTorchGradScaler", + "Shim", + "TensorFlowShim", + "TorchScriptShim", + "maybe_handshake_model", + "keras_model_fns", +] +# fmt: on diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c8feba9b0afa6d5aff90de73691434e464a8bdb Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/mxnet.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/mxnet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9b449d4adbf5db3a7837b3e83a76cae8a21ad1c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/mxnet.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9b9837e009f33c1d2e45e61fdee91b771c9a526 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch_grad_scaler.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch_grad_scaler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58e47df692a1d672a0458cb6472744431fc9cce4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/pytorch_grad_scaler.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/shim.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/shim.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f7ccc2822b1bf5631fa7c330a8a65021e4015a9 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/shim.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/tensorflow.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/tensorflow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a959ebafe48192be078d1df5f974e538abcd5de1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/tensorflow.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/torchscript.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/torchscript.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4865428af7bf80f77cf91e9a3488277da2cb29eb Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/__pycache__/torchscript.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/mxnet.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/mxnet.py new file mode 100644 index 0000000000000000000000000000000000000000..5c8d3dc3ac1e21fe84ee78413b4005da69186325 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/mxnet.py @@ -0,0 +1,123 @@ +# mypy: ignore-errors +import copy +from typing import Any, cast + +import srsly + +from ..compat import mxnet as mx +from ..optimizers import Optimizer +from ..types import ArgsKwargs, FloatsXd +from ..util import ( + convert_recursive, + get_array_module, + make_tempfile, + mxnet2xp, + xp2mxnet, +) +from .shim import Shim + + +class MXNetShim(Shim): + """Interface between a MXNet model and a Thinc Model. This container is + *not* a Thinc Model subclass itself. + """ + + def __call__(self, inputs, is_train): + if is_train: + return self.begin_update(inputs) + else: + return self.predict(inputs), lambda a: ... + + def predict(self, inputs: ArgsKwargs) -> Any: + """Pass inputs through to the underlying MXNet model, and return the + output. No conversions are performed. The MXNet model is set into + evaluation mode. + """ + mx.autograd.set_training(train_mode=False) + with mx.autograd.pause(): + outputs = self._model(*inputs.args, **inputs.kwargs) + mx.autograd.set_training(train_mode=True) + return outputs + + def begin_update(self, inputs: ArgsKwargs): + """Pass the inputs through to the underlying MXNet model, keeping + track of which items in the input are tensors requiring gradients. + If the model returns a single value, it is converted into a one-element + tuple. Return the outputs and a callback to backpropagate. + """ + mx.autograd.set_training(train_mode=True) + mx.autograd.set_recording(True) + output = self._model(*inputs.args, **inputs.kwargs) + + def backprop(grads): + mx.autograd.set_recording(False) + mx.autograd.backward(*grads.args, **grads.kwargs) + return convert_recursive( + lambda x: hasattr(x, "grad"), lambda x: x.grad, inputs + ) + + return output, backprop + + def finish_update(self, optimizer: Optimizer): + params = [] + grads = [] + shapes = [] + ctx = mx.current_context() + for key, value in self._model.collect_params().items(): + grad = cast(FloatsXd, mxnet2xp(value.grad(ctx))) + param = cast(FloatsXd, mxnet2xp(value.data(ctx))) + params.append(param.ravel()) + grads.append(grad.ravel()) + shapes.append((param.size, param.shape)) + if not params: + return + xp = get_array_module(params[0]) + flat_params, flat_grads = optimizer( + (self.id, "mxnet-shim"), xp.concatenate(params), xp.concatenate(grads) + ) + start = 0 + for key, value in self._model.collect_params().items(): + size, shape = shapes.pop(0) + param = flat_params[start : start + size].reshape(shape) + value.set_data(xp2mxnet(param)) + value.zero_grad() + start += size + + def copy(self, ctx: "mx.context.Context" = None): + if ctx is None: + ctx = mx.current_context() + model_bytes = self.to_bytes() + copied = copy.deepcopy(self) + copied._model.initialize(ctx=ctx) + copied.from_bytes(model_bytes) + return copied + + def to_device(self, device_type: str, device_id: int): + if device_type == "cpu": + self._model = self.copy(mx.cpu()) + elif device_type == "gpu": + self._model = self.copy(mx.gpu()) + else: + msg = f"Unexpected device_type: {device_type}. Try 'cpu' or 'gpu'." + raise ValueError(msg) + + def to_bytes(self): + # MXNet doesn't implement save/load without a filename + with make_tempfile("w+b") as temp: + self._model.save_parameters(temp.name) + temp.seek(0) + weights_bytes = temp.read() + msg = {"config": self.cfg, "state": weights_bytes} + return srsly.msgpack_dumps(msg) + + def from_bytes(self, bytes_data): + msg = srsly.msgpack_loads(bytes_data) + self.cfg = msg["config"] + self._load_params(msg["state"]) + return self + + def _load_params(self, params): + # MXNet doesn't implement save/load without a filename :( + with make_tempfile("w+b") as temp: + temp.write(params) + self._model.load_parameters(temp.name, ctx=mx.current_context()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..a9cbd1dc826f6cb59b88152d7e309bcc0fe0c94f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch.py @@ -0,0 +1,263 @@ +import contextlib +import itertools +from io import BytesIO +from typing import Any, Callable, Dict, Optional, cast + +import srsly + +from ..backends import CupyOps, context_pools, get_current_ops, set_gpu_allocator +from ..compat import torch +from ..optimizers import Optimizer +from ..types import ArgsKwargs, FloatsXd +from ..util import ( + convert_recursive, + get_torch_default_device, + iterate_recursive, + torch2xp, + xp2torch, +) +from .pytorch_grad_scaler import PyTorchGradScaler +from .shim import Shim + + +class PyTorchShim(Shim): + """Interface between a PyTorch model and a Thinc Model. This container is + *not* a Thinc Model subclass itself. + + mixed_precision: + Enable mixed-precision. This changes whitelisted ops to run + in half precision for better performance and lower memory use. + grad_scaler: + The gradient scaler to use for mixed-precision training. If this + argument is set to "None" and mixed precision is enabled, a gradient + scaler with the default configuration is used. + device: + The PyTorch device to run the model on. When this argument is + set to "None", the default device for the currently active Thinc + ops is used. + serialize_model: + Callback that receives the wrapped PyTorch model as its argument and + returns a "bytes" representation of the same. The representation should + contain all the necessary information to fully deserialize the model. + deserialize_model: + Callback that receives the default PyTorch model (passed to the constructor), the + serialized "bytes" representation and a PyTorch device. It should return a + fully deserialized model on the target device as its result. + """ + + def __init__( + self, + model: Any, + config=None, + optimizer: Any = None, + mixed_precision: bool = False, + grad_scaler: Optional[PyTorchGradScaler] = None, + device: Optional["torch.device"] = None, + serialize_model: Optional[Callable[[Any], bytes]] = None, + deserialize_model: Optional[Callable[[Any, bytes, "torch.device"], Any]] = None, + ): + super().__init__(model, config, optimizer) + + if device is None: + device = get_torch_default_device() + if model is not None: + model.to(device) + + if grad_scaler is None: + grad_scaler = PyTorchGradScaler(mixed_precision) + + grad_scaler.to_(device) + + self._grad_scaler = grad_scaler + self._mixed_precision = mixed_precision + + self._serialize_model = ( + serialize_model + if serialize_model is not None + else default_serialize_torch_model + ) + self._deserialize_model = ( + deserialize_model + if deserialize_model is not None + else default_deserialize_torch_model + ) + + if CupyOps.xp is not None and isinstance(get_current_ops(), CupyOps): + pools = context_pools.get() + if "pytorch" not in pools: + from cupy import get_default_memory_pool + + set_gpu_allocator("pytorch") + get_default_memory_pool().free_all_blocks() + + def __call__(self, inputs, is_train): + if is_train: + return self.begin_update(inputs) + else: + return self.predict(inputs), lambda a: ... + + @property + def device(self): + p = next(self._model.parameters(), None) + if p is None: + return get_torch_default_device() + else: + return p.device + + def predict(self, inputs: ArgsKwargs) -> Any: + """Pass inputs through to the underlying PyTorch model, and return the + output. No conversions are performed. The PyTorch model is set into + evaluation mode. + """ + self._model.eval() + with torch.no_grad(): + # NB: Previously this was torch.cuda.amp.autocast, passing a boolean + # for mixed_precision. That doesn't seem to match the docs, and now + # it raises an error when moving from the deprecated function. So + # I've removed the argument but I'm not certain it's correct. + with torch.autocast(device_type="cuda", enabled=self._mixed_precision): + outputs = self._model(*inputs.args, **inputs.kwargs) + self._model.train() + return outputs + + def begin_update(self, inputs: ArgsKwargs): + """Pass the inputs through to the underlying PyTorch model, keeping + track of which items in the input are tensors requiring gradients. + If the model returns a single value, it is converted into a one-element tuple. + Return the outputs and a callback to backpropagate. + """ + self._model.train() + + # Note: mixed-precision autocast must not be applied to backprop. + # NB: Previously this was torch.cuda.amp.autocast, passing a boolean + # for mixed_precision. That doesn't seem to match the docs, and now + # it raises an error when moving from the deprecated function. So + # I've removed the argument but I'm not certain it's correct. + with torch.autocast("cuda", enabled=self._mixed_precision): + output = self._model(*inputs.args, **inputs.kwargs) + + def backprop(grads): + # Normally, gradient scaling is applied to the loss of a model. However, + # since regular thinc layers do not use mixed-precision, we perform scaling + # locally in this shim. Scaling the loss by a factor, scales the gradients + # by the same factor (see the chain rule). Therefore, we scale the gradients + # backprop'ed through the succeeding layer to get the same effect as loss + # scaling. + grads.kwargs["grad_tensors"] = self._grad_scaler.scale( + grads.kwargs["grad_tensors"], inplace=True + ) + + torch.autograd.backward(*grads.args, **grads.kwargs) + + # Unscale weights and check for overflows during backprop. + grad_tensors = [] + for torch_data in itertools.chain( + self._model.parameters(), + iterate_recursive(lambda x: hasattr(x, "grad"), inputs), + ): + if torch_data.grad is not None: + grad_tensors.append(torch_data.grad) + found_inf = self._grad_scaler.unscale(grad_tensors) + + # If there was an over/underflow, return zeroed-out gradients. + if found_inf: + grad_get = lambda x: x.grad.zero_() if x.grad is not None else x.grad + else: + grad_get = lambda x: x.grad + + return convert_recursive(lambda x: hasattr(x, "grad"), grad_get, inputs) + + return output, backprop + + def finish_update(self, optimizer: Optimizer): + for name, torch_data in self._model.named_parameters(): + if torch_data.grad is not None: + if ( + not self._grad_scaler.found_inf + ): # Skip weight update if any gradient overflowed. + param, grad = optimizer( + (self.id, name), + cast(FloatsXd, torch2xp(torch_data.data)), + cast(FloatsXd, torch2xp(torch_data.grad)), + ) + torch_data.data = xp2torch( + param, requires_grad=True, device=torch_data.device + ) + torch_data.grad.zero_() + + self._grad_scaler.update() + + @contextlib.contextmanager + def use_params(self, params): + key_prefix = f"pytorch_{self.id}_" + state_dict = {} + for k, v in params.items(): + if hasattr(k, "startswith") and k.startswith(key_prefix): + state_dict[k.replace(key_prefix, "")] = xp2torch(v, device=self.device) + if state_dict: + backup = {k: v.clone() for k, v in self._model.state_dict().items()} + self._model.load_state_dict(state_dict) + yield + self._model.load_state_dict(backup) + else: + yield + + def to_device(self, device_type: str, device_id: int): # pragma: no cover + if device_type == "cpu": + self._model.cpu() + elif device_type == "gpu": + self._model.cuda(device_id) + else: + msg = f"Invalid device_type: {device_type}. Try 'cpu' or 'gpu'" + raise ValueError(msg) + + def to_bytes(self): + model_bytes = self._serialize_model(self._model) + msg = {"config": self.cfg, "state": model_bytes} + return srsly.msgpack_dumps(msg) + + def from_bytes(self, bytes_data): + device = get_torch_default_device() + msg = srsly.msgpack_loads(bytes_data) + self.cfg = msg["config"] + self._model = self._deserialize_model(self._model, msg["state"], device) + self._grad_scaler.to_(device) + return self + + +def default_serialize_torch_model(model: Any) -> bytes: + """Serializes the parameters of the wrapped PyTorch model to bytes. + + model: + Wrapped PyTorch model. + + Returns: + A `bytes` object that encapsulates the serialized model parameters. + """ + filelike = BytesIO() + torch.save(model.state_dict(), filelike) + filelike.seek(0) + return filelike.getvalue() + + +def default_deserialize_torch_model( + model: Any, state_bytes: bytes, device: "torch.device" +) -> Any: + """Deserializes the parameters of the wrapped PyTorch model and + moves it to the specified device. + + model: + Wrapped PyTorch model. + state_bytes: + Serialized parameters as a byte stream. + device: + PyTorch device to which the model is bound. + + Returns: + The deserialized model. + """ + filelike = BytesIO(state_bytes) + filelike.seek(0) + model.load_state_dict(torch.load(filelike, map_location=device)) + model.to(device) + return model diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch_grad_scaler.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch_grad_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..822c9ed32f320b2f33c536a14f5aa7b637c64829 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/pytorch_grad_scaler.py @@ -0,0 +1,181 @@ +from typing import Dict, Iterable, List, Union, cast + +from ..compat import has_torch_amp, torch +from ..util import is_torch_array + + +class PyTorchGradScaler: + """ + Gradient scaler for the PyTorch shim. + + Gradients with small magnitudes are not representable in half-precision and + will underflow to zero. A gradient scaler counters this issue by scaling + up the loss before backpropagation, increasing the gradients by the same + magnitude. A large enough scale will avoid that the gradients underflow. + The gradients are unscaled in single precision after backpropagation, to + provide the unscaled gradients to the optimizer. + """ + + def __init__( + self, + enabled: bool = False, + init_scale: float = 2.0**16, + backoff_factor: float = 0.5, + growth_factor: float = 2.0, + growth_interval: int = 2000, + ): + """ + Construct a gradient scaler for the PyTorch shim. + + enabled (bool): + Sets whether the gradient scalar is enabled. If it is disabled, the + methods of the grad scaler are no-ops. + + init_scale (float): + The initial scale used to increase the gradient magnitude. + + backoff_factor (float): + The scale will be multiplied by this factor if any of the gradients + overflows. + + growth_factor (float): + The scale will be multiplied by this factor when none of the gradients + overflowed for "growth_interval" steps. + + growth_interval (int): + When no overflows were found for this number of steps, the scale will + be multiplied by "growth_factor". + """ + self._enabled = enabled + self._growth_factor = growth_factor + self._backoff_factor = backoff_factor + self._growth_interval = growth_interval + + self._growth_tracker = torch.full((1,), 0, dtype=torch.int) + self._scale = torch.full((1,), init_scale) + self._found_inf = False + + def to_(self, device): + self._growth_tracker = self._growth_tracker.to(device) + self._scale = self._scale.to(device) + + def scale( + self, tensors: Union["torch.Tensor", Iterable["torch.Tensor"]], inplace=False + ) -> Union["torch.Tensor", List["torch.Tensor"]]: + """Scale up the values in the given tensors.""" + if not self._enabled: + return cast("torch.Tensor", tensors) + + incorrect_type = ValueError( + "Input to gradient scaling must be a Tensor or Iterable[Tensor]" + ) + + # Cache per-device scales to avoid unnecessary d2d copies of the current scale. + scale_per_device: Dict["torch.device", "torch.Tensor"] = dict() + + if is_torch_array(tensors): + tensor = cast("torch.Tensor", tensors) + return self._scale_tensor(tensor, scale_per_device, inplace) + elif isinstance(tensors, Iterable): + scaled_tensors = [] + + for tensor in tensors: + if not is_torch_array(tensor): + raise incorrect_type + + scaled_tensors.append( + self._scale_tensor(tensor, scale_per_device, inplace) + ) + + return scaled_tensors + + raise incorrect_type + + def _scale_tensor( + self, + tensor: "torch.Tensor", + scale_per_device: Dict["torch.device", "torch.Tensor"], + inplace: bool, + ): + if not has_torch_amp: + raise ValueError( + "Gradient scaling is not supported, requires capable GPU and torch>=1.9.0" + ) + + if not tensor.is_cuda: + msg = ( + "Gradient scaling is only supported for CUDA tensors. " + "If you are using PyTorch models, you can avoid this " + "error by disabling mixed-precision support." + ) + raise ValueError(msg) + + device = tensor.device + + if device not in scale_per_device: + scale_per_device[device] = self._scale.to(device=device) + + scale = scale_per_device[device] + if inplace: + return tensor.mul_(scale) + else: + return tensor * scale + + def _tensors_per_device(self, tensors): + tensors_per_device = dict() + for tensor in tensors: + device_tensors = tensors_per_device.setdefault(tensor.device, []) + device_tensors.append(tensor) + + return tensors_per_device + + @property + def found_inf(self): + return self._found_inf + + def unscale(self, tensors): + """Unscale the given tensors. Returns True if any of the gradients were infinite.""" + if not self._enabled: + return False + + # Invert scale (in higher precision). + inv_scale = self._scale.double().reciprocal().float() + + # Apply unscaling to tensors, per device. + tensors_per_device = self._tensors_per_device(tensors) + for device, device_tensors in tensors_per_device.items(): + found_inf_device = torch.full((1,), 0.0, device=device) + inv_scale_device = inv_scale.to(device=device) + + torch._amp_foreach_non_finite_check_and_unscale_( + device_tensors, found_inf_device, inv_scale_device + ) + + if bool(found_inf_device != 0): + self._found_inf = True + + return self._found_inf + + def update(self): + """ + Update the scale factor and clear information about infinities. + + This method should be called after each optimization step. + """ + if not self._enabled: + return + + found_inf_device = torch.full( + (1,), 1.0 if self._found_inf else 0.0, device=self._scale.device + ) + torch._amp_update_scale_( + self._scale, + self._growth_tracker, + found_inf_device, + self._growth_factor, + self._backoff_factor, + self._growth_interval, + ) + + # Clear infinity found status + self._found_inf = False diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/shim.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/shim.py new file mode 100644 index 0000000000000000000000000000000000000000..c25a4fdbcc6c79de01f74330b641688d071b2a4a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/shim.py @@ -0,0 +1,74 @@ +import contextlib +import copy +import threading +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Tuple, Union + + +class Shim: # pragma: no cover + """Define a basic interface for external models. Users can create subclasses + of 'shim' to wrap external libraries. We provide shims for PyTorch. + + The Thinc Model class treats Shim objects as a sort of special type of + sublayer: it knows they're not actual Thinc Model instances, but it also + knows to talk to the shim instances when doing things like using transferring + between devices, loading in parameters, optimization. It also knows Shim + objects need to be serialized and deserialized with to/from bytes/disk, + rather than expecting that they'll be msgpack-serializable. + """ + + global_id: int = 0 + global_id_lock: threading.Lock = threading.Lock() + cfg: Dict + _model: Any + _optimizer: Optional[Any] + + def __init__(self, model: Any, config=None, optimizer: Any = None): + with Shim.global_id_lock: + Shim.global_id += 1 + self.id = Shim.global_id + + self.cfg = dict(config) if config is not None else {} + self._model = model + self._optimizer = optimizer + + def __call__(self, inputs, is_train: bool) -> Tuple[Any, Callable[..., Any]]: + raise NotImplementedError + + def predict(self, fwd_args: Any) -> Any: + Y, backprop = self(fwd_args, is_train=False) + return Y + + def begin_update(self, fwd_args: Any) -> Tuple[Any, Callable[..., Any]]: + return self(fwd_args, is_train=True) + + def finish_update(self, optimizer): + raise NotImplementedError + + @contextlib.contextmanager + def use_params(self, params): + yield + + def copy(self): + return copy.deepcopy(self) + + def to_device(self, device_type: str, device_id: int): + raise NotImplementedError + + def to_disk(self, path: Union[str, Path]): + bytes_data = self.to_bytes() + path = Path(path) if isinstance(path, str) else path + with path.open("wb") as file_: + file_.write(bytes_data) + + def from_disk(self, path: Union[str, Path]) -> "Shim": + path = Path(path) if isinstance(path, str) else path + with path.open("rb") as file_: + bytes_data = file_.read() + return self.from_bytes(bytes_data) + + def to_bytes(self): + raise NotImplementedError + + def from_bytes(self, data) -> "Shim": + raise NotImplementedError diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/tensorflow.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/tensorflow.py new file mode 100644 index 0000000000000000000000000000000000000000..3a1f98bec7065500bcc5ec429add4930988b4a52 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/tensorflow.py @@ -0,0 +1,277 @@ +# mypy: ignore-errors +import contextlib +import copy +from io import BytesIO +from typing import Any, Dict, List, Optional + +import catalogue +import numpy + +from ..backends import Ops, get_current_ops +from ..compat import cupy, h5py +from ..compat import tensorflow as tf +from ..optimizers import Optimizer +from ..types import ArgsKwargs, ArrayXd +from ..util import get_array_module +from .shim import Shim + +keras_model_fns = catalogue.create("thinc", "keras", entry_points=True) + + +def maybe_handshake_model(keras_model): + """Call the required predict/compile/build APIs to initialize a model if it + is a subclass of tf.keras.Model. This is required to be able to call set_weights + on subclassed layers.""" + try: + keras_model.get_config() + return keras_model + except (AttributeError, NotImplementedError): + # Subclassed models don't implement get_config + pass + + for prop_name in ["catalogue_name", "eg_x", "eg_y", "eg_shape"]: + if not hasattr(keras_model, prop_name): + raise ValueError( + "Keras subclassed models are not whole-model serializable by " + "TensorFlow. To work around this, you must decorate your keras " + "model subclasses with the 'keras_subclass' decorator. The decorator " + "requires a single X/Y input of fake-data that can be used to initialize " + "your subclass model properly when loading the saved version." + ) + + ops: Ops = get_current_ops() + if ops.device_type == "cpu": + device = "CPU" + else: # pragma: no cover + device = tf.test.gpu_device_name() + + compile_args = keras_model.eg_compile + + with tf.device(device): + # Calling predict creates layers and weights for subclassed models + keras_model.compile(**compile_args) + keras_model.build(keras_model.eg_shape) + keras_model.predict(keras_model.eg_x) + # Made public in 2.2.x + if hasattr(keras_model, "_make_train_function"): + keras_model._make_train_function() + else: + keras_model.make_train_function() + return keras_model + + +class TensorFlowShim(Shim): + """Interface between a TensorFlow model and a Thinc Model. This container is + *not* a Thinc Model subclass itself. + + Reference for custom training: + https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough + """ + + gradients: Optional[List["tf.Tensor"]] + + def __init__(self, model: Any, config=None, optimizer: Any = None): + super().__init__(model, config, optimizer) + self.gradients = None + + def __str__(self): + lines: List[str] = [] + + def accumulate(line: str): + lines.append(line) + + self._model.summary(print_fn=accumulate) + return "\n".join(lines) + + def __call__(self, X: ArgsKwargs, is_train: bool): + if is_train: + return self.begin_update(X) + else: + return self.predict(X) + + def predict(self, X: ArgsKwargs): + old_phase = tf.keras.backend.learning_phase() + tf.keras.backend.set_learning_phase(0) + Y = self._model(*X.args, **X.kwargs) + tf.keras.backend.set_learning_phase(old_phase) + return Y + + def begin_update(self, X: ArgsKwargs): + tf.keras.backend.set_learning_phase(1) + tape = tf.GradientTape() + tape.__enter__() + tape.watch(X.args) # watch the input layers + output = self._model(*X.args, **X.kwargs) + + def backprop(d_output): + # d_args[0] contains derivative of loss wrt output (d_loss/d_output) + tape.__exit__(None, None, None) + # We need to handle a tuple of inputs + if len(X.args) == 1: + wrt_tensors = [X.args[0]] # add the input layer also for d_loss/d_input + else: + wrt_tensors = list(X.args[0]) + wrt_tensors.extend(self._model.trainable_variables) + all_gradients = tape.gradient( + output, wrt_tensors, output_gradients=d_output + ) + dX = all_gradients[: len(X.args)] + opt_grads = all_gradients[1:] + # Accumulate gradients + if self.gradients is not None: + assert len(opt_grads) == len(self.gradients), "gradients must match" + variable: tf.Variable + for variable, new_variable in zip(self.gradients, opt_grads): + variable.assign_add(new_variable) + else: + # Create variables from the grads to allow accumulation + self.gradients = [tf.Variable(f) for f in opt_grads] + return ArgsKwargs(args=tuple(dX), kwargs={}) + + return output, backprop + + def finish_update(self, optimizer: Optimizer): + if self.gradients is None: + raise ValueError( + "There are no gradients for optimization. Be sure to call begin_update" + " before calling finish_update." + ) + assert len(self.gradients) == len(self._model.trainable_variables) + grad: tf.Tensor + variable: tf.Variable + params = [] + grads = [] + shapes = [] + + for grad, variable in zip(self.gradients, self._model.trainable_variables): + param = variable.numpy() + grad = grad.numpy() + shapes.append((param.size, param.shape)) + params.append(param.ravel()) + grads.append(grad.ravel()) + xp = get_array_module(params[0]) + flat_params, flat_grads = optimizer( + (self.id, "tensorflow-shim"), xp.concatenate(params), xp.concatenate(grads) + ) + start = 0 + for grad, variable in zip(self.gradients, self._model.trainable_variables): + size, shape = shapes.pop(0) + param = flat_params[start : start + size].reshape(shape) + variable.assign(param) + start += size + self.gradients = None + + def _load_weights_from_state_dict( + self, state_dict: Optional[Dict[str, ArrayXd]] = None + ): + if state_dict is None: + state_dict = self._create_state_dict() + for layer in self._model.layers: + current_layer_weights = [] + for weight in layer.weights: + current_layer_weights.append(state_dict[weight.name]) + layer.set_weights(current_layer_weights) + + # Create a state dict similar to PyTorch + def _create_state_dict(self): + # key as variable name and value as numpy arrays + state_dict = {} + for layer in self._model.layers: + for weight in layer.weights: + state_dict[weight.name] = weight.numpy() + return state_dict + + @contextlib.contextmanager + def use_params(self, params): + key_prefix = f"tensorflow_{self.id}_" + # state dict stores key as name and value as numpy array + state_dict = {} + for k, v in params.items(): + if hasattr(k, "startswith") and k.startswith(key_prefix): + if cupy is None: + assert isinstance(v, numpy.ndarray) + else: # pragma: no cover + if isinstance(v, cupy.core.core.ndarray): + v = cupy.asnumpy(v) + assert isinstance(v, numpy.ndarray) + state_dict[k.replace(key_prefix, "")] = v + if state_dict: + backup = self._create_state_dict() + self._load_weights_from_state_dict(state_dict) + yield + self._load_weights_from_state_dict(backup) + else: + yield + + def _clone_model(self): + """similar to tf.keras.models.clone_model() + But the tf.keras.models.clone_model changes the names of tf.Variables. + This method even preserves that + """ + model_json_config = self._model.to_json() + tf.keras.backend.clear_session() + self._model = tf.keras.models.model_from_json(model_json_config) + self._load_weights_from_state_dict() + + def copy(self): + model_json_config = self._model.to_json() + self._model = None + tf.keras.backend.clear_session() + copied = copy.deepcopy(self) + copied._model = tf.keras.models.model_from_json(model_json_config) + copied._load_weights_from_state_dict() + return copied + + def to_device(self, device_type: str, device_id: int): # pragma: no cover + if device_type == "cpu": + with tf.device("/CPU"): # pragma: no cover + self._clone_model() + elif device_type == "gpu": + with tf.device("/GPU:{}".format(device_id)): + self._clone_model() + + def to_bytes(self): + filelike = BytesIO() + try: + with h5py.File(filelike, "w") as f: + self._model.save(f, save_format="h5") + return filelike.getvalue() + except NotImplementedError: + if not hasattr(self._model, "catalogue_name"): + raise ValueError( + "Couldn't serialize to h5, and model has no factory " + "function for component serialization." + ) + # Check the factory function and throw ValueError if it doesn't exist + keras_model_fns.get(self._model.catalogue_name) + return self._model.catalogue_name, self._model.get_weights() + + def from_bytes(self, data): + ops: Ops = get_current_ops() + if ops.device_type == "cpu": + device = "CPU" + else: # pragma: no cover + device = tf.test.gpu_device_name() + + # Plain bytes + if isinstance(data, (str, bytes)): + tf.keras.backend.clear_session() + filelike = BytesIO(data) + filelike.seek(0) + with h5py.File(filelike, "r") as f: + with tf.device(device): + self._model = tf.keras.models.load_model(f) + return + # We only have to create the model if it doesn't already exist. + catalogue_name, model_weights = data + if self._model is None: + model_fn = keras_model_fns.get(catalogue_name) + tf.keras.backend.clear_session() + with tf.device(device): + if hasattr(self._model, "eg_args"): + ak: ArgsKwargs = self._model.eg_args + new_model = model_fn(*ak.args, **ak.kwargs) + else: + new_model = model_fn() + self._model_initialized = maybe_handshake_model(new_model) + self._model.set_weights(model_weights) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/torchscript.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/torchscript.py new file mode 100644 index 0000000000000000000000000000000000000000..11df738830d28f4ccb9faa063c55c9b5dfcd3642 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/shims/torchscript.py @@ -0,0 +1,67 @@ +from io import BytesIO +from typing import Any, Optional + +import srsly + +from ..compat import torch +from ..util import get_torch_default_device +from .pytorch import PyTorchShim +from .pytorch_grad_scaler import PyTorchGradScaler + + +class TorchScriptShim(PyTorchShim): + """A Thinc shim that wraps a TorchScript module. + + model: + The TorchScript module. A value of `None` is also possible to + construct a shim to deserialize into. + mixed_precision: + Enable mixed-precision. This changes whitelisted ops to run + in half precision for better performance and lower memory use. + grad_scaler: + The gradient scaler to use for mixed-precision training. If this + argument is set to "None" and mixed precision is enabled, a gradient + scaler with the default configuration is used. + device: + The PyTorch device to run the model on. When this argument is + set to "None", the default device for the currently active Thinc + ops is used. + """ + + def __init__( + self, + model: Optional["torch.jit.ScriptModule"], + config=None, + optimizer: Any = None, + mixed_precision: bool = False, + grad_scaler: Optional[PyTorchGradScaler] = None, + device: Optional["torch.device"] = None, + ): + if model is not None and not isinstance(model, torch.jit.ScriptModule): + raise ValueError( + "PyTorchScriptShim must be initialized with ScriptModule or None (for deserialization)" + ) + + super().__init__(model, config, optimizer, mixed_precision, grad_scaler, device) + + def to_bytes(self): + filelike = BytesIO() + torch.jit.save(self._model, filelike) + filelike.seek(0) + model_bytes = filelike.getvalue() + msg = {"config": self.cfg, "model": model_bytes} + return srsly.msgpack_dumps(msg) + + def from_bytes(self, bytes_data): + device = get_torch_default_device() + msg = srsly.msgpack_loads(bytes_data) + self.cfg = msg["config"] + filelike = BytesIO(msg["model"]) + filelike.seek(0) + # As of Torch 2.0.0, loading TorchScript models directly to + # an MPS device is not supported. + map_location = torch.device("cpu") if device.type == "mps" else device + self._model = torch.jit.load(filelike, map_location=map_location) + self._model.to(device) + self._grad_scaler.to_(device) + return self diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0d79401622d82d2d272b35de7bb8aa518fb8834 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/conftest.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/conftest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae0298f3d17ef48ed52f29444977cf8b8391e267 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/conftest.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_mxnet.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_mxnet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..274b4b2ef1b8723d3ecd5d22e8f864fd842395dc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_mxnet.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_tensorflow.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_tensorflow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..10bfade6fba212e0007f3f6c8fa18d3fd3f91368 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/enable_tensorflow.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/strategies.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/strategies.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e8b1154bd4ab3597d8ec8d05131dba225e82980 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/strategies.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_config.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3a06f99a71f389173805f55112a4f786bb53bde Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_config.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_examples.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_examples.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63dbbbfe306161f2ab3cb2aea28af958bb16b9b1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_examples.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_import__all__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_import__all__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21ad198cfb5bbace4c02cea364f801a88ffc6442 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_import__all__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_indexing.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_indexing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f733463abed008910fa17fa7d40ae7683b2d2ce5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_indexing.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_initializers.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_initializers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74086f5f138514c704a6760678bd5da1e27ee502 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_initializers.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_loss.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_loss.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7ffd65eb9718e790e0b10f62caab4eeb398c697 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_loss.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_optimizers.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_optimizers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5dc5652b0bf679108c764246c12f30648e6d732e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_optimizers.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_schedules.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_schedules.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6626c59bdda21d365f44d8dd26fb2b3e45f8fddd Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_schedules.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_serialize.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_serialize.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..879e9c2b85b86972c95f50c5758c4d517ac125b0 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_serialize.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_types.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2521562a8c5963008484d80141bb049f7f7dd59 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_types.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_util.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bcafaedfdf77de7caf8b507aa357f756ca46c212 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/test_util.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/util.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..095aa18d696953cd57fdd23f0148110cf5c9da0c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/__pycache__/util.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..802ecfce334844ac0236d69b0eb6bcc864da473d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_mem.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_mem.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f12c88859e9082b88704df62d451533d8687550f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_mem.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_ops.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e47eda170bf86374ea0a18a7280d13a14a76a199 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/__pycache__/test_ops.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_mem.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_mem.py new file mode 100644 index 0000000000000000000000000000000000000000..ffa8de891da95f3006a678d776dbf1315158fc6b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_mem.py @@ -0,0 +1,12 @@ +import numpy + +from thinc.backends._param_server import ParamServer + + +def test_param_server_init(): + array = numpy.zeros((5,), dtype="f") + params = {("a", 1): array, ("b", 2): array} + grads = {("a", 1): array, ("c", 3): array} + ps = ParamServer(params, grads) + assert ps.param_keys == (("a", 1), ("b", 2)) + assert ps.grad_keys == (("a", 1),) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f6ec6377c6970cde94cf82f93306dd5dac43f4e7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/backends/test_ops.py @@ -0,0 +1,1610 @@ +import inspect +import platform +from typing import Tuple, cast + +import numpy +import pytest +from hypothesis import given, settings +from hypothesis.strategies import composite, integers +from numpy.testing import assert_allclose +from packaging.version import Version + +from thinc.api import ( + LSTM, + CupyOps, + NumpyOps, + Ops, + fix_random_seed, + get_current_ops, + get_ops, + use_ops, +) +from thinc.backends._custom_kernels import KERNELS, KERNELS_LIST, compile_mmh +from thinc.compat import has_cupy_gpu, has_torch, torch_version +from thinc.types import Floats2d +from thinc.util import torch2xp, xp2torch + +from .. import strategies +from ..strategies import arrays_BI, ndarrays_of_shape + +MAX_EXAMPLES = 10 + +VANILLA_OPS = Ops(numpy) # type:ignore +NUMPY_OPS = NumpyOps() +BLIS_OPS = NumpyOps(use_blis=True) +CPU_OPS = [NUMPY_OPS, VANILLA_OPS] +XP_OPS = [NUMPY_OPS] +if has_cupy_gpu: + XP_OPS.append(CupyOps()) +ALL_OPS = XP_OPS + [VANILLA_OPS] + +FLOAT_TYPES = ["float32", "float64"] +INT_TYPES = ["int32", "int64"] + +REDUCTIONS = ["reduce_first", "reduce_last", "reduce_max", "reduce_mean", "reduce_sum"] + +REDUCE_ZERO_LENGTH_RAISES = [ + ("reduce_first", True), + ("reduce_last", True), + ("reduce_max", True), + # From a mathematical perspective we'd want mean reduction to raise for + # zero-length sequences, since floating point numbers are not a monoid + # under averaging. However, floret relies on reduce_mean to return a + # zero-vector in this case. + ("reduce_mean", False), + ("reduce_sum", False), +] + + +def create_pytorch_funcs(): + import math + + import torch + + def torch_relu(x): + return torch.nn.functional.relu(x) + + def torch_relu_k(x): + return torch.nn.functional.relu6(x) + + def torch_hard_sigmoid(x): + return torch.clip(x * 0.2 + 0.5, 0, 1) + + def torch_hard_tanh(x): + return torch.nn.functional.hardtanh(x) + + def torch_mish(x): + return torch.nn.functional.mish(x) + + def torch_swish(x): + return torch.nn.functional.silu(x) + + def torch_hard_swish(x): + return x * torch_hard_sigmoid(x) + + def torch_hard_swish_mobilenet(x): + return torch.nn.functional.hardswish(x) + + def torch_sigmoid(x): + return torch.sigmoid(x) + + def torch_dish(x): + return 0.5 * x * (x / (1 + x * x).sqrt() + 1) + + # https://github.com/huggingface/transformers/blob/master/src/transformers/activations.py#L37 + def torch_gelu_approx(x): + return ( + 0.5 + * x + * ( + 1.0 + + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)) + ) + ) + ) + + def torch_gelu(x): + return torch.nn.functional.gelu(x) + + return [ + ("relu", torch_relu), + ("relu_k", torch_relu_k), + ("hard_sigmoid", torch_hard_sigmoid), + ("hard_tanh", torch_hard_tanh), + ("mish", torch_mish), + ("swish", torch_swish), + ("hard_swish", torch_hard_swish), + ("hard_swish_mobilenet", torch_hard_swish_mobilenet), + ("dish", torch_dish), + ("gelu_approx", torch_gelu_approx), + ("gelu", torch_gelu), + ("sigmoid", torch_sigmoid), + ] + + +if has_torch: + TORCH_FUNCS = create_pytorch_funcs() +else: + TORCH_FUNCS = [] + + +@pytest.mark.parametrize("op", [NumpyOps, CupyOps]) +def test_ops_consistency(op): + """Test that specific ops don't define any methods that are not on the + Ops base class and that all ops methods define the exact same arguments.""" + attrs = [m for m in dir(op) if not m.startswith("_")] + for attr in attrs: + assert hasattr(Ops, attr) + method = getattr(op, attr) + if hasattr(method, "__call__"): + sig = inspect.signature(method) + params = [p for p in sig.parameters][1:] + base_sig = inspect.signature(getattr(Ops, attr)) + print(base_sig) + base_params = [p for p in base_sig.parameters][1:] + assert params == base_params, attr + defaults = [p.default for p in sig.parameters.values()][1:] + base_defaults = [p.default for p in base_sig.parameters.values()][1:] + assert defaults == base_defaults, attr + # If args are type annotated, their types should be the same + annots = [p.annotation for p in sig.parameters.values()][1:] + base_annots = [p.annotation for p in base_sig.parameters.values()][1:] + for i, (p1, p2) in enumerate(zip(annots, base_annots)): + if p1 != inspect.Parameter.empty and p2 != inspect.Parameter.empty: + # TODO: This used to work, but not longer does in Cython 3. + # We're getting spurious differences. + # Need to check string value to handle TypeVars etc. + # assert str(p1) == str(p2), attr + pass + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.filterwarnings("ignore::RuntimeWarning") +def test_adam_incorrect_inputs(ops): + one = ops.xp.zeros(1, dtype="f") + two = ops.xp.zeros(2, dtype="f") + + ops.adam(one, one, one, one, 0.0, 0.0, 0.0, 0.0) + with pytest.raises(ValueError): + ops.adam(two, one, one, one, 0.0, 0.0, 0.0, 0.0) + with pytest.raises(ValueError): + ops.adam(one, two, one, one, 0.0, 0.0, 0.0, 0.0) + with pytest.raises(ValueError): + ops.adam(one, one, two, one, 0.0, 0.0, 0.0, 0.0) + with pytest.raises(ValueError): + ops.adam(one, one, one, two, 0.0, 0.0, 0.0, 0.0) + + +@pytest.mark.parametrize("ops", ALL_OPS) +def test_alloc(ops): + float_methods = (ops.alloc1f, ops.alloc2f, ops.alloc3f, ops.alloc4f) + for i, method in enumerate(float_methods): + shape = (1,) * (i + 1) + arr = method(*shape) + assert arr.dtype == numpy.float32 + assert arr.ndim == len(shape) + arr = ops.alloc_f(shape) + assert arr.dtype == numpy.float32 + assert arr.ndim == len(shape) + int_methods = (ops.alloc1i, ops.alloc2i, ops.alloc3i, ops.alloc4i) + for i, method in enumerate(int_methods): + shape = (1,) * (i + 1) + arr = method(*shape) + assert arr.dtype == numpy.int32 + assert arr.ndim == len(shape) + arr = ops.alloc_i(shape) + assert arr.dtype == numpy.int32 + assert arr.ndim == len(shape) + assert ops.alloc(1).ndim == 1 + + +@pytest.mark.parametrize("ops", XP_OPS) +def test_hash_gives_distinct_keys(ops): + ids = ops.alloc1f(5, dtype="uint64") + keys = ops.hash(ids, 0) + assert keys.shape == (5, 4) + assert keys.dtype == "uint32" + for i in range(len(ids)): + for j in range(keys.shape[1]): + assert keys[i, j] != 0 + + +@pytest.mark.parametrize("ops", XP_OPS) +def test_get_dropout_empty(ops): + shape = (2, 2) + drop = 0.0 + mask = ops.get_dropout_mask(shape, drop) + if drop <= 0.0: + assert mask[mask == 1.0].all() + else: + assert mask[mask != 1.0].all() + + +@pytest.mark.parametrize("ops", XP_OPS) +def test_get_dropout_not_empty(ops): + shape = (200, 200) + drop = 0.5 + mask = ops.get_dropout_mask(shape, drop) + assert (mask > 1.0).any() + assert (mask == 0.0).any() + assert mask.shape == shape + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@pytest.mark.parametrize("index_dtype", ["int32", "uint32"]) +def test_gather_add(ops, dtype, index_dtype): + table = ops.xp.arange(12, dtype=dtype).reshape(4, 3) + indices = ops.xp.array([[0, 2], [3, 1], [0, 1]], dtype=index_dtype) + gathered = ops.gather_add(table, indices) + ops.xp.testing.assert_allclose( + gathered, [[6.0, 8.0, 10.0], [12.0, 14.0, 16.0], [3.0, 5.0, 7.0]] + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@given(table=strategies.arrays_BI()) +def test_gather_add_against_numpy(ops, table): + table = ops.asarray(table) + indices = ops.xp.arange(100, dtype="i").reshape(25, 4) % table.shape[0] + ops.xp.testing.assert_allclose( + ops.gather_add(table, indices), + table[indices].sum(1), + atol=1e-5, + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +def test_gather_add_oob_raises(ops): + table = ops.xp.arange(12, dtype="f").reshape(4, 3) + indices = ops.xp.array([[0, 2], [3, 1], [5, 1]], dtype="i") + with pytest.raises(IndexError): + ops.gather_add(table, indices) + + +@pytest.mark.parametrize("ops", CPU_OPS) +def test_seq2col_window_one_small(ops): + seq = ops.asarray([[1.0], [3.0], [4.0], [5]], dtype="float32") + cols = ops.seq2col(seq, 1) + if hasattr(cols, "get"): + cols = cols.get() + assert_allclose(cols[0], [0.0, 1.0, 3.0]) + assert_allclose(cols[1], [1.0, 3.0, 4.0]) + assert_allclose(cols[2], [3.0, 4.0, 5.0]) + assert_allclose(cols[3], [4.0, 5.0, 0.0]) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BOP()) +def test_maxout(ops, dtype, X): + X = ops.asarray(X, dtype=dtype) + expected_best = X.max(axis=-1).astype(dtype) + predicted_best, which = ops.maxout(X) + assert predicted_best.dtype == dtype + ops.xp.testing.assert_allclose( + expected_best, predicted_best, rtol=0.001, atol=0.001 + ) + + # Can't compare 'which' directly, as sort order might be different. + # So, instead we use 'which' to extract elements from X and then + # check the result against the expected output. + ops.xp.testing.assert_allclose( + ops.xp.take_along_axis(X, ops.xp.expand_dims(which, -1), axis=-1), + ops.xp.expand_dims(expected_best, -1), + atol=1e-10, + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_maxout(ops, dtype): + dX = ops.backprop_maxout( + ops.asarray2f([[1.0, 2.0], [3.0, 4.0]], dtype=dtype), + ops.asarray2i([[1, 0], [2, 1]]), + 3, + ) + assert dX.dtype == dtype + ops.xp.testing.assert_allclose( + dX, + [[[0.0, 1.0, 0.0], [2.0, 0.0, 0.0]], [[0.0, 0.0, 3.0], [0.0, 4.0, 0.0]]], + ) + + with pytest.raises(IndexError): + ops.backprop_maxout( + ops.asarray2f([[1.0, 2.0], [3.0, 4.0]]), ops.asarray2i([[1, 0], [3, 1]]), 3 + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_seq2col_window_one(ops, X): + X = ops.asarray(X) + base_ops = Ops() + base_ops.xp = ops.xp + baseX = base_ops.alloc(X.shape) + X + target = base_ops.seq2col(base_ops.asarray(baseX), nW=1) + predicted = ops.seq2col(X, nW=1) + ops.xp.testing.assert_allclose(target, predicted, atol=0.001, rtol=0.001) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_lengths_all_zero(ops, dtype): + # Empty batch + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.seq2col( + ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.xp.zeros((0,), dtype="int32") + ), + ) + + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.backprop_seq2col( + ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.xp.zeros((0,), dtype="int32") + ), + ) + + # Zero-length sequence + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.seq2col(ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.asarray1i([0])), + ) + + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.backprop_seq2col( + ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.asarray1i([0]) + ), + ) + + # Multiple zero-length sequences + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.seq2col(ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.asarray1i([0, 0])), + ) + + ops.xp.testing.assert_allclose( + ops.alloc((0, 0), dtype=dtype), + ops.backprop_seq2col( + ops.alloc((0, 0), dtype=dtype), 1, lengths=ops.asarray1i([0, 0]) + ), + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_lengths_zero_first_last(ops, dtype): + cols_check = ops.asarray2f( + [ + [0, 0, 0, 1, 2, 3, 4, 5, 6], + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [4, 5, 6, 7, 8, 9, 10, 11, 12], + [7, 8, 9, 10, 11, 12, 13, 14, 15], + [10, 11, 12, 13, 14, 15, 0, 0, 0], + ], + dtype=dtype, + ) + + grad_check = ops.asarray2f( + [[2, 4, 6], [12, 15, 18], [21, 24, 27], [30, 33, 36], [26, 28, 30]], dtype=dtype + ) + + # Initial zero-length sequence + ops.xp.testing.assert_allclose( + cols_check, + ops.seq2col( + ops.xp.arange(1.0, 16.0, dtype=dtype).reshape(5, 3), + 1, + lengths=ops.asarray1i([0, 5]), + ), + ) + + ops.xp.testing.assert_allclose( + grad_check, + ops.backprop_seq2col( + cols_check, + 1, + lengths=ops.asarray1i([0, 5]), + ), + ) + + # Final zero-length sequence. + ops.xp.testing.assert_allclose( + cols_check, + ops.seq2col( + ops.xp.arange(1.0, 16.0, dtype=dtype).reshape(5, 3), + 1, + lengths=ops.asarray1i([5, 0]), + ), + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_lengths_zero_between(ops, dtype): + cols_check = ops.asarray2f( + [ + [0, 0, 0, 1, 2, 3, 4, 5, 6], + [1, 2, 3, 4, 5, 6, 7, 8, 9], + [4, 5, 6, 7, 8, 9, 10, 11, 12], + [7, 8, 9, 10, 11, 12, 13, 14, 15], + [10, 11, 12, 13, 14, 15, 0, 0, 0], + [0, 0, 0, 16, 17, 18, 19, 20, 21], + [16, 17, 18, 19, 20, 21, 0, 0, 0], + ], + dtype=dtype, + ) + + grad_check = ops.asarray2f( + [ + [2, 4, 6], + [12, 15, 18], + [21, 24, 27], + [30, 33, 36], + [26, 28, 30], + [32, 34, 36], + [38, 40, 42], + ], + dtype=dtype, + ) + + # Zero-length between. + ops.xp.testing.assert_allclose( + cols_check, + ops.seq2col( + ops.xp.arange(1.0, 22.0, dtype=dtype).reshape(7, 3), + 1, + lengths=ops.asarray1i([5, 0, 2]), + ), + ) + + ops.xp.testing.assert_allclose( + grad_check, + ops.backprop_seq2col( + cols_check, + 1, + lengths=ops.asarray1i([5, 0, 2]), + ), + ) + + # Zero-length between twice. + ops.xp.testing.assert_allclose( + cols_check, + ops.seq2col( + ops.xp.arange(1.0, 22.0, dtype=dtype).reshape(7, 3), + 1, + lengths=ops.asarray1i([5, 0, 0, 2]), + ), + ) + + ops.xp.testing.assert_allclose( + grad_check, + ops.backprop_seq2col( + cols_check, + 1, + lengths=ops.asarray1i([5, 0, 0, 2]), + ), + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_window_one_lengths(ops, dtype): + X = ops.xp.arange(1.0, 16.0, dtype=dtype).reshape(5, 3) + lengths = ops.asarray1i([1, 3, 1]) + cols = ops.seq2col(X, 1, lengths=lengths) + ops.xp.testing.assert_allclose( + ops.asarray2f( + [ + [0, 0, 0, 1, 2, 3, 0, 0, 0], + [0, 0, 0, 4, 5, 6, 7, 8, 9], + [4, 5, 6, 7, 8, 9, 10, 11, 12], + [7, 8, 9, 10, 11, 12, 0, 0, 0], + [0, 0, 0, 13, 14, 15, 0, 0, 0], + ], + dtype=dtype, + ), + cols, + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_window_two_lengths(ops, dtype): + X = ops.xp.arange(1.0, 16.0, dtype=dtype).reshape(5, 3) + lengths = ops.asarray1i([1, 3, 1]) + cols = ops.seq2col(X, 2, lengths=lengths) + ops.xp.testing.assert_allclose( + ops.asarray2f( + [ + [0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12], + [0, 0, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 0, 0], + [4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 13, 14, 15, 0, 0, 0, 0, 0, 0], + ], + dtype=dtype, + ), + cols, + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_seq2col_window_one_small(ops, dtype): + cols = ops.asarray( + [[0.0, 0.0, 0.0], [-1.0, 0.0, 1.0], [2.0, 0.0, 0.0]], dtype=dtype + ) + expected = [[-1.0], [2.0], [1.0]] + seq = ops.backprop_seq2col(cols, 1) + if not isinstance(seq, numpy.ndarray): + seq = seq.get() + assert_allclose(seq, expected, atol=0.001, rtol=0.001) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_backprop_seq2col_window_one(ops, dtype, X): + if X.shape[1] % 3: + return None + X = ops.asarray(X, dtype=dtype) + if ops.xp.abs(X).max() >= 30: + return None + base_ops = Ops() + base_ops.xp = ops.xp + target = base_ops.backprop_seq2col(X, nW=1) + predicted = ops.backprop_seq2col(X, nW=1) + for row in range(target.shape[0]): + diff = target[row].sum() - predicted[row].sum() + if diff < -0.1 or diff > 0.1: + print(row, diff) + print(target[row]) + print(predicted[row]) + ops.xp.testing.assert_allclose(target, predicted, atol=0.001, rtol=0.001) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_seq2col_window_one_lengths(ops, dtype): + d_y = ops.xp.arange(0.1, 4.6, step=0.1, dtype=dtype).reshape(5, 9) + lengths = ops.asarray1i([1, 3, 1]) + d_seqs = ops.backprop_seq2col(d_y, 1, lengths=lengths) + + ops.xp.testing.assert_allclose( + ops.asarray2f( + [ + [0.4, 0.5, 0.6], + [3.2, 3.4, 3.6], + [6.6, 6.9, 7.2], + [5.6, 5.8, 6.0], + [4.0, 4.1, 4.2], + ], + dtype=dtype, + ), + d_seqs, + atol=1e-6, + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_seq2col_window_two(ops, dtype): + seq = ops.asarray([[1.0], [2.0], [3.0], [4]], dtype=dtype) + cols = ops.seq2col(seq, 2) + if not isinstance(cols, numpy.ndarray): + cols = cols.get() + assert_allclose(cols[0], [0.0, 0.0, 1.0, 2.0, 3.0]) + assert_allclose(cols[1], [0.0, 1.0, 2.0, 3.0, 4.0]) + assert_allclose(cols[2], [1.0, 2.0, 3.0, 4.0, 0.0]) + assert_allclose(cols[3], [2.0, 3.0, 4.0, 0.0, 0.0]) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_seq2col_window_two_lengths(ops, dtype): + d_y = ops.xp.arange(0.1, 7.6, step=0.1, dtype=dtype).reshape(5, 15) + lengths = ops.asarray1i([1, 3, 1]) + d_seqs = ops.backprop_seq2col(d_y, 2, lengths=lengths) + + ops.xp.testing.assert_allclose( + ops.asarray2f( + [ + [0.7, 0.8, 0.9], + [10.2, 10.5, 10.8], + [11.1, 11.4, 11.7], + [12.0, 12.3, 12.6], + [6.7, 6.8, 6.9], + ], + dtype=dtype, + ), + d_seqs, + ) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_seq2col_window_two(ops, dtype): + cols = ops.asarray( + [ + [0.0, 0.0, 1.0, 2.0, 3.0], + [0.0, 1.0, 2.0, 3.0, 4.0], + [1.0, 2.0, 3.0, 4.0, 0.0], + [2.0, 3.0, 4.0, 0.0, 0.0], + ], + dtype=dtype, + ) + # We're summing the values that each row + # was used as a feature. So row 0 had a + # gradient of 1 in row 0, 1 in row 2, and + # 1 in row 3. + expected = ops.asarray( + [ + [1 + 1 + 1.0 + 0.0], + [2.0 + 2.0 + 2.0 + 2.0], + [3.0 + 3.0 + 3.0 + 3.0], + [0.0 + 4.0 + 4.0 + 4.0], + ], + dtype=dtype, + ) + seq = ops.backprop_seq2col(cols, 2) + ops.xp.testing.assert_allclose(seq, expected, atol=0.001, rtol=0.001) + + +@pytest.mark.skipif(not has_cupy_gpu, reason="needs GPU/CuPy") +@pytest.mark.parametrize("nW", [1, 2]) +def test_large_seq2col_gpu_against_cpu(nW): + cupy_ops = CupyOps() + numpy_ops = NumpyOps() + + # Use array with a large enough batch to require multiple + # CUDA grids. + batch_size = 128 * 128 * 2 # threads per block * blocks * 2 + X = numpy_ops.xp.random.randn(batch_size * 2).astype("float32").reshape(-1, 2) + X_gpu = cupy_ops.asarray2f(X) + + # Use somewhat interesting sequence lengths. + lengths = numpy_ops.asarray1i([1, 4, 2, 1] * (batch_size // 8)) + lengths_gpu = cupy_ops.asarray1i(lengths) + + cols = numpy_ops.seq2col(X, nW=nW, lengths=lengths) + cols_gpu = cupy_ops.seq2col(X_gpu, nW=nW, lengths=lengths_gpu) + + assert_allclose(cols, cols_gpu.get()) + + +@pytest.mark.skipif(not has_cupy_gpu, reason="needs GPU/CuPy") +@pytest.mark.parametrize("nW", [1, 2]) +def test_large_backprop_seq2col_gpu_against_cpu(nW): + cupy_ops = CupyOps() + numpy_ops = NumpyOps() + + # Use array with a large enough batch to require multiple + # CUDA grids. + batch_size = 128 * 128 * 2 # threads per block * blocks * 2 + nF = 2 * nW + 1 + d_cols = ( + numpy_ops.xp.random.randn(batch_size * nF).astype("float32").reshape(-1, nF) + ) + d_cols_gpu = cupy_ops.asarray2f(d_cols) + + # Use somewhat interesting sequence lengths. + lengths = numpy_ops.asarray1i([1, 4, 2, 1] * (batch_size // 8)) + lengths_gpu = cupy_ops.asarray1i(lengths) + + d_seqs = numpy_ops.backprop_seq2col(d_cols, nW=nW, lengths=lengths) + d_seqs_gpu = cupy_ops.backprop_seq2col(d_cols_gpu, nW=nW, lengths=lengths_gpu) + + assert_allclose(d_seqs, d_seqs_gpu.get()) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_backprop_reduce_sum(ops, dtype, X): + X = ops.asarray(X, dtype=dtype) + if ops.xp.abs(X).max() >= 5: + return None + lengths = ops.asarray([3] * len(X), dtype="i") + out = ops.backprop_reduce_sum(X, lengths) + assert out.dtype == dtype + assert out.shape == (sum(lengths), X.shape[1]) + start = 0 + for i, length in enumerate(lengths): + ops.xp.testing.assert_allclose( + out[start : start + length].sum(axis=0), X[i] * length, rtol=0.01, atol=0.01 + ) + start += length + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_softmax_sums_to_one(ops, X): + y = ops.softmax(ops.asarray(X)) + for row in y: + assert 0.99999 <= row.sum() <= 1.0001 + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_softmax_works_inplace(ops, X): + X = ops.asarray(X) + X = ops.softmax(X, inplace=True) + for row in X: + assert 0.99999 <= row.sum() <= 1.00001 + + +def torch_softmax_with_temperature( + X: Floats2d, dY: Floats2d, temperature: float +) -> Tuple[Floats2d, Floats2d]: + import torch + + Xt = xp2torch(X, requires_grad=True) + dYt = xp2torch(dY) + + Xt_temp = Xt / temperature + + Yt = torch.nn.functional.softmax(Xt_temp, dim=-1) + Yt.backward(dYt) + + return cast(Floats2d, torch2xp(Yt)), cast( + Floats2d, torch2xp(cast(torch.Tensor, Xt.grad)) + ) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("temperature", [0.5, 1.0, 2.0]) +def test_softmax_temperature(ops, temperature): + X = ops.xp.arange(-10, 10, 0.2, dtype="f").reshape(10, 10) + dY = ops.xp.eye(10, dtype="f") + + Y = ops.softmax(X, temperature=temperature) + dX = ops.backprop_softmax(Y, dY, temperature=temperature) + + Yt, dXt = torch_softmax_with_temperature(X, dY, temperature) + + ops.xp.testing.assert_allclose(Y, Yt, atol=1e-6) + ops.xp.testing.assert_allclose(dX, dXt, atol=1e-6) + + +@pytest.mark.parametrize("cpu_ops", [*CPU_OPS, BLIS_OPS]) +def test_gemm_computes_correctly(cpu_ops): + W = numpy.zeros((3, 2), dtype="f") + X = numpy.zeros((4, 2), dtype="f") + W += numpy.random.uniform(size=W.size).reshape(W.shape) + X += numpy.random.uniform(size=X.size).reshape(X.shape) + Y = cpu_ops.gemm(X, W, trans2=True) + expected = numpy.dot(X, W.T) + assert_allclose(expected, Y, atol=1e-4, rtol=1e-4) + W = numpy.zeros((2, 3), dtype="f") + X = numpy.zeros((2, 4), dtype="f") + W += numpy.random.uniform(size=W.size).reshape(W.shape) + X += numpy.random.uniform(size=X.size).reshape(X.shape) + Y = cpu_ops.gemm(X, W, trans1=True) + expected = numpy.dot(X.T, W) + assert_allclose(expected, Y, atol=1e-4, rtol=1e-4) + cpu_ops.gemm(X, W, trans1=True, out=Y) + + +@pytest.mark.parametrize("cpu_ops", [*CPU_OPS, BLIS_OPS]) +def test_gemm_out_used(cpu_ops): + a = b = numpy.zeros((2, 2), dtype="f") + c = numpy.ones((2, 2), dtype="f") + cpu_ops.gemm(a, b, out=c) + assert numpy.array_equal(c, numpy.zeros((2, 2))) + + +@pytest.mark.parametrize("cpu_ops", CPU_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_flatten_unflatten_roundtrip(cpu_ops, X): + flat = cpu_ops.flatten([x for x in X]) + assert flat.ndim == 1 + unflat = cpu_ops.unflatten(flat, [len(x) for x in X]) + assert_allclose(X, unflat) + flat2 = cpu_ops.flatten([x for x in X], pad=1, dtype="f") + assert len(flat2) > len(flat) + unflat2 = cpu_ops.unflatten(flat2, [len(x) for x in X], pad=1) + assert_allclose(X, unflat2) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES + INT_TYPES) +def test_pad(ops, dtype): + X = [ops.xp.arange(1, 3, dtype=dtype), ops.xp.arange(1, 5, dtype=dtype)] + ops.xp.testing.assert_allclose(ops.pad(X), [[1, 2, 0, 0], [1, 2, 3, 4]]) + ops.xp.testing.assert_allclose( + ops.pad(X, round_to=8), [[1, 2, 0, 0, 0, 0, 0, 0], [1, 2, 3, 4, 0, 0, 0, 0]] + ) + + X = [ + ops.xp.arange(1, 5, dtype=dtype).reshape(2, 2), + ops.xp.arange(1, 9, dtype=dtype).reshape(4, 2), + ] + ops.xp.testing.assert_allclose( + ops.pad(X), + [ + [[1, 2], [3, 4], [0, 0], [0, 0]], + [[1, 2], [3, 4], [5, 6], [7, 8]], + ], + ) + + ops.xp.testing.assert_allclose( + ops.pad(X, round_to=5), + [ + [[1, 2], [3, 4], [0, 0], [0, 0], [0, 0]], + [[1, 2], [3, 4], [5, 6], [7, 8], [0, 0]], + ], + ) + + with pytest.raises(ValueError, match=r"Rounding for padding must at least be 1"): + ops.pad(X, round_to=0) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_sum(ops, dtype): + X = ops.asarray2f( + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [1.0, 2.0], [3.0, 4.0]], dtype=dtype + ) + lengths = ops.asarray1i([3, 2]) + ops.xp.testing.assert_allclose( + ops.reduce_sum(X, lengths), [[9.0, 12.0], [4.0, 6.0]] + ) + + # Zero-length array + lengths = ops.asarray1i([3, 0, 2]) + ops.xp.testing.assert_allclose( + ops.reduce_sum(X, lengths), [[9.0, 12.0], [0.0, 0.0], [4.0, 6.0]] + ) + + with pytest.raises(IndexError): + ops.reduce_sum(X, ops.xp.array([5, 5, 5, 5], dtype="i")) + + with pytest.raises(ValueError): + ops.reduce_sum(X, ops.xp.array([-1, 10, 5, 5], dtype="i")) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_fails_with_incorrect_length(ops, dtype): + with pytest.raises(ValueError, match=r"lengths must be"): + ops.backprop_reduce_sum( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([-1, 2], dtype="int32"), + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_first(ops, dtype): + X = ops.asarray2f( + [[1.0, 6.0], [2.0, 7.0], [3.0, 8.0], [4.0, 9.0], [5.0, 10.0]], dtype=dtype + ) + lengths = ops.asarray1i([3, 2]) + Y, starts_ends = ops.reduce_first(X, lengths) + ops.xp.testing.assert_array_equal(starts_ends, ops.asarray1i([0, 3, 5])) + ops.xp.testing.assert_allclose(Y, [[1.0, 6.0], [4.0, 9.0]]) + + lengths = ops.asarray1i([3, 0, 2]) + with pytest.raises(ValueError, match=r"all sequence lengths must be > 0"): + ops.reduce_last(X, lengths) + + lengths = ops.asarray1i([3, 2, 1]) + with pytest.raises(IndexError, match=r"lengths must sum up to the number of rows"): + ops.reduce_last(X, lengths) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_reduce_first(ops, dtype): + dY = ops.asarray2f([[1.0, 3.0], [2.0, 4.0]], dtype=dtype) + starts_ends = ops.asarray1i([0, 3, 5]) + dX = ops.backprop_reduce_first(dY, starts_ends) + ops.xp.testing.assert_allclose( + dX, [[1.0, 3.0], [0.0, 0.0], [0.0, 0.0], [2.0, 4.0], [0.0, 0.0]] + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_last(ops, dtype): + X = ops.asarray2f( + [[1.0, 6.0], [2.0, 7.0], [3.0, 8.0], [4.0, 9.0], [5.0, 10.0]], dtype=dtype + ) + lengths = ops.asarray1i([3, 2]) + Y, lasts = ops.reduce_last(X, lengths) + ops.xp.testing.assert_array_equal(lasts, ops.asarray1i([2, 4])) + ops.xp.testing.assert_allclose(Y, [[3.0, 8.0], [5.0, 10.0]]) + + lengths = ops.asarray1i([3, 0, 2]) + with pytest.raises(ValueError, match=r"all sequence lengths must be > 0"): + ops.reduce_last(X, lengths) + + lengths = ops.asarray1i([3, 2, 1]) + with pytest.raises(IndexError, match=r"lengths must sum up to the number of rows"): + ops.reduce_last(X, lengths) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_reduce_last(ops, dtype): + dY = ops.asarray2f([[1.0, 3.0], [2.0, 4.0]], dtype=dtype) + lasts = ops.asarray1i([2, 4]) + dX = ops.backprop_reduce_last(dY, lasts) + ops.xp.testing.assert_allclose( + dX, [[0.0, 0.0], [0.0, 0.0], [1.0, 3.0], [0.0, 0.0], [2.0, 4.0]] + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_max_sm(ops, dtype): + X = ops.xp.zeros((6, 3), dtype=dtype) + X += ops.xp.random.uniform(-1, 1, X.shape) + lengths = ops.xp.array([2, 2, 2], dtype="i") + maxes, which = ops.reduce_max(X, lengths) + assert maxes.dtype == dtype + assert ops.xp.all(which >= 0) + assert ops.xp.all(which < X.shape[0]) + + start = 0 + for i, length in enumerate(lengths): + truth = X[start : start + length].max(axis=0) + ops.xp.testing.assert_allclose(maxes[i], truth) + start += length + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_max(ops, dtype): + m = ops.xp.zeros((19, 5), dtype=dtype) + m += ops.xp.random.uniform(-1, 1, m.shape) + lengths = ops.xp.array([5, 5, 3, 6], dtype="i") + # m[4, 0] = 1 + # m[0, 1] = 2 + # m[1, 3] = 3 + maxes, which = ops.reduce_max(m, lengths) + assert maxes.dtype == dtype + assert ops.xp.all(which >= 0) + assert ops.xp.all(which < m.shape[0]) + + start = 0 + for i, length in enumerate(lengths): + truth = m[start : start + length].max(axis=0) + ops.xp.testing.assert_allclose(maxes[i], truth) + start += length + + with pytest.raises(IndexError): + ops.reduce_max(m, ops.xp.array([5, 5, 5, 5], dtype="i")) + + with pytest.raises(ValueError): + ops.reduce_max(m, ops.xp.array([-1, 10, 5, 5], dtype="i")) + + with pytest.raises(ValueError): + ops.reduce_max(m, ops.xp.array([5, 5, 0, 3, 6], dtype="i")) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_reduce_max(ops, dtype): + dX = ops.backprop_reduce_max( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([[2, 1, 0], [1, 0, 1]]).astype("int32"), + ops.xp.array([3, 2], dtype="int32"), + ) + assert dX.dtype == dtype + ops.xp.testing.assert_allclose( + dX, + [ + [0.0, 0.0, 3.0], + [0.0, 2.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 5.0, 0.0], + [4.0, 0.0, 6.0], + ], + ) + + with pytest.raises(IndexError): + ops.backprop_reduce_max( + ops.xp.arange(1, 7, dtype="f").reshape(2, 3), + ops.xp.array([[2, 3, 0], [1, 0, 1]]).astype("int32"), + ops.xp.array([3, 2], dtype="int32"), + ) + + with pytest.raises(ValueError): + ops.backprop_reduce_max( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([[2, 1, 0], [1, 0, 1]]).astype("int32"), + ops.xp.array([-3, 2], dtype="int32"), + ) + + with pytest.raises(ValueError): + ops.backprop_reduce_max( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([[2, 1, 0], [1, 0, 1], [1, 0, 1]]).astype("int32"), + ops.xp.array([3, 0, 2], dtype="int32"), + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_reduce_mean(ops, dtype): + X = ops.asarray2f( + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [1.0, 2.0], [3.0, 4.0]], dtype=dtype + ) + lengths = ops.asarray1i([3, 2]) + ops.xp.testing.assert_allclose( + ops.reduce_mean(X, lengths), [[3.0, 4.0], [2.0, 3.0]] + ) + + # Zero-length array + lengths = ops.asarray1i([3, 0, 2]) + ops.xp.testing.assert_allclose( + ops.reduce_mean(X, lengths), [[3.0, 4.0], [0.0, 0.0], [2.0, 3.0]] + ) + + # Zero-length array last. + X = ops.asarray2f([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=dtype) + lengths = ops.asarray1i([3, 0]) + ops.xp.testing.assert_allclose( + ops.reduce_mean(X, lengths), [[3.0, 4.0], [0.0, 0.0]] + ) + + with pytest.raises(IndexError): + ops.reduce_mean(X, ops.xp.array([3, 3], dtype="i")) + + with pytest.raises(ValueError): + ops.reduce_mean(X, ops.xp.array([-1, 5], dtype="i")) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +def test_backprop_reduce_mean(ops, dtype): + dX = ops.backprop_reduce_mean( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([4, 2], dtype="int32"), + ) + assert dX.dtype == dtype + ops.xp.testing.assert_allclose( + dX, + [ + [0.25, 0.5, 0.75], + [0.25, 0.5, 0.75], + [0.25, 0.5, 0.75], + [0.25, 0.5, 0.75], + [2.0, 2.5, 3.0], + [2.0, 2.5, 3.0], + ], + ) + + with pytest.raises(ValueError, match=r"lengths must be"): + ops.backprop_reduce_mean( + ops.xp.arange(1, 7, dtype=dtype).reshape(2, 3), + ops.xp.array([-1, 2], dtype="int32"), + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@pytest.mark.parametrize("reduction", REDUCTIONS) +def test_reduce_empty_batch(ops, dtype, reduction): + func = getattr(ops, reduction) + backprop_func = getattr(ops, f"backprop_{reduction}") + + lengths = ops.asarray1i([]) + Y = func(ops.alloc((0, 10), dtype=dtype), lengths) + + if reduction == "reduce_max": + Y, which = Y + dX = backprop_func(Y, which, lengths) + elif isinstance(Y, tuple): + Y, extra = Y + dX = backprop_func(Y, extra) + else: + dX = backprop_func(Y, lengths) + + assert Y.shape == (0, 10) + assert dX.shape == (0, 10) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@pytest.mark.parametrize("reduction", REDUCTIONS) +def test_reduce_empty_hidden(ops, dtype, reduction): + func = getattr(ops, reduction) + backprop_func = getattr(ops, f"backprop_{reduction}") + + lengths = ops.asarray1i([2, 3]) + Y = func(ops.alloc((5, 0), dtype=dtype), lengths) + + if reduction == "reduce_max": + Y, which = Y + dX = backprop_func(Y, which, lengths) + elif isinstance(Y, tuple): + Y, extra = Y + dX = backprop_func(Y, extra) + else: + dX = backprop_func(Y, lengths) + + assert Y.shape == (2, 0) + assert dX.shape == (5, 0) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@pytest.mark.parametrize("reduction_raises", REDUCE_ZERO_LENGTH_RAISES) +def test_reduce_zero_seq_length(ops, dtype, reduction_raises): + reduction_str, raises = reduction_raises + reduction = getattr(ops, reduction_str) + X = ops.asarray2f( + [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [1.0, 2.0], [3.0, 4.0]], dtype=dtype + ) + lengths = ops.asarray1i([3, 0, 2]) + + if raises: + with pytest.raises(ValueError): + reduction(X, lengths) + else: + # All non-raising reductions have zero as their identity element. + ops.xp.testing.assert_allclose(reduction(X, lengths)[1], [0.0, 0.0]) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_mish(ops, X): + X = ops.asarray(X) + Y = ops.mish(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize("dtype", FLOAT_TYPES) +@pytest.mark.parametrize( + "op", + [ + "backprop_clipped_linear", + "backprop_dish", + "backprop_gelu", + "backprop_gelu_approx", + "backprop_hard_sigmoid", + "backprop_hard_swish", + "backprop_hard_swish_mobilenet", + "backprop_hard_tanh", + "backprop_mish", + "backprop_relu", + "backprop_relu_k", + "backprop_softmax", + "backprop_swish", + ], +) +def test_eltwise_backprop_rejects_incorrect_shapes(ops, dtype, op): + backprop = getattr(ops, op) + positional_args = [ + p + for p in inspect.signature(backprop).parameters.values() + if p.default == inspect.Parameter.empty + ] + if len(positional_args) == 3: + with pytest.raises(ValueError): + backprop( + ops.xp.zeros(10, dtype=dtype), + ops.xp.zeros(5, dtype=dtype), + ops.xp.zeros(10, dtype=dtype), + ) + with pytest.raises(ValueError): + backprop( + ops.xp.zeros(10, dtype=dtype), + ops.xp.zeros(10, dtype=dtype), + ops.xp.zeros(5, dtype=dtype), + ) + else: + with pytest.raises(ValueError): + backprop( + ops.xp.arange(-10, 10, dtype=dtype), + ops.xp.arange(5, -5, -1, dtype=dtype), + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_relu_k(ops, X): + X = ops.asarray(X) + Y = ops.relu_k(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + assert (Y >= 0).sum() == Y.size + assert (Y <= 6.0).sum() == Y.size + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_swish(ops, X): + X = ops.asarray(X) + Y = ops.swish(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_hard_sigmoid(ops, X): + X = ops.asarray(X) + Y = ops.hard_sigmoid(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + assert (Y >= 0).sum() == Y.size + assert (Y <= 1.0).sum() == Y.size + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_hard_tanh(ops, X): + X = ops.asarray(X) + Y = ops.hard_tanh(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + assert (Y >= -1.0).sum() == Y.size + assert (Y <= 1.0).sum() == Y.size + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_hard_swish(ops, X): + X = ops.asarray(X) + Y = ops.hard_swish(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_hard_swish_mobilenet(ops, X): + X = ops.asarray(X) + Y = ops.hard_swish_mobilenet(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_gelu_approx(ops, X): + X = ops.asarray(X) + Y = ops.gelu_approx(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_dish(ops, X): + X = ops.asarray(X) + Y = ops.dish(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_gelu(ops, X): + X = ops.asarray(X) + Y = ops.gelu(X) + assert Y.shape == X.shape + assert not ops.xp.isnan(Y).any() + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(X=strategies.arrays_BI()) +def test_backprop_mish(ops, X): + X = ops.asarray(X) + # Test zero gradients result in 0 dX + zeros = ops.alloc(X.shape) + dX = ops.backprop_mish(zeros, X) + assert dX.shape == X.shape + assert (dX == 0).all() + + +def get_lstm_args(depth, dirs, nO, batch_size, nI, draw=None): + + if dirs == 1: + n_params = (nO * 4) * nI + nO * 4 + nO * 4 * nO + nO * 4 + for _ in range(1, depth): + n_params += nO * 4 * nO + nO * 4 + nO * 4 * nO + nO * 4 + else: + n_params = (nO * 2) * nI + nO * 2 + nO * 2 * (nO // 2) + nO * 2 + for _ in range(1, depth): + n_params += nO * 2 * nO + nO * 2 + nO * 2 * (nO // 2) + nO * 2 + n_params *= 2 + lstm = LSTM(nO, nI, depth=depth, bi=dirs >= 2).initialize() + assert lstm.get_param("LSTM").size == n_params + if draw: + params = draw(ndarrays_of_shape(n_params)) + # For some reason this is crashing hypothesis? + # size_at_t = draw(ndarrays_of_shape(shape=(batch_size,), lo=1, dtype="int32")) + size_at_t = numpy.ones(shape=(batch_size,), dtype="int32") + X = draw(ndarrays_of_shape((int(size_at_t.sum()), nI))) + else: + params = numpy.ones((n_params,), dtype="f") + size_at_t = numpy.ones(shape=(batch_size,), dtype="int32") + X = numpy.zeros(((int(size_at_t.sum()), nI))) + H0 = numpy.zeros((depth, dirs, nO // dirs)) + C0 = numpy.zeros((depth, dirs, nO // dirs)) + return (params, H0, C0, X, size_at_t) + + +@composite +def draw_lstm_args(draw): + depth = draw(integers(1, 4)) + dirs = draw(integers(1, 2)) + nO = draw(integers(1, 16)) * dirs + batch_size = draw(integers(1, 6)) + nI = draw(integers(1, 16)) + return get_lstm_args(depth, dirs, nO, batch_size, nI, draw=draw) + + +@pytest.mark.parametrize("ops", XP_OPS) +@pytest.mark.parametrize( + "depth,dirs,nO,batch_size,nI", + [ + (1, 1, 1, 1, 1), + (1, 1, 2, 1, 1), + (1, 1, 2, 1, 2), + (2, 1, 1, 1, 1), + (2, 1, 2, 2, 2), + (1, 2, 2, 1, 1), + (2, 2, 2, 2, 2), + ], +) +def test_lstm_forward_training(ops, depth, dirs, nO, batch_size, nI): + reference_ops = Ops() + params, H0, C0, X, size_at_t = get_lstm_args(depth, dirs, nO, batch_size, nI) + reference = reference_ops.lstm_forward_training(params, H0, C0, X, size_at_t) + Y, fwd_state = ops.lstm_forward_training(params, H0, C0, X, size_at_t) + assert_allclose(fwd_state[2], reference[1][2], atol=1e-4, rtol=1e-3) + assert_allclose(fwd_state[1], reference[1][1], atol=1e-4, rtol=1e-3) + assert_allclose(Y, reference[0], atol=1e-4, rtol=1e-3) + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@pytest.mark.parametrize("ops", XP_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(args=draw_lstm_args()) +def test_lstm_forward_training_fuzz(ops, args): + params, H0, C0, X, size_at_t = args + reference_ops = Ops() + reference = reference_ops.lstm_forward_training(params, H0, C0, X, size_at_t) + Y, fwd_state = ops.lstm_forward_training(params, H0, C0, X, size_at_t) + assert_allclose(fwd_state[2], reference[1][2], atol=1e-4, rtol=1e-3) + assert_allclose(fwd_state[1], reference[1][1], atol=1e-4, rtol=1e-3) + assert_allclose(Y, reference[0], atol=1e-4, rtol=1e-3) + + +def test_get_ops(): + assert isinstance(get_ops("numpy"), NumpyOps) + assert isinstance(get_ops("cupy"), CupyOps) + # If Apple ops are available, "cpu" should return AppleOps or + # NumpyOps otherwise. + try: + from thinc_apple_ops import AppleOps + + assert isinstance(get_ops("cpu"), AppleOps) + except ImportError: + assert isinstance(get_ops("cpu"), NumpyOps) + # If BigEndian ops are available, "cpu" should return BigEndianOps or + # NumpyOps otherwise. + try: + from thinc_bigendian_ops import BigEndianOps + + assert isinstance(get_ops("cpu"), BigEndianOps) + except ImportError: + assert isinstance(get_ops("cpu"), NumpyOps) + with pytest.raises(ValueError): + get_ops("blah") + ops = Ops(numpy) + assert ops.xp == numpy + + +def test_use_ops(): + class_ops = get_current_ops() + with use_ops("numpy"): + new_ops = get_current_ops() + assert new_ops.name == "numpy" + with use_ops("cupy"): + new_ops = get_current_ops() + assert new_ops.name == "cupy" + new_ops = get_current_ops() + assert class_ops.name == new_ops.name + + +def test_minibatch(): + fix_random_seed(0) + ops = get_current_ops() + items = [1, 2, 3, 4, 5, 6] + batches = ops.minibatch(3, items) + assert list(batches) == [[1, 2, 3], [4, 5, 6]] + batches = ops.minibatch((i for i in (3, 2, 1)), items) + assert list(batches) == [[1, 2, 3], [4, 5], [6]] + batches = list(ops.minibatch(3, numpy.asarray(items))) + assert isinstance(batches[0], numpy.ndarray) + assert numpy.array_equal(batches[0], numpy.asarray([1, 2, 3])) + assert numpy.array_equal(batches[1], numpy.asarray([4, 5, 6])) + batches = list(ops.minibatch((i for i in (3, 2, 1)), items, shuffle=True)) + assert batches != [[1, 2, 3], [4, 5], [6]] + assert len(batches[0]) == 3 + assert len(batches[1]) == 2 + assert len(batches[2]) == 1 + with pytest.raises(ValueError): + ops.minibatch(10, (i for i in range(100))) + with pytest.raises(ValueError): + ops.minibatch(10, True) + + +def test_multibatch(): + fix_random_seed(0) + ops = get_current_ops() + arr1 = numpy.asarray([1, 2, 3, 4]) + arr2 = numpy.asarray([5, 6, 7, 8]) + batches = list(ops.multibatch(2, arr1, arr2)) + assert numpy.concatenate(batches).tolist() == [[1, 2], [5, 6], [3, 4], [7, 8]] + batches = list(ops.multibatch(2, arr1, arr2, shuffle=True)) + assert len(batches) == 2 + assert len(batches[0]) == 2 + assert len(batches[1]) == 2 + batches = list(ops.multibatch(2, [1, 2, 3, 4], [5, 6, 7, 8])) + assert batches == [[[1, 2], [5, 6]], [[3, 4], [7, 8]]] + with pytest.raises(ValueError): + ops.multibatch(10, (i for i in range(100)), (i for i in range(100))) + with pytest.raises(ValueError): + ops.multibatch(10, arr1, (i for i in range(100)), arr2) + + +def test_ngrams(): + ops = get_current_ops() + arr1 = numpy.asarray([1, 2, 3, 4, 5], dtype=numpy.uint64) + for n in range(1, 10): + assert len(ops.ngrams(n, arr1)) == max(0, arr1.shape[0] - (n - 1)) + assert len(ops.ngrams(-1, arr1)) == 0 + assert len(ops.ngrams(arr1.shape[0] + 1, arr1)) == 0 + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif(torch_version < Version("1.9.0"), reason="needs PyTorch 1.9.0") +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("dtype", ["float32", "float64"]) +@pytest.mark.parametrize("torch_func", TORCH_FUNCS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given( + x=strategies.floats(min_value=-30, max_value=30), + dY=strategies.floats(min_value=-1, max_value=1), +) +def test_compare_activations_to_torch(ops, dtype, x, dY, torch_func): + import torch + + func_name, pytorch_func = torch_func + forward = getattr(ops, func_name) + backward = getattr(ops, "backprop_" + func_name) + # The tolerance of isclose is set to 1e-06 instead of + # the default 1e-08 due to the GELU + x_thinc = ops.asarray([x], dtype=dtype) + x_torch = xp2torch(x_thinc, requires_grad=True) + y = pytorch_func(x_torch) + y_thinc = forward(x_thinc) + y.backward() + assert x_thinc.dtype == y_thinc.dtype + assert y_thinc is not x_thinc + y_think_inplace = forward(x_thinc, inplace=True) + assert y_think_inplace is x_thinc + assert ops.xp.isclose(y_thinc, y_think_inplace, atol=1e-06) + assert ops.xp.isclose(y_thinc, y.detach(), atol=1e-05) + x_thinc = ops.asarray([x], dtype=dtype) + dY_thinc = ops.asarray([dY], dtype=dtype) + dY_thinc_inplace = dY_thinc.copy() + + s = inspect.signature(backward) + params = {p for p in s.parameters if p in ["dY", "X", "Y"]} + + if params == {"dY", "X", "Y"}: + dx_thinc = backward(dY_thinc, Y=y_thinc, X=x_thinc) + assert dx_thinc.dtype == x_thinc.dtype + assert dx_thinc is not dY_thinc + dx_thinc_inplace = backward( + dY=dY_thinc_inplace, Y=y_thinc, X=x_thinc, inplace=True + ) + assert dx_thinc_inplace is dY_thinc_inplace + assert ops.xp.isclose(dx_thinc, dx_thinc_inplace) + assert ops.xp.isclose(x_torch.grad.item() * dY, float(dx_thinc), atol=1e-06) + elif params == {"Y", "dY"}: + dx_thinc = backward(dY_thinc, Y=y_thinc) + assert dx_thinc.dtype == x_thinc.dtype + assert ops.xp.isclose( + dx_thinc, + backward(dY=dY_thinc_inplace, Y=y_thinc, inplace=True), + ) + assert ops.xp.isclose(x_torch.grad.item() * dY, float(dx_thinc), atol=1e-06) + elif params == {"dY", "X"}: + dx_thinc = backward(dY_thinc, X=x_thinc) + assert dx_thinc.dtype == x_thinc.dtype + assert ops.xp.isclose( + dx_thinc, backward(dY=dY_thinc_inplace, X=x_thinc, inplace=True) + ) + assert ops.xp.isclose( + x_torch.grad.item() * dY, float(backward(dY_thinc, X=x_thinc)), atol=1e-06 + ) + else: + raise NotImplementedError( + f"No PyTorch comparison implemented for parameter set: {params}" + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(x=strategies.floats(min_value=-10, max_value=10)) +def test_clipped_linear(ops, x): + x_thinc = ops.xp.asarray([x]) + assert ops.xp.isclose(ops.clipped_linear(x_thinc, max_val=6.0), ops.relu_k(x_thinc)) + assert ops.xp.isclose( + ops.backprop_clipped_linear(ops.asarray1f([1.0]), x_thinc, max_val=6.0), + ops.backprop_relu_k(ops.asarray1f([1.0]), x_thinc), + ) + assert ops.xp.isclose( + ops.clipped_linear(x_thinc, slope=0.2, offset=0.5), ops.hard_sigmoid(x_thinc) + ) + assert ops.xp.isclose( + ops.backprop_clipped_linear( + ops.asarray1f([1.0]), x_thinc, slope=0.2, offset=0.5 + ), + ops.backprop_hard_sigmoid(ops.asarray1f([1.0]), x_thinc), + ) + + +@pytest.mark.parametrize("ops", ALL_OPS) +@pytest.mark.parametrize("byte_order", (">", "<", "=", "|")) +@settings(max_examples=MAX_EXAMPLES, deadline=None) +@given(x=strategies.floats(min_value=-10, max_value=10)) +def test_to_numpy_byteorder(ops, byte_order, x): + x = ops.xp.asarray([x]) + y = ops.to_numpy(x, byte_order=byte_order) + assert numpy.array_equal(ops.to_numpy(x), ops.to_numpy(y)) + if byte_order in (">", "<"): + # hack from: https://stackoverflow.com/a/49740663 + assert y.dtype.newbyteorder("S").newbyteorder("S").byteorder == byte_order + else: + assert x.dtype.byteorder == y.dtype.byteorder + + +@pytest.mark.skipif(not has_cupy_gpu, reason="needs GPU/CuPy") +def test_custom_kernel_compilation(): + for kernel_name in KERNELS_LIST: + compiled_kernel = KERNELS.get_function(kernel_name) + assert compiled_kernel is not None + + assert compile_mmh() is not None + + +@pytest.mark.parametrize("ops", ALL_OPS) +def test_asarray_from_list_uint64(ops): + # list contains int values both above and below int64.max + uint64_list = [16, 11648197037703959513] + assert uint64_list == list(ops.asarray(uint64_list, dtype="uint64")) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/conftest.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..a80df930d1113ba1e508ace8e64792da2539d421 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/conftest.py @@ -0,0 +1,68 @@ +import pytest +from hypothesis import settings + +# Functionally disable deadline settings for tests +# to prevent spurious test failures in CI builds. +settings.register_profile("no_deadlines", deadline=2 * 60 * 1000) # in ms +settings.load_profile("no_deadlines") + + +def pytest_sessionstart(session): + # If Tensorflow is installed, attempt to enable memory growth + # to prevent it from allocating all of the GPU's free memory + # to its internal memory pool(s). + try: + import tensorflow as tf + + physical_devices = tf.config.list_physical_devices("GPU") + for device in physical_devices: + try: + tf.config.experimental.set_memory_growth(device, True) + except: + # Invalid device or cannot modify virtual devices once initialized. + print(f"failed to enable Tensorflow memory growth on {device}") + except ImportError: + pass + + +def pytest_addoption(parser): + try: + parser.addoption("--slow", action="store_true", help="include slow tests") + # Options are already added, e.g. if conftest is copied in a build pipeline + # and runs twice + except ValueError: + pass + + +def pytest_runtest_setup(item): + def getopt(opt): + # When using 'pytest --pyargs thinc' to test an installed copy of + # thinc, pytest skips running our pytest_addoption() hook. Later, when + # we call getoption(), pytest raises an error, because it doesn't + # recognize the option we're asking about. To avoid this, we need to + # pass a default value. We default to False, i.e., we act like all the + # options weren't given. + return item.config.getoption(f"--{opt}", False) + + for opt in ["slow"]: + if opt in item.keywords and not getopt(opt): + pytest.skip(f"need --{opt} option to run") + + +@pytest.fixture() +def pathy_fixture(): + pytest.importorskip("pathy") + import shutil + import tempfile + + from pathy import Pathy, use_fs + + temp_folder = tempfile.mkdtemp(prefix="thinc-pathy") + use_fs(temp_folder) + + root = Pathy("gs://test-bucket") + root.mkdir(exist_ok=True) + + yield root + use_fs(False) + shutil.rmtree(temp_folder) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_mxnet.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_mxnet.py new file mode 100644 index 0000000000000000000000000000000000000000..3126d7d9c5f5c5ba822858cc1618cfd85ea09e22 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_mxnet.py @@ -0,0 +1,6 @@ +from thinc.compat import enable_mxnet + +try: + enable_mxnet() +except ImportError: + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_tensorflow.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_tensorflow.py new file mode 100644 index 0000000000000000000000000000000000000000..523d6059c8f0646a640acdb99c9d7072e2345a40 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/enable_tensorflow.py @@ -0,0 +1,6 @@ +from thinc.compat import enable_tensorflow + +try: + enable_tensorflow() +except ImportError: + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53093518a8977cc873ab30006d41a909f6199a77 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/test_beam_search.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/test_beam_search.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57e03807be2a6fcd22acf23160c660ee81965b01 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/__pycache__/test_beam_search.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/test_beam_search.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/test_beam_search.py new file mode 100644 index 0000000000000000000000000000000000000000..ef67af5317c5da30049cca273fa0376fcc5d555d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/extra/test_beam_search.py @@ -0,0 +1,5 @@ +from thinc.extra.search import MaxViolation + + +def test_init_violn(): + MaxViolation() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97ef51574efc525eb78297a4048a988df6719e80 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_basic_tagger.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_basic_tagger.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..748548ff37ab103705893ad381b75ac25cb4f43b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_basic_tagger.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_combinators.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_combinators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a372e9b155071a147529895efecc0b320071ae77 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_combinators.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_feed_forward.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_feed_forward.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccfb9d0887c5ff48b6f4f8a463a3f778fb15f8ca Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_feed_forward.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_hash_embed.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_hash_embed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69e250e497e773250ce463c6101b78a52bbe049b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_hash_embed.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_layers_api.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_layers_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..befc8dd76f13ca7c117db3aa375c4dfc63940091 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_layers_api.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_linear.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_linear.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..54d3a944ce201cc7c118821d548757414b81b74f Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_linear.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_lstm.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_lstm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1435b1dae8ecd338d83ee5df421412fdd34c182 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_lstm.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mappers.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mappers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..506b609a5d7dff811f357734348f4c6c5fdc4e41 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mappers.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mnist.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mnist.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..621f35429386b982ee844d96cd96f4aaa8cbed33 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mnist.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mxnet_wrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mxnet_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..054963a41af24b8c6d5d3b77dc9d13db5b09e7d7 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_mxnet_wrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_parametric_attention_v2.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_parametric_attention_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e82aeb17fbec3dfa0eb2e0e0a4a557b65210a4c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_parametric_attention_v2.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_pytorch_wrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_pytorch_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..682fb240bfb9c41d4e330b26c6c4aa14ad2ebd18 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_pytorch_wrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_reduce.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_reduce.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c65bdae0985e32197c9c41e386d2d7a163f759c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_reduce.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_resizable.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_resizable.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5bbb7aeeba1281b3543f88ec3ee2d5de39108f1b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_resizable.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_shim.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_shim.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9927a18b17ae4895edc76c44f0656079b8216098 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_shim.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_softmax.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_softmax.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3e0044968c255db90419ebe8499ecb27a7f65c6 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_softmax.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_sparse_linear.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_sparse_linear.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e50727f838c4e1d04e4298a39534266c07db17e8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_sparse_linear.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_tensorflow_wrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_tensorflow_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0c06335b3375f98f277bd512673f145b5191121 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_tensorflow_wrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_torchscriptwrapper.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_torchscriptwrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..156e83108f0fab326c60fd74357ffbe46a79fa3e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_torchscriptwrapper.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_transforms.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_transforms.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..574e2cc627171b42ac2e790d16cef74230c677cd Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_transforms.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_uniqued.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_uniqued.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73eff0fc5a297166e7ca53638dabd78418594dfc Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_uniqued.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_debug.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_debug.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1ec4aa06bda51df0ecd01cc6cb22b12079bd5ca Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_debug.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_flatten.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_flatten.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e61d9d8d49d0fef30ac7a21b8e5481a91adf9443 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_flatten.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_transforms.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_transforms.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da782ae48ddffab5d6c9551ef7f35a9de1f0b467 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/__pycache__/test_with_transforms.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_basic_tagger.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_basic_tagger.py new file mode 100644 index 0000000000000000000000000000000000000000..ada2cacfde92a69db56cb3e289176fdcf9f93fdf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_basic_tagger.py @@ -0,0 +1,85 @@ +import random + +import pytest + +from thinc.api import ( + Adam, + HashEmbed, + Model, + Relu, + Softmax, + chain, + expand_window, + strings2arrays, + with_array, +) + + +@pytest.fixture(scope="module") +def ancora(): + pytest.importorskip("ml_datasets") + import ml_datasets + + return ml_datasets.ud_ancora_pos_tags() + + +def create_embed_relu_relu_softmax(depth, width, vector_length): + with Model.define_operators({">>": chain}): + model = strings2arrays() >> with_array( + HashEmbed(width, vector_length, column=0) + >> expand_window(window_size=1) + >> Relu(width, width * 3) + >> Relu(width, width) + >> Softmax(17, width) + ) + return model + + +@pytest.fixture(params=[create_embed_relu_relu_softmax]) +def create_model(request): + return request.param + + +def evaluate_tagger(model, dev_X, dev_Y, batch_size): + correct = 0.0 + total = 0.0 + for i in range(0, len(dev_X), batch_size): + Yh = model.predict(dev_X[i : i + batch_size]) + Y = dev_Y[i : i + batch_size] + for j in range(len(Yh)): + correct += (Yh[j].argmax(axis=1) == Y[j].argmax(axis=1)).sum() + total += Yh[j].shape[0] + return correct / total + + +def get_shuffled_batches(Xs, Ys, batch_size): + zipped = list(zip(Xs, Ys)) + random.shuffle(zipped) + for i in range(0, len(zipped), batch_size): + batch_X, batch_Y = zip(*zipped[i : i + batch_size]) + yield list(batch_X), list(batch_Y) + + +@pytest.mark.parametrize( + ("depth", "width", "vector_width", "nb_epoch"), [(2, 32, 16, 5)] +) +def test_small_end_to_end(depth, width, vector_width, nb_epoch, create_model, ancora): + (train_X, train_Y), (dev_X, dev_Y) = ancora + batch_size = 8 + model = create_model(depth, width, vector_width).initialize() + optimizer = Adam(0.001) + losses = [] + scores = [] + for _ in range(nb_epoch): + losses.append(0.0) + for X, Y in get_shuffled_batches(train_X, train_Y, batch_size): + Yh, backprop = model.begin_update(X) + d_loss = [] + for i in range(len(Yh)): + d_loss.append(Yh[i] - Y[i]) + losses[-1] += ((Yh[i] - Y[i]) ** 2).sum() + backprop(d_loss) + model.finish_update(optimizer) + scores.append(evaluate_tagger(model, dev_X, dev_Y, batch_size)) + assert losses[-1] < losses[0] + assert scores[-1] > scores[0] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_combinators.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_combinators.py new file mode 100644 index 0000000000000000000000000000000000000000..8cc273e58e22ce5a78c3a357dfbab2b5eb4522c0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_combinators.py @@ -0,0 +1,298 @@ +import numpy +import pytest +from numpy.testing import assert_allclose + +from thinc.api import ( + Dropout, + Linear, + Model, + NumpyOps, + add, + clone, + concatenate, + map_list, + noop, +) +from thinc.layers import chain, tuplify + + +@pytest.fixture(params=[1, 2, 9]) +def nB(request): + return request.param + + +@pytest.fixture(params=[1, 6]) +def nI(request): + return request.param + + +@pytest.fixture(params=[1, 5, 3]) +def nH(request): + return request.param + + +@pytest.fixture(params=[1, 2, 7, 9]) +def nO(request): + return request.param + + +@pytest.fixture +def model1(nH, nI): + return Linear(nH, nI) + + +@pytest.fixture +def model2(nO, nH): + return Linear(nO, nH) + + +@pytest.fixture +def model3(nO): + return Linear(nO, nO) + + +def test_tuplify_zero(): + with pytest.raises(TypeError): + tuplify() + + +def test_tuplify_one(model1): + with pytest.raises(TypeError): + tuplify(model1) + + +def test_tuplify_two(model1, model2): + model = tuplify(model1, model2) + assert len(model.layers) == 2 + + +def test_tuplify_operator_two(model1, model2): + with Model.define_operators({"&": tuplify}): + model = model1 & model2 + assert len(model.layers) == 2 + + +def test_tuplify_dulicates_input(): + model = tuplify(noop(), noop()) + ones = numpy.ones([10]) + out = model.predict(ones) + assert out == (ones, ones) + + +def test_tuplify_initialize(nI, nO): + linear = Linear(nO) + model = tuplify(linear, linear) + ones = numpy.ones((1, nI), dtype="float") + model.initialize(X=ones) + + +def test_tuplify_three(model1, model2, model3): + model = tuplify(model1, model2, model3) + assert len(model.layers) == 3 + + +def test_tuplify_operator_three(model1, model2, model3): + # Previously we 'flattened' these nested calls. We might opt to do so + # again, especially for the operators. + with Model.define_operators({"&": tuplify}): + model = model1 & model2 & model3 + assert len(model.layers) == 2 + assert len(model.layers[0].layers) == 2 + + +def test_chain_zero(): + with pytest.raises(TypeError): + chain() + + +def test_chain_one(model1): + with pytest.raises(TypeError): + chain(model1) + + +def test_chain_two(model1, model2): + model = chain(model1, model2) + assert len(model.layers) == 2 + + +def test_chain_operator_two(model1, model2): + with Model.define_operators({">>": chain}): + model = model1 >> model2 + assert len(model.layers) == 2 + + +def test_chain_three(model1, model2, model3): + model = chain(model1, model2, model3) + assert len(model.layers) == 3 + + +def test_chain_operator_three(model1, model2, model3): + # Previously we 'flattened' these nested calls. We might opt to do so + # again, especially for the operators. + with Model.define_operators({">>": chain}): + model = model1 >> model2 >> model3 + assert len(model.layers) == 2 + assert len(model.layers[0].layers) == 2 + + +def test_chain_right_branch(model1, model2, model3): + # Previously we 'flattened' these nested calls. We might opt to do so + # again, especially for the operators. + merge1 = chain(model1, model2) + merge2 = chain(merge1, model3) + assert len(merge1.layers) == 2 + assert len(merge2.layers) == 2 + + +@pytest.mark.parametrize("ops", [NumpyOps(), NumpyOps(use_blis=True)]) +def test_chain(ops): + data = numpy.asarray([[1, 2, 3, 4]], dtype="f") + model = chain(Linear(1), Dropout(), Linear(1)) + model.ops = ops + model.initialize(data, data) + Y, backprop = model(data, is_train=True) + backprop(Y) + # Layers with and without nO/nI + model = chain(Linear(1), Dropout(), Linear(1, 1)) + model.initialize(data, data) + # Setting dim on model + model = chain(Linear(1), Dropout(), Linear(1)) + model.set_dim("nO", 1) + model.initialize(data, None) + model = chain(Linear(1, 1), Dropout(), Linear(1, 1)) + model.set_dim("nI", 1) + model.initialize(None, data) + # Not enough arguments + with pytest.raises(TypeError): + chain(Linear()) + with pytest.raises(TypeError): + chain() + + +def test_concatenate_one(model1): + model = concatenate(model1) + assert isinstance(model, Model) + + +def test_concatenate_two(model1, model2): + model = concatenate(model1, model2) + assert len(model.layers) == 2 + + +def test_concatenate_operator_two(model1, model2): + with Model.define_operators({"|": concatenate}): + model = model1 | model2 + assert len(model.layers) == 2 + + +def test_concatenate_three(model1, model2, model3): + model = concatenate(model1, model2, model3) + assert len(model.layers) == 3 + + +def test_concatenate_operator_three(model1, model2, model3): + with Model.define_operators({"|": concatenate}): + model = model1 | model2 | model3 + assert len(model.layers) == 3 + + +def test_clone_changes_predictions(nH, nI): + model1 = Linear(nH) + model = clone(model1, 10) + ones = numpy.ones((10, nI), dtype="f") + model.initialize(X=ones) + output_from_cloned = model.predict(ones) + output_from_orig = model1.predict(ones) + assert output_from_cloned.sum() != output_from_orig.sum() + + +def test_clone_gives_distinct_ids(nH, nI): + model = clone(Linear(nH), 5) + assert len(model.layers) == 5 + seen_ids = set() + for node in model.walk(): + assert node.id not in seen_ids + seen_ids.add(node.id) + assert len(seen_ids) == 6 + + +def test_clone_noop(): + model = clone(Linear(), 0) + assert len(model.layers) == 0 + assert model.name == "noop" + + +def test_concatenate_noop(): + model = concatenate() + assert len(model.layers) == 0 + assert model.name == "noop" + + +def test_noop(): + data = numpy.asarray([1, 2, 3], dtype="f") + model = noop(Linear(), Linear()) + model.initialize(data, data) + Y, backprop = model(data, is_train=True) + assert numpy.array_equal(Y, data) + dX = backprop(Y) + assert numpy.array_equal(dX, data) + + +def test_add(): + data = numpy.asarray([[1, 2, 3, 4]], dtype="f") + model = add(Linear(), Linear()) + model.initialize(data, data) + Y, backprop = model(data, is_train=True) + Y2 = sum(layer.predict(data) for layer in model.layers) + assert numpy.array_equal(Y, Y2) + dX = backprop(Y) + assert dX.shape == data.shape + # Test that nesting works + model2 = add(model, Linear()) + assert len(model2.layers) == 3 + model.initialize(data, data) + Y = model2.predict(data) + Y2 = sum(layer.predict(data) for layer in model2.layers) + assert numpy.array_equal(Y, Y2) + + +def test_add_edge_cases(): + data = numpy.asarray([[1, 2, 3, 4]], dtype="f") + with pytest.raises(TypeError): + add() + model = add(Linear(), Linear()) + model._layers = [] + Y, backprop = model(data, is_train=True) + assert numpy.array_equal(data, Y) + dX = backprop(Y) + assert numpy.array_equal(dX, data) + + +def test_concatenate(): + data = numpy.asarray([[1, 2, 3], [4, 5, 6]], dtype="f") + model = concatenate(Linear(), Linear()) + model.initialize(data, data) + Y, backprop = model(data, is_train=True) + assert Y.shape[1] == sum([layer.predict(data).shape[1] for layer in model.layers]) + dX = backprop(Y) + assert dX.shape == data.shape + + +def test_map_list(): + nI = 4 + nO = 9 + Xs = [numpy.zeros((6, nI), dtype="f"), numpy.ones((3, nI), dtype="f")] + Y_shapes = [(x.shape[0], nO) for x in Xs] + model = map_list(Linear()) + model.initialize(X=Xs, Y=[numpy.zeros(shape, dtype="f") for shape in Y_shapes]) + Ys, backprop = model(Xs, is_train=True) + assert isinstance(Ys, list) + assert len(Ys) == len(Xs) + layer = model.layers[0] + for X, Y in zip(Xs, Ys): + assert_allclose(layer.predict(X), Y) + dXs = backprop(Ys) + assert isinstance(dXs, list) + assert len(dXs) == len(Xs) + assert dXs[0].shape == Xs[0].shape + assert dXs[1].shape == Xs[1].shape diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_feed_forward.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_feed_forward.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b2edca84502f221d3c0978ab3f9e5fb8437f35 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_feed_forward.py @@ -0,0 +1,196 @@ +from functools import partial + +import numpy +import pytest +from numpy.testing import assert_allclose + +from thinc.api import Linear, NumpyOps, Relu, chain + + +@pytest.fixture(params=[1, 2, 9]) +def nB(request): + return request.param + + +@pytest.fixture(params=[1, 6]) +def nI(request): + return request.param + + +@pytest.fixture(params=[1, 5, 3]) +def nH(request): + return request.param + + +@pytest.fixture(params=[1, 2, 7, 9]) +def nO(request): + return request.param + + +@pytest.fixture +def model1(nH, nI): + model = Relu(nH, nI).initialize() + return model + + +@pytest.fixture +def model2(nO, nH): + model = Linear(nO, nH).initialize() + return model + + +@pytest.fixture +def input_data(nB, nI): + return numpy.ones((nB, nI), dtype="f") + 1.0 + + +@pytest.fixture +def gradient_data(nB, nO): + return numpy.zeros((nB, nO), dtype="f") - 1.0 + + +@pytest.fixture +def model(model1, model2): + return chain(model1, model2).initialize() + + +def get_expected_predict(input_data, Ws, bs): + numpy_ops = NumpyOps() + X = input_data + for i, (W, b) in enumerate(zip(Ws, bs)): + X = numpy_ops.asarray(X) + if i > 0: + X *= X > 0 + X = numpy.tensordot(X, W, axes=[[1], [1]]) + b + return X + + +def numeric_gradient(predict, weights, epsilon=1e-4): + out1 = predict(weights + epsilon) + out2 = predict(weights - epsilon) + return (out1 - out2) / (2 * epsilon) + + +def test_models_have_shape(model1, model2, nI, nH, nO): + assert model1.get_param("W").shape == (nH, nI) + assert model1.get_param("b").shape == (nH,) + assert model2.get_param("W").shape == (nO, nH) + assert model2.get_param("b").shape == (nO,) + + +def test_model_shape(model, model1, model2, nI, nH, nO): + assert model.get_dim("nI") == model1.get_dim("nI") + assert model.get_dim("nO") == model2.get_dim("nO") + + +def test_infer_output_shape(): + model = Relu(dropout=0.2) + X = model.ops.alloc2f(4, 5) + Y = model.ops.alloc2f(4, 2) + assert model.has_dim("nI") is None + assert model.has_dim("nO") is None + model.initialize(X=X, Y=Y) + assert model.get_dim("nI") == 5 + assert model.get_dim("nO") == 2 + + +def test_predict_and_begin_update_match(model, model1, model2, input_data): + model = chain(model1, model2) + via_predict = model.predict(input_data) + via_update, _ = model.begin_update(input_data) + assert_allclose(via_predict, via_update) + expected = get_expected_predict( + input_data, + [model1.get_param("W"), model2.get_param("W")], + [model1.get_param("b"), model2.get_param("b")], + ) + assert_allclose(via_update, expected, atol=1e-2, rtol=1e-4) + + +def test_init_functions_are_called(): + init_was_called = {} + + def register_init(name, model, X=None, Y=None): + init_was_called[name] = True + + layer1 = Linear(5) + layer2 = Linear(5) + layer3 = Linear(5) + layer1.init = partial(register_init, "one") + layer2.init = partial(register_init, "two") + layer3.init = partial(register_init, "three") + # This is the nesting we'll get from operators. + model = chain(layer1, chain(layer2, layer3)) + assert not init_was_called + model.initialize() + assert init_was_called["one"] + assert init_was_called["two"] + assert init_was_called["three"] + + +class GradientSpy(object): + def __init__(self): + self.weights = None + self.d_weights = None + + def __call__(self, weights, grad): + self.weights = weights + self.d_weights = grad + + +# I don't know how to get this working properly after the refactor. It's a numeric +# gradient check. I suspect the test is the problem, not the code. +@pytest.mark.skip +# This is the actual definition -- it's just annoying to see tonnes of skips. +# def test_gradient(model, input_data, nB, nH, nI, nO): +def test_gradient(): + truth = numpy.zeros((nB, nO), dtype="float32") + truth[0] = 1.0 + + guess, backprop = model.begin_update(input_data) + backprop(guess - truth) + + for layer in model.layers: + for name in layer.param_names: + agrad = layer.get_grad(name).ravel() # Should have grads for all params. + predict = get_predict(layer, name, input_data) + ngrad = get_numeric_gradient(predict, agrad.size, truth) + assert_allclose(agrad, ngrad, atol=0.2, rtol=0.2) + + +def get_predict(layer, param_name, inputs): + """Helper for gradient check. To do the numeric gradient check, we have + to be able to wiggle one value in a parameter, and check the prediction + before and after. So we need to get a callback that gives an output + given a change to one weight. + """ + + def predict(i, epsilon): + param = layer.get_param(param_name) + shape = param.shape + param = param.ravel() + param[i] += epsilon + layer.set_param(param_name, param.reshape(shape)) + outputs = layer.predict(inputs) + param[i] -= epsilon + layer.set_param(param_name, param.reshape(shape)) + return outputs.reshape(shape) + + return predict + + +def get_numeric_gradient(predict, n, target): + gradient = numpy.zeros(n) + for i in range(n): + out1 = predict(i, 1e-4) + out2 = predict(i, -1e-4) + + err1 = _get_loss(out1, target) + err2 = _get_loss(out2, target) + gradient[i] = (err1 - err2) / (2 * 1e-4) + print("NGrad", i, err1, err2) + return gradient + + +def _get_loss(truth, guess): + return numpy.sum(numpy.sum(0.5 * numpy.square(truth - guess), 1)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_hash_embed.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_hash_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..313f055e0383b958c1ad4a85ad081fbffbf3d896 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_hash_embed.py @@ -0,0 +1,19 @@ +import numpy + +from thinc.api import HashEmbed + + +def test_init(): + model = HashEmbed(64, 1000).initialize() + assert model.get_dim("nV") == 1000 + assert model.get_dim("nO") == 64 + assert model.get_param("E").shape == (1000, 64) + + +def test_seed_changes_bucket(): + model1 = HashEmbed(64, 1000, seed=2).initialize() + model2 = HashEmbed(64, 1000, seed=1).initialize() + arr = numpy.ones((1,), dtype="uint64") + vector1 = model1.predict(arr) + vector2 = model2.predict(arr) + assert vector1.sum() != vector2.sum() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_layers_api.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_layers_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c249b023df82b5faf4b1616bf76e9cbaf2ed00f4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_layers_api.py @@ -0,0 +1,236 @@ +from typing import List, Optional + +import numpy +import pytest +import srsly +from numpy.testing import assert_almost_equal + +from thinc.api import Dropout, Model, NumpyOps, registry, with_padded +from thinc.backends import NumpyOps +from thinc.compat import has_torch +from thinc.types import Array2d, Floats2d, FloatsXd, Padded, Ragged, Shape +from thinc.util import data_validation, get_width + +OPS = NumpyOps() + + +class NoDropoutOps(NumpyOps): + def get_dropout_mask(self, shape: Shape, drop: Optional[float]) -> FloatsXd: + if drop is None or drop <= 0: + return self.xp.ones(shape, dtype="f") + else: + raise ValueError("During prediction, dropout should not be applied") + + +array1d = OPS.xp.asarray([1, 2, 3], dtype="f") +array1dint = OPS.xp.asarray([1, 2, 3], dtype="i") +array2d = OPS.xp.asarray([[4, 2, 3, 4], [1, 5, 3, 1], [9, 8, 5, 7]], dtype="f") +array2dint = OPS.xp.asarray([[1, 2, 3], [4, 5, 6]], dtype="i") +array3d = OPS.xp.zeros((3, 3, 3), dtype="f") +ragged = Ragged(array2d, OPS.xp.asarray([2, 1], dtype="i")) +padded = Padded( + array3d, array1d, OPS.asarray1i([1, 2, 3, 4]), OPS.asarray1i([1, 2, 3, 4]) +) +width = array2d.shape[1] +vectors = numpy.zeros((array2dint.max(), 1), dtype="f") + + +def assert_data_match(Y, out_data): + assert type(Y) == type(out_data) + if isinstance(out_data, OPS.xp.ndarray): + assert isinstance(Y, OPS.xp.ndarray) + assert out_data.ndim == Y.ndim + elif isinstance(out_data, Ragged): + assert isinstance(Y, Ragged) + assert out_data.data.ndim == Y.data.ndim + assert out_data.lengths.ndim == Y.lengths.ndim + elif isinstance(out_data, Padded): + assert isinstance(Y, Padded) + assert out_data.data.ndim == Y.data.ndim + assert out_data.size_at_t.ndim == Y.size_at_t.ndim + assert len(out_data.lengths) == len(Y.lengths) + assert len(out_data.indices) == len(Y.indices) + elif isinstance(out_data, (list, tuple)): + assert isinstance(Y, (list, tuple)) + assert all(isinstance(x, numpy.ndarray) for x in Y) + else: + pytest.fail(f"wrong output of {type(Y)}: {Y}") + + +TEST_CASES_SUMMABLE = [ + # Array to array + ("Dish.v1", {}, array2d, array2d), + ("Dish.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Dropout.v1", {}, array2d, array2d), + ("LayerNorm.v1", {}, array2d, array2d), + ("Linear.v1", {}, array2d, array2d), + ("Logistic.v1", {}, array2d, array2d), + ("Maxout.v1", {}, array2d, array2d), + ("Maxout.v1", {"normalize": True, "dropout": 0.2}, array2d, array2d), + ("Maxout.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Mish.v1", {}, array2d, array2d), + ("Mish.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Mish.v1", {"normalize": True, "dropout": 0.2}, array2d, array2d), + ("Relu.v1", {}, array2d, array2d), + ("Relu.v1", {"normalize": True, "dropout": 0.2}, array2d, array2d), + ("Sigmoid.v1", {}, array2d, array2d), + ("Sigmoid.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("ClippedLinear.v1", {}, array2d, array2d), + ("ClippedLinear.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("ReluK.v1", {}, array2d, array2d), + ("ReluK.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("HardSigmoid.v1", {}, array2d, array2d), + ("HardSigmoid.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("HardTanh.v1", {}, array2d, array2d), + ("HardTanh.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("HardSwish.v1", {}, array2d, array2d), + ("HardSwish.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("HardSwishMobilenet.v1", {}, array2d, array2d), + ("HardSwishMobilenet.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Swish.v1", {}, array2d, array2d), + ("Swish.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Gelu.v1", {}, array2d, array2d), + ("Gelu.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("sigmoid_activation.v1", {}, array2d, array2d), + ("softmax_activation.v1", {}, array2d, array2d), + ("Softmax.v1", {}, array2d, array2d), + ("Softmax.v1", {"nO": 4, "nI": 4}, array2d, array2d), + ("Softmax.v2", {}, array2d, array2d), + ("Softmax.v2", {"nO": 4, "nI": 4}, array2d, array2d), + # fmt: off + # List to list + ("LSTM.v1", {"bi": False}, [array2d, array2d], [array2d, array2d]), + pytest.param("PyTorchLSTM.v1", {"bi": False, "nO": width, "nI": width}, [array2d, array2d], [array2d, array2d], marks=pytest.mark.skipif(not has_torch, reason="needs PyTorch")), + # fmt: on +] + +TEST_CASES = [ + *TEST_CASES_SUMMABLE, + pytest.param( + "PyTorchLSTM.v1", + {"bi": True, "nO": width * 2, "nI": width}, + [array2d, array2d], + [array2d, array2d], + marks=pytest.mark.skipif(not has_torch, reason="needs PyTorch"), + ), + ("LSTM.v1", {"bi": True}, [array2d, array2d], [array2d, array2d]), + # Ragged to array + ("reduce_max.v1", {}, ragged, array2d), + ("reduce_mean.v1", {}, ragged, array2d), + ("reduce_sum.v1", {}, ragged, array2d), + # fmt: off + # Other + ("expand_window.v1", {}, array2d, array2d), + ("expand_window.v1", {}, ragged, ragged), + ("Embed.v1", {"nO": 4, "nV": int(array2dint.max() + 1), "column": 0, "dropout": 0.2}, array2dint, array2d), + ("Embed.v1", {"nO": 4, "nV": int(array1dint.max() + 1)}, array1dint, array2d), + ("HashEmbed.v1", {"nO": 1, "nV": int(array2dint.max()), "column": 0, "dropout": 0.2}, array2dint, array2d), + ("HashEmbed.v1", {"nO": 1, "nV": 2}, array1dint, array2d), + ("MultiSoftmax.v1", {"nOs": (1, 3)}, array2d, array2d), + # ("CauchySimilarity.v1", {}, (array2d, array2d), array1d), + ("ParametricAttention.v1", {}, ragged, ragged), + ("ParametricAttention.v2", {}, ragged, ragged), + ("ParametricAttention.v2", {"key_transform": {"@layers": "Gelu.v1"}}, ragged, ragged), + ("SparseLinear.v1", {}, (numpy.asarray([1, 2, 3], dtype="uint64"), array1d, numpy.asarray([1, 1], dtype="i")), array2d), + ("SparseLinear.v2", {}, (numpy.asarray([1, 2, 3], dtype="uint64"), array1d, numpy.asarray([1, 1], dtype="i")), array2d), + ("remap_ids.v1", {"dtype": "f"}, ["a", 1, 5.0], array2dint), + ("remap_ids.v2", {"mapping_table": {}, "column": 1}, numpy.array([[1, 2, 3], [4, 5, 6]]).T, array2dint), + ("premap_ids.v1", {"mapping_table": {}, "column": 1}, numpy.array([[1, 2, 3], [4, 5, 6]]).T, array2dint), + # fmt: on +] + + +@pytest.mark.parametrize("name,kwargs,in_data,out_data", TEST_CASES) +def test_layers_from_config(name, kwargs, in_data, out_data): + cfg = {"@layers": name, **kwargs} + filled_cfg = registry.fill({"config": cfg}) + assert srsly.is_json_serializable(filled_cfg) + model = registry.resolve({"config": cfg})["config"] + if "LSTM" in name: + model = with_padded(model) + valid = True + with data_validation(valid): + model.initialize(in_data, out_data) + Y, backprop = model(in_data, is_train=True) + if model.has_dim("nO"): + assert get_width(Y) == model.get_dim("nO") + assert_data_match(Y, out_data) + dX = backprop(Y) + assert_data_match(dX, in_data) + # Test that during predictions, no dropout is applied + model._to_ops(NoDropoutOps()) + model.predict(in_data) + + +@pytest.mark.parametrize("name,kwargs,in_data,out_data", TEST_CASES_SUMMABLE) +def test_layers_with_residual(name, kwargs, in_data, out_data): + cfg = {"@layers": "residual.v1", "layer": {"@layers": name, **kwargs}} + model = registry.resolve({"config": cfg})["config"] + if "LSTM" in name: + model = with_padded(model) + model.initialize(in_data, out_data) + Y, backprop = model(in_data, is_train=True) + assert_data_match(Y, out_data) + dX = backprop(Y) + assert_data_match(dX, in_data) + + +@pytest.mark.parametrize("data", [array2d, ragged, padded, [array2d, array2d]]) +def test_dropout(data): + model = Dropout(0.2) + model.initialize(data, data) + Y, backprop = model(data, is_train=False) + assert_data_match(Y, data) + dX = backprop(Y) + assert_data_match(dX, data) + + +@pytest.mark.parametrize("name,kwargs,in_data,out_data", TEST_CASES) +def test_layers_batching_all(name, kwargs, in_data, out_data): + cfg = {"@layers": name, **kwargs} + model = registry.resolve({"config": cfg})["config"] + if "expand_window" in name: + return + if "LSTM" in name: + model = with_padded(model) + util_batch_unbatch_list(model, in_data, out_data) + else: + if isinstance(in_data, OPS.xp.ndarray) and in_data.ndim == 2: + if isinstance(out_data, OPS.xp.ndarray) and out_data.ndim == 2: + util_batch_unbatch_array(model, in_data, out_data) + if isinstance(in_data, Ragged): + if isinstance(out_data, OPS.xp.ndarray) and out_data.ndim == 2: + util_batch_unbatch_ragged(model, in_data, out_data) + + +def util_batch_unbatch_array( + model: Model[Floats2d, Array2d], in_data: Floats2d, out_data: Array2d +): + unbatched = [model.ops.reshape2f(a, 1, -1) for a in in_data] + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data).tolist() + Y_not_batched = [model.predict(u)[0].tolist() for u in unbatched] + assert_almost_equal(Y_batched, Y_not_batched, decimal=4) + + +def util_batch_unbatch_list( + model: Model[List[Array2d], List[Array2d]], + in_data: List[Array2d], + out_data: List[Array2d], +): + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data) + Y_not_batched = [model.predict([u])[0] for u in in_data] + assert_almost_equal(Y_batched, Y_not_batched, decimal=4) # type: ignore + + +def util_batch_unbatch_ragged( + model: Model[Ragged, Array2d], in_data: Ragged, out_data: Array2d +): + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data) + Y_not_batched = [model.predict(in_data[i])[0] for i in range(len(in_data))] + assert_almost_equal(Y_batched, Y_not_batched, decimal=4) # type: ignore diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_linear.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..9d07e21bb4deccbcca39559e6309e129828a892a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_linear.py @@ -0,0 +1,245 @@ +import numpy +import pytest +from hypothesis import given, settings +from mock import MagicMock +from numpy.testing import assert_allclose + +from thinc.api import SGD, Dropout, Linear, chain + +from ..strategies import arrays_OI_O_BI +from ..util import get_model, get_shape + + +@pytest.fixture +def model(): + model = Linear() + return model + + +def test_linear_default_name(model): + assert model.name == "linear" + + +def test_linear_dimensions_on_data(): + X = MagicMock(shape=(5, 10), spec=numpy.ndarray) + X.ndim = 2 + X.dtype = "float32" + y = MagicMock(shape=(8,), spec=numpy.ndarray) + y.ndim = 2 + y.dtype = "float32" + y.max = MagicMock() + model = Linear() + model.initialize(X, y) + assert model.get_dim("nI") is not None + y.max.assert_called_with() + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@given(arrays_OI_O_BI(max_batch=8, max_out=8, max_in=8)) +def test_begin_update_matches_predict(W_b_input): + model = get_model(W_b_input) + nr_batch, nr_out, nr_in = get_shape(W_b_input) + W, b, input_ = W_b_input + fwd_via_begin_update, finish_update = model.begin_update(input_) + fwd_via_predict_batch = model.predict(input_) + assert_allclose(fwd_via_begin_update, fwd_via_predict_batch) + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@given(arrays_OI_O_BI(max_batch=8, max_out=8, max_in=8)) +def test_finish_update_calls_optimizer_with_weights(W_b_input): + model = get_model(W_b_input) + nr_batch, nr_out, nr_in = get_shape(W_b_input) + W, b, input_ = W_b_input + output, finish_update = model.begin_update(input_) + + seen_keys = set() + + def sgd(key, data, gradient, **kwargs): + seen_keys.add(key) + assert data.shape == gradient.shape + return data, gradient + + grad_BO = numpy.ones((nr_batch, nr_out), dtype="f") + grad_BI = finish_update(grad_BO) # noqa: F841 + model.finish_update(sgd) + for name in model.param_names: + assert (model.id, name) in seen_keys + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@settings(max_examples=100) +@given(arrays_OI_O_BI(max_batch=8, max_out=8, max_in=8)) +def test_predict_small(W_b_input): + W, b, input_ = W_b_input + nr_out, nr_in = W.shape + model = Linear(nr_out, nr_in) + model.set_param("W", W) + model.set_param("b", b) + + einsummed = numpy.einsum( + "oi,bi->bo", + numpy.asarray(W, dtype="float64"), + numpy.asarray(input_, dtype="float64"), + optimize=False, + ) + + expected_output = einsummed + b + + predicted_output = model.predict(input_) + assert_allclose(predicted_output, expected_output, rtol=0.01, atol=0.01) + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@given(arrays_OI_O_BI(max_batch=20, max_out=30, max_in=30)) +@settings(deadline=None) +def test_predict_extensive(W_b_input): + W, b, input_ = W_b_input + nr_out, nr_in = W.shape + model = Linear(nr_out, nr_in) + model.set_param("W", W) + model.set_param("b", b) + + einsummed = numpy.einsum( + "bi,oi->bo", + numpy.asarray(input_, dtype="float32"), + numpy.asarray(W, dtype="float32"), + optimize=False, + ) + + expected_output = einsummed + b + + predicted_output = model.predict(input_) + assert_allclose(predicted_output, expected_output, rtol=1e-04, atol=0.0001) + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@given(arrays_OI_O_BI(max_batch=8, max_out=8, max_in=8)) +def test_dropout_gives_zero_activations(W_b_input): + model = chain(get_model(W_b_input), Dropout(1.0)) + nr_batch, nr_out, nr_in = get_shape(W_b_input) + W, b, input_ = W_b_input + fwd_dropped, _ = model.begin_update(input_) + assert all(val == 0.0 for val in fwd_dropped.flatten()) + + +@pytest.mark.skip(reason="Flaky, skip temporarily") +@given(arrays_OI_O_BI(max_batch=8, max_out=8, max_in=8)) +def test_dropout_gives_zero_gradients(W_b_input): + model = chain(get_model(W_b_input), Dropout(1.0)) + nr_batch, nr_out, nr_in = get_shape(W_b_input) + W, b, input_ = W_b_input + for node in model.walk(): + if node.name == "dropout": + node.attrs["dropout_rate"] = 1.0 + fwd_dropped, finish_update = model.begin_update(input_) + grad_BO = numpy.ones((nr_batch, nr_out), dtype="f") + grad_BI = finish_update(grad_BO) + assert all(val == 0.0 for val in grad_BI.flatten()) + + +@pytest.fixture +def model2(): + model = Linear(2, 2).initialize() + return model + + +def test_init(model2): + assert model2.get_dim("nO") == 2 + assert model2.get_dim("nI") == 2 + assert model2.get_param("W") is not None + assert model2.get_param("b") is not None + + +def test_predict_bias(model2): + input_ = model2.ops.alloc2f(1, model2.get_dim("nI")) + target_scores = model2.ops.alloc2f(1, model2.get_dim("nI")) + scores = model2.predict(input_) + assert_allclose(scores[0], target_scores[0]) + # Set bias for class 0 + model2.get_param("b")[0] = 2.0 + target_scores[0, 0] = 2.0 + scores = model2.predict(input_) + assert_allclose(scores, target_scores) + # Set bias for class 1 + model2.get_param("b")[1] = 5.0 + target_scores[0, 1] = 5.0 + scores = model2.predict(input_) + assert_allclose(scores, target_scores) + + +@pytest.mark.parametrize( + "X,expected", + [ + (numpy.asarray([0.0, 0.0], dtype="f"), [0.0, 0.0]), + (numpy.asarray([1.0, 0.0], dtype="f"), [1.0, 0.0]), + (numpy.asarray([0.0, 1.0], dtype="f"), [0.0, 1.0]), + (numpy.asarray([1.0, 1.0], dtype="f"), [1.0, 1.0]), + ], +) +def test_predict_weights(X, expected): + W = numpy.asarray([1.0, 0.0, 0.0, 1.0], dtype="f").reshape((2, 2)) + bias = numpy.asarray([0.0, 0.0], dtype="f") + + model = Linear(W.shape[0], W.shape[1]) + model.set_param("W", W) + model.set_param("b", bias) + + scores = model.predict(X.reshape((1, -1))) + assert_allclose(scores.ravel(), expected) + + +def test_update(): + W = numpy.asarray([1.0, 0.0, 0.0, 1.0], dtype="f").reshape((2, 2)) + bias = numpy.asarray([0.0, 0.0], dtype="f") + + model = Linear(2, 2) + model.set_param("W", W) + model.set_param("b", bias) + sgd = SGD(1.0, L2=0.0, grad_clip=0.0) + sgd.averages = None + + ff = numpy.asarray([[0.0, 0.0]], dtype="f") + tf = numpy.asarray([[1.0, 0.0]], dtype="f") + ft = numpy.asarray([[0.0, 1.0]], dtype="f") # noqa: F841 + tt = numpy.asarray([[1.0, 1.0]], dtype="f") # noqa: F841 + + # ff, i.e. 0, 0 + scores, backprop = model.begin_update(ff) + assert_allclose(scores[0, 0], scores[0, 1]) + # Tell it the answer was 'f' + gradient = numpy.asarray([[-1.0, 0.0]], dtype="f") + backprop(gradient) + for key, (param, d_param) in model.get_gradients().items(): + param, d_param = sgd(key, param, d_param) + model.set_param(key[1], param) + model.set_grad(key[1], d_param) + + b = model.get_param("b") + W = model.get_param("W") + assert b[0] == 1.0 + assert b[1] == 0.0 + # Unchanged -- input was zeros, so can't get gradient for weights. + assert W[0, 0] == 1.0 + assert W[0, 1] == 0.0 + assert W[1, 0] == 0.0 + assert W[1, 1] == 1.0 + + # tf, i.e. 1, 0 + scores, finish_update = model.begin_update(tf) + # Tell it the answer was 'T' + gradient = numpy.asarray([[0.0, -1.0]], dtype="f") + finish_update(gradient) + for key, (W, dW) in model.get_gradients().items(): + sgd(key, W, dW) + b = model.get_param("b") + W = model.get_param("W") + assert b[0] == 1.0 + assert b[1] == 1.0 + # Gradient for weights should have been outer(gradient, input) + # so outer([0, -1.], [1., 0.]) + # = [[0., 0.], [-1., 0.]] + assert W[0, 0] == 1.0 - 0.0 + assert W[0, 1] == 0.0 - 0.0 + assert W[1, 0] == 0.0 - -1.0 + assert W[1, 1] == 1.0 - 0.0 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_lstm.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_lstm.py new file mode 100644 index 0000000000000000000000000000000000000000..1e97eec1fdcfb9adec749284b2ac32ca63061c83 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_lstm.py @@ -0,0 +1,178 @@ +import timeit + +import numpy +import pytest + +from thinc.api import LSTM, NumpyOps, Ops, PyTorchLSTM, fix_random_seed, with_padded +from thinc.compat import has_torch + + +@pytest.fixture(params=[1, 6]) +def nI(request): + return request.param + + +@pytest.fixture(params=[1, 2, 7, 9]) +def nO(request): + return request.param + + +def test_list2padded(): + ops = NumpyOps() + seqs = [numpy.zeros((5, 4)), numpy.zeros((8, 4)), numpy.zeros((2, 4))] + padded = ops.list2padded(seqs) + arr = padded.data + size_at_t = padded.size_at_t + assert arr.shape == (8, 3, 4) + assert size_at_t[0] == 3 + assert size_at_t[1] == 3 + assert size_at_t[2] == 2 + assert size_at_t[3] == 2 + assert size_at_t[4] == 2 + assert size_at_t[5] == 1 + assert size_at_t[6] == 1 + assert size_at_t[7] == 1 + unpadded = ops.padded2list(padded) + assert unpadded[0].shape == (5, 4) + assert unpadded[1].shape == (8, 4) + assert unpadded[2].shape == (2, 4) + + +@pytest.mark.parametrize("ops", [Ops(), NumpyOps()]) +@pytest.mark.parametrize("nO,nI", [(1, 2), (2, 2), (100, 200), (9, 6)]) +def test_LSTM_init_with_sizes(ops, nO, nI): + model = with_padded(LSTM(nO, nI, depth=1)).initialize() + for node in model.walk(): + model.ops = ops + # Check no unallocated params. + assert node.has_param("LSTM") is not None + assert node.has_param("HC0") is not None + for node in model.walk(): + # Check param sizes. + if node.has_param("LSTM"): + params = node.get_param("LSTM") + assert params.shape == ( + ((nO * 4 * nI)) + (nO * 4) + (nO * 4 * nO + nO * 4), + ) + if node.has_param("HC0"): + params = node.get_param("HC0") + assert params.shape == (2, 1, 1, nO) + + +@pytest.mark.parametrize("ops", [Ops(), NumpyOps()]) +def test_LSTM_fwd_bwd_shapes_simple(ops, nO, nI): + nO = 1 + nI = 2 + X = numpy.asarray([[0.1, 0.1], [-0.1, -0.1], [1.0, 1.0]], dtype="f") + model = with_padded(LSTM(nO, nI)).initialize(X=[X]) + for node in model.walk(): + node.ops = ops + ys, backprop_ys = model([X], is_train=True) + dXs = backprop_ys(ys) + assert numpy.vstack(dXs).shape == numpy.vstack([X]).shape + + +@pytest.mark.parametrize("ops", [Ops(), NumpyOps()]) +@pytest.mark.parametrize( + "nO,nI,depth,bi,lengths", + [ + (1, 1, 1, False, [1]), + (12, 32, 1, False, [3, 1]), + (2, 2, 1, True, [2, 5, 7]), + (2, 2, 2, False, [7, 2, 4]), + (2, 2, 2, True, [1]), + (32, 16, 1, True, [5, 1, 10, 2]), + (32, 16, 2, True, [3, 3, 5]), + (32, 16, 3, True, [9, 2, 4]), + ], +) +def test_BiLSTM_fwd_bwd_shapes(ops, nO, nI, depth, bi, lengths): + Xs = [numpy.ones((length, nI), dtype="f") for length in lengths] + model = with_padded(LSTM(nO, nI, depth=depth, bi=bi)).initialize(X=Xs) + for node in model.walk(): + node.ops = ops + ys, backprop_ys = model(Xs, is_train=True) + dXs = backprop_ys(ys) + assert numpy.vstack(dXs).shape == numpy.vstack(Xs).shape + + +def test_LSTM_learns(): + fix_random_seed(0) + + nO = 2 + nI = 2 + + def sgd(key, weights, gradient): + weights -= 0.001 * gradient + return weights, gradient * 0 + + model = with_padded(LSTM(nO, nI)) + X = [[0.1, 0.1], [0.2, 0.2], [0.3, 0.3]] + Y = [[0.2, 0.2], [0.3, 0.3], [0.4, 0.4]] + X = [model.ops.asarray(x, dtype="f").reshape((1, -1)) for x in X] + Y = [model.ops.asarray(y, dtype="f").reshape((1, -1)) for y in Y] + model = model.initialize(X, Y) + Yhs, bp_Yhs = model.begin_update(X) + loss1 = sum([((yh - y) ** 2).sum() for yh, y in zip(Yhs, Y)]) + Yhs, bp_Yhs = model.begin_update(X) + dYhs = [yh - y for yh, y in zip(Yhs, Y)] + dXs = bp_Yhs(dYhs) + model.finish_update(sgd) + Yhs, bp_Yhs = model.begin_update(X) + dYhs = [yh - y for yh, y in zip(Yhs, Y)] + dXs = bp_Yhs(dYhs) # noqa: F841 + loss2 = sum([((yh - y) ** 2).sum() for yh, y in zip(Yhs, Y)]) + assert loss1 > loss2, (loss1, loss2) + + +@pytest.mark.skip +def test_benchmark_LSTM_fwd(): + nO = 128 + nI = 128 + n_batch = 1000 + batch_size = 30 + seq_len = 30 + lengths = numpy.random.normal(scale=10, loc=30, size=n_batch * batch_size) + lengths = numpy.maximum(lengths, 1) + batches = [] + uniform_lengths = False + model = with_padded(LSTM(nO, nI)).initialize() + for batch_lengths in model.ops.minibatch(batch_size, lengths): + batch_lengths = list(batch_lengths) + if uniform_lengths: + seq_len = max(batch_lengths) + batch = [ + numpy.asarray( + numpy.random.uniform(0.0, 1.0, (int(seq_len), nI)), dtype="f" + ) + for _ in batch_lengths + ] + else: + batch = [ + numpy.asarray( + numpy.random.uniform(0.0, 1.0, (int(seq_len), nI)), dtype="f" + ) + for seq_len in batch_lengths + ] + batches.append(batch) + start = timeit.default_timer() + for Xs in batches: + ys, bp_ys = model.begin_update(list(Xs)) + # _ = bp_ys(ys) + end = timeit.default_timer() + n_samples = n_batch * batch_size + print( + "--- %i samples in %s seconds (%f samples/s, %.7f s/sample) ---" + % (n_samples, end - start, n_samples / (end - start), (end - start) / n_samples) + ) + + +def test_lstm_init(): + model = with_padded(LSTM(2, 2, bi=True)).initialize() + model.initialize() + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +def test_pytorch_lstm_init(): + model = with_padded(PyTorchLSTM(2, 2, depth=0)).initialize() + assert model.name == "with_padded(noop)" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mappers.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mappers.py new file mode 100644 index 0000000000000000000000000000000000000000..ad8035e6b2787d177c88dbb66d812b88f55ede1e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mappers.py @@ -0,0 +1,56 @@ +import numpy +import pytest + +from thinc.layers import premap_ids, remap_ids, remap_ids_v2 + + +@pytest.fixture +def keys(): + return numpy.array([4, 2, 6, 1, 8, 7, 9, 3, 30]) + + +@pytest.fixture +def mapper(keys): + return {int(k): int(v) for v, k in enumerate(keys)} + + +def test_premap(keys, mapper): + premap = premap_ids(mapper, default=99) + values, _ = premap(keys, False) + numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) + + +def test_remap(keys, mapper): + remap = remap_ids(mapper, default=99) + values, _ = remap(keys, False) + numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) + + +def test_remap_v2(keys, mapper): + remap = remap_ids_v2(mapper, default=99) + values, _ = remap(keys, False) + numpy.testing.assert_equal(values.squeeze(), numpy.asarray(range(len(keys)))) + + +def test_remap_premap_eq(keys, mapper): + remap = remap_ids(mapper, default=99) + remap_v2 = remap_ids_v2(mapper, default=99) + premap = premap_ids(mapper, default=99) + values1, _ = remap(keys, False) + values2, _ = remap_v2(keys, False) + values3, _ = premap(keys, False) + numpy.testing.assert_equal(values1, values2) + numpy.testing.assert_equal(values2, values3) + + +def test_column(keys, mapper): + idx = numpy.zeros((len(keys), 4), dtype="int") + idx[:, 3] = keys + remap_v2 = remap_ids_v2(mapper, column=3) + premap = premap_ids(mapper, column=3) + numpy.testing.assert_equal( + remap_v2(idx, False)[0].squeeze(), numpy.asarray(range(len(keys))) + ) + numpy.testing.assert_equal( + premap(idx, False)[0].squeeze(), numpy.asarray(range(len(keys))) + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mnist.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..0473d25659c013d1edb3d916738fadf5baccf29b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mnist.py @@ -0,0 +1,116 @@ +import pytest + +from thinc.api import ( + Adam, + PyTorchWrapper, + Relu, + Softmax, + TensorFlowWrapper, + chain, + clone, + get_current_ops, +) +from thinc.compat import has_tensorflow, has_torch + + +@pytest.fixture(scope="module") +def mnist(limit=5000): + pytest.importorskip("ml_datasets") + import ml_datasets + + (train_X, train_Y), (dev_X, dev_Y) = ml_datasets.mnist() + return (train_X[:limit], train_Y[:limit]), (dev_X[:limit], dev_Y[:limit]) + + +def create_relu_softmax(width, dropout, nI, nO): + return chain(clone(Relu(nO=width, dropout=dropout), 2), Softmax(10, width)) + + +def create_wrapped_pytorch(width, dropout, nI, nO): + import torch + import torch.nn + import torch.nn.functional as F + + class PyTorchModel(torch.nn.Module): + def __init__(self, width, nO, nI, dropout): + super(PyTorchModel, self).__init__() + self.dropout1 = torch.nn.Dropout2d(dropout) + self.dropout2 = torch.nn.Dropout2d(dropout) + self.fc1 = torch.nn.Linear(nI, width) + self.fc2 = torch.nn.Linear(width, nO) + + def forward(self, x): + x = F.relu(x) + x = self.dropout1(x) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + return PyTorchWrapper(PyTorchModel(width, nO, nI, dropout)) + + +def create_wrapped_tensorflow(width, dropout, nI, nO): + from tensorflow.keras.layers import Dense, Dropout + from tensorflow.keras.models import Sequential + + tf_model = Sequential() + tf_model.add(Dense(width, activation="relu", input_shape=(nI,))) + tf_model.add(Dropout(dropout)) + tf_model.add(Dense(width, activation="relu")) + tf_model.add(Dropout(dropout)) + tf_model.add(Dense(nO, activation=None)) + return TensorFlowWrapper(tf_model) + + +@pytest.fixture( + # fmt: off + params=[ + create_relu_softmax, + pytest.param(create_wrapped_pytorch, marks=pytest.mark.skipif(not has_torch, reason="needs PyTorch")), + pytest.param(create_wrapped_tensorflow, marks=pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow")) + ] + # fmt: on +) +def create_model(request): + return request.param + + +@pytest.mark.slow +@pytest.mark.parametrize(("width", "nb_epoch", "min_score"), [(32, 20, 0.8)]) +def test_small_end_to_end(width, nb_epoch, min_score, create_model, mnist): + batch_size = 128 + dropout = 0.2 + (train_X, train_Y), (dev_X, dev_Y) = mnist + model = create_model(width, dropout, nI=train_X.shape[1], nO=train_Y.shape[1]) + model.initialize(X=train_X[:5], Y=train_Y[:5]) + optimizer = Adam(0.001) + losses = [] + scores = [] + ops = get_current_ops() + + for i in range(nb_epoch): + for X, Y in model.ops.multibatch(batch_size, train_X, train_Y, shuffle=True): + Yh, backprop = model.begin_update(X) + # Ensure that the tensor is type-compatible with the current backend. + Yh = ops.asarray(Yh) + + backprop(Yh - Y) + model.finish_update(optimizer) + losses.append(((Yh - Y) ** 2).sum()) + correct = 0 + total = 0 + for X, Y in model.ops.multibatch(batch_size, dev_X, dev_Y): + Yh = model.predict(X) + Yh = ops.asarray(Yh) + + correct += (Yh.argmax(axis=1) == Y.argmax(axis=1)).sum() + total += Yh.shape[0] + score = correct / total + scores.append(score) + assert losses[-1] < losses[0], losses + if scores[0] < 1.0: + assert scores[-1] > scores[0], scores + assert any([score > min_score for score in scores]), scores diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mxnet_wrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mxnet_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..02e7b3de828c40b4fc7428fb5f5925223ac6f6c9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_mxnet_wrapper.py @@ -0,0 +1,216 @@ +from typing import cast + +import numpy +import pytest + +from thinc.api import ( + Adam, + ArgsKwargs, + Model, + MXNetWrapper, + Ops, + get_current_ops, + mxnet2xp, + xp2mxnet, +) +from thinc.compat import has_cupy_gpu, has_mxnet +from thinc.types import Array1d, Array2d, IntsXd +from thinc.util import to_categorical + +from ..util import check_input_converters, make_tempdir + + +@pytest.fixture +def n_hidden() -> int: + return 12 + + +@pytest.fixture +def input_size() -> int: + return 784 + + +@pytest.fixture +def n_classes() -> int: + return 10 + + +@pytest.fixture +def answer() -> int: + return 1 + + +@pytest.fixture +def X(input_size: int) -> Array2d: + ops: Ops = get_current_ops() + return cast(Array2d, ops.alloc(shape=(1, input_size))) + + +@pytest.fixture +def Y(answer: int, n_classes: int) -> Array2d: + ops: Ops = get_current_ops() + return cast( + Array2d, + to_categorical(cast(IntsXd, ops.asarray([answer])), n_classes=n_classes), + ) + + +@pytest.fixture +def mx_model(n_hidden: int, input_size: int, X: Array2d): + import mxnet as mx + + mx_model = mx.gluon.nn.Sequential() + mx_model.add( + mx.gluon.nn.Dense(n_hidden), + mx.gluon.nn.LayerNorm(), + mx.gluon.nn.Dense(n_hidden, activation="relu"), + mx.gluon.nn.LayerNorm(), + mx.gluon.nn.Dense(10, activation="softrelu"), + ) + mx_model.initialize() + return mx_model + + +@pytest.fixture +def model(mx_model) -> Model[Array2d, Array2d]: + return MXNetWrapper(mx_model) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_roundtrip_conversion(): + import mxnet as mx + + xp_tensor = numpy.zeros((2, 3), dtype="f") + mx_tensor = xp2mxnet(xp_tensor) + assert isinstance(mx_tensor, mx.nd.NDArray) + new_xp_tensor = mxnet2xp(mx_tensor) + assert numpy.array_equal(xp_tensor, new_xp_tensor) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_gluon_sequential(): + import mxnet as mx + + mx_model = mx.gluon.nn.Sequential() + mx_model.add(mx.gluon.nn.Dense(12)) + wrapped = MXNetWrapper(mx_model) + assert isinstance(wrapped, Model) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_built_model( + model: Model[Array2d, Array2d], X: Array2d, Y: Array1d +): + # built models are validated more and can perform useful operations: + assert model.predict(X) is not None + # They can de/serialized + assert model.from_bytes(model.to_bytes()) is not None + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_predict(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_train_overfits( + model: Model[Array2d, Array2d], X: Array2d, Y: Array1d, answer: int +): + optimizer = Adam() + for i in range(100): + guesses, backprop = model(X, is_train=True) + d_guesses = (guesses - Y) / guesses.shape[0] + backprop(d_guesses) + model.finish_update(optimizer) + predicted = model.predict(X).argmax() + assert predicted == answer + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_can_copy_model(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + copy: Model[Array2d, Array2d] = model.copy() + assert copy is not None + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_to_bytes(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + # And can be serialized + model_bytes = model.to_bytes() + assert model_bytes is not None + model.from_bytes(model_bytes) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_to_from_disk(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + with make_tempdir() as tmp_path: + model_file = tmp_path / "model.bytes" + model.to_disk(model_file) + another_model = model.from_disk(model_file) + assert another_model is not None + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_from_bytes(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + model_bytes = model.to_bytes() + another_model = model.from_bytes(model_bytes) + assert another_model is not None + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_to_cpu(mx_model, X: Array2d): + model = MXNetWrapper(mx_model) + model.predict(X) + model.to_cpu() + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +@pytest.mark.skipif(not has_cupy_gpu, reason="needs GPU/cupy") +def test_mxnet_wrapper_to_gpu(model: Model[Array2d, Array2d], X: Array2d): + model.predict(X) + model.to_gpu(0) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +@pytest.mark.parametrize( + "data,n_args,kwargs_keys", + [ + # fmt: off + (numpy.zeros((2, 3), dtype="f"), 1, []), + ([numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")], 2, []), + ((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), 2, []), + ({"a": numpy.zeros((2, 3), dtype="f"), "b": numpy.zeros((2, 3), dtype="f")}, 0, ["a", "b"]), + (ArgsKwargs((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), {"c": numpy.zeros((2, 3), dtype="f")}), 2, ["c"]), + # fmt: on + ], +) +def test_mxnet_wrapper_convert_inputs(data, n_args, kwargs_keys): + import mxnet as mx + + mx_model = mx.gluon.nn.Sequential() + mx_model.add(mx.gluon.nn.Dense(12)) + mx_model.initialize() + model = MXNetWrapper(mx_model) + convert_inputs = model.attrs["convert_inputs"] + Y, backprop = convert_inputs(model, data, is_train=True) + check_input_converters(Y, backprop, data, n_args, kwargs_keys, mx.nd.NDArray) + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_thinc_model_subclass(mx_model): + class CustomModel(Model): + def fn(self) -> int: + return 1337 + + model = MXNetWrapper(mx_model, model_class=CustomModel) + assert isinstance(model, CustomModel) + assert model.fn() == 1337 + + +@pytest.mark.skipif(not has_mxnet, reason="needs MXNet") +def test_mxnet_wrapper_thinc_set_model_name(mx_model): + model = MXNetWrapper(mx_model, model_name="cool") + assert model.name == "cool" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_parametric_attention_v2.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_parametric_attention_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..d939d8903874024136de72e49aadf76066340ae3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_parametric_attention_v2.py @@ -0,0 +1,10 @@ +from thinc.layers.gelu import Gelu +from thinc.layers.parametricattention_v2 import ( + KEY_TRANSFORM_REF, + ParametricAttention_v2, +) + + +def test_key_transform_used(): + attn = ParametricAttention_v2(key_transform=Gelu()) + assert attn.get_ref(KEY_TRANSFORM_REF).name == "gelu" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_pytorch_wrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_pytorch_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..b80c3c6bf0e4e99e4027b856b89a5e5e1c83dee3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_pytorch_wrapper.py @@ -0,0 +1,215 @@ +import numpy +import pytest + +from thinc.api import ( + SGD, + ArgsKwargs, + CupyOps, + Linear, + MPSOps, + NumpyOps, + PyTorchWrapper, + PyTorchWrapper_v2, + PyTorchWrapper_v3, + Relu, + chain, + get_current_ops, + torch2xp, + use_ops, + xp2torch, +) +from thinc.backends import context_pools +from thinc.compat import has_cupy_gpu, has_torch, has_torch_amp, has_torch_mps_gpu +from thinc.layers.pytorchwrapper import PyTorchWrapper_v3 +from thinc.shims.pytorch import ( + default_deserialize_torch_model, + default_serialize_torch_model, +) +from thinc.shims.pytorch_grad_scaler import PyTorchGradScaler +from thinc.util import get_torch_default_device + +from ..util import check_input_converters, make_tempdir + +XP_OPS = [NumpyOps()] +if has_cupy_gpu: + XP_OPS.append(CupyOps()) +if has_torch_mps_gpu: + XP_OPS.append(MPSOps()) + + +if has_torch_amp: + TORCH_MIXED_PRECISION = [False, True] +else: + TORCH_MIXED_PRECISION = [False] + +XP_OPS_MIXED = [ + (ops, mixed) + for ops in XP_OPS + for mixed in TORCH_MIXED_PRECISION + if not mixed or isinstance(ops, CupyOps) +] + + +def check_learns_zero_output(model, sgd, X, Y): + """Check we can learn to output a zero vector""" + Yh, get_dX = model.begin_update(X) + dYh = (Yh - Y) / Yh.shape[0] + dX = get_dX(dYh) + model.finish_update(sgd) + prev = numpy.abs(Yh.sum()) + for i in range(100): + Yh, get_dX = model.begin_update(X) + total = numpy.abs(Yh.sum()) + dX = get_dX(Yh - Y) # noqa: F841 + model.finish_update(sgd) + assert total < prev + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("nN,nI,nO", [(2, 3, 4)]) +def test_pytorch_unwrapped(nN, nI, nO): + model = Linear(nO, nI).initialize() + X = numpy.zeros((nN, nI), dtype="f") + X += numpy.random.uniform(size=X.size).reshape(X.shape) + sgd = SGD(0.01) + Y = numpy.zeros((nN, nO), dtype="f") + check_learns_zero_output(model, sgd, X, Y) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("nN,nI,nO", [(2, 3, 4)]) +def test_pytorch_wrapper(nN, nI, nO): + import torch.nn + + model = PyTorchWrapper(torch.nn.Linear(nI, nO)).initialize() + sgd = SGD(0.001) + X = numpy.zeros((nN, nI), dtype="f") + X += numpy.random.uniform(size=X.size).reshape(X.shape) + Y = numpy.zeros((nN, nO), dtype="f") + Yh, get_dX = model.begin_update(X) + assert isinstance(Yh, numpy.ndarray) + assert Yh.shape == (nN, nO) + dYh = (Yh - Y) / Yh.shape[0] + dX = get_dX(dYh) + model.finish_update(sgd) + assert dX.shape == (nN, nI) + check_learns_zero_output(model, sgd, X, Y) + assert isinstance(model.predict(X), numpy.ndarray) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("ops_mixed", XP_OPS_MIXED) +@pytest.mark.parametrize("nN,nI,nO", [(2, 3, 4)]) +def test_pytorch_wrapper_thinc_input(ops_mixed, nN, nI, nO): + import torch.nn + + ops, mixed_precision = ops_mixed + + with use_ops(ops.name): + ops = get_current_ops() + pytorch_layer = torch.nn.Linear(nO, nO) + # Initialize with large weights to trigger overflow of FP16 in + # mixed-precision training. + torch.nn.init.uniform_(pytorch_layer.weight, 9.0, 11.0) + device = get_torch_default_device() + model = chain( + Relu(), + PyTorchWrapper_v2( + pytorch_layer.to(device), + mixed_precision=mixed_precision, + grad_scaler=PyTorchGradScaler( + enabled=mixed_precision, init_scale=2.0**16 + ), + ).initialize(), + ) + # pytorch allocator is set in PyTorchShim + if isinstance(ops, CupyOps): + assert "pytorch" in context_pools.get() + sgd = SGD(0.001) + X = ops.xp.zeros((nN, nI), dtype="f") + X += ops.xp.random.uniform(size=X.size).reshape(X.shape) + Y = ops.xp.zeros((nN, nO), dtype="f") + model.initialize(X, Y) + Yh, get_dX = model.begin_update(X) + assert isinstance(Yh, ops.xp.ndarray) + assert Yh.shape == (nN, nO) + dYh = (Yh - Y) / Yh.shape[0] + dX = get_dX(dYh) + model.finish_update(sgd) + assert dX.shape == (nN, nI) + check_learns_zero_output(model, sgd, X, Y) + assert isinstance(model.predict(X), ops.xp.ndarray) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +def test_pytorch_roundtrip_conversion(): + import torch + + xp_tensor = numpy.zeros((2, 3), dtype="f") + torch_tensor = xp2torch(xp_tensor) + assert isinstance(torch_tensor, torch.Tensor) + new_xp_tensor = torch2xp(torch_tensor) + assert numpy.array_equal(xp_tensor, new_xp_tensor) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +def test_pytorch_wrapper_roundtrip(): + import torch.nn + + model = PyTorchWrapper(torch.nn.Linear(2, 3)) + model_bytes = model.to_bytes() + PyTorchWrapper(torch.nn.Linear(2, 3)).from_bytes(model_bytes) + with make_tempdir() as path: + model_path = path / "model" + model.to_disk(model_path) + new_model = PyTorchWrapper(torch.nn.Linear(2, 3)).from_bytes(model_bytes) + new_model.from_disk(model_path) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize( + "data,n_args,kwargs_keys", + [ + # fmt: off + (numpy.zeros((2, 3), dtype="f"), 1, []), + ([numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")], 2, []), + ((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), 2, []), + ({"a": numpy.zeros((2, 3), dtype="f"), "b": numpy.zeros((2, 3), dtype="f")}, 0, ["a", "b"]), + (ArgsKwargs((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), {"c": numpy.zeros((2, 3), dtype="f")}), 2, ["c"]), + # fmt: on + ], +) +def test_pytorch_convert_inputs(data, n_args, kwargs_keys): + import torch.nn + + model = PyTorchWrapper(torch.nn.Linear(3, 4)) + convert_inputs = model.attrs["convert_inputs"] + Y, backprop = convert_inputs(model, data, is_train=True) + check_input_converters(Y, backprop, data, n_args, kwargs_keys, torch.Tensor) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +def test_pytorch_wrapper_custom_serde(): + import torch.nn + + def serialize(model): + return default_serialize_torch_model(model) + + def deserialize(model, state_bytes, device): + return default_deserialize_torch_model(model, state_bytes, device) + + def get_model(): + return PyTorchWrapper_v3( + torch.nn.Linear(2, 3), + serialize_model=serialize, + deserialize_model=deserialize, + ) + + model = get_model() + model_bytes = model.to_bytes() + get_model().from_bytes(model_bytes) + with make_tempdir() as path: + model_path = path / "model" + model.to_disk(model_path) + new_model = get_model().from_bytes(model_bytes) + new_model.from_disk(model_path) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_reduce.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_reduce.py new file mode 100644 index 0000000000000000000000000000000000000000..14081b35c7c1bfe1e59348bc400966f76472c424 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_reduce.py @@ -0,0 +1,122 @@ +import numpy +import pytest + +from thinc.api import reduce_first, reduce_last, reduce_max, reduce_mean, reduce_sum +from thinc.types import Ragged + + +@pytest.fixture +def Xs(): + seqs = [numpy.zeros((10, 8), dtype="f"), numpy.zeros((4, 8), dtype="f")] + for x in seqs: + x[0] = 1 + x[1] = 2 # so max != first + x[-1] = -1 + return seqs + + +def test_init_reduce_first(): + model = reduce_first() + + +def test_init_reduce_last(): + model = reduce_last() + + +def test_init_reduce_mean(): + model = reduce_mean() + + +def test_init_reduce_max(): + model = reduce_max() + + +def test_init_reduce_sum(): + model = reduce_sum() + + +def test_reduce_first(Xs): + model = reduce_first() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + assert isinstance(Y, numpy.ndarray) + assert Y.shape == (len(Xs), Xs[0].shape[1]) + assert Y.dtype == Xs[0].dtype + assert list(Y[0]) == list(Xs[0][0]) + assert list(Y[1]) == list(Xs[1][0]) + dX = backprop(Y) + assert dX.dataXd.shape == X.dataXd.shape + + +def test_reduce_last(Xs): + model = reduce_last() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + assert isinstance(Y, numpy.ndarray) + assert Y.shape == (len(Xs), Xs[0].shape[1]) + assert Y.dtype == Xs[0].dtype + assert list(Y[0]) == list(Xs[0][-1]) + assert list(Y[1]) == list(Xs[1][-1]) + dX = backprop(Y) + assert dX.dataXd.shape == X.dataXd.shape + + +def test_reduce_max(Xs): + model = reduce_max() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + assert isinstance(Y, numpy.ndarray) + assert Y.shape == (len(Xs), Xs[0].shape[1]) + assert Y.dtype == Xs[0].dtype + assert list(Y[0]) == list(Xs[0][1]) + assert list(Y[1]) == list(Xs[1][1]) + dX = backprop(Y) + assert dX.dataXd.shape == X.dataXd.shape + + +def test_reduce_mean(Xs): + Xs = [x * 1000 for x in Xs] # use large numbers for numeric stability + model = reduce_mean() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + assert isinstance(Y, numpy.ndarray) + assert Y.shape == (len(Xs), Xs[0].shape[1]) + assert Y.dtype == Xs[0].dtype + assert numpy.all(Y[0] == Y[0][0]) # all values in row should be equal + assert Y[0][0] == Xs[0].mean() + assert numpy.all(Y[1] == Y[1][0]) + assert Y[1][0] == Xs[1].mean() + dX = backprop(Y) + assert dX.dataXd.shape == X.dataXd.shape + + +def test_reduce_sum(Xs): + model = reduce_sum() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + assert isinstance(Y, numpy.ndarray) + assert Y.shape == (len(Xs), Xs[0].shape[1]) + assert Y.dtype == Xs[0].dtype + assert Y[0][0] == Xs[0][:, 0].sum() + assert numpy.all(Y[0] == Y[0][0]) + assert Y[1][-1] == Xs[1][:, 0].sum() + assert numpy.all(Y[1] == Y[1][0]) + dX = backprop(Y) + assert dX.dataXd.shape == X.dataXd.shape + + +def test_size_mismatch(Xs): + for reduce in [reduce_first, reduce_last, reduce_max, reduce_mean, reduce_sum]: + model = reduce() + lengths = model.ops.asarray([x.shape[0] for x in Xs], dtype="i") + X = Ragged(model.ops.flatten(Xs), lengths) + Y, backprop = model(X, is_train=True) + + Y_bad = Y[:-1] + with pytest.raises(ValueError): + backprop(Y_bad) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_resizable.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_resizable.py new file mode 100644 index 0000000000000000000000000000000000000000..101a9acd3c15540c5ec18423569c87ccf9f50eaf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_resizable.py @@ -0,0 +1,34 @@ +from functools import partial + +import pytest + +from thinc.api import Linear, resizable +from thinc.layers.resizable import resize_linear_weighted, resize_model + + +@pytest.fixture +def model(): + output_layer = Linear(nO=None, nI=None) + fill_defaults = {"b": 0, "W": 0} + model = resizable( + output_layer, + resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults), + ) + return model + + +def test_resizable_linear_default_name(model): + assert model.name == "resizable(linear)" + + +def test_resize_model(model): + """Test that resizing the model doesn't cause an exception.""" + resize_model(model, new_nO=10) + resize_model(model, new_nO=11) + + model.set_dim("nO", 0, force=True) + resize_model(model, new_nO=10) + + model.set_dim("nI", 10, force=True) + model.set_dim("nO", 0, force=True) + resize_model(model, new_nO=10) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_shim.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_shim.py new file mode 100644 index 0000000000000000000000000000000000000000..50f60a05d709320a94e3622fcac2f7f025161b60 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_shim.py @@ -0,0 +1,35 @@ +from typing import List + +from thinc.shims.shim import Shim + +from ..util import make_tempdir + + +class MockShim(Shim): + def __init__(self, data: List[int]): + super().__init__(None, config=None, optimizer=None) + self.data = data + + def to_bytes(self): + return bytes(self.data) + + def from_bytes(self, data: bytes) -> "MockShim": + return MockShim(data=list(data)) + + +def test_shim_can_roundtrip_with_path(): + + with make_tempdir() as path: + shim_path = path / "cool_shim.data" + shim = MockShim([1, 2, 3]) + shim.to_disk(shim_path) + copy_shim = shim.from_disk(shim_path) + assert copy_shim.to_bytes() == shim.to_bytes() + + +def test_shim_can_roundtrip_with_path_subclass(pathy_fixture): + shim_path = pathy_fixture / "cool_shim.data" + shim = MockShim([1, 2, 3]) + shim.to_disk(shim_path) + copy_shim = shim.from_disk(shim_path) + assert copy_shim.to_bytes() == shim.to_bytes() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_softmax.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_softmax.py new file mode 100644 index 0000000000000000000000000000000000000000..024232b4d6707b95d14202985e7eec7501485c79 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_softmax.py @@ -0,0 +1,92 @@ +from typing import Tuple, cast + +import numpy +import pytest +from numpy.testing import assert_allclose + +from thinc.api import Model, NumpyOps, Softmax_v2 +from thinc.types import Floats2d, Ints1d +from thinc.util import has_torch, torch2xp, xp2torch + +OPS = NumpyOps() + +inputs = OPS.xp.asarray([[4, 2, 3, 4], [1, 5, 3, 1], [9, 8, 5, 7]], dtype="f") +outputs = OPS.xp.asarray( + [ + [0.39948627, 0.05406459, 0.14696279, 0.39948627], + [0.01562812, 0.8532666, 0.11547707, 0.01562812], + [0.657233, 0.24178252, 0.01203764, 0.08894681], + ], + dtype="f", +) + + +def test_unnormalized_softmax_backprop(): + model = Softmax_v2(normalize_outputs=False) + model.initialize(inputs, outputs) + _, backprop = model(inputs, is_train=False) + with pytest.raises(ValueError, match="backprop is not supported"): + backprop(OPS.xp.zeros_like(outputs)) + + # Backprop should not fail when training. + _, backprop = model(inputs, is_train=True) + dX = backprop(OPS.xp.zeros_like(outputs)) + assert OPS.xp.all(dX == 0.0) + + +def torch_softmax_with_temperature( + model: Model, X: Floats2d, targets: Ints1d +) -> Tuple[Floats2d, Floats2d]: + import torch + + Wt = xp2torch(model.get_param("W")) + bt = xp2torch(model.get_param("b")) + temperature = model.attrs["softmax_temperature"] + + Xt = xp2torch(X, requires_grad=True) + Yt_gold = xp2torch(targets).long() + + XWbt = (Xt @ Wt) + bt + XWbt_temp = XWbt / temperature + + loss = torch.nn.CrossEntropyLoss() + output = loss(XWbt_temp, Yt_gold) + output.backward() + + return cast( + Floats2d, torch2xp(torch.nn.functional.softmax(XWbt_temp, dim=-1)) + ), cast(Floats2d, torch2xp(cast(torch.Tensor, Xt.grad))) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("temperature", [0.5, 1.0, 2.0]) +def test_softmax_temperature(temperature): + model = Softmax_v2( + temperature=temperature, + init_W=lambda ops, shape: ops.xp.eye(shape[1], dtype="f"), + init_b=lambda ops, shape: ops.xp.zeros(shape, dtype="f"), + ) + + X = OPS.xp.arange(-1, 1, 0.2, dtype="f").reshape(1, 10) + targets = OPS.asarray1i([4]) + Y_gold = OPS.xp.eye(10, dtype="f")[targets] + model.initialize(X, Y_gold) + + Yt, dXt = torch_softmax_with_temperature(model, X, targets) + + Y, backprop = model(X, is_train=True) + dX = backprop(Y - Y_gold) + + assert_allclose(Y, Yt, atol=1e-4) + assert_allclose(dX, dXt, atol=1e-4) + + +def test_reject_incorrect_temperature(): + with pytest.raises(ValueError, match=r"softmax temperature.*zero"): + Softmax_v2(normalize_outputs=False, temperature=0.0) + + model = Softmax_v2(normalize_outputs=False) + model.attrs["softmax_temperature"] = 0.0 + model.initialize(inputs, outputs) + with pytest.raises(ValueError, match=r"softmax temperature.*zero"): + model(inputs, is_train=False) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_sparse_linear.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_sparse_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..8f16b62aacc004b4e7a44bd9ab3db770edbba3fd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_sparse_linear.py @@ -0,0 +1,79 @@ +import math + +import numpy +import pytest + +from thinc.api import SGD, SparseLinear, SparseLinear_v2, to_categorical + + +@pytest.fixture +def instances(): + lengths = numpy.asarray([5, 4], dtype="int32") + keys = numpy.arange(9, dtype="uint64") + values = numpy.ones(9, dtype="float32") + X = (keys, values, lengths) + y = numpy.asarray([0, 2], dtype="int32") + return X, to_categorical(y, n_classes=3) + + +@pytest.fixture +def sgd(): + return SGD(0.001) + + +def test_basic(instances, sgd): + X, y = instances + nr_class = 3 + model = SparseLinear(nr_class).initialize() + yh, backprop = model.begin_update(X) + loss1 = ((yh - y) ** 2).sum() + backprop(yh - y) + model.finish_update(sgd) + yh, backprop = model.begin_update(X) + loss2 = ((yh - y) ** 2).sum() + assert loss2 < loss1 + + +def test_init(): + model = SparseLinear(3).initialize() + keys = numpy.ones((5,), dtype="uint64") + values = numpy.ones((5,), dtype="f") + lengths = numpy.zeros((2,), dtype="int32") + lengths[0] = 3 + lengths[1] = 2 + scores, backprop = model.begin_update((keys, values, lengths)) + assert scores.shape == (2, 3) + d_feats = backprop(scores) + assert len(d_feats) == 3 + + +def test_distribution(): + n_class = 10 + length = 2**18 + model = SparseLinear_v2(nO=n_class, length=length).initialize() + + ii64 = numpy.iinfo(numpy.uint64) + lengths = numpy.zeros((2,), dtype="int32") + + for p_nonzero in range(1, 12): + # Clear gradients from the previous iterarion. + model.set_grad("W", 0.0) + + n = 2**p_nonzero + keys = numpy.random.randint(ii64.min, ii64.max, size=(n,), dtype=numpy.uint64) + values = numpy.ones((n,), dtype="f") + lengths[0] = n // 2 + lengths[1] = n // 2 + + # Probability that a bit is set (2 because we use 2 hashes). + p_nonzero = 1 - math.exp(-2 * n / length) + + Y, backprop = model.begin_update((keys, values, lengths)) + backprop(numpy.ones_like(Y)) + + # Check that for each class we have the expected rate of non-zeros. + dW = model.get_grad("W").reshape(n_class, -1) + nonzero_empirical = numpy.count_nonzero(dW, axis=1) / dW.shape[1] + numpy.testing.assert_allclose( + nonzero_empirical, p_nonzero, rtol=1e-4, atol=1e-4 + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_tensorflow_wrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_tensorflow_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..f23be8163125371783fb90e5b3d93c56b3657baa --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_tensorflow_wrapper.py @@ -0,0 +1,414 @@ +import numpy +import pytest + +from thinc.api import ( + Adam, + ArgsKwargs, + Linear, + Model, + TensorFlowWrapper, + get_current_ops, + keras_subclass, + tensorflow2xp, + xp2tensorflow, +) +from thinc.compat import has_cupy_gpu, has_tensorflow +from thinc.util import to_categorical + +from ..util import check_input_converters, make_tempdir + + +@pytest.fixture +def n_hidden(): + return 12 + + +@pytest.fixture +def input_size(): + return 784 + + +@pytest.fixture +def n_classes(): + return 10 + + +@pytest.fixture +def answer(): + return 1 + + +@pytest.fixture +def X(input_size): + ops = get_current_ops() + return ops.alloc(shape=(1, input_size)) + + +@pytest.fixture +def Y(answer, n_classes): + ops = get_current_ops() + return to_categorical(ops.asarray1i([answer]), n_classes=n_classes) + + +@pytest.fixture +def tf_model(n_hidden, input_size): + import tensorflow as tf + + tf_model = tf.keras.Sequential( + [ + tf.keras.layers.Dense(n_hidden, input_shape=(input_size,)), + tf.keras.layers.LayerNormalization(), + tf.keras.layers.Dense(n_hidden, activation="relu"), + tf.keras.layers.LayerNormalization(), + tf.keras.layers.Dense(10, activation="softmax"), + ] + ) + return tf_model + + +@pytest.fixture +def model(tf_model): + return TensorFlowWrapper(tf_model) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_roundtrip_conversion(): + import tensorflow as tf + + ops = get_current_ops() + xp_tensor = ops.alloc2f(2, 3, zeros=True) + tf_tensor = xp2tensorflow(xp_tensor) + assert isinstance(tf_tensor, tf.Tensor) + new_xp_tensor = tensorflow2xp(tf_tensor, ops=ops) + assert ops.xp.array_equal(xp_tensor, new_xp_tensor) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_construction_requires_keras_model(): + import tensorflow as tf + + keras_model = tf.keras.Sequential([tf.keras.layers.Dense(12, input_shape=(12,))]) + assert isinstance(TensorFlowWrapper(keras_model), Model) + with pytest.raises(ValueError): + TensorFlowWrapper(Linear(2, 3)) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_built_model(model, X, Y): + # built models are validated more and can perform useful operations: + assert model.predict(X) is not None + # Can print a keras summary + assert str(model.shims[0]) != "" + # They can de/serialized + assert model.from_bytes(model.to_bytes()) is not None + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_predict(model, X): + model.predict(X) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_train_overfits(model, X, Y, answer): + optimizer = Adam() + ops = get_current_ops() + for i in range(100): + guesses, backprop = model(X, is_train=True) + # Ensure that the tensor is type-compatible with the current backend. + guesses = ops.asarray(guesses) + + d_guesses = (guesses - Y) / guesses.shape[0] + backprop(d_guesses) + model.finish_update(optimizer) + predicted = model.predict(X).argmax() + assert predicted == answer + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_accumulate_gradients(model, X, Y, answer): + import tensorflow as tf + + optimizer = Adam() + gradients = [] + ops = get_current_ops() + for i in range(3): + guesses, backprop = model(X, is_train=True) + # Ensure that the tensor is type-compatible with the current backend. + guesses = ops.asarray(guesses) + + d_guesses = (guesses - Y) / guesses.shape[0] + backprop(d_guesses) + shim_grads = [tf.identity(var) for var in model.shims[0].gradients] + gradients.append(shim_grads) + + # Apply the gradients + model.finish_update(optimizer) + assert model.shims[0].gradients is None + + # Compare prev/next pairs and ensure their gradients have changed + for i in range(len(gradients)): + # Skip the first one + if i == 0: + continue + found_diff = False + curr_grads = gradients[i] + prev_grads = gradients[i - 1] + for curr, prev in zip(curr_grads, prev_grads): + if (prev != curr).numpy().any(): + found_diff = True + assert found_diff is True + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_serialize_model_subclass( + X, Y, input_size, n_classes, answer +): + import tensorflow as tf + + input_shape = (1, input_size) + ops = get_current_ops() + + @keras_subclass( + "foo.v1", + X=ops.alloc2f(*input_shape), + Y=to_categorical(ops.asarray1i([1]), n_classes=n_classes), + input_shape=input_shape, + ) + class CustomKerasModel(tf.keras.Model): + def __init__(self, **kwargs): + super(CustomKerasModel, self).__init__(**kwargs) + self.in_dense = tf.keras.layers.Dense( + 12, name="in_dense", input_shape=input_shape + ) + self.out_dense = tf.keras.layers.Dense( + n_classes, name="out_dense", activation="softmax" + ) + + def call(self, inputs) -> tf.Tensor: + x = self.in_dense(inputs) + return self.out_dense(x) + + model = TensorFlowWrapper(CustomKerasModel()) + # Train the model to predict the right single answer + optimizer = Adam() + for i in range(50): + guesses, backprop = model(X, is_train=True) + # Ensure that the tensor is type-compatible with the current backend. + guesses = ops.asarray(guesses) + + d_guesses = (guesses - Y) / guesses.shape[0] + backprop(d_guesses) + model.finish_update(optimizer) + predicted = model.predict(X).argmax() + assert predicted == answer + + # Save then Load the model from bytes + model.from_bytes(model.to_bytes()) + + # The from_bytes model gets the same answer + assert model.predict(X).argmax() == answer + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_keras_subclass_decorator_compile_args(): + import tensorflow as tf + + class UndecoratedModel(tf.keras.Model): + def call(self, inputs): + return inputs + + # Can't wrap an undecorated keras subclass model + with pytest.raises(ValueError): + TensorFlowWrapper(UndecoratedModel()) + + @keras_subclass( + "TestModel", + X=numpy.array([0.0, 0.0]), + Y=numpy.array([0.5]), + input_shape=(2,), + compile_args={"loss": "binary_crossentropy"}, + ) + class TestModel(tf.keras.Model): + def call(self, inputs): + return inputs + + model = TensorFlowWrapper(TestModel()) + model = model.from_bytes(model.to_bytes()) + + assert model.shims[0]._model.loss == "binary_crossentropy" + assert isinstance(model, Model) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_keras_subclass_decorator(): + import tensorflow as tf + + class UndecoratedModel(tf.keras.Model): + def call(self, inputs): + return inputs + + # Can't wrap an undecorated keras subclass model + with pytest.raises(ValueError): + TensorFlowWrapper(UndecoratedModel()) + + @keras_subclass( + "TestModel", X=numpy.array([0.0, 0.0]), Y=numpy.array([0.5]), input_shape=(2,) + ) + class TestModel(tf.keras.Model): + def call(self, inputs): + return inputs + + # Can wrap an decorated keras subclass model + assert isinstance(TensorFlowWrapper(TestModel()), Model) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_keras_subclass_decorator_capture_args_kwargs( + X, Y, input_size, n_classes, answer +): + import tensorflow as tf + + @keras_subclass( + "TestModel", X=numpy.array([0.0, 0.0]), Y=numpy.array([0.5]), input_shape=(2,) + ) + class TestModel(tf.keras.Model): + def __init__(self, custom=False, **kwargs): + super().__init__(self) + # This is to force the mode to pass the captured arguments + # or fail. + assert custom is True + assert kwargs.get("other", None) is not None + + def call(self, inputs): + return inputs + + # Can wrap an decorated keras subclass model + model = TensorFlowWrapper(TestModel(True, other=1337)) + + assert hasattr(model.shims[0]._model, "eg_args") + args_kwargs = model.shims[0]._model.eg_args + assert True in args_kwargs.args + assert "other" in args_kwargs.kwargs + + # Raises an error if the args/kwargs is not serializable + obj = {} + obj["key"] = obj + with pytest.raises(ValueError): + TensorFlowWrapper(TestModel(True, other=obj)) + + # Provides the same arguments when copying a capture model + model = model.from_bytes(model.to_bytes()) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_can_copy_model(model): + copy = model.copy() + assert copy is not None + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_print_summary(model, X): + summary = str(model.shims[0]) + # Summary includes the layers of our model + assert "layer_normalization" in summary + assert "dense" in summary + # And counts of params + assert "Total params" in summary + assert "Trainable params" in summary + assert "Non-trainable params" in summary + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_to_bytes(model, X): + # And can be serialized + model_bytes = model.to_bytes() + assert model_bytes is not None + model.from_bytes(model_bytes) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_to_from_disk(model, X, Y, answer): + with make_tempdir() as tmp_path: + model_file = tmp_path / "model.h5" + model.to_disk(model_file) + another_model = model.from_disk(model_file) + assert another_model is not None + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_from_bytes(model, X): + model.predict(X) + model_bytes = model.to_bytes() + another_model = model.from_bytes(model_bytes) + assert another_model is not None + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_use_params(model, X, Y, answer): + optimizer = Adam() + ops = get_current_ops() + with model.use_params(optimizer.averages): + assert model.predict(X).argmax() is not None + for i in range(10): + guesses, backprop = model.begin_update(X) + # Ensure that the tensor is type-compatible with the current backend. + guesses = ops.asarray(guesses) + + d_guesses = (guesses - Y) / guesses.shape[0] + backprop(d_guesses) + model.finish_update(optimizer) + with model.use_params(optimizer.averages): + predicted = model.predict(X).argmax() + assert predicted == answer + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_to_cpu(tf_model): + model = TensorFlowWrapper(tf_model) + model.to_cpu() + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +@pytest.mark.skipif(not has_cupy_gpu, reason="needs GPU/cupy") +def test_tensorflow_wrapper_to_gpu(model, X): + model.to_gpu(0) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +@pytest.mark.parametrize( + "data,n_args,kwargs_keys", + [ + # fmt: off + (numpy.zeros((2, 3), dtype="f"), 1, []), + ([numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")], 2, []), + ((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), 2, []), + ({"a": numpy.zeros((2, 3), dtype="f"), "b": numpy.zeros((2, 3), dtype="f")}, 0, ["a", "b"]), + (ArgsKwargs((numpy.zeros((2, 3), dtype="f"), numpy.zeros((2, 3), dtype="f")), {"c": numpy.zeros((2, 3), dtype="f")}), 2, ["c"]), + # fmt: on + ], +) +def test_tensorflow_wrapper_convert_inputs(data, n_args, kwargs_keys): + import tensorflow as tf + + keras_model = tf.keras.Sequential([tf.keras.layers.Dense(12, input_shape=(12,))]) + model = TensorFlowWrapper(keras_model) + convert_inputs = model.attrs["convert_inputs"] + Y, backprop = convert_inputs(model, data, is_train=True) + check_input_converters(Y, backprop, data, n_args, kwargs_keys, tf.Tensor) + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_thinc_model_subclass(tf_model): + class CustomModel(Model): + def fn(self): + return 1337 + + model = TensorFlowWrapper(tf_model, model_class=CustomModel) + assert isinstance(model, CustomModel) + assert model.fn() == 1337 + + +@pytest.mark.skipif(not has_tensorflow, reason="needs TensorFlow") +def test_tensorflow_wrapper_thinc_set_model_name(tf_model): + model = TensorFlowWrapper(tf_model, model_name="cool") + assert model.name == "cool" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_torchscriptwrapper.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_torchscriptwrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..e543b51f660eb05c2278ab9f4e3a1a872fbe69ca --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_torchscriptwrapper.py @@ -0,0 +1,28 @@ +import numpy +import pytest + +from thinc.api import ( + PyTorchWrapper_v2, + TorchScriptWrapper_v1, + pytorch_to_torchscript_wrapper, +) +from thinc.compat import has_torch, torch + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.parametrize("nN,nI,nO", [(2, 3, 4)]) +def test_pytorch_script(nN, nI, nO): + + model = PyTorchWrapper_v2(torch.nn.Linear(nI, nO)).initialize() + script_model = pytorch_to_torchscript_wrapper(model) + + X = numpy.random.randn(nN, nI).astype("f") + Y = model.predict(X) + Y_script = script_model.predict(X) + numpy.testing.assert_allclose(Y, Y_script) + + serialized = script_model.to_bytes() + script_model2 = TorchScriptWrapper_v1() + script_model2.from_bytes(serialized) + + numpy.testing.assert_allclose(Y, script_model2.predict(X)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_transforms.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..724a9fa5086e84acd4269acd7697f3ed522310e9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_transforms.py @@ -0,0 +1,79 @@ +import numpy +import pytest + +from thinc.api import NumpyOps, Ragged, registry, strings2arrays + +from ..util import get_data_checker + + +@pytest.fixture(params=[[], [(10, 2)], [(5, 3), (1, 3)], [(2, 3), (0, 3), (1, 3)]]) +def shapes(request): + return request.param + + +@pytest.fixture +def ops(): + return NumpyOps() + + +@pytest.fixture +def list_data(shapes): + return [numpy.zeros(shape, dtype="f") for shape in shapes] + + +@pytest.fixture +def ragged_data(ops, list_data): + lengths = numpy.array([len(x) for x in list_data], dtype="i") + if not list_data: + return Ragged(ops.alloc2f(0, 0), lengths) + else: + return Ragged(ops.flatten(list_data), lengths) + + +@pytest.fixture +def padded_data(ops, list_data): + return ops.list2padded(list_data) + + +@pytest.fixture +def array_data(ragged_data): + return ragged_data.data + + +def check_transform(transform, in_data, out_data): + model = registry.resolve({"config": {"@layers": transform}})["config"] + input_checker = get_data_checker(in_data) + output_checker = get_data_checker(out_data) + model.initialize(in_data, out_data) + Y, backprop = model(in_data, is_train=True) + output_checker(Y, out_data) + dX = backprop(Y) + input_checker(dX, in_data) + + +def test_list2array(list_data, array_data): + check_transform("list2array.v1", list_data, array_data) + + +def test_list2ragged(list_data, ragged_data): + check_transform("list2ragged.v1", list_data, ragged_data) + + +def test_list2padded(list_data, padded_data): + check_transform("list2padded.v1", list_data, padded_data) + + +def test_ragged2list(ragged_data, list_data): + check_transform("ragged2list.v1", ragged_data, list_data) + + +def test_padded2list(padded_data, list_data): + check_transform("padded2list.v1", padded_data, list_data) + + +def test_strings2arrays(): + strings = ["hello", "world"] + model = strings2arrays() + Y, backprop = model.begin_update(strings) + assert len(Y) == len(strings) + assert backprop([]) == [] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_uniqued.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_uniqued.py new file mode 100644 index 0000000000000000000000000000000000000000..0a977999342fa6c93d3610a91ea0c986f8d48dd2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_uniqued.py @@ -0,0 +1,66 @@ +import numpy +import pytest +from hypothesis import given, settings +from hypothesis.strategies import composite, integers, lists +from numpy.testing import assert_allclose + +from thinc.layers import Embed +from thinc.layers.uniqued import uniqued + +ROWS = 10 + +# This test uses a newer hypothesis feature than the skanky flatmap-style +# I used previously. This is much nicer, although it still takes some getting +# used to. The key feature is this composite decorator. It injects a function, +# 'draw'. + + +@composite +def lists_of_integers(draw, columns=2, lo=0, hi=ROWS - 1): + # We call draw to get example values, which we can manipulate. + # Here we get a list of integers, where each member of the list + # should be between a min and max value. + int_list = draw(lists(integers(min_value=lo, max_value=hi))) + # Now we can use this int list to make an array, and it'll be the arrays + # that our functions receive. + # We trim the list, so we're of length divisible by columns. + int_list = int_list[len(int_list) % columns :] + # And make the array and reshape it. + array = numpy.array(int_list, dtype="uint64") + return array.reshape((-1, columns)) + + +@pytest.fixture(scope="module") +def model(nO=128): + return Embed(nO, ROWS, column=0).initialize() + + +def test_uniqued_calls_init(): + calls = [] + embed = Embed(5, 5, column=0) + embed.init = lambda *args, **kwargs: calls.append(True) + embed.initialize() + assert calls == [True] + uembed = uniqued(embed) + uembed.initialize() + assert calls == [True, True] + + +@given(X=lists_of_integers(lo=0, hi=ROWS - 1)) +@settings(deadline=None) +def test_uniqued_doesnt_change_result(model, X): + umodel = uniqued(model, column=model.attrs["column"]).initialize() + Y, bp_Y = model(X, is_train=True) + Yu, bp_Yu = umodel(X, is_train=True) + assert_allclose(Y, Yu) + dX = bp_Y(Y) + dXu = bp_Yu(Yu) + assert_allclose(dX, dXu) + if X.size: + pass + # TODO: This test is a problem, because we exceed the embedding table. + # Fix it with a better cap. + # Check that different inputs do give different results + # Z, bp_Z = model(X + 1, is_train=True) + # with pytest.raises(AssertionError): + # assert_allclose(Y, Z) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_debug.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_debug.py new file mode 100644 index 0000000000000000000000000000000000000000..fb4da46881417614b9808614c85278cfa80cdff1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_debug.py @@ -0,0 +1,26 @@ +from mock import MagicMock + +from thinc.api import Linear, with_debug + + +def test_with_debug(): + on_init = MagicMock() + on_forward = MagicMock() + on_backprop = MagicMock() + model = with_debug( + Linear(), on_init=on_init, on_forward=on_forward, on_backprop=on_backprop + ) + on_init.assert_not_called() + on_forward.assert_not_called() + on_backprop.assert_not_called() + X = model.ops.alloc2f(1, 1) + Y = model.ops.alloc2f(1, 1) + model.initialize(X=X, Y=Y) + on_init.assert_called_once_with(model, X, Y) + on_forward.assert_not_called() + on_backprop.assert_not_called() + Yh, backprop = model(X, is_train=True) + on_forward.assert_called_once_with(model, X, True) + on_backprop.assert_not_called() + backprop(Y) + on_backprop.assert_called_once_with(Y) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_flatten.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_flatten.py new file mode 100644 index 0000000000000000000000000000000000000000..27db1c43693c4b56a480a530a3b78f74281dfae6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_flatten.py @@ -0,0 +1,31 @@ +from typing import List + +from thinc.api import Model, with_flatten_v2 + +INPUT = [[1, 2, 3], [4, 5], [], [6, 7, 8]] +INPUT_FLAT = [1, 2, 3, 4, 5, 6, 7, 8] +OUTPUT = [[2, 3, 4], [5, 6], [], [7, 8, 9]] +BACKPROP_OUTPUT = [[3, 4, 5], [6, 7], [], [8, 9, 10]] + + +def _memoize_input() -> Model[List[int], List[int]]: + return Model(name="memoize_input", forward=_memoize_input_forward) + + +def _memoize_input_forward( + model: Model[List[int], List[int]], X: List[int], is_train: bool +): + model.attrs["last_input"] = X + + def backprop(dY: List[int]): + return [v + 2 for v in dY] + + return [v + 1 for v in X], backprop + + +def test_with_flatten(): + model = with_flatten_v2(_memoize_input()) + Y, backprop = model(INPUT, is_train=True) + assert Y == OUTPUT + assert model.layers[0].attrs["last_input"] == INPUT_FLAT + assert backprop(INPUT) == BACKPROP_OUTPUT diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_transforms.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..faa674940709c34d488bcd3e5db311be2232bcad --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/layers/test_with_transforms.py @@ -0,0 +1,299 @@ +import numpy +import numpy.testing +import pytest + +from thinc.api import ( + Linear, + Model, + NumpyOps, + noop, + with_array, + with_array2d, + with_getitem, + with_list, + with_padded, + with_ragged, +) +from thinc.types import Padded, Ragged + +from ..util import get_data_checker + + +@pytest.fixture(params=[[], [(10, 2)], [(5, 3), (1, 3)], [(2, 3), (0, 3), (1, 3)]]) +def shapes(request): + return request.param + + +@pytest.fixture +def ops(): + return NumpyOps() + + +@pytest.fixture +def list_input(shapes): + data = [numpy.zeros(shape, dtype="f") for shape in shapes] + for i, x in enumerate(data): + # Give values that make it easy to see where rows or columns mismatch. + x += i * 100 + x += numpy.arange(x.shape[0]).reshape((-1, 1)) * 10 + x += numpy.arange(x.shape[1]).reshape((1, -1)) + return data + + +@pytest.fixture +def ragged_input(ops, list_input): + lengths = numpy.array([len(x) for x in list_input], dtype="i") + if not list_input: + return Ragged(ops.alloc2f(0, 0), lengths) + else: + return Ragged(ops.flatten(list_input), lengths) + + +@pytest.fixture +def padded_input(ops, list_input): + return ops.list2padded(list_input) + + +@pytest.fixture +def array_input(ragged_input): + return ragged_input.data + + +@pytest.fixture +def padded_data_input(padded_input): + x = padded_input + return (x.data, x.size_at_t, x.lengths, x.indices) + + +@pytest.fixture +def ragged_data_input(ragged_input): + return (ragged_input.data, ragged_input.lengths) + + +@pytest.fixture +def noop_models(): + return [ + with_padded(noop()), + with_array(noop()), + with_array2d(noop()), + with_list(noop()), + with_ragged(noop()), + ] + + +# As an example operation, lets just trim the last dimension. That +# should catch stuff that confuses the input and output. + + +def get_array_model(): + def _trim_array_forward(model, X, is_train): + def backprop(dY): + return model.ops.alloc2f(dY.shape[0], dY.shape[1] + 1) + + return X[:, :-1], backprop + + return with_array2d(Model("trimarray", _trim_array_forward)) + + +def get_list_model(): + def _trim_list_forward(model, Xs, is_train): + def backprop(dYs): + dXs = [] + for dY in dYs: + dXs.append(model.ops.alloc2f(dY.shape[0], dY.shape[1] + 1)) + return dXs + + Ys = [X[:, :-1] for X in Xs] + return Ys, backprop + + return with_list(Model("trimlist", _trim_list_forward)) + + +def get_padded_model(): + def _trim_padded_forward(model, Xp, is_train): + def backprop(dYp): + dY = dYp.data + dX = model.ops.alloc3f(dY.shape[0], dY.shape[1], dY.shape[2] + 1) + return Padded(dX, dYp.size_at_t, dYp.lengths, dYp.indices) + + assert isinstance(Xp, Padded) + X = Xp.data + X = X.reshape((X.shape[0] * X.shape[1], X.shape[2])) + X = X[:, :-1] + X = X.reshape((Xp.data.shape[0], Xp.data.shape[1], X.shape[1])) + return Padded(X, Xp.size_at_t, Xp.lengths, Xp.indices), backprop + + return with_padded(Model("trimpadded", _trim_padded_forward)) + + +def get_ragged_model(): + def _trim_ragged_forward(model, Xr, is_train): + def backprop(dYr): + dY = dYr.data + dX = model.ops.alloc2f(dY.shape[0], dY.shape[1] + 1) + return Ragged(dX, dYr.lengths) + + return Ragged(Xr.data[:, :-1], Xr.lengths), backprop + + return with_ragged(Model("trimragged", _trim_ragged_forward)) + + +def check_initialize(model, inputs): + # Just check that these run and don't hit errors. I guess we should add a + # spy and check that model.layers[0].initialize gets called, but shrug? + model.initialize() + model.initialize(X=inputs) + model.initialize(X=inputs, Y=model.predict(inputs)) + + +def check_transform_produces_correct_output_type_forward(model, inputs, checker): + # It's pretty redundant to check these three assertions, so if the tests + # get slow this could be removed. I think it should be fine though? + outputs = model.predict(inputs) + assert checker(inputs, outputs) + outputs, _ = model(inputs, is_train=True) + assert checker(inputs, outputs) + outputs, _ = model(inputs, is_train=False) + assert checker(inputs, outputs) + + +def check_transform_produces_correct_output_type_backward(model, inputs, checker): + # It's pretty redundant to check these three assertions, so if the tests + # get slow this could be removed. I think it should be fine though? + outputs, backprop = model.begin_update(inputs) + d_inputs = backprop(outputs) + assert checker(inputs, d_inputs) + + +def check_transform_doesnt_change_noop_values(model, inputs, d_outputs): + # Check that if we're wrapping a noop() layer in the transform, we don't + # change the output values. + outputs, backprop = model.begin_update(inputs) + d_inputs = backprop(d_outputs) + if isinstance(outputs, list): + for i in range(len(outputs)): + numpy.testing.assert_equal(inputs[i], outputs[i]) + numpy.testing.assert_equal(d_outputs[i], d_inputs[i]) + elif isinstance(outputs, numpy.ndarray): + numpy.testing.assert_equal(inputs, outputs) + numpy.testing.assert_equal(d_outputs, d_inputs) + elif isinstance(outputs, Ragged): + numpy.testing.assert_equal(inputs.data, outputs.data) + numpy.testing.assert_equal(d_outputs.data, d_inputs.data) + elif isinstance(outputs, Padded): + numpy.testing.assert_equal(inputs.data, outputs.data) + numpy.testing.assert_equal(d_inputs.data, d_inputs.data) + + +def test_noop_transforms(noop_models, ragged_input, padded_input, list_input): + # Make distinct backprop values, + # to check that the gradients get passed correctly + d_ragged = Ragged(ragged_input.data + 1, ragged_input.lengths) + d_padded = padded_input.copy() + d_padded.data += 1 + d_list = [dx + 1 for dx in list_input] + for model in noop_models: + print(model.name) + check_transform_doesnt_change_noop_values(model, padded_input, d_padded) + check_transform_doesnt_change_noop_values(model, list_input, d_list) + check_transform_doesnt_change_noop_values(model, ragged_input, d_ragged) + + +def test_with_array_initialize(ragged_input, padded_input, list_input, array_input): + for inputs in (ragged_input, padded_input, list_input, array_input): + check_initialize(get_array_model(), inputs) + + +def test_with_padded_initialize( + ragged_input, padded_input, list_input, padded_data_input +): + for inputs in (ragged_input, padded_input, list_input, padded_data_input): + check_initialize(get_padded_model(), inputs) + + +def test_with_list_initialize(ragged_input, padded_input, list_input): + for inputs in (ragged_input, padded_input, list_input): + check_initialize(get_list_model(), inputs) + + +def test_with_ragged_initialize( + ragged_input, padded_input, list_input, ragged_data_input +): + for inputs in (ragged_input, padded_input, list_input, ragged_data_input): + check_initialize(get_ragged_model(), inputs) + + +def test_with_array_forward(ragged_input, padded_input, list_input, array_input): + for inputs in (ragged_input, padded_input, list_input, array_input): + checker = get_data_checker(inputs) + model = get_array_model() + check_transform_produces_correct_output_type_forward(model, inputs, checker) + + +def test_with_list_forward(ragged_input, padded_input, list_input): + for inputs in (ragged_input, padded_input, list_input): + checker = get_data_checker(inputs) + model = get_list_model() + check_transform_produces_correct_output_type_forward(model, inputs, checker) + + +def test_with_padded_forward(ragged_input, padded_input, list_input, padded_data_input): + for inputs in (ragged_input, padded_input, list_input, padded_data_input): + checker = get_data_checker(inputs) + model = get_padded_model() + check_transform_produces_correct_output_type_forward(model, inputs, checker) + + +def test_with_ragged_forward(ragged_input, padded_input, list_input, ragged_data_input): + for inputs in (ragged_input, padded_input, list_input, ragged_data_input): + checker = get_data_checker(inputs) + model = get_ragged_model() + check_transform_produces_correct_output_type_forward(model, inputs, checker) + + +def test_with_array_backward(ragged_input, padded_input, list_input, array_input): + for inputs in (ragged_input, padded_input, list_input, array_input): + checker = get_data_checker(inputs) + model = get_array_model() + check_transform_produces_correct_output_type_backward(model, inputs, checker) + + +def test_with_list_backward(ragged_input, padded_input, list_input): + for inputs in (ragged_input, padded_input, list_input): + checker = get_data_checker(inputs) + model = get_list_model() + check_transform_produces_correct_output_type_backward(model, inputs, checker) + + +def test_with_ragged_backward( + ragged_input, padded_input, list_input, ragged_data_input +): + for inputs in (ragged_input, padded_input, list_input, ragged_data_input): + checker = get_data_checker(inputs) + model = get_ragged_model() + check_transform_produces_correct_output_type_backward(model, inputs, checker) + + +def test_with_padded_backward( + ragged_input, padded_input, list_input, padded_data_input +): + for inputs in (ragged_input, padded_input, list_input, padded_data_input): + checker = get_data_checker(inputs) + model = get_padded_model() + check_transform_produces_correct_output_type_backward(model, inputs, checker) + + +def test_with_getitem(): + data = ( + numpy.asarray([[1, 2, 3, 4]], dtype="f"), + numpy.asarray([[5, 6, 7, 8]], dtype="f"), + ) + model = with_getitem(1, Linear()) + model.initialize(data, data) + Y, backprop = model.begin_update(data) + assert len(Y) == len(data) + assert numpy.array_equal(Y[0], data[0]) # the other item stayed the same + assert not numpy.array_equal(Y[1], data[1]) + dX = backprop(Y) + assert numpy.array_equal(dX[0], data[0]) + assert not numpy.array_equal(dX[1], data[1]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d0f4079ac72579938b3bf1ee21d1d73e7b6d038 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_model.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb1f6864f96547d62d42ab35382224666a0fda99 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_model.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_validation.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_validation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e1f7148014837a97dcfec10abc4740082028cc3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/__pycache__/test_validation.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_model.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0cbe156659c0cee819342b5a47b5d32f0acf0e64 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_model.py @@ -0,0 +1,669 @@ +import threading +import time +from collections import Counter + +import numpy +import pytest + +from thinc.api import ( + Adam, + CupyOps, + Dropout, + Linear, + Model, + Relu, + Shim, + Softmax, + chain, + change_attr_values, + concatenate, + set_dropout_rate, + use_ops, + with_debug, + wrap_model_recursive, +) +from thinc.compat import has_cupy_gpu + +from ..util import make_tempdir + + +@pytest.fixture +def model_with_no_args(): + return Linear() + + +def create_model(name): + return Model(name, lambda X: (X, lambda dY: dY)) + + +def test_model_defaults_to_cpu(model_with_no_args): + assert not isinstance(model_with_no_args.ops, CupyOps) + + +def test_models_get_different_ids(model_with_no_args): + model1 = Linear() + model2 = Linear() + assert model1.id != model2.id + + +def test_model_init(): + class MyShim(Shim): + name = "testshim" + + model_a = create_model("a") + model = Model( + "test", + lambda X: (X, lambda dY: dY), + dims={"nI": 10, "nO": None}, + params={"W": numpy.zeros((10,)), "b": None}, + refs={"a": model_a, "b": None}, + attrs={"foo": "bar"}, + shims=[MyShim(None)], + layers=[model_a, model_a], + ) + assert model.has_param("W") + assert model.get_param("W").shape == (10,) + assert model.has_param("b") is None + with pytest.raises(KeyError): + model.get_param("b") + with pytest.raises(KeyError): + model.get_param("X") + model.set_param("X", numpy.zeros((10,))) + assert model.has_param("X") + assert model.get_param("X").shape == (10,) + with model.use_params({(model.id, "X"): numpy.ones((10,))}): + assert numpy.array_equal(model.get_param("X"), numpy.ones((10,))) + assert numpy.array_equal(model.get_param("X"), numpy.zeros((10,))) + assert not model.has_grad("W") + assert not model.has_grad("xyz") + with pytest.raises(KeyError): + model.get_grad("b") + model.set_param("W", model.ops.alloc1f(10)) + model.set_grad("W", model.ops.alloc1f(10)) + with pytest.raises(ValueError): + model.inc_grad("W", numpy.zeros((5, 0))) + assert model.has_dim("nI") + assert model.get_dim("nI") == 10 + with pytest.raises(KeyError): + model.get_dim("xyz") + with pytest.raises(ValueError): + model.get_dim("nO") + assert model.has_ref("a") + assert model.get_ref("a").name == "a" + assert not model.has_ref("xyz") + with pytest.raises(KeyError): + model.get_ref("xyz") + assert model.has_ref("b") is None + with pytest.raises(ValueError): + model.get_ref("b") + model.set_ref("c", model_a) + assert model.has_ref("c") + assert model.get_ref("c").name == "a" + with pytest.raises(ValueError): + model.set_ref("c", create_model("c")) + assert "foo" in model.attrs + assert "bar" not in model.attrs + assert model.attrs["foo"] == "bar" + with pytest.raises(KeyError): + model.attrs["bar"] + model.attrs["bar"] = "baz" + model_copy = model.copy() + assert model_copy.name == "test" + + +def test_model_set_dim(): + class MyShim(Shim): + name = "testshim" + + model_a = create_model("a") + model = Model( + "test", + lambda X: (X, lambda dY: dY), + dims={"nI": 5, "nO": None}, + params={"W": None, "b": None}, + refs={"a": model_a, "b": None}, + attrs={"foo": "bar"}, + shims=[MyShim(None)], + layers=[model_a, model_a], + ) + with pytest.raises(ValueError): + model.set_dim("nI", 10) + # force can be used before any parameters are set + model.set_dim("nI", 10, force=True) + model.set_param("W", model.ops.alloc1f(10)) + model.set_grad("W", model.ops.alloc1f(10)) + assert model.has_dim("nI") + assert model.get_dim("nI") == 10 + with pytest.raises(KeyError): + model.set_dim("xyz", 20) + with pytest.raises(ValueError): + model.set_dim("nI", 20) + # force can't be used after any parameter is set + with pytest.raises(ValueError): + model.set_dim("nI", 20, force=True) + + +def test_param_names(): + model = create_model("tmp") + assert model.param_names == tuple() + model.set_param("param1", None) + assert model.param_names == ("param1",) + model.set_param("param2", None) + assert model.param_names == ("param1", "param2") + + +def test_grad_names(): + model = create_model("tmp") + assert model.grad_names == tuple() + model.set_param("param1", model.ops.alloc2f(4, 4)) + model.set_grad("param1", model.ops.alloc2f(4, 4) + 1) + assert model.grad_names == ("param1",) + + +def test_dim_names(): + model = Linear(5, 3) + assert model.dim_names == ("nO", "nI") + + +def test_model_set_reference(): + parent = create_model("parent") + child = create_model("child") + grandchild = create_model("child") + parent.layers.append(child) + assert parent.ref_names == tuple() + parent.set_ref("kid", child) + assert parent.ref_names == ("kid",) + assert parent.get_ref("kid") is child + child.layers.append(grandchild) + with pytest.raises(KeyError): + parent.get_ref("grandkid") + parent.set_ref("grandkid", grandchild) + assert parent.get_ref("grandkid") is grandchild + parent.remove_node(grandchild) + assert grandchild not in child.layers + assert not parent.has_ref("grandkind") + + +def test_maybe_methods(): + model = Linear(5) + assert model.maybe_get_dim("nI") is None + model.set_dim("nI", 4) + assert model.maybe_get_dim("nI") == 4 + assert model.maybe_get_ref("boo") is None + assert model.maybe_get_param("W") is None + model.initialize() + assert model.maybe_get_param("W") is not None + + +def test_model_can_save_to_disk(model_with_no_args): + with make_tempdir() as path: + model_with_no_args.to_disk(path / "thinc_model") + + +def test_model_can_load_from_disk(model_with_no_args): + with make_tempdir() as path: + model_with_no_args.to_disk(path / "thinc_model") + m2 = model_with_no_args.from_disk(path / "thinc_model") + assert model_with_no_args.to_bytes() == m2.to_bytes() + + +def test_model_can_roundtrip_with_path_subclass(model_with_no_args, pathy_fixture): + path = pathy_fixture / "thinc_model" + model_with_no_args.to_disk(path) + m2 = model_with_no_args.from_disk(path) + assert model_with_no_args.to_bytes() == m2.to_bytes() + + +def test_change_attr_values(model_with_no_args): + model = model_with_no_args + model.name = "target" + model.attrs["has_var"] = False + change_attr_values(model, {"target": {"has_var": True, "error": True}}) + assert model.attrs["has_var"] is True + assert "error" not in model.attrs + + +def test_set_dropout(): + model = Dropout() + assert model.attrs["dropout_rate"] == 0.0 + set_dropout_rate(model, 0.2) + assert model.attrs["dropout_rate"] == 0.2 + + +def test_set_dropout_2(model_with_no_args): + model = model_with_no_args + model.name = "dropout" + model.attrs["dropout_rate"] = 0.0 + set_dropout_rate(model, 0.2) + assert model.attrs["dropout_rate"] == 0.2 + + +def test_bind_plus(): + with Model.define_operators({"+": lambda a, b: (a.name, b.name)}): + m = create_model(name="a") + create_model(name="b") + assert m == ("a", "b") + + +def test_plus_chain(): + with Model.define_operators({"+": lambda a, b: a}): + m = ( + create_model(name="a") + + create_model(name="b") + + create_model(name="c") + + create_model(name="d") + ) + assert m.name == "a" + + +def test_overload_operators_in_subthread(): + """Test we can create a model in a child thread with overloaded operators.""" + # Worker1 will start and run, while worker 2 sleeps after Model.define_operators. + # Without thread-safety, worker2 will find that its operator definitions + # have been removed, causing an error. + worker1 = threading.Thread(target=_overload_plus, args=("+", 0)) + worker2 = threading.Thread(target=_overload_plus, args=("*", 1)) + worker2.start() + worker1.start() + worker1.join() + worker2.join() + + worker1 = threading.Thread(target=_overload_plus, args=("+", 1)) + worker2 = threading.Thread(target=_overload_plus, args=("*", 0)) + worker2.start() + worker1.start() + worker1.join() + worker2.join() + + +def _overload_plus(operator, sleep): + m1 = create_model(name="a") + m2 = create_model(name="b") + with Model.define_operators({operator: lambda a, b: a.name + b.name}): + time.sleep(sleep) + if operator == "+": + value = m1 + m2 + else: + value = m1 * m2 + assert value == "ab" + assert Model._context_operators.get() == {} + + +def test_nested_operator_contexts(): + m1 = create_model(name="a") + m2 = create_model(name="b") + assert Model._context_operators.get() == {} + with Model.define_operators({"+": lambda a, b: a.name + b.name}): + value = m1 + m2 + with pytest.raises(TypeError): + value = m1 * m2 + with Model.define_operators({"*": lambda a, b: a.name + b.name}): + with pytest.raises(TypeError): + value = m1 + m2 + value = m1 * m2 + with Model.define_operators({"-": lambda a, b: a.name + b.name}): + with pytest.raises(TypeError): + value = m1 + m2 + value = m1 - m2 + with pytest.raises(TypeError): + value = m1 + m2 + value = m1 * m2 + value = m1 + m2 + with pytest.raises(TypeError): + value = m1 * m2 + assert value == "ab" + assert Model._context_operators.get() == {} + + +@pytest.mark.parametrize("op", "+ - * @ / // % ** << >> & ^ |".split()) +def test_all_operators(op): + m1 = Linear() + m2 = Linear() + with Model.define_operators({op: lambda a, b: a.name + b.name}): + if op == "+": + value = m1 + m2 + else: + with pytest.raises(TypeError): + value = m1 + m2 + if op == "-": + value = m1 - m2 + else: + with pytest.raises(TypeError): + value = m1 - m2 + + if op == "*": + value = m1 * m2 + else: + with pytest.raises(TypeError): + value = m1 * m2 + + if op == "@": + value = m1.__matmul__(m2) # Be kind to Python 2... + else: + with pytest.raises(TypeError): + value = m1.__matmul__(m2) + + if op == "/": + value = m1 / m2 + else: + with pytest.raises(TypeError): + value = m1 / m2 + + if op == "//": + value = m1 // m2 + else: + with pytest.raises(TypeError): + value = m1 // m2 + if op == "^": + value = m1 ^ m2 + else: + with pytest.raises(TypeError): + value = m1 ^ m2 + if op == "%": + value = m1 % m2 + else: + with pytest.raises(TypeError): + value = m1 % m2 + if op == "**": + value = m1**m2 + else: + with pytest.raises(TypeError): + value = m1**m2 + if op == "<<": + value = m1 << m2 + else: + with pytest.raises(TypeError): + value = m1 << m2 + if op == ">>": + value = m1 >> m2 + else: + with pytest.raises(TypeError): + value = m1 >> m2 + if op == "&": + value = m1 & m2 + else: + with pytest.raises(TypeError): + value = m1 & m2 + if op == "^": + value = m1 ^ m2 + else: + with pytest.raises(TypeError): + value = m1 ^ m2 + if op == "|": + value = m1 | m2 + else: + with pytest.raises(TypeError): + value = m1 | m2 # noqa: F841 + assert Model._context_operators.get() == {} + + +def test_unique_id_multithreading(): + """Create a bunch of threads and assert they all get unique IDs""" + + list_of_ids = [] + + def get_model_id(id_list, index): + id_list.append(create_model(name=f"worker{index}").id) + + counter = 0 + while len(list_of_ids) < 1000: + workers = [] + for i in range(50): + w = threading.Thread(target=get_model_id, args=(list_of_ids, counter)) + workers.append(w) + counter += 1 + for w in workers: + w.start() + for w in workers: + w.join() + + assert len(list_of_ids) == len(list(set(list_of_ids))) + + +@pytest.mark.skipif(not has_cupy_gpu, reason="needs CuPy GPU") +def test_model_gpu(): + pytest.importorskip("ml_datasets") + import ml_datasets + + with use_ops("cupy"): + n_hidden = 32 + dropout = 0.2 + (train_X, train_Y), (dev_X, dev_Y) = ml_datasets.mnist() + model = chain( + Relu(nO=n_hidden, dropout=dropout), + Relu(nO=n_hidden, dropout=dropout), + Softmax(), + ) + # make sure the data is on the right device + train_X = model.ops.asarray(train_X) + train_Y = model.ops.asarray(train_Y) + dev_X = model.ops.asarray(dev_X) + dev_Y = model.ops.asarray(dev_Y) + + model.initialize(X=train_X[:5], Y=train_Y[:5]) + optimizer = Adam(0.001) + batch_size = 128 + + for i in range(2): + batches = model.ops.multibatch(batch_size, train_X, train_Y, shuffle=True) + for X, Y in batches: + Yh, backprop = model.begin_update(X) + backprop(Yh - Y) + model.finish_update(optimizer) + # Evaluate and print progress + correct = 0 + total = 0 + for X, Y in model.ops.multibatch(batch_size, dev_X, dev_Y): + Yh = model.predict(X) + correct += (Yh.argmax(axis=1) == Y.argmax(axis=1)).sum() + total += Yh.shape[0] + + +def test_replace_node(): + relu1 = Relu(5) + relu2 = Relu(5) + relu_chain = chain(relu1, relu2) + relu1_debug = with_debug(relu1) + debug = Model( + "test", + lambda X: (X, lambda dY: dY), + layers=[relu1, relu2, relu1, relu_chain], + refs={"relu1": relu1, "relu2": relu2, "relu3": relu1}, + ) + debug.replace_node(relu1, relu1_debug) + assert debug.layers[0] == relu1_debug + assert debug.layers[1] == relu2 + assert debug.layers[2] == relu1_debug + assert debug.get_ref("relu1") == relu1_debug + assert debug.get_ref("relu2") == relu2 + assert debug.get_ref("relu3") == relu1_debug + + # Check that nodes are replaced recursively + assert debug.layers[3] == relu_chain + assert debug.layers[3].layers[0] == relu1_debug + assert debug.layers[3].layers[1] == relu2 + + +def test_replace_node_with_indirect_node_ref(): + # a + # / \ + # x b[y=y] + # | | + # y x + # | + # y + + def dummy_model(name, layers): + return Model(name, lambda model, X, is_train: ..., layers=layers) + + y = dummy_model("y", []) + x = dummy_model("x", [y]) + + y_debug = with_debug(y) + + b = dummy_model("b", [x]) + b.set_ref("y", y) + + a = chain(x, b) + a.name = "a" + + a.replace_node(y, y_debug) + + assert a.layers[0].layers[0] == y_debug + assert a.layers[1].layers[0].layers[0] == y_debug + assert a.layers[1].get_ref("y") == y_debug + + +def test_with_debug(): + pytest.importorskip("ml_datasets") + import ml_datasets + + (train_X, train_Y), (dev_X, dev_Y) = ml_datasets.mnist() + + counts = Counter() + + def on_init(*_): + counts["init"] += 1 + + def on_forward(*_): + counts["forward"] += 1 + + def on_backprop(*_): + counts["backprop"] += 1 + + relu = Relu() + relu2 = with_debug( + Relu(), on_init=on_init, on_forward=on_forward, on_backprop=on_backprop + ) + chained = chain(relu, relu2, relu2) + chained.initialize(X=train_X[:5], Y=train_Y[:5]) + _, backprop = chained(X=train_X[:5], is_train=False) + + # Not real loss gradients, but we don't care for testing. + backprop(train_Y[:5]) + + # Four times forward, because initialization also applies forward for + # validation. + assert counts == {"init": 2, "forward": 4, "backprop": 2} + + +def test_recursive_wrap(): + def dummy_model(name, layers): + return Model(name, lambda model, X, is_train: ..., layers=layers) + + # Check: + # + # * Recursion: chain -> relu + # * Multiple sublayers: chain -> [relu, relu] + + relu = Relu(5) + chained = chain(relu, relu) + chained_debug = wrap_model_recursive( + chained, lambda model: dummy_model(f"dummy({model.name})", [model]) + ) + + assert chained_debug.name == "dummy(relu>>relu)" + assert chained_debug.layers[0] is chained + assert chained_debug.layers[0].layers[0].name == "dummy(relu)" + assert chained_debug.layers[0].layers[0].layers[0] is relu + assert chained_debug.layers[0].layers[1].name == "dummy(relu)" + assert chained_debug.layers[0].layers[1].layers[0] is relu + + +def test_recursive_double_wrap(): + def dummy_model(name, layers): + return Model(name, lambda model, X, is_train: ..., layers=layers) + + relu = Relu(5) + chained = chain(relu, relu) + concat = concatenate(chained, chained, relu) + concat_wrapped = wrap_model_recursive( + concat, lambda model: dummy_model(f"dummy({model.name})", [model]) + ) + + n_debug = 0 + for model in concat_wrapped.walk(): + if model.name.startswith("dummy"): + n_debug += 1 + + # There should be 3 unique dummy wrappers: + # * Around concatenate. + # * Around chain. + # * Around relu. + assert n_debug == 3 + + assert concat_wrapped.layers[0].layers[0].layers[0].layers[0].name == "dummy(relu)" + assert concat_wrapped.layers[0].layers[0].layers[0].layers[1].name == "dummy(relu)" + assert concat_wrapped.layers[0].layers[1].layers[0].layers[0].name == "dummy(relu)" + assert concat_wrapped.layers[0].layers[1].layers[0].layers[1].name == "dummy(relu)" + assert concat_wrapped.layers[0].layers[2].name == "dummy(relu)" + + +def test_wrap_non_child_references(): + relu = Relu(5) + relu2 = Relu(5) + chained = chain(relu, relu) + chained2 = chain(relu2, chained) + chained2.set_ref("relu", relu) + # Fails in case non-child references cannot be set. + wrap_model_recursive(chained2, with_debug) + + +def test_walk_dfs(): + relu = Relu(5) + relu2 = Relu(5) + inner_chain = chain(relu, relu2) + chained = chain(inner_chain, inner_chain) + assert list(chained.walk(order="dfs_pre")) == [chained, inner_chain, relu, relu2] + assert list(chained.walk(order="dfs_post")) == [ + relu, + relu2, + inner_chain, + chained, + ] + + +def test_walk_bfs_post_order_fails(): + relu = Relu(5) + with pytest.raises(ValueError, match="Invalid order"): + relu.walk(order="dfs_post_order") + + +def test_model_copy_with_loop(): + class MyShim(Shim): + name = "testshim" + + def to_bytes(self): + return test_replace_node_with_indirect_node_ref + + def from_bytes(self, bytes): + pass + + model_a = create_model("a") + working_shim = MyShim(None) + layer = Model( + "test", + lambda X: (X, lambda dY: dY), + dims={"nI": 5, "nO": 5}, + params={"W": numpy.zeros((10,)), "b": None}, + refs={"a": model_a, "b": None}, + attrs={"foo": "bar"}, + shims=[working_shim], + layers=[model_a, model_a], + ) + layer2 = Model( + "test2", + lambda X: (X, lambda dY: dY), + dims={"nI": 5, "nO": 5}, + params={"W": numpy.zeros((10,)), "b": None}, + refs={"a": model_a, "b": None}, + attrs={"foo": "bar"}, + shims=[working_shim], + layers=[model_a, model_a], + ) + relu = Relu(5) + model = chain(layer, relu, layer, layer2) + model2 = model.copy() + model.from_dict(model2.to_dict()) + assert model2.name == "test>>relu>>test>>test2" + assert model2.layers[0] == model2.layers[2] + assert id(model2.layers[0].shims[0]) == id(model2.layers[3].shims[0]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_validation.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_validation.py new file mode 100644 index 0000000000000000000000000000000000000000..718f12498b9cd3a7ee74e3c388ccfc89c12b1004 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/model/test_validation.py @@ -0,0 +1,46 @@ +import pytest + +from thinc.api import ( + ParametricAttention, + Relu, + Softmax, + chain, + list2ragged, + reduce_max, + reduce_sum, + with_ragged, +) +from thinc.util import DataValidationError, data_validation + + +@pytest.mark.xfail(reason="Validation currently disabled for Pydantic 2 changes0") +def test_validation(): + model = chain(Relu(10), Relu(10), with_ragged(reduce_max()), Softmax()) + with data_validation(True): + with pytest.raises(DataValidationError): + model.initialize(X=model.ops.alloc2f(1, 10), Y=model.ops.alloc2f(1, 10)) + with pytest.raises(DataValidationError): + model.initialize(X=model.ops.alloc3f(1, 10, 1), Y=model.ops.alloc2f(1, 10)) + with pytest.raises(DataValidationError): + model.initialize(X=[model.ops.alloc2f(1, 10)], Y=model.ops.alloc2f(1, 10)) + + +@pytest.mark.xfail(reason="Validation currently disabled for Pydantic 2 changes0") +def test_validation_complex(): + good_model = chain(list2ragged(), reduce_sum(), Relu(12, dropout=0.5), Relu(1)) + X = [good_model.ops.xp.zeros((4, 75), dtype="f")] + Y = good_model.ops.xp.zeros((1,), dtype="f") + good_model.initialize(X, Y) + good_model.predict(X) + + bad_model = chain( + list2ragged(), + reduce_sum(), + Relu(12, dropout=0.5), + # ERROR: Why can't I attach a Relu to an attention layer? + ParametricAttention(12), + Relu(1), + ) + with data_validation(True): + with pytest.raises(DataValidationError): + bad_model.initialize(X, Y) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02606b31026e4fc35fd5d7d0d1a6c4341d4185f8 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/test_mypy.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/test_mypy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30a7e6d74bb09c485a4de78bccc745aa1e3a6074 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/__pycache__/test_mypy.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-default.ini b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-default.ini new file mode 100644 index 0000000000000000000000000000000000000000..86fb4986cf5e7ad255ca853b000248a7f2272631 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-default.ini @@ -0,0 +1,8 @@ +[mypy] +follow_imports = silent +strict_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +# disallow_any_generics = True +check_untyped_defs = True +disallow_untyped_defs = True diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-plugin.ini b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-plugin.ini new file mode 100644 index 0000000000000000000000000000000000000000..959a65dd7ca6dbf60f0c915fcadd12fc98951495 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/configs/mypy-plugin.ini @@ -0,0 +1,10 @@ +[mypy] +plugins = thinc.mypy + +follow_imports = silent +strict_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +# disallow_any_generics = True +check_untyped_defs = True +disallow_untyped_defs = True diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..652ee9f887cb6bfec995579290ceecf065302f05 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_no_plugin.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_no_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa98b7c3c43132b8e5efcd71b24ca95caa99fce4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_no_plugin.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_plugin.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88fc3401ad3a0d57979865347a02c5ccd7e5d38b Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/fail_plugin.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_no_plugin.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_no_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c565e28de2c39b552e84aabc9ff3683bc077fcb4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_no_plugin.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_plugin.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2df8dea74564cd723dba6e50b44206d2fecd42a3 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/__pycache__/success_plugin.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_no_plugin.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_no_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..6468fcd4e2b4061d009abd02a5c74b91bf92fdb5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_no_plugin.py @@ -0,0 +1,5 @@ +from thinc.api import Relu, Softmax, add, chain, reduce_max + +bad_model = chain(Relu(10), reduce_max(), Softmax()) + +bad_model2 = add(Relu(10), reduce_max(), Softmax()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_plugin.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..5c2ff12bce77a2a1d2b39e39dae6a7057b990516 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/fail_plugin.py @@ -0,0 +1,20 @@ +from thinc.api import Relu, Softmax, add, chain, concatenate, reduce_max + +bad_model = chain(Relu(10), reduce_max(), Softmax()) + +bad_model2 = add(Relu(10), reduce_max(), Softmax()) + +bad_model_only_plugin = chain( + Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() +) + +bad_model_only_plugin2 = add( + Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() +) +reveal_type(bad_model_only_plugin2) + +bad_model_only_plugin3 = concatenate( + Relu(10), Relu(10), Relu(10), Relu(10), reduce_max(), Softmax() +) + +reveal_type(bad_model_only_plugin3) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_no_plugin.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_no_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..0873589ebda568c98aa9dad2dd9173e3e30e1be4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_no_plugin.py @@ -0,0 +1,13 @@ +from thinc.api import Relu, Softmax, add, chain, reduce_max + +good_model = chain(Relu(10), Relu(10), Softmax()) +reveal_type(good_model) + +good_model2 = add(Relu(10), Relu(10), Softmax()) +reveal_type(good_model2) + +bad_model_undetected = chain(Relu(10), Relu(10), reduce_max(), Softmax()) +reveal_type(bad_model_undetected) + +bad_model_undetected2 = add(Relu(10), Relu(10), reduce_max(), Softmax()) +reveal_type(bad_model_undetected2) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_plugin.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..af44c170f4f5e2a0eef93b273ddd3dfdf0ff6336 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/modules/success_plugin.py @@ -0,0 +1,34 @@ +from typing import Any, TypeVar + +from thinc.api import Model, Relu, Softmax, add, chain, reduce_max + +good_model = chain(Relu(10), Relu(10), Softmax()) +reveal_type(good_model) + +good_model2 = add(Relu(10), Relu(10), Softmax()) +reveal_type(good_model2) + +bad_model_undetected = chain(Relu(10), Relu(10), Relu(10), Relu(10), Softmax()) +reveal_type(bad_model_undetected) + +bad_model_undetected2 = add(Relu(10), Relu(10), Relu(10), Relu(10), Softmax()) +reveal_type(bad_model_undetected2) + + +def forward() -> None: + pass + + +OtherType = TypeVar("OtherType") + + +def other_function( + layer1: Model, layer2: Model, *layers: Model +) -> Model[Any, OtherType]: + return Model("some_model", forward) + + +non_combinator_model = other_function( + Model("x", forward), Model("y", forward), Model("z", forward) +) +reveal_type(non_combinator_model) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-no-plugin.txt b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-no-plugin.txt new file mode 100644 index 0000000000000000000000000000000000000000..cf9832ab083621529aba72a8541dde53e485dfdf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-no-plugin.txt @@ -0,0 +1,2 @@ +3: error: Cannot infer type argument 2 of "chain" [misc] +5: error: Cannot infer type argument 1 of "add" [misc] \ No newline at end of file diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-plugin.txt b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-plugin.txt new file mode 100644 index 0000000000000000000000000000000000000000..7865d8db09277c9b795421533192bf566c46e498 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/fail-plugin.txt @@ -0,0 +1,20 @@ +3: error: Cannot infer type argument 2 of "chain" [misc] +3: error: Layer outputs type (thinc.types.Floats2d) but the next layer expects (thinc.types.Ragged) as an input [layer-mismatch-output] +3: error: Layer input type (thinc.types.Ragged) is not compatible with output (thinc.types.Floats2d) from previous layer [layer-mismatch-input] +5: error: Cannot infer type argument 1 of "add" [misc] +5: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] +5: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] +5: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] +5: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] +8: error: Layer outputs type (thinc.types.Floats2d) but the next layer expects (thinc.types.Ragged) as an input [layer-mismatch-output] +8: error: Layer input type (thinc.types.Ragged) is not compatible with output (thinc.types.Floats2d) from previous layer [layer-mismatch-input] +12: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] +12: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] +12: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] +12: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] +14: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" +17: error: Layer input (thinc.types.Floats2d) not compatible with next layer input (thinc.types.Ragged) [layer-mismatch-input] +17: error: Layer input (thinc.types.Ragged) not compatible with previous layer input (thinc.types.Floats2d) [layer-mismatch-input] +17: error: Layer input (thinc.types.Ragged) not compatible with next layer input (thinc.types.Floats2d) [layer-mismatch-input] +17: error: Layer input (thinc.types.Floats2d) not compatible with previous layer input (thinc.types.Ragged) [layer-mismatch-input] +20: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-no-plugin.txt b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-no-plugin.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff16ff8c1ce1f03fad5a8e7d98fe28836ebb8b58 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-no-plugin.txt @@ -0,0 +1,8 @@ +3: error: Need type annotation for "good_model" [var-annotated] +4: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" +6: error: Need type annotation for "good_model2" [var-annotated] +7: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" +9: error: Need type annotation for "bad_model_undetected" [var-annotated] +10: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" +12: error: Need type annotation for "bad_model_undetected2" [var-annotated] +13: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, Any]" \ No newline at end of file diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-plugin.txt b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-plugin.txt new file mode 100644 index 0000000000000000000000000000000000000000..202e078baf408d141828f59d13f9d1f86690a915 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/outputs/success-plugin.txt @@ -0,0 +1,6 @@ +6: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" +9: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" +12: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" +15: note: Revealed type is "thinc.model.Model[thinc.types.Floats2d, thinc.types.Floats2d]" +31: error: Need type annotation for "non_combinator_model" [var-annotated] +34: note: Revealed type is "thinc.model.Model[Any, Any]" \ No newline at end of file diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/test_mypy.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/test_mypy.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d19e47942ede3ef863f3b69600004f253c0c77 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/mypy/test_mypy.py @@ -0,0 +1,91 @@ +import os +import re +import shutil +import sys +from pathlib import Path + +import pytest + +mypy = pytest.importorskip("mypy") + +# You can change the following variable to True during development to overwrite expected output with generated output +GENERATE = False + +cases = [ + ("mypy-plugin.ini", "success_plugin.py", "success-plugin.txt"), + ("mypy-plugin.ini", "fail_plugin.py", "fail-plugin.txt"), + ("mypy-default.ini", "success_no_plugin.py", "success-no-plugin.txt"), + ("mypy-default.ini", "fail_no_plugin.py", "fail-no-plugin.txt"), +] + + +@pytest.mark.skipif( + mypy.__file__.endswith(".py"), reason="Non-compiled mypy is too slow" +) +@pytest.mark.parametrize("config_filename,python_filename,output_filename", cases) +def test_mypy_results( + config_filename, python_filename, output_filename, tmpdir, monkeypatch +): + from mypy import api as mypy_api + + os.chdir(tmpdir) + root_dir = Path(__file__).parent + thinc_root_dir = Path(__file__).parent.parent.parent.parent + if "--pyargs" not in sys.argv: + monkeypatch.setenv("MYPYPATH", str(thinc_root_dir)) + tmpdir_path = Path(tmpdir) + + full_config_path: Path = root_dir / f"configs/{config_filename}" + full_module_path: Path = root_dir / f"modules/{python_filename}" + full_output_path: Path = root_dir / f"outputs/{output_filename}" + + full_tmp_config_path: Path = tmpdir_path / config_filename + full_tmp_module_path: Path = tmpdir_path / python_filename + + shutil.copy(str(full_config_path), tmpdir) + shutil.copy(str(full_module_path), tmpdir) + + expected_out = "" + expected_err = "" + expected_returncode = 1 + expected_out = full_output_path.read_text() + + # Specifying a different cache dir for each configuration dramatically speeds up subsequent execution + # It also prevents cache-invalidation-related bugs in the tests + cache_dir = tmpdir_path / f".mypy_cache/test-{config_filename[:-4]}" + command = [ + str(full_tmp_module_path), + "--config-file", + str(full_tmp_config_path), + "--cache-dir", + str(cache_dir), + "--show-error-codes", + ] + print( + f"\nExecuting: mypy {' '.join(command)}" + ) # makes it easier to debug as necessary + actual_result = mypy_api.run(command) + actual_out, actual_err, actual_returncode = actual_result + # Need to strip filenames due to differences in formatting by OS + actual_out = "\n".join( + [".py:".join(line.split(".py:")[1:]) for line in actual_out.split("\n") if line] + ).strip() + actual_out = re.sub(r"\n\s*\n", r"\n", actual_out) + + if GENERATE and output_filename is not None: + full_output_path.write_text(actual_out) + else: + assert actual_out.strip() == expected_out.strip(), actual_out + + assert actual_err == expected_err + assert actual_returncode == expected_returncode + + +@pytest.mark.skipif( + mypy.__file__.endswith(".py"), reason="Non-compiled mypy is too slow" +) +def test_generation_is_disabled(): + """ + Makes sure we don't accidentally leave generation on + """ + assert not GENERATE diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..279257279faf1f2290feb010be59fb8f351c5a9d Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue208.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue208.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..073070c109c031d3181718cf1e02cc7638f68128 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue208.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue564.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue564.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..524617f5f1918e50b19d1c835040aacb6978cbcb Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/__pycache__/test_issue564.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b1a29351f2f7304ef2abfa4a376054d64554a7e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/program.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/program.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4622db7735381d3cad912105b26038bf1f2c0a1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/program.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/test_issue519.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/test_issue519.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9522295877e04f1b19acfde99cb49f41682bf28 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/__pycache__/test_issue519.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/program.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/program.py new file mode 100644 index 0000000000000000000000000000000000000000..f482e6b4aadd0c43e05ddcbb83bfe8c4c13127f6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/program.py @@ -0,0 +1,16 @@ +from thinc.api import Relu, Softmax, chain, concatenate +from thinc.model import Model +from thinc.types import Floats2d + +n_hidden = 32 +dropout = 0.2 + +model1: Model[Floats2d, Floats2d] = chain( + Relu(nO=n_hidden, dropout=dropout), Relu(nO=n_hidden, dropout=dropout), Softmax() +) + +model2: Model[Floats2d, Floats2d] = chain( + Relu(nO=n_hidden, dropout=dropout), Relu(nO=n_hidden, dropout=dropout), Softmax() +) + +model3: Model[Floats2d, Floats2d] = concatenate(*[model1, model2]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/test_issue519.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/test_issue519.py new file mode 100644 index 0000000000000000000000000000000000000000..fb8980180ba8e3cb2e192878c8831349b6a9d50a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/issue519/test_issue519.py @@ -0,0 +1,29 @@ +import subprocess +import sys + +try: + import importlib.resources as importlib_resources +except ImportError: + import importlib_resources # type: ignore + + +import pytest + + +@pytest.mark.slow +def test_issue519(): + """ + Test ability of Thinc mypy plugin to handle variadic arguments. + + This test can take up to 45 seconds, and is thus marked as slow. + """ + # Determine the name of the parent module (which contains the test program) + parent_module_name = __name__[: __name__.rfind(".")] + + # Load test program that calls a Thinc API with variadic arguments + program_text = importlib_resources.read_text(parent_module_name, "program.py") + + # Ask Mypy to type-check the loaded program text + subprocess.run( + [sys.executable, "-m", "mypy", "--command", program_text], check=True + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue208.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue208.py new file mode 100644 index 0000000000000000000000000000000000000000..db4274fca567944b3f42da97b77651022ef627ab --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue208.py @@ -0,0 +1,10 @@ +from thinc.api import Linear, chain + + +def test_issue208(): + """Test issue that was caused by trying to flatten nested chains.""" + layer1 = Linear(nO=9, nI=3) + layer2 = Linear(nO=12, nI=9) + layer3 = Linear(nO=5, nI=12) + model = chain(layer1, chain(layer2, layer3)).initialize() + assert model.get_dim("nO") == 5 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue564.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue564.py new file mode 100644 index 0000000000000000000000000000000000000000..32152f29bc104b8d3cfe01f0f8f60867fce0a368 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/regression/test_issue564.py @@ -0,0 +1,21 @@ +import pytest + +from thinc.api import CupyOps +from thinc.compat import has_torch, has_torch_cuda_gpu + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif(not has_torch_cuda_gpu, reason="needs a GPU") +def test_issue564(): + import torch + + if CupyOps.xp is not None: + ops = CupyOps() + t = torch.zeros((10, 2)).cuda() + a = ops.asarray(t) + + assert a.shape == t.shape + ops.xp.testing.assert_allclose( + a, + ops.alloc2f(10, 2), + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2db724a1e0850f38cb049f23ee9622e727dd28e1 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/test_pytorch_grad_scaler.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/test_pytorch_grad_scaler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0777204bff17aaa28c87ad72ef848be87e40e702 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/__pycache__/test_pytorch_grad_scaler.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/test_pytorch_grad_scaler.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/test_pytorch_grad_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..337ec7e50c8fc5a8bf5ae9e5830029d31a0043e2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/shims/test_pytorch_grad_scaler.py @@ -0,0 +1,109 @@ +import pytest +from hypothesis import given, settings +from hypothesis.strategies import lists, one_of, tuples + +from thinc.api import PyTorchGradScaler +from thinc.compat import has_torch, has_torch_amp, has_torch_cuda_gpu, torch +from thinc.util import is_torch_array + +from ..strategies import ndarrays + + +def tensors(): + return ndarrays().map(lambda a: torch.tensor(a).cuda()) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif(not has_torch_cuda_gpu, reason="needs a GPU") +@pytest.mark.skipif( + not has_torch_amp, reason="requires PyTorch with mixed-precision support" +) +@given(X=one_of(tensors(), lists(tensors()), tuples(tensors()))) +@settings(deadline=None) +def test_scale_random_inputs(X): + import torch + + device_id = torch.cuda.current_device() + scaler = PyTorchGradScaler(enabled=True) + scaler.to_(device_id) + + if is_torch_array(X): + assert torch.allclose(scaler.scale(X), X * 2.0**16) + else: + scaled1 = scaler.scale(X) + scaled2 = [t * 2.0**16 for t in X] + for t1, t2 in zip(scaled1, scaled2): + assert torch.allclose(t1, t2) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif(not has_torch_cuda_gpu, reason="needs a GPU") +@pytest.mark.skipif( + not has_torch_amp, reason="requires PyTorch with mixed-precision support" +) +def test_grad_scaler(): + import torch + + device_id = torch.cuda.current_device() + + scaler = PyTorchGradScaler(enabled=True) + scaler.to_(device_id) + + # Test that scaling works as expected. + t = torch.tensor([1.0], device=device_id) + assert scaler.scale([torch.tensor([1.0], device=device_id)]) == [ + torch.tensor([2.0**16], device=device_id) + ] + assert scaler.scale(torch.tensor([1.0], device=device_id)) == torch.tensor( + [2.0**16], device=device_id + ) + with pytest.raises(ValueError): + scaler.scale("bogus") + with pytest.raises(ValueError): + scaler.scale(42) + + # Test infinity detection. + g = [ + torch.tensor([2.0**16], device=device_id), + torch.tensor([float("Inf")], device=device_id), + ] + + # Check that infinity was found. + assert scaler.unscale(g) + + # Check whether unscale was successful. + assert g[0] == torch.tensor([1.0]).cuda() + + scaler.update() + + # Since infinity was found, the scale should be halved from 2**16 + # to 2**15 for the next step. + assert scaler.scale([torch.tensor([1.0], device=device_id)]) == [ + torch.tensor([2.0**15], device=device_id) + ] + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif( + has_torch_amp, reason="needs PyTorch without gradient scaling support" +) +def test_raises_on_old_pytorch(): + import torch + + scaler = PyTorchGradScaler(enabled=True) + with pytest.raises(ValueError, match=r"not supported.*1.9.0"): + scaler.scale([torch.tensor([1.0], device="cpu")]) + + +@pytest.mark.skipif(not has_torch, reason="needs PyTorch") +@pytest.mark.skipif( + not has_torch_amp, reason="needs PyTorch with gradient scaling support" +) +def test_raises_with_cpu_tensor(): + import torch + + scaler = PyTorchGradScaler(enabled=True) + with pytest.raises( + ValueError, match=r"Gradient scaling is only supported for CUDA tensors." + ): + scaler.scale([torch.tensor([1.0], device="cpu")]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/strategies.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/strategies.py new file mode 100644 index 0000000000000000000000000000000000000000..40f279fccd6ea89751bc87b79f7c3851701d0f25 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/strategies.py @@ -0,0 +1,125 @@ +import numpy +from hypothesis.extra.numpy import arrays +from hypothesis.strategies import floats, integers, just, tuples + +from thinc.api import Linear, NumpyOps + + +def get_ops(): + return NumpyOps() + + +def get_model(W_values, b_values): + model = Linear(W_values.shape[0], W_values.shape[1], ops=NumpyOps()) + model.initialize() + model.set_param("W", W_values) + model.set_param("b", b_values) + return model + + +def get_output(input_, W_values, b_values): + return numpy.einsum("oi,bi->bo", W_values, input_) + b_values + + +def get_input(nr_batch, nr_in): + ops = NumpyOps() + return ops.alloc2f(nr_batch, nr_in) + + +def lengths(lo=1, hi=10): + return integers(min_value=lo, max_value=hi) + + +def shapes(min_rows=1, max_rows=100, min_cols=1, max_cols=100): + return tuples(lengths(lo=min_rows, hi=max_rows), lengths(lo=min_cols, hi=max_cols)) + + +def ndarrays_of_shape(shape, lo=-10.0, hi=10.0, dtype="float32", width=32): + if dtype.startswith("float"): + return arrays( + dtype, shape=shape, elements=floats(min_value=lo, max_value=hi, width=width) + ) + else: + return arrays(dtype, shape=shape, elements=integers(min_value=lo, max_value=hi)) + + +def ndarrays(min_len=0, max_len=10, min_val=-10.0, max_val=10.0): + return lengths(lo=1, hi=2).flatmap( + lambda n: ndarrays_of_shape(n, lo=min_val, hi=max_val) + ) + + +def arrays_BI(min_B=1, max_B=10, min_I=1, max_I=100): + shapes = tuples(lengths(lo=min_B, hi=max_B), lengths(lo=min_I, hi=max_I)) + return shapes.flatmap(ndarrays_of_shape) + + +def arrays_BOP(min_B=1, max_B=10, min_O=1, max_O=100, min_P=1, max_P=5): + shapes = tuples( + lengths(lo=min_B, hi=max_B), + lengths(lo=min_O, hi=max_O), + lengths(lo=min_P, hi=max_P), + ) + return shapes.flatmap(ndarrays_of_shape) + + +def arrays_BOP_BO(min_B=1, max_B=10, min_O=1, max_O=100, min_P=1, max_P=5): + shapes = tuples( + lengths(lo=min_B, hi=max_B), + lengths(lo=min_O, hi=max_O), + lengths(lo=min_P, hi=max_P), + ) + return shapes.flatmap( + lambda BOP: tuples(ndarrays_of_shape(BOP), ndarrays_of_shape(BOP[:-1])) + ) + + +def arrays_BI_BO(min_B=1, max_B=10, min_I=1, max_I=100, min_O=1, max_O=100): + shapes = tuples( + lengths(lo=min_B, hi=max_B), + lengths(lo=min_I, hi=max_I), + lengths(lo=min_O, hi=max_O), + ) + return shapes.flatmap( + lambda BIO: tuples( + ndarrays_of_shape((BIO[0], BIO[1])), ndarrays_of_shape((BIO[0], BIO[2])) + ) + ) + + +def arrays_OI_O_BI( + min_batch=1, max_batch=16, min_out=1, max_out=16, min_in=1, max_in=16 +): + shapes = tuples( + lengths(lo=min_batch, hi=max_batch), + lengths(lo=min_in, hi=max_out), + lengths(lo=min_in, hi=max_in), + ) + + def W_b_inputs(shape): + batch_size, nr_out, nr_in = shape + W = ndarrays_of_shape((nr_out, nr_in)) + b = ndarrays_of_shape((nr_out,)) + input_ = ndarrays_of_shape((batch_size, nr_in)) + return tuples(W, b, input_) + + return shapes.flatmap(W_b_inputs) + + +def arrays_OPFI_BI_lengths(max_B=5, max_P=3, max_F=5, max_I=8): + shapes = tuples( + lengths(hi=max_B), + lengths(hi=max_P), + lengths(hi=max_F), + lengths(hi=max_I), + arrays("int32", shape=(5,), elements=integers(min_value=1, max_value=10)), + ) + + strat = shapes.flatmap( + lambda opfi_lengths: tuples( + ndarrays_of_shape(opfi_lengths[:-1]), + ndarrays_of_shape((sum(opfi_lengths[-1]), opfi_lengths[-2])), + just(opfi_lengths[-1]), + ) + ) + return strat diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_config.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..721b93e62960babb78a4f5f3ba7fcf767633d024 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_config.py @@ -0,0 +1,229 @@ +import inspect +import pickle +from types import GeneratorType +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + +import catalogue +import numpy +import pytest + +try: + from pydantic.v1 import BaseModel, PositiveInt, StrictBool, StrictFloat, constr +except ImportError: + from pydantic import BaseModel, PositiveInt, StrictBool, StrictFloat, constr # type: ignore + +import thinc.config +from thinc.api import Config, Model, NumpyOps, RAdam +from thinc.config import ConfigValidationError +from thinc.types import Generator, Ragged +from thinc.util import partial + +from .util import make_tempdir + +EXAMPLE_CONFIG = """ +[optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +use_averages = true + +[optimizer.learn_rate] +@schedules = "warmup_linear.v1" +initial_rate = 0.1 +warmup_steps = 10000 +total_steps = 100000 + +[pipeline] + +[pipeline.parser] +name = "parser" +factory = "parser" + +[pipeline.parser.model] +@layers = "spacy.ParserModel.v1" +hidden_depth = 1 +hidden_width = 64 +token_vector_width = 128 + +[pipeline.parser.model.tok2vec] +@layers = "Tok2Vec.v1" +width = ${pipeline.parser.model:token_vector_width} + +[pipeline.parser.model.tok2vec.embed] +@layers = "spacy.MultiFeatureHashEmbed.v1" +width = ${pipeline.parser.model.tok2vec:width} + +[pipeline.parser.model.tok2vec.embed.hidden] +@layers = "MLP.v1" +depth = 1 +pieces = 3 +layer_norm = true +outputs = ${pipeline.parser.model.tok2vec.embed:width} + +[pipeline.parser.model.tok2vec.encode] +@layers = "spacy.MaxoutWindowEncoder.v1" +depth = 4 +pieces = 3 +window_size = 1 + +[pipeline.parser.model.lower] +@layers = "spacy.ParserLower.v1" + +[pipeline.parser.model.upper] +@layers = "thinc.Linear.v1" +""" + +OPTIMIZER_CFG = """ +[optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +use_averages = true + +[optimizer.learn_rate] +@schedules = "warmup_linear.v1" +initial_rate = 0.1 +warmup_steps = 10000 +total_steps = 100000 +""" + + +class my_registry(thinc.config.registry): + cats = catalogue.create("thinc", "tests", "cats", entry_points=False) + + +class HelloIntsSchema(BaseModel): + hello: int + world: int + + class Config: + extra = "forbid" + + +class DefaultsSchema(BaseModel): + required: int + optional: str = "default value" + + class Config: + extra = "forbid" + + +class ComplexSchema(BaseModel): + outer_req: int + outer_opt: str = "default value" + + level2_req: HelloIntsSchema + level2_opt: DefaultsSchema = DefaultsSchema(required=1) + + +@my_registry.cats.register("catsie.v1") +def catsie_v1(evil: StrictBool, cute: bool = True) -> str: + if evil: + return "scratch!" + else: + return "meow" + + +@my_registry.cats.register("catsie.v2") +def catsie_v2(evil: StrictBool, cute: bool = True, cute_level: int = 1) -> str: + if evil: + return "scratch!" + else: + if cute_level > 2: + return "meow <3" + return "meow" + + +good_catsie = {"@cats": "catsie.v1", "evil": False, "cute": True} +ok_catsie = {"@cats": "catsie.v1", "evil": False, "cute": False} +bad_catsie = {"@cats": "catsie.v1", "evil": True, "cute": True} +worst_catsie = {"@cats": "catsie.v1", "evil": True, "cute": False} + + +def test_make_config_positional_args_dicts(): + cfg = { + "hyper_params": {"n_hidden": 512, "dropout": 0.2, "learn_rate": 0.001}, + "model": { + "@layers": "chain.v1", + "*": { + "relu1": {"@layers": "Relu.v1", "nO": 512, "dropout": 0.2}, + "relu2": {"@layers": "Relu.v1", "nO": 512, "dropout": 0.2}, + "softmax": {"@layers": "Softmax.v1"}, + }, + }, + "optimizer": {"@optimizers": "Adam.v1", "learn_rate": 0.001}, + } + resolved = my_registry.resolve(cfg) + model = resolved["model"] + X = numpy.ones((784, 1), dtype="f") + model.initialize(X=X, Y=numpy.zeros((784, 1), dtype="f")) + model.begin_update(X) + model.finish_update(resolved["optimizer"]) + + +def test_objects_from_config(): + config = { + "optimizer": { + "@optimizers": "my_cool_optimizer.v1", + "beta1": 0.2, + "learn_rate": { + "@schedules": "my_cool_repetitive_schedule.v1", + "base_rate": 0.001, + "repeat": 4, + }, + } + } + + @thinc.registry.optimizers.register("my_cool_optimizer.v1") + def make_my_optimizer(learn_rate: List[float], beta1: float): + return RAdam(learn_rate, beta1=beta1) + + @thinc.registry.schedules("my_cool_repetitive_schedule.v1") + def decaying(base_rate: float, repeat: int) -> List[float]: + return repeat * [base_rate] + + optimizer = my_registry.resolve(config)["optimizer"] + assert optimizer.b1 == 0.2 + assert "learn_rate" in optimizer.schedules + assert optimizer.learn_rate == 0.001 + + +def test_handle_generic_model_type(): + """Test that validation can handle checks against arbitrary generic + types in function argument annotations.""" + + @my_registry.layers("my_transform.v1") + def my_transform(model: Model[int, int]): + model.name = "transformed_model" + return model + + cfg = {"@layers": "my_transform.v1", "model": {"@layers": "Linear.v1"}} + model = my_registry.resolve({"test": cfg})["test"] + assert isinstance(model, Model) + assert model.name == "transformed_model" + + +def test_arg_order_is_preserved(): + str_cfg = """ + [model] + + [model.chain] + @layers = "chain.v1" + + [model.chain.*.hashembed] + @layers = "HashEmbed.v1" + nO = 8 + nV = 8 + + [model.chain.*.expand_window] + @layers = "expand_window.v1" + window_size = 1 + """ + + cfg = Config().from_str(str_cfg) + resolved = my_registry.resolve(cfg) + model = resolved["model"]["chain"] + + # Fails when arguments are sorted, because expand_window + # is sorted before hashembed. + assert model.name == "hashembed>>expand_window" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_examples.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..4b1ef6401447a35b3a68b2bdce109c85542b9941 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_examples.py @@ -0,0 +1,59 @@ +import os +from pathlib import Path + +import pytest + + +@pytest.fixture +def test_files(nb_file): + pytest.importorskip("nbconvert") + pytest.importorskip("nbformat") + import nbconvert + import nbformat + from nbconvert.preprocessors import ExecutePreprocessor + + if not Path(nb_file).exists(): + return + kernel_name = os.environ.get("NOTEBOOK_KERNEL", "python3") + with open(nb_file) as f: + nb = nbformat.read(f, as_version=4) + proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) + proc.allow_errors = True + proc.preprocess(nb, {"metadata": {"path": "/"}}) + cells_with_outputs = [c for c in nb.cells if "outputs" in c] + for cell in cells_with_outputs: + for output in cell["outputs"]: + if output.output_type == "error": + for l in output.traceback: + print(l) + raise Exception(f"{output.ename}: {output.evalue}") + + +@pytest.mark.skip(reason="Notebooks should be updated") +@pytest.mark.parametrize( + "nb_file", + ( + "examples/01_intro_model_definition_methods.ipynb", + "examples/05_benchmarking_layers.ipynb", + ), +) +def test_ipython_notebooks(test_files: None): + ... + + +@pytest.mark.skip(reason="these notebooks need special software or hardware") +@pytest.mark.parametrize( + "nb_file", + ( + "examples/00_intro_to_thinc.ipynb", + "examples/02_transformers_tagger_bert.ipynb", + "examples/03_pos_tagger_basic_cnn.ipynb", + "examples/03_textcat_basic_neural_bow.ipynb", + "examples/04_configure_gpu_memory.ipynb", + "examples/04_parallel_training_ray.ipynb", + "examples/05_visualizing_models.ipynb", + "examples/06_predicting_like_terms.ipynb", + ), +) +def test_ipython_notebooks_slow(test_files: None): + ... diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_import__all__.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_import__all__.py new file mode 100644 index 0000000000000000000000000000000000000000..c23cf2298982b51b6dde244d2761f5e85d8e30da --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_import__all__.py @@ -0,0 +1,53 @@ +import ast +import importlib +from collections import namedtuple +from typing import List, Tuple + +import pytest + +_Import = namedtuple("_Import", ["module", "name", "alias"]) + + +def get_imports(path: str) -> Tuple[_Import, ...]: + """Parse Python file at path, retrieve import statements. + Adapted from https://stackoverflow.com/a/9049549. + path (str): Path to Python file. + RETURNS (Tuple[_Import]): All imports found in file at path. + """ + with open(path) as fh: + root = ast.parse(fh.read(), path) + + imports: List[_Import] = [] + for node in ast.walk(root): + if isinstance(node, ast.Import): + module: List[str] = [] + elif isinstance(node, ast.ImportFrom) and node.module: + module = node.module.split(".") + else: + continue + + assert isinstance(node, (ast.Import, ast.ImportFrom)) + imports.extend( + [_Import(module, n.name.split("."), n.asname) for n in node.names] + ) + + return tuple(imports) + + +@pytest.mark.parametrize("module_name", ["thinc.api", "thinc.shims", "thinc.layers"]) +def test_import_reexport_equivalency(module_name: str): + """Tests whether a module's __all__ is equivalent to its imports. This assumes that this module is supposed to + re-export all imported values. + module_name (str): Module to load. + """ + mod = importlib.import_module(module_name) + + assert set(mod.__all__) == { + k + for k in set(n for i in get_imports(str(mod.__file__)) for n in i.name) + if ( + # Ignore all values prefixed with _, as we expect those not to be re-exported. + # However, __version__ should be reexported in thinc/__init__.py. + (not k.startswith("_") or module_name == "thinc" and k == "__version__") + ) + } diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_indexing.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_indexing.py new file mode 100644 index 0000000000000000000000000000000000000000..55f3454abbc0143affe00ab63b979825ffb955fc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_indexing.py @@ -0,0 +1,66 @@ +import numpy +import pytest +from numpy.testing import assert_allclose + +from thinc.types import Pairs, Ragged + + +@pytest.fixture +def ragged(): + data = numpy.zeros((20, 4), dtype="f") + lengths = numpy.array([4, 2, 8, 1, 4], dtype="i") + data[0] = 0 + data[1] = 1 + data[2] = 2 + data[3] = 3 + data[4] = 4 + data[5] = 5 + return Ragged(data, lengths) + + +def test_ragged_empty(): + data = numpy.zeros((0, 4), dtype="f") + lengths = numpy.array([], dtype="i") + ragged = Ragged(data, lengths) + assert_allclose(ragged[0:0].data, ragged.data) + assert_allclose(ragged[0:0].lengths, ragged.lengths) + assert_allclose(ragged[0:2].data, ragged.data) + assert_allclose(ragged[0:2].lengths, ragged.lengths) + assert_allclose(ragged[1:2].data, ragged.data) + assert_allclose(ragged[1:2].lengths, ragged.lengths) + + +def test_ragged_starts_ends(ragged): + starts = ragged._get_starts() + ends = ragged._get_ends() + assert list(starts) == [0, 4, 6, 14, 15] + assert list(ends) == [4, 6, 14, 15, 19] + + +def test_ragged_simple_index(ragged, i=1): + r = ragged[i] + assert_allclose(r.data, ragged.data[4:6]) + assert_allclose(r.lengths, ragged.lengths[i : i + 1]) + + +def test_ragged_slice_index(ragged, start=0, end=2): + r = ragged[start:end] + size = ragged.lengths[start:end].sum() + assert r.data.shape == (size, r.data.shape[1]) + assert_allclose(r.lengths, ragged.lengths[start:end]) + + +def test_ragged_array_index(ragged): + arr = numpy.array([2, 1, 4], dtype="i") + r = ragged[arr] + assert r.data.shape[0] == ragged.lengths[arr].sum() + + +def test_pairs_arrays(): + one = numpy.zeros((128, 45), dtype="f") + two = numpy.zeros((128, 12), dtype="f") + pairs = Pairs(one, two) + assert pairs[:2].one.shape == (2, 45) + assert pairs[0].two.shape == (12,) + assert pairs[-1:].one.shape == (1, 45) + assert pairs[-1:].two.shape == (1, 12) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_initializers.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_initializers.py new file mode 100644 index 0000000000000000000000000000000000000000..49fd136c854ad81eb3cd21adc8fbe1f0ac7cea0d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_initializers.py @@ -0,0 +1,38 @@ +import numpy +import pytest + +from thinc import registry +from thinc.api import ( + NumpyOps, + glorot_uniform_init, + normal_init, + uniform_init, + zero_init, +) + + +@pytest.mark.parametrize( + "init_func", [glorot_uniform_init, zero_init, uniform_init, normal_init] +) +def test_initializer_func_setup(init_func): + ops = NumpyOps() + data = numpy.ndarray([1, 2, 3, 4], dtype="f") + result = init_func(ops, data.shape) + assert not numpy.array_equal(data, result) + + +@pytest.mark.parametrize( + "name,kwargs", + [ + ("glorot_uniform_init.v1", {}), + ("zero_init.v1", {}), + ("uniform_init.v1", {"lo": -0.5, "hi": 0.5}), + ("normal_init.v1", {"mean": 0.1}), + ], +) +def test_initializer_from_config(name, kwargs): + """Test that initializers are loaded and configured correctly from registry + (as partials).""" + cfg = {"test": {"@initializers": name, **kwargs}} + func = registry.resolve(cfg)["test"] + func(NumpyOps(), (1, 2, 3, 4)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_loss.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..f8183017c81f21fe72d651caa31983a404f8f9c3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_loss.py @@ -0,0 +1,332 @@ +import numpy +import pytest + +from thinc import registry +from thinc.api import ( + CategoricalCrossentropy, + CosineDistance, + L2Distance, + SequenceCategoricalCrossentropy, +) + +# some simple arrays +scores0 = numpy.zeros((3, 3), dtype="f") +labels0 = numpy.asarray([0, 1, 1], dtype="i") + +# a few more diverse ones to test realistic values +guesses1 = numpy.asarray([[0.1, 0.5, 0.6], [0.4, 0.6, 0.3], [1, 1, 1], [0, 0, 0]]) +labels1 = numpy.asarray([2, 1, 0, 2]) +labels1_full = numpy.asarray([[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 0, 1]]) +labels1_strings = ["C", "B", "A", "C"] + +guesses2 = numpy.asarray([[0.2, 0.3, 0.0]]) +labels2 = numpy.asarray([1]) +labels2_strings = ["B"] + +eps = 0.0001 + + +def test_loss(): + d_scores = CategoricalCrossentropy().get_grad(scores0, labels0) + assert d_scores.dtype == "float32" + assert d_scores.shape == scores0.shape + d_scores = SequenceCategoricalCrossentropy().get_grad([scores0], [labels0]) + assert d_scores[0].dtype == "float32" + assert d_scores[0].shape == scores0.shape + assert SequenceCategoricalCrossentropy().get_grad([], []) == [] + + +@pytest.mark.parametrize( + "dist", [CategoricalCrossentropy(), CosineDistance(ignore_zeros=True), L2Distance()] +) +@pytest.mark.parametrize("vect", [scores0, guesses1, guesses2]) +def test_equality(dist, vect): + assert int(dist.get_grad(vect, vect)[0][0]) == pytest.approx(0, eps) + assert dist.get_loss(vect, vect) == pytest.approx(0, eps) + + +@pytest.mark.parametrize( + "guesses, labels", [(guesses1, labels1), (guesses1, labels1_full)] +) +def test_categorical_crossentropy(guesses, labels): + d_scores = CategoricalCrossentropy(normalize=True).get_grad(guesses, labels) + assert d_scores.shape == guesses.shape + + # The normalization divides the difference (e.g. 0.4) by the number of vectors (4) + assert d_scores[1][0] == pytest.approx(0.1, eps) + assert d_scores[1][1] == pytest.approx(-0.1, eps) + + # The third vector predicted all labels, but only the first one was correct + assert d_scores[2][0] == pytest.approx(0, eps) + assert d_scores[2][1] == pytest.approx(0.25, eps) + assert d_scores[2][2] == pytest.approx(0.25, eps) + + # The fourth vector predicted no labels but should have predicted the last one + assert d_scores[3][0] == pytest.approx(0, eps) + assert d_scores[3][1] == pytest.approx(0, eps) + assert d_scores[3][2] == pytest.approx(-0.25, eps) + + loss = CategoricalCrossentropy(normalize=True).get_loss(guesses, labels) + assert loss == pytest.approx(0.239375, eps) + + +def test_crossentropy_incorrect_scores_targets(): + labels = numpy.asarray([2]) + + guesses_neg = numpy.asarray([[-0.1, 0.5, 0.6]]) + with pytest.raises(ValueError, match=r"Cannot calculate.*guesses"): + CategoricalCrossentropy(normalize=True).get_grad(guesses_neg, labels) + + guesses_larger_than_one = numpy.asarray([[1.1, 0.5, 0.6]]) + with pytest.raises(ValueError, match=r"Cannot calculate.*guesses"): + CategoricalCrossentropy(normalize=True).get_grad( + guesses_larger_than_one, labels + ) + + guesses_ok = numpy.asarray([[0.1, 0.4, 0.5]]) + targets_neg = numpy.asarray([[-0.1, 0.5, 0.6]]) + with pytest.raises(ValueError, match=r"Cannot calculate.*truth"): + CategoricalCrossentropy(normalize=True).get_grad(guesses_ok, targets_neg) + + targets_larger_than_one = numpy.asarray([[2.0, 0.5, 0.6]]) + with pytest.raises(ValueError, match=r"Cannot calculate.*truth"): + CategoricalCrossentropy(normalize=True).get_grad( + guesses_ok, targets_larger_than_one + ) + + +@pytest.mark.parametrize( + "guesses, labels", + [(guesses1, [2, 1, 0, 2])], +) +def test_categorical_crossentropy_int_list_missing(guesses, labels): + d_scores = CategoricalCrossentropy(normalize=True, missing_value=0).get_grad( + guesses, labels + ) + assert d_scores.shape == guesses.shape + + # The normalization divides the difference (e.g. 0.4) by the number of vectors (4) + assert d_scores[1][0] == pytest.approx(0.1, eps) + assert d_scores[1][1] == pytest.approx(-0.1, eps) + + # Label 0 is masked, because it represents the missing value + assert d_scores[2][0] == 0.0 + assert d_scores[2][1] == 0.0 + assert d_scores[2][2] == 0.0 + + # The fourth vector predicted no labels but should have predicted the last one + assert d_scores[3][0] == pytest.approx(0, eps) + assert d_scores[3][1] == pytest.approx(0, eps) + assert d_scores[3][2] == pytest.approx(-0.25, eps) + + loss = CategoricalCrossentropy(normalize=True, missing_value=0).get_loss( + guesses, labels + ) + assert loss == pytest.approx(0.114375, eps) + + +@pytest.mark.parametrize( + "guesses, labels", [(guesses1, labels1), (guesses1, labels1_full)] +) +def test_categorical_crossentropy_missing(guesses, labels): + d_scores = CategoricalCrossentropy(normalize=True, missing_value=0).get_grad( + guesses, labels + ) + assert d_scores.shape == guesses.shape + + # The normalization divides the difference (e.g. 0.4) by the number of vectors (4) + assert d_scores[1][0] == pytest.approx(0.1, eps) + assert d_scores[1][1] == pytest.approx(-0.1, eps) + + # Label 0 is masked, because it represents the missing value + assert d_scores[2][0] == 0.0 + assert d_scores[2][1] == 0.0 + assert d_scores[2][2] == 0.0 + + # The fourth vector predicted no labels but should have predicted the last one + assert d_scores[3][0] == pytest.approx(0, eps) + assert d_scores[3][1] == pytest.approx(0, eps) + assert d_scores[3][2] == pytest.approx(-0.25, eps) + + loss = CategoricalCrossentropy(normalize=True, missing_value=0).get_loss( + guesses, labels + ) + assert loss == pytest.approx(0.114375, eps) + + +@pytest.mark.parametrize( + "guesses, labels, names", + [ + ([guesses1, guesses2], [labels1, labels2], []), + ([guesses1, guesses2], [labels1_full, labels2], []), + ([guesses1, guesses2], [labels1_strings, labels2_strings], ["A", "B", "C"]), + ], +) +def test_sequence_categorical_crossentropy(guesses, labels, names): + d_scores = SequenceCategoricalCrossentropy(normalize=False, names=names).get_grad( + guesses, labels + ) + d_scores1 = d_scores[0] + d_scores2 = d_scores[1] + assert d_scores1.shape == guesses1.shape + assert d_scores2.shape == guesses2.shape + assert d_scores1[1][0] == pytest.approx(0.4, eps) + assert d_scores1[1][1] == pytest.approx(-0.4, eps) + # The normalization divides the difference (e.g. 0.4) by the number of seqs + d_scores = SequenceCategoricalCrossentropy(normalize=True, names=names).get_grad( + guesses, labels + ) + d_scores1 = d_scores[0] + d_scores2 = d_scores[1] + + assert d_scores1[1][0] == pytest.approx(0.2, eps) + assert d_scores1[1][1] == pytest.approx(-0.2, eps) + + # The third vector predicted all labels, but only the first one was correct + assert d_scores1[2][0] == pytest.approx(0, eps) + assert d_scores1[2][1] == pytest.approx(0.5, eps) + assert d_scores1[2][2] == pytest.approx(0.5, eps) + + # The fourth vector predicted no labels but should have predicted the last one + assert d_scores1[3][0] == pytest.approx(0, eps) + assert d_scores1[3][1] == pytest.approx(0, eps) + assert d_scores1[3][2] == pytest.approx(-0.5, eps) + + # Test the second batch + assert d_scores2[0][0] == pytest.approx(0.1, eps) + assert d_scores2[0][1] == pytest.approx(-0.35, eps) + + loss = SequenceCategoricalCrossentropy(normalize=True, names=names).get_loss( + guesses, labels + ) + assert loss == pytest.approx(1.09, eps) + + +@pytest.mark.parametrize( + "guesses, labels, names", + [ + ([guesses1], [["A", "!A", "", "!C"]], ["A", "B", "C"]), + ], +) +def test_sequence_categorical_missing_negative(guesses, labels, names): + d_scores = SequenceCategoricalCrossentropy( + normalize=False, names=names, neg_prefix="!", missing_value="" + ).get_grad(guesses, labels) + d_scores0 = d_scores[0] + + # [0.1, 0.5, 0.6] should be A + assert d_scores0[0][0] == pytest.approx(-0.9, eps) + assert d_scores0[0][1] == pytest.approx(0.5, eps) + assert d_scores0[0][2] == pytest.approx(0.6, eps) + + # [0.4, 0.6, 0.3] should NOT be A + assert d_scores0[1][0] == pytest.approx(0.4, eps) + assert d_scores0[1][1] == pytest.approx(0.0, eps) + assert d_scores0[1][2] == pytest.approx(0.0, eps) + + # [1, 1, 1] has missing gold label + assert d_scores0[2][0] == pytest.approx(0.0, eps) + assert d_scores0[2][1] == pytest.approx(0.0, eps) + assert d_scores0[2][2] == pytest.approx(0.0, eps) + + # [0.0, 0.0, 0.0] should NOT be C + assert d_scores0[3][0] == pytest.approx(0.0, eps) + assert d_scores0[3][1] == pytest.approx(0.0, eps) + assert d_scores0[3][2] == pytest.approx(0.0, eps) + + +def test_L2(): + # L2 loss = 2²+4²=20 (or normalized: 1²+2²=5) + vec1 = numpy.asarray([[1, 2], [8, 9]]) + vec2 = numpy.asarray([[1, 2], [10, 5]]) + d_vecs = L2Distance().get_grad(vec1, vec2) + assert d_vecs.shape == vec1.shape + numpy.testing.assert_allclose( + d_vecs[0], numpy.zeros(d_vecs[0].shape), rtol=eps, atol=eps + ) + + loss_not_normalized = L2Distance(normalize=False).get_loss(vec1, vec2) + assert loss_not_normalized == pytest.approx(20, eps) + + loss_normalized = L2Distance(normalize=True).get_loss(vec1, vec2) + assert loss_normalized == pytest.approx(5, eps) + + +def test_cosine_orthogonal(): + # These are orthogonal, i.e. loss is 1 + vec1 = numpy.asarray([[0, 2], [0, 5]]) + vec2 = numpy.asarray([[8, 0], [7, 0]]) + + d_vecs = CosineDistance(normalize=True).get_grad(vec1, vec2) + assert d_vecs.shape == vec1.shape + assert d_vecs[0][0] < 0 + assert d_vecs[0][1] > 0 + assert d_vecs[1][0] < 0 + assert d_vecs[1][1] > 0 + + loss_not_normalized = CosineDistance(normalize=False).get_loss(vec1, vec2) + assert loss_not_normalized == pytest.approx(2, eps) + + loss_normalized = CosineDistance(normalize=True).get_loss(vec1, vec2) + assert loss_normalized == pytest.approx(1, eps) + + +def test_cosine_equal(): + # These 3 vectors are equal when measured with Cosine similarity, i.e. loss is 0 + vec1 = numpy.asarray([[1, 2], [8, 9], [3, 3]]) + vec2 = numpy.asarray([[1, 2], [80, 90], [300, 300]]) + + d_vec1 = CosineDistance().get_grad(vec1, vec2) + assert d_vec1.shape == vec1.shape + numpy.testing.assert_allclose(d_vec1, numpy.zeros(d_vec1.shape), rtol=eps, atol=eps) + + loss_not_normalized = CosineDistance(normalize=False).get_loss(vec1, vec2) + assert loss_not_normalized == pytest.approx(0, eps) + + loss_normalized = CosineDistance(normalize=True).get_loss(vec1, vec2) + assert loss_normalized == pytest.approx(0, eps) + + +def test_cosine_unmatched(): + vec1 = numpy.asarray([[1, 2, 3]]) + vec2 = numpy.asarray([[1, 2]]) + with pytest.raises(ValueError): + CosineDistance().get_grad(vec1, vec2) + + +@pytest.mark.parametrize( + "name,kwargs,args", + [ + ("CategoricalCrossentropy.v1", {}, (scores0, labels0)), + ("SequenceCategoricalCrossentropy.v1", {}, ([scores0], [labels0])), + ("CategoricalCrossentropy.v2", {"neg_prefix": "!"}, (scores0, labels0)), + ("CategoricalCrossentropy.v3", {"neg_prefix": "!"}, (scores0, labels0)), + ( + "SequenceCategoricalCrossentropy.v2", + {"neg_prefix": "!"}, + ([scores0], [labels0]), + ), + ( + "SequenceCategoricalCrossentropy.v3", + {"neg_prefix": "!"}, + ([scores0], [labels0]), + ), + ("L2Distance.v1", {}, (scores0, scores0)), + ( + "CosineDistance.v1", + {"normalize": True, "ignore_zeros": True}, + (scores0, scores0), + ), + ], +) +def test_loss_from_config(name, kwargs, args): + """Test that losses are loaded and configured correctly from registry + (as partials).""" + cfg = {"test": {"@losses": name, **kwargs}} + func = registry.resolve(cfg)["test"] + loss = func.get_grad(*args) + if isinstance(loss, (list, tuple)): + loss = loss[0] + assert loss.ndim == 2 + func.get_loss(*args) + func(*args) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_optimizers.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_optimizers.py new file mode 100644 index 0000000000000000000000000000000000000000..25c37e1a392ece4bc699321d97b3334f597d6d2b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_optimizers.py @@ -0,0 +1,100 @@ +import numpy +import pytest + +from thinc.api import Optimizer, registry + + +def _test_schedule_valid(): + while True: + yield 0.456 + + +def _test_schedule_invalid(): + yield from [] + + +@pytest.fixture( + params=[ + (lambda: 0.123, 0.123, 0.123, 0.123), + (lambda: _test_schedule_valid(), 0.456, 0.456, 0.456), + (lambda: (i for i in [0.2, 0.1, 0.4, 0.5, 0.6, 0.7, 0.8]), 0.2, 0.1, 0.4), + (lambda: (i for i in [0.333, 0.666]), 0.333, 0.666, 0.666), + (lambda: [0.9, 0.8, 0.7], 0.9, 0.8, 0.7), + (lambda: [0.0, 0.123], 0.0, 0.123, 0.123), + ], + scope="function", +) +def schedule_valid(request): + # Use lambda to prevent iterator from being consumed by first test + r_func, r1, r2, r3 = request.param + return r_func(), r1, r2, r3 + + +@pytest.fixture( + params=[ + (lambda: "hello"), + (lambda: _test_schedule_invalid()), + (lambda: (_ for _ in [])), + (lambda: []), + ], + scope="function", +) +def schedule_invalid(request): + # Use lambda to prevent iterator from being consumed by first test + r_func = request.param + return r_func() + + +@pytest.mark.parametrize("name", ["RAdam.v1", "Adam.v1", "SGD.v1"]) +def test_optimizers_from_config(name): + learn_rate = 0.123 + cfg = {"@optimizers": name, "learn_rate": learn_rate} + optimizer = registry.resolve({"config": cfg})["config"] + assert optimizer.learn_rate == learn_rate + + +def test_optimizer_schedules_from_config(schedule_valid): + lr, lr_next1, lr_next2, lr_next3 = schedule_valid + cfg = {"@optimizers": "Adam.v1", "learn_rate": lr} + optimizer = registry.resolve({"cfg": cfg})["cfg"] + assert optimizer.learn_rate == lr_next1 + optimizer.step_schedules() + assert optimizer.learn_rate == lr_next2 + optimizer.step_schedules() + assert optimizer.learn_rate == lr_next3 + optimizer.learn_rate = 1.0 + assert optimizer.learn_rate == 1.0 + + +def test_optimizer_schedules_valid(schedule_valid): + lr, lr_next1, lr_next2, lr_next3 = schedule_valid + optimizer = Optimizer(learn_rate=lr) + assert optimizer.learn_rate == lr_next1 + optimizer.step_schedules() + assert optimizer.learn_rate == lr_next2 + optimizer.step_schedules() + assert optimizer.learn_rate == lr_next3 + optimizer.learn_rate = 1.0 + assert optimizer.learn_rate == 1.0 + + +def test_optimizer_schedules_invalid(schedule_invalid): + with pytest.raises(ValueError): + Optimizer(learn_rate=schedule_invalid) + + +def test_optimizer_init(): + optimizer = Optimizer( + learn_rate=0.123, + use_averages=False, + use_radam=True, + L2=0.1, + L2_is_weight_decay=False, + ) + _, gradient = optimizer((0, "x"), numpy.zeros((1, 2)), numpy.zeros(0)) + assert numpy.array_equal(gradient, numpy.zeros(0)) + W = numpy.asarray([1.0, 0.0, 0.0, 1.0], dtype="f").reshape((4,)) + dW = numpy.asarray([[-1.0, 0.0, 0.0, 1.0]], dtype="f").reshape((4,)) + optimizer((0, "x"), W, dW) + optimizer = Optimizer(learn_rate=0.123, beta1=0.1, beta2=0.1) + optimizer((1, "x"), W, dW) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_schedules.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_schedules.py new file mode 100644 index 0000000000000000000000000000000000000000..df27f795a5aaf7ec28b0622186590733971161d1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_schedules.py @@ -0,0 +1,70 @@ +from thinc.api import ( + compounding, + constant, + constant_then, + cyclic_triangular, + decaying, + slanted_triangular, + warmup_linear, +) + + +def test_decaying_rate(): + rates = decaying(0.001, 1e-4) + rate = next(rates) + assert rate == 0.001 + next_rate = next(rates) + assert next_rate < rate + assert next_rate > 0 + assert next_rate > next(rates) + + +def test_compounding_rate(): + rates = compounding(1, 16, 1.01) + rate0 = next(rates) + assert rate0 == 1.0 + rate1 = next(rates) + rate2 = next(rates) + rate3 = next(rates) + assert rate3 > rate2 > rate1 > rate0 + assert (rate3 - rate2) > (rate2 - rate1) > (rate1 - rate0) + + +def test_slanted_triangular_rate(): + rates = slanted_triangular(1.0, 20.0, ratio=10) + rate0 = next(rates) + assert rate0 < 1.0 + rate1 = next(rates) + assert rate1 > rate0 + rate2 = next(rates) + assert rate2 < rate1 + rate3 = next(rates) + assert rate0 < rate3 < rate2 + + +def test_constant_then_schedule(): + rates = constant_then(1.0, 2, [100, 200]) + assert next(rates) == 1.0 + assert next(rates) == 1.0 + assert next(rates) == 100 + assert next(rates) == 200 + + +def test_constant(): + rates = constant(123) + assert next(rates) == 123 + assert next(rates) == 123 + + +def test_warmup_linear(): + rates = warmup_linear(1.0, 2, 10) + expected = [0.0, 0.5, 1.0, 0.875, 0.75, 0.625, 0.5, 0.375, 0.25, 0.125, 0.0] + for i in range(11): + assert next(rates) == expected[i] + + +def test_cyclic_triangular(): + rates = cyclic_triangular(0.1, 1.0, 2) + expected = [0.55, 1.0, 0.55, 0.1, 0.55, 1.0, 0.55, 0.1, 0.55, 1.0] + for i in range(10): + assert next(rates) == expected[i] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_serialize.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_serialize.py new file mode 100644 index 0000000000000000000000000000000000000000..5f61e26e7e13b2b6ee63b8788275595e9cdf2c6a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_serialize.py @@ -0,0 +1,209 @@ +import pytest +import srsly + +from thinc.api import ( + Linear, + Maxout, + Model, + Shim, + chain, + deserialize_attr, + serialize_attr, + with_array, +) + + +@pytest.fixture +def linear(): + return Linear(5, 3) + + +class SerializableAttr: + value = "foo" + + def to_bytes(self): + return self.value.encode("utf8") + + def from_bytes(self, data): + self.value = f"{data.decode('utf8')} from bytes" + return self + + +class SerializableShim(Shim): + name = "testshim" + value = "shimdata" + + def to_bytes(self): + return self.value.encode("utf8") + + def from_bytes(self, data): + self.value = f"{data.decode('utf8')} from bytes" + return self + + +def test_pickle_with_flatten(linear): + Xs = [linear.ops.alloc2f(2, 3), linear.ops.alloc2f(4, 3)] + model = with_array(linear).initialize() + pickled = srsly.pickle_dumps(model) + loaded = srsly.pickle_loads(pickled) + Ys = loaded.predict(Xs) + assert len(Ys) == 2 + assert Ys[0].shape == (Xs[0].shape[0], linear.get_dim("nO")) + assert Ys[1].shape == (Xs[1].shape[0], linear.get_dim("nO")) + + +def test_simple_model_roundtrip_bytes(): + model = Maxout(5, 10, nP=2).initialize() + b = model.get_param("b") + b += 1 + data = model.to_bytes() + b = model.get_param("b") + b -= 1 + model = model.from_bytes(data) + assert model.get_param("b")[0, 0] == 1 + + +def test_simple_model_roundtrip_bytes_length(): + """Ensure that serialization of non-initialized weight matrices goes fine""" + model1 = Maxout(5, 10, nP=2) + model2 = Maxout(5, 10, nP=2) + + data1 = model1.to_bytes() + model2 = model2.from_bytes(data1) + data2 = model2.to_bytes() + + assert data1 == data2 + assert len(data1) == len(data2) + + +def test_simple_model_roundtrip_bytes_serializable_attrs(): + fwd = lambda model, X, is_train: (X, lambda dY: dY) + attr = SerializableAttr() + assert attr.value == "foo" + assert attr.to_bytes() == b"foo" + model = Model("test", fwd, attrs={"test": attr}) + model.initialize() + + @serialize_attr.register(SerializableAttr) + def serialize_attr_custom(_, value, name, model): + return value.to_bytes() + + @deserialize_attr.register(SerializableAttr) + def deserialize_attr_custom(_, value, name, model): + return SerializableAttr().from_bytes(value) + + model_bytes = model.to_bytes() + model = model.from_bytes(model_bytes) + assert "test" in model.attrs + assert model.attrs["test"].value == "foo from bytes" + + +def test_multi_model_roundtrip_bytes(): + model = chain(Maxout(5, 10, nP=2), Maxout(2, 3)).initialize() + b = model.layers[0].get_param("b") + b += 1 + b = model.layers[1].get_param("b") + b += 2 + data = model.to_bytes() + b = model.layers[0].get_param("b") + b -= 1 + b = model.layers[1].get_param("b") + b -= 2 + model = model.from_bytes(data) + assert model.layers[0].get_param("b")[0, 0] == 1 + assert model.layers[1].get_param("b")[0, 0] == 2 + + +def test_multi_model_load_missing_dims(): + model = chain(Maxout(5, 10, nP=2), Maxout(2, 3)).initialize() + b = model.layers[0].get_param("b") + b += 1 + b = model.layers[1].get_param("b") + b += 2 + data = model.to_bytes() + + model2 = chain(Maxout(5, nP=None), Maxout(nP=None)) + model2 = model2.from_bytes(data) + assert model2.layers[0].get_param("b")[0, 0] == 1 + assert model2.layers[1].get_param("b")[0, 0] == 2 + + +def test_serialize_model_shims_roundtrip_bytes(): + fwd = lambda model, X, is_train: (X, lambda dY: dY) + test_shim = SerializableShim(None) + shim_model = Model("shimmodel", fwd, shims=[test_shim]) + model = chain(Linear(2, 3), shim_model, Maxout(2, 3)) + model.initialize() + assert model.layers[1].shims[0].value == "shimdata" + model_bytes = model.to_bytes() + with pytest.raises(ValueError): + Linear(2, 3).from_bytes(model_bytes) + test_shim = SerializableShim(None) + shim_model = Model("shimmodel", fwd, shims=[test_shim]) + new_model = chain(Linear(2, 3), shim_model, Maxout(2, 3)).from_bytes(model_bytes) + assert new_model.layers[1].shims[0].value == "shimdata from bytes" + + +def test_serialize_refs_roundtrip_bytes(): + fwd = lambda model, X, is_train: (X, lambda dY: dY) + model_a = Model("a", fwd) + model = Model("test", fwd, refs={"a": model_a, "b": None}).initialize() + with pytest.raises(ValueError): # ref not in nodes + model.to_bytes() + model = Model("test", fwd, refs={"a": model_a, "b": None}, layers=[model_a]) + assert model.ref_names == ("a", "b") + model_bytes = model.to_bytes() + with pytest.raises(ValueError): + Model("test", fwd).from_bytes(model_bytes) + new_model = Model("test", fwd, layers=[model_a]) + new_model.from_bytes(model_bytes) + assert new_model.ref_names == ("a", "b") + + +def test_serialize_attrs(): + fwd = lambda model, X, is_train: (X, lambda dY: dY) + attrs = {"test": "foo"} + model1 = Model("test", fwd, attrs=attrs).initialize() + bytes_attr = serialize_attr(model1.attrs["test"], attrs["test"], "test", model1) + assert bytes_attr == srsly.msgpack_dumps("foo") + model2 = Model("test", fwd, attrs={"test": ""}) + result = deserialize_attr(model2.attrs["test"], bytes_attr, "test", model2) + assert result == "foo" + + # Test objects with custom serialization functions + @serialize_attr.register(SerializableAttr) + def serialize_attr_custom(_, value, name, model): + return value.to_bytes() + + @deserialize_attr.register(SerializableAttr) + def deserialize_attr_custom(_, value, name, model): + return SerializableAttr().from_bytes(value) + + attrs = {"test": SerializableAttr()} + model3 = Model("test", fwd, attrs=attrs) + bytes_attr = serialize_attr(model3.attrs["test"], attrs["test"], "test", model3) + assert bytes_attr == b"foo" + model4 = Model("test", fwd, attrs=attrs) + assert model4.attrs["test"].value == "foo" + result = deserialize_attr(model4.attrs["test"], bytes_attr, "test", model4) + assert result.value == "foo from bytes" + + +def test_simple_model_can_from_dict(): + model = Maxout(5, 10, nP=2).initialize() + model_dict = model.to_dict() + assert model.can_from_dict(model_dict) + # Test check without initialize + assert Maxout(5, 10, nP=2).can_from_dict(model_dict) + # Test not-strict check + assert not Maxout(10, 5, nP=2).can_from_dict(model_dict) + assert Maxout(5, nP=2).can_from_dict(model_dict) + + +def test_multi_model_can_from_dict(): + model = chain(Maxout(5, 10, nP=2), Maxout(2, 3)).initialize() + model_dict = model.to_dict() + assert model.can_from_dict(model_dict) + assert chain(Maxout(5, 10, nP=2), Maxout(2, 3)).can_from_dict(model_dict) + resized = chain(Maxout(5, 10, nP=3), Maxout(2, 3)) + assert not resized.can_from_dict(model_dict) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_types.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_types.py new file mode 100644 index 0000000000000000000000000000000000000000..9aff540e467aa76903525eb9cb6bfafe0b7f1236 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_types.py @@ -0,0 +1,53 @@ +import numpy +import pytest + +try: + from pydantic.v1 import ValidationError, create_model +except ImportError: + from pydantic import ValidationError, create_model # type: ignore + + +from thinc.types import ( + Floats1d, + Floats2d, + Floats3d, + Floats4d, + Ints1d, + Ints2d, + Ints3d, + Ints4d, +) + + +@pytest.mark.xfail(reason="Validation currently disabled") +@pytest.mark.parametrize( + "arr,arr_type", + [ + (numpy.zeros(0, dtype=numpy.float32), Floats1d), + (numpy.zeros((0, 0), dtype=numpy.float32), Floats2d), + (numpy.zeros((0, 0, 0), dtype=numpy.float32), Floats3d), + (numpy.zeros((0, 0, 0, 0), dtype=numpy.float32), Floats4d), + (numpy.zeros(0, dtype=numpy.int32), Ints1d), + (numpy.zeros((0, 0), dtype=numpy.int32), Ints2d), + (numpy.zeros((0, 0, 0), dtype=numpy.int32), Ints3d), + (numpy.zeros((0, 0, 0, 0), dtype=numpy.int32), Ints4d), + ], +) +def test_array_validation_valid(arr, arr_type): + test_model = create_model("TestModel", arr=(arr_type, ...)) + result = test_model(arr=arr) + assert numpy.array_equal(arr, result.arr) + + +@pytest.mark.xfail(reason="Validation currently disabled") +@pytest.mark.parametrize( + "arr,arr_type", + [ + (numpy.zeros((0, 0), dtype=numpy.float32), Floats1d), + (numpy.zeros((0, 0), dtype=numpy.float32), Ints2d), + ], +) +def test_array_validation_invalid(arr, arr_type): + test_model = create_model("TestModel", arr=(arr_type, ...)) + with pytest.raises(ValidationError): + test_model(arr=arr) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_util.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_util.py new file mode 100644 index 0000000000000000000000000000000000000000..ae588bd12462d5d6c9b3a45a15aeb265e293a53a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/test_util.py @@ -0,0 +1,165 @@ +import numpy +import pytest +from hypothesis import given + +from thinc.api import Padded, Ragged, get_width +from thinc.types import ArgsKwargs +from thinc.util import ( + convert_recursive, + get_array_module, + is_cupy_array, + is_numpy_array, + to_categorical, +) + +from . import strategies + +ALL_XP = [numpy] +try: + import cupy + + ALL_XP.append(cupy) +except ImportError: + pass + + +@pytest.mark.parametrize( + "obj,width", + [ + (numpy.zeros((1, 2, 3, 4)), 4), + (numpy.array(1), 0), + (numpy.array([1, 2]), 3), + ([numpy.zeros((1, 2)), numpy.zeros((1))], 2), + (Ragged(numpy.zeros((1, 2)), numpy.zeros(1)), 2), # type:ignore + ( + Padded( + numpy.zeros((2, 1, 2)), # type:ignore + numpy.zeros(2), # type:ignore + numpy.array([1, 0]), # type:ignore + numpy.array([0, 1]), # type:ignore + ), + 2, + ), + ([], 0), + ], +) +def test_get_width(obj, width): + assert get_width(obj) == width + + +@pytest.mark.parametrize("obj", [1234, "foo", {"a": numpy.array(0)}]) +def test_get_width_fail(obj): + with pytest.raises(ValueError): + get_width(obj) + + +@pytest.mark.parametrize("xp", ALL_XP) +def test_array_module_cpu_gpu_helpers(xp): + error = ( + "Only numpy and cupy arrays are supported" + ", but found instead. If " + "get_array_module module wasn't called " + "directly, this might indicate a bug in Thinc." + ) + with pytest.raises(ValueError, match=error): + get_array_module(0) + zeros = xp.zeros((1, 2)) + xp_ = get_array_module(zeros) + assert xp_ == xp + if xp == numpy: + assert is_numpy_array(zeros) + assert not is_numpy_array((1, 2)) + else: + assert is_cupy_array(zeros) + assert not is_cupy_array((1, 2)) + + +@given( + label_smoothing=strategies.floats(min_value=0.0, max_value=0.5, exclude_max=True) +) +def test_to_categorical(label_smoothing): + # Test without n_classes + one_hot = to_categorical(numpy.asarray([1, 2], dtype="i")) + assert one_hot.shape == (2, 3) + # From keras + # https://github.com/keras-team/keras/blob/master/tests/keras/utils/np_utils_test.py + nc = 5 + shapes = [(1,), (3,), (4, 3), (5, 4, 3), (3, 1), (3, 2, 1)] + expected_shapes = [ + (1, nc), + (3, nc), + (4, 3, nc), + (5, 4, 3, nc), + (3, 1, nc), + (3, 2, 1, nc), + ] + labels = [numpy.random.randint(0, nc, shape) for shape in shapes] + one_hots = [to_categorical(label, nc) for label in labels] + smooths = [ + to_categorical(label, nc, label_smoothing=label_smoothing) for label in labels + ] + for i in range(len(expected_shapes)): + label = labels[i] + one_hot = one_hots[i] + expected_shape = expected_shapes[i] + smooth = smooths[i] + assert one_hot.shape == expected_shape + assert smooth.shape == expected_shape + assert numpy.array_equal(one_hot, one_hot.astype(bool)) + assert numpy.all(one_hot.sum(axis=-1) == 1) + assert numpy.all(numpy.argmax(one_hot, -1).reshape(label.shape) == label) + assert numpy.all(smooth.argmax(axis=-1) == one_hot.argmax(axis=-1)) + assert numpy.all(numpy.isclose(numpy.sum(smooth, axis=-1), 1.0)) + assert numpy.isclose(numpy.max(smooth), 1 - label_smoothing) + assert numpy.isclose( + numpy.min(smooth), label_smoothing / (smooth.shape[-1] - 1) + ) + + # At least one class is required without label smoothing. + numpy.testing.assert_allclose( + to_categorical(numpy.asarray([0, 0, 0]), 1), [[1.0], [1.0], [1.0]] + ) + numpy.testing.assert_allclose( + to_categorical(numpy.asarray([0, 0, 0])), [[1.0], [1.0], [1.0]] + ) + with pytest.raises(ValueError, match=r"n_classes should be at least 1"): + to_categorical(numpy.asarray([0, 0, 0]), 0) + + # At least two classes are required with label smoothing. + numpy.testing.assert_allclose( + to_categorical(numpy.asarray([0, 1, 0]), 2, label_smoothing=0.01), + [[0.99, 0.01], [0.01, 0.99], [0.99, 0.01]], + ) + numpy.testing.assert_allclose( + to_categorical(numpy.asarray([0, 1, 0]), label_smoothing=0.01), + [[0.99, 0.01], [0.01, 0.99], [0.99, 0.01]], + ) + with pytest.raises( + ValueError, match=r"n_classes should be greater than 1.*label smoothing.*but 1" + ): + to_categorical(numpy.asarray([0, 1, 0]), 1, label_smoothing=0.01), + with pytest.raises( + ValueError, match=r"n_classes should be greater than 1.*label smoothing.*but 1" + ): + to_categorical(numpy.asarray([0, 0, 0]), label_smoothing=0.01), + + with pytest.raises(ValueError, match=r"label_smoothing parameter"): + to_categorical(numpy.asarray([0, 1, 2, 3, 4]), label_smoothing=0.8) + + with pytest.raises(ValueError, match=r"label_smoothing parameter"): + to_categorical(numpy.asarray([0, 1, 2, 3, 4]), label_smoothing=0.88) + + +def test_convert_recursive(): + is_match = lambda obj: obj == "foo" + convert_item = lambda obj: obj.upper() + obj = { + "a": {("b", "foo"): {"c": "foo", "d": ["foo", {"e": "foo", "f": (1, "foo")}]}} + } + result = convert_recursive(is_match, convert_item, obj) + assert result["a"][("b", "FOO")]["c"] == "FOO" + assert result["a"][("b", "FOO")]["d"] == ["FOO", {"e": "FOO", "f": (1, "FOO")}] + obj = {"a": ArgsKwargs(("foo", [{"b": "foo"}]), {"a": ["x", "foo"]})} + result = convert_recursive(is_match, convert_item, obj) + assert result["a"].args == ("FOO", [{"b": "FOO"}]) + assert result["a"].kwargs == {"a": ["x", "FOO"]} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/util.py b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/util.py new file mode 100644 index 0000000000000000000000000000000000000000..6a2786bdface62a47ac21a2e4cbd94b0637421b9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/thinc/tests/util.py @@ -0,0 +1,121 @@ +import contextlib +import shutil +import tempfile +from pathlib import Path + +import numpy +import pytest + +from thinc.api import ArgsKwargs, Linear, Padded, Ragged +from thinc.util import has_cupy, is_cupy_array, is_numpy_array + + +@contextlib.contextmanager +def make_tempdir(): + d = Path(tempfile.mkdtemp()) + yield d + shutil.rmtree(str(d)) + + +def get_model(W_b_input, cls=Linear): + W, b, input_ = W_b_input + nr_out, nr_in = W.shape + model = cls(nr_out, nr_in) + model.set_param("W", W) + model.set_param("b", b) + model.initialize() + return model + + +def get_shape(W_b_input): + W, b, input_ = W_b_input + return input_.shape[0], W.shape[0], W.shape[1] + + +def get_data_checker(inputs): + if isinstance(inputs, Ragged): + return assert_raggeds_match + elif isinstance(inputs, Padded): + return assert_paddeds_match + elif isinstance(inputs, list): + return assert_lists_match + elif isinstance(inputs, tuple) and len(inputs) == 4: + return assert_padded_data_match + elif isinstance(inputs, tuple) and len(inputs) == 2: + return assert_ragged_data_match + else: + return assert_arrays_match + + +def assert_arrays_match(X, Y): + assert X.dtype == Y.dtype + # Transformations are allowed to change last dimension, but not batch size. + assert X.shape[0] == Y.shape[0] + return True + + +def assert_lists_match(X, Y): + assert isinstance(X, list) + assert isinstance(Y, list) + assert len(X) == len(Y) + for x, y in zip(X, Y): + assert_arrays_match(x, y) + return True + + +def assert_raggeds_match(X, Y): + assert isinstance(X, Ragged) + assert isinstance(Y, Ragged) + assert_arrays_match(X.lengths, Y.lengths) + assert_arrays_match(X.data, Y.data) + return True + + +def assert_paddeds_match(X, Y): + assert isinstance(X, Padded) + assert isinstance(Y, Padded) + assert_arrays_match(X.size_at_t, Y.size_at_t) + assert assert_arrays_match(X.lengths, Y.lengths) + assert assert_arrays_match(X.indices, Y.indices) + assert X.data.dtype == Y.data.dtype + assert X.data.shape[1] == Y.data.shape[1] + assert X.data.shape[0] == Y.data.shape[0] + return True + + +def assert_padded_data_match(X, Y): + return assert_paddeds_match(Padded(*X), Padded(*Y)) + + +def assert_ragged_data_match(X, Y): + return assert_raggeds_match(Ragged(*X), Ragged(*Y)) + + +def check_input_converters(Y, backprop, data, n_args, kwargs_keys, type_): + assert isinstance(Y, ArgsKwargs) + assert len(Y.args) == n_args + assert list(Y.kwargs.keys()) == kwargs_keys + assert all(isinstance(arg, type_) for arg in Y.args) + assert all(isinstance(arg, type_) for arg in Y.kwargs.values()) + dX = backprop(Y) + + def is_supported_backend_array(arr): + return is_cupy_array(arr) or is_numpy_array(arr) + + input_type = type(data) if not isinstance(data, list) else tuple + assert isinstance(dX, input_type) or is_supported_backend_array(dX) + + if isinstance(data, dict): + assert list(dX.keys()) == kwargs_keys + assert all(is_supported_backend_array(arr) for arr in dX.values()) + elif isinstance(data, (list, tuple)): + assert isinstance(dX, tuple) + assert all(is_supported_backend_array(arr) for arr in dX) + elif isinstance(data, ArgsKwargs): + assert len(dX.args) == n_args + assert list(dX.kwargs.keys()) == kwargs_keys + + assert all(is_supported_backend_array(arg) for arg in dX.args) + assert all(is_supported_backend_array(arg) for arg in dX.kwargs.values()) + elif not isinstance(data, numpy.ndarray): + pytest.fail(f"Bad data type: {dX}") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1b1e2fe7409eed12ff80ad806df1466cd9cf96e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cf42c8857f49955d727da350aab2599a12ac2873 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.py @@ -0,0 +1,15 @@ +from .. import decoders + + +Decoder = decoders.Decoder +ByteLevel = decoders.ByteLevel +Replace = decoders.Replace +WordPiece = decoders.WordPiece +ByteFallback = decoders.ByteFallback +Fuse = decoders.Fuse +Strip = decoders.Strip +Metaspace = decoders.Metaspace +BPEDecoder = decoders.BPEDecoder +CTC = decoders.CTC +Sequence = decoders.Sequence +DecodeStream = decoders.DecodeStream diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2b72e00f9fb4f5591a03a255e48894fbdf100ff3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__init__.pyi @@ -0,0 +1,569 @@ +# Generated content DO NOT EDIT +class DecodeStream: + """ + Class needed for streaming decode + + """ + def __init__(self, ids=None, skip_special_tokens=False): + pass + + def __getstate__(self, /): + """ + Helper for pickle. + """ + pass + + def step(self, tokenizer, id): + """ + Streaming decode step + + Args: + tokenizer (:class:`~tokenizers.Tokenizer`): + The tokenizer to use for decoding + id (:obj:`int` or `List[int]`): + The next token id or list of token ids to add to the stream + + + Returns: + :obj:`Optional[str]`: The next decoded string chunk, or None if not enough + tokens have been provided yet. + """ + pass + +class Decoder: + """ + Base class for all decoders + + This class is not supposed to be instantiated directly. Instead, any implementation of + a Decoder will return an instance of this class when instantiated. + """ + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class BPEDecoder(Decoder): + """ + BPEDecoder Decoder + + Args: + suffix (:obj:`str`, `optional`, defaults to :obj:``): + The suffix that was used to characterize an end-of-word. This suffix will + be replaced by whitespaces during the decoding + """ + def __init__(self, suffix=""): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + + @property + def suffix(self): + """ """ + pass + + @suffix.setter + def suffix(self, value): + """ """ + pass + +class ByteFallback(Decoder): + """ + ByteFallback Decoder + ByteFallback is a simple trick which converts tokens looking like `<0x61>` + to pure bytes, and attempts to make them into a string. If the tokens + cannot be decoded you will get � instead for each inconvertible byte token + + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class ByteLevel(Decoder): + """ + ByteLevel Decoder + + This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.ByteLevel` + :class:`~tokenizers.pre_tokenizers.PreTokenizer`. + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class CTC(Decoder): + """ + CTC Decoder + + Args: + pad_token (:obj:`str`, `optional`, defaults to :obj:``): + The pad token used by CTC to delimit a new token. + word_delimiter_token (:obj:`str`, `optional`, defaults to :obj:`|`): + The word delimiter token. It will be replaced by a + cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to cleanup some tokenization artifacts. + Mainly spaces before punctuation, and some abbreviated english forms. + """ + def __init__(self, pad_token="", word_delimiter_token="|", cleanup=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def cleanup(self): + """ """ + pass + + @cleanup.setter + def cleanup(self, value): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + + @property + def pad_token(self): + """ """ + pass + + @pad_token.setter + def pad_token(self, value): + """ """ + pass + + @property + def word_delimiter_token(self): + """ """ + pass + + @word_delimiter_token.setter + def word_delimiter_token(self, value): + """ """ + pass + +class Fuse(Decoder): + """ + Fuse Decoder + Fuse simply fuses every token into a single string. + This is the last step of decoding, this decoder exists only if + there is need to add other decoders *after* the fusion + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class Metaspace(Decoder): + """ + Metaspace Decoder + + Args: + replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): + The replacement character. Must be exactly one character. By default we + use the `▁` (U+2581) meta symbol (Same as in SentencePiece). + + prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): + Whether to add a space to the first word if there isn't already one. This + lets us treat `hello` exactly like `say hello`. + Choices: "always", "never", "first". First means the space is only added on the first + token (relevant when special tokens are used or other pre_tokenizer are used). + """ + def __init__(self, replacement="▁", prepend_scheme="always", split=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + + @property + def prepend_scheme(self): + """ """ + pass + + @prepend_scheme.setter + def prepend_scheme(self, value): + """ """ + pass + + @property + def replacement(self): + """ """ + pass + + @replacement.setter + def replacement(self, value): + """ """ + pass + + @property + def split(self): + """ """ + pass + + @split.setter + def split(self, value): + """ """ + pass + +class Replace(Decoder): + """ + Replace Decoder + + This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.Replace` + :class:`~tokenizers.pre_tokenizers.PreTokenizer`. + """ + def __init__(self, pattern, content): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class Sequence(Decoder): + """ + Sequence Decoder + + Args: + decoders (:obj:`List[Decoder]`) + The decoders that need to be chained + """ + def __init__(self, decoders): + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + +class Strip(Decoder): + """ + Strip normalizer + Strips n left characters of each token, or n right characters of each token + """ + def __init__(self, content=" ", left=0, right=0): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def content(self): + """ """ + pass + + @content.setter + def content(self, value): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + + @property + def start(self): + """ """ + pass + + @start.setter + def start(self, value): + """ """ + pass + + @property + def stop(self): + """ """ + pass + + @stop.setter + def stop(self, value): + """ """ + pass + +class WordPiece(Decoder): + """ + WordPiece Decoder + + Args: + prefix (:obj:`str`, `optional`, defaults to :obj:`##`): + The prefix to use for subwords that are not a beginning-of-word + + cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation, + and some abbreviated english forms. + """ + def __init__(self, prefix="##", cleanup=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def cleanup(self): + """ """ + pass + + @cleanup.setter + def cleanup(self, value): + """ """ + pass + + @staticmethod + def custom(decoder): + """ """ + pass + + def decode(self, tokens): + """ + Decode the given list of tokens to a final string + + Args: + tokens (:obj:`List[str]`): + The list of tokens to decode + + Returns: + :obj:`str`: The decoded string + """ + pass + + @property + def prefix(self): + """ """ + pass + + @prefix.setter + def prefix(self, value): + """ """ + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88b9ca449d18da60c828c8d46cd39acb6ba7b736 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/decoders/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..710505ed6627e3322ba06ad8547769b0934ff537 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__init__.py @@ -0,0 +1,6 @@ +from .base_tokenizer import BaseTokenizer +from .bert_wordpiece import BertWordPieceTokenizer +from .byte_level_bpe import ByteLevelBPETokenizer +from .char_level_bpe import CharBPETokenizer +from .sentencepiece_bpe import SentencePieceBPETokenizer +from .sentencepiece_unigram import SentencePieceUnigramTokenizer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49a000918070a3e2051e3c5896766e7ba1dcf9ea Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/base_tokenizer.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/base_tokenizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77b1c0fc3d873b4742b9d79bbb24bc943fcd1650 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/base_tokenizer.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/bert_wordpiece.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/bert_wordpiece.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7204dcf6ffe57134655b50624ca854881fa29a5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/bert_wordpiece.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/byte_level_bpe.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/byte_level_bpe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fea732fde8374f2b6ea5841c8ef72ee49db30886 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/byte_level_bpe.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/char_level_bpe.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/char_level_bpe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7378c08ad71dd12b5c5bc26390a4a23d2e41810 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/char_level_bpe.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_bpe.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_bpe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62d0547d7b7afb69ac8c89d8d3b90d046c08b68e Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_bpe.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_unigram.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_unigram.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64438185e873958f769e62ffaa9b2eb6dd567145 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/__pycache__/sentencepiece_unigram.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/base_tokenizer.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/base_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..6e6cec744f1aa9c0e698dfa968c145d885868dd7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/base_tokenizer.py @@ -0,0 +1,459 @@ +from typing import Dict, List, Optional, Tuple, Union + +from tokenizers import AddedToken, EncodeInput, Encoding, InputSequence, Tokenizer +from tokenizers.decoders import Decoder +from tokenizers.models import Model +from tokenizers.normalizers import Normalizer +from tokenizers.pre_tokenizers import PreTokenizer +from tokenizers.processors import PostProcessor + + +Offsets = Tuple[int, int] + + +class BaseTokenizer: + def __init__(self, tokenizer: Tokenizer, parameters=None): + self._tokenizer = tokenizer + self._parameters = parameters if parameters is not None else {} + + def __repr__(self): + return "Tokenizer(vocabulary_size={}, {})".format( + self._tokenizer.get_vocab_size(), + ", ".join(k + "=" + str(v) for k, v in self._parameters.items()), + ) + + def num_special_tokens_to_add(self, is_pair: bool) -> int: + """ + Return the number of special tokens that would be added for single/pair sentences. + :param is_pair: Boolean indicating if the input would be a single sentence or a pair + :return: + """ + return self._tokenizer.num_special_tokens_to_add(is_pair) + + def get_vocab(self, with_added_tokens: bool = True) -> Dict[str, int]: + """Returns the vocabulary + + Args: + with_added_tokens: boolean: + Whether to include the added tokens in the vocabulary + + Returns: + The vocabulary + """ + return self._tokenizer.get_vocab(with_added_tokens=with_added_tokens) + + def get_added_tokens_decoder(self) -> Dict[int, AddedToken]: + """Returns the added reverse vocabulary + + Returns: + The added vocabulary mapping ints to AddedTokens + """ + return self._tokenizer.get_added_tokens_decoder() + + def get_vocab_size(self, with_added_tokens: bool = True) -> int: + """Return the size of vocabulary, with or without added tokens. + + Args: + with_added_tokens: (`optional`) bool: + Whether to count in added special tokens or not + + Returns: + Size of vocabulary + """ + return self._tokenizer.get_vocab_size(with_added_tokens=with_added_tokens) + + def enable_padding( + self, + direction: Optional[str] = "right", + pad_to_multiple_of: Optional[int] = None, + pad_id: Optional[int] = 0, + pad_type_id: Optional[int] = 0, + pad_token: Optional[str] = "[PAD]", + length: Optional[int] = None, + ): + """Change the padding strategy + + Args: + direction: (`optional`) str: + Can be one of: `right` or `left` + + pad_to_multiple_of: (`optional`) unsigned int: + If specified, the padding length should always snap to the next multiple of + the given value. For example if we were going to pad with a length of 250 but + `pad_to_multiple_of=8` then we will pad to 256. + + pad_id: (`optional`) unsigned int: + The indice to be used when padding + + pad_type_id: (`optional`) unsigned int: + The type indice to be used when padding + + pad_token: (`optional`) str: + The pad token to be used when padding + + length: (`optional`) unsigned int: + If specified, the length at which to pad. If not specified + we pad using the size of the longest sequence in a batch + """ + return self._tokenizer.enable_padding( + direction=direction, + pad_to_multiple_of=pad_to_multiple_of, + pad_id=pad_id, + pad_type_id=pad_type_id, + pad_token=pad_token, + length=length, + ) + + def no_padding(self): + """Disable padding""" + return self._tokenizer.no_padding() + + @property + def padding(self) -> Optional[dict]: + """Get the current padding parameters + + Returns: + None if padding is disabled, a dict with the currently set parameters + if the padding is enabled. + """ + return self._tokenizer.padding + + def enable_truncation(self, max_length: int, stride: Optional[int] = 0, strategy: Optional[str] = "longest_first"): + """Change the truncation options + + Args: + max_length: unsigned int: + The maximum length at which to truncate + + stride: (`optional`) unsigned int: + The length of the previous first sequence to be included + in the overflowing sequence + + strategy: (`optional`) str: + Can be one of `longest_first`, `only_first` or `only_second` + """ + return self._tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy) + + def no_truncation(self): + """Disable truncation""" + return self._tokenizer.no_truncation() + + @property + def truncation(self) -> Optional[dict]: + """Get the current truncation parameters + + Returns: + None if truncation is disabled, a dict with the current truncation parameters if + truncation is enabled + """ + return self._tokenizer.truncation + + def add_tokens(self, tokens: List[Union[str, AddedToken]]) -> int: + """Add the given tokens to the vocabulary + + Args: + tokens: List[Union[str, AddedToken]]: + A list of tokens to add to the vocabulary. Each token can either be + a string, or an instance of AddedToken + + Returns: + The number of tokens that were added to the vocabulary + """ + return self._tokenizer.add_tokens(tokens) + + def add_special_tokens(self, special_tokens: List[Union[str, AddedToken]]) -> int: + """Add the given special tokens to the vocabulary, and treat them as special tokens. + + The special tokens will never be processed by the model, and will be + removed while decoding. + + Args: + tokens: List[Union[str, AddedToken]]: + A list of special tokens to add to the vocabulary. Each token can either be + a string, or an instance of AddedToken + + Returns: + The number of tokens that were added to the vocabulary + """ + return self._tokenizer.add_special_tokens(special_tokens) + + def normalize(self, sequence: str) -> str: + """Normalize the given sequence + + Args: + sequence: str: + The sequence to normalize + + Returns: + The normalized string + """ + return self._tokenizer.normalizer.normalize_str(sequence) + + def encode( + self, + sequence: InputSequence, + pair: Optional[InputSequence] = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> Encoding: + """Encode the given sequence and pair. This method can process raw text sequences as well + as already pre-tokenized sequences. + + Args: + sequence: InputSequence: + The sequence we want to encode. This sequence can be either raw text or + pre-tokenized, according to the `is_pretokenized` argument: + + - If `is_pretokenized=False`: `InputSequence` is expected to be `str` + - If `is_pretokenized=True`: `InputSequence` is expected to be + `Union[List[str], Tuple[str]]` + + is_pretokenized: bool: + Whether the input is already pre-tokenized. + + add_special_tokens: bool: + Whether to add the special tokens while encoding. + + Returns: + An Encoding + """ + if sequence is None: + raise ValueError("encode: `sequence` can't be `None`") + + return self._tokenizer.encode(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + inputs: List[EncodeInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> List[Encoding]: + """Encode the given inputs. This method accept both raw text sequences as well as already + pre-tokenized sequences. + + Args: + inputs: List[EncodeInput]: + A list of single sequences or pair sequences to encode. Each `EncodeInput` is + expected to be of the following form: + `Union[InputSequence, Tuple[InputSequence, InputSequence]]` + + Each `InputSequence` can either be raw text or pre-tokenized, + according to the `is_pretokenized` argument: + + - If `is_pretokenized=False`: `InputSequence` is expected to be `str` + - If `is_pretokenized=True`: `InputSequence` is expected to be + `Union[List[str], Tuple[str]]` + + is_pretokenized: bool: + Whether the input is already pre-tokenized. + + add_special_tokens: bool: + Whether to add the special tokens while encoding. + + Returns: + A list of Encoding + """ + + if inputs is None: + raise ValueError("encode_batch: `inputs` can't be `None`") + + return self._tokenizer.encode_batch(inputs, is_pretokenized, add_special_tokens) + + async def async_encode_batch( + self, + inputs: List[EncodeInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> List[Encoding]: + """Asynchronously encode a batch (tracks character offsets). + + Args: + inputs: A list of single or pair sequences to encode. + is_pretokenized: Whether inputs are already pre-tokenized. + add_special_tokens: Whether to add special tokens. + + Returns: + A list of Encoding. + """ + if inputs is None: + raise ValueError("async_encode_batch: `inputs` can't be `None`") + # Exposed by the Rust bindings via pyo3_async_runtimes::tokio::future_into_py + return await self._tokenizer.async_encode_batch(inputs, is_pretokenized, add_special_tokens) + + async def async_encode_batch_fast( + self, + inputs: List[EncodeInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> List[Encoding]: + """Asynchronously encode a batch (no character offsets, faster). + + Args: + inputs: A list of single or pair sequences to encode. + is_pretokenized: Whether inputs are already pre-tokenized. + add_special_tokens: Whether to add special tokens. + + Returns: + A list of Encoding. + """ + if inputs is None: + raise ValueError("async_encode_batch_fast: `inputs` can't be `None`") + return await self._tokenizer.async_encode_batch_fast(inputs, is_pretokenized, add_special_tokens) + + def decode(self, ids: List[int], skip_special_tokens: Optional[bool] = True) -> str: + """Decode the given list of ids to a string sequence + + Args: + ids: List[unsigned int]: + A list of ids to be decoded + + skip_special_tokens: (`optional`) boolean: + Whether to remove all the special tokens from the output string + + Returns: + The decoded string + """ + if ids is None: + raise ValueError("None input is not valid. Should be a list of integers.") + + return self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch(self, sequences: List[List[int]], skip_special_tokens: Optional[bool] = True) -> str: + """Decode the list of sequences to a list of string sequences + + Args: + sequences: List[List[unsigned int]]: + A list of sequence of ids to be decoded + + skip_special_tokens: (`optional`) boolean: + Whether to remove all the special tokens from the output strings + + Returns: + A list of decoded strings + """ + if sequences is None: + raise ValueError("None input is not valid. Should be list of list of integers.") + + return self._tokenizer.decode_batch(sequences, skip_special_tokens=skip_special_tokens) + + def token_to_id(self, token: str) -> Optional[int]: + """Convert the given token to its corresponding id + + Args: + token: str: + The token to convert + + Returns: + The corresponding id if it exists, None otherwise + """ + return self._tokenizer.token_to_id(token) + + def id_to_token(self, id: int) -> Optional[str]: + """Convert the given token id to its corresponding string + + Args: + token: id: + The token id to convert + + Returns: + The corresponding string if it exists, None otherwise + """ + return self._tokenizer.id_to_token(id) + + def save_model(self, directory: str, prefix: Optional[str] = None): + """Save the current model to the given directory + + Args: + directory: str: + A path to the destination directory + + prefix: (Optional) str: + An optional prefix, used to prefix each file name + """ + return self._tokenizer.model.save(directory, prefix=prefix) + + def save(self, path: str, pretty: bool = True): + """Save the current Tokenizer at the given path + + Args: + path: str: + A path to the destination Tokenizer file + """ + return self._tokenizer.save(path, pretty) + + def to_str(self, pretty: bool = False): + """Get a serialized JSON version of the Tokenizer as a str + + Args: + pretty: bool: + Whether the JSON string should be prettified + + Returns: + str + """ + return self._tokenizer.to_str(pretty) + + def post_process( + self, encoding: Encoding, pair: Optional[Encoding] = None, add_special_tokens: bool = True + ) -> Encoding: + """Apply all the post-processing steps to the given encodings. + + The various steps are: + 1. Truncate according to global params (provided to `enable_truncation`) + 2. Apply the PostProcessor + 3. Pad according to global params. (provided to `enable_padding`) + + Args: + encoding: Encoding: + The main Encoding to post process + + pair: Optional[Encoding]: + An optional pair Encoding + + add_special_tokens: bool: + Whether to add special tokens + + Returns: + The resulting Encoding + """ + return self._tokenizer.post_process(encoding, pair, add_special_tokens) + + @property + def model(self) -> Model: + return self._tokenizer.model + + @model.setter + def model(self, model: Model): + self._tokenizer.model = model + + @property + def normalizer(self) -> Normalizer: + return self._tokenizer.normalizer + + @normalizer.setter + def normalizer(self, normalizer: Normalizer): + self._tokenizer.normalizer = normalizer + + @property + def pre_tokenizer(self) -> PreTokenizer: + return self._tokenizer.pre_tokenizer + + @pre_tokenizer.setter + def pre_tokenizer(self, pre_tokenizer: PreTokenizer): + self._tokenizer.pre_tokenizer = pre_tokenizer + + @property + def post_processor(self) -> PostProcessor: + return self._tokenizer.post_processor + + @post_processor.setter + def post_processor(self, post_processor: PostProcessor): + self._tokenizer.post_processor = post_processor + + @property + def decoder(self) -> Decoder: + return self._tokenizer.decoder + + @decoder.setter + def decoder(self, decoder: Decoder): + self._tokenizer.decoder = decoder diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/bert_wordpiece.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/bert_wordpiece.py new file mode 100644 index 0000000000000000000000000000000000000000..ed98c626537d2ad7ec881dc237d91fddaffda9ae --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/bert_wordpiece.py @@ -0,0 +1,151 @@ +from typing import Dict, Iterator, List, Optional, Union + +from tokenizers import AddedToken, Tokenizer, decoders, trainers +from tokenizers.models import WordPiece +from tokenizers.normalizers import BertNormalizer +from tokenizers.pre_tokenizers import BertPreTokenizer +from tokenizers.processors import BertProcessing + +from .base_tokenizer import BaseTokenizer + + +class BertWordPieceTokenizer(BaseTokenizer): + """Bert WordPiece Tokenizer""" + + def __init__( + self, + vocab: Optional[Union[str, Dict[str, int]]] = None, + unk_token: Union[str, AddedToken] = "[UNK]", + sep_token: Union[str, AddedToken] = "[SEP]", + cls_token: Union[str, AddedToken] = "[CLS]", + pad_token: Union[str, AddedToken] = "[PAD]", + mask_token: Union[str, AddedToken] = "[MASK]", + clean_text: bool = True, + handle_chinese_chars: bool = True, + strip_accents: Optional[bool] = None, + lowercase: bool = True, + wordpieces_prefix: str = "##", + ): + if vocab is not None: + tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(unk_token))) + else: + tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token))) + + # Let the tokenizer know about special tokens if they are part of the vocab + if tokenizer.token_to_id(str(unk_token)) is not None: + tokenizer.add_special_tokens([str(unk_token)]) + if tokenizer.token_to_id(str(sep_token)) is not None: + tokenizer.add_special_tokens([str(sep_token)]) + if tokenizer.token_to_id(str(cls_token)) is not None: + tokenizer.add_special_tokens([str(cls_token)]) + if tokenizer.token_to_id(str(pad_token)) is not None: + tokenizer.add_special_tokens([str(pad_token)]) + if tokenizer.token_to_id(str(mask_token)) is not None: + tokenizer.add_special_tokens([str(mask_token)]) + + tokenizer.normalizer = BertNormalizer( + clean_text=clean_text, + handle_chinese_chars=handle_chinese_chars, + strip_accents=strip_accents, + lowercase=lowercase, + ) + tokenizer.pre_tokenizer = BertPreTokenizer() + + if vocab is not None: + sep_token_id = tokenizer.token_to_id(str(sep_token)) + if sep_token_id is None: + raise TypeError("sep_token not found in the vocabulary") + cls_token_id = tokenizer.token_to_id(str(cls_token)) + if cls_token_id is None: + raise TypeError("cls_token not found in the vocabulary") + + tokenizer.post_processor = BertProcessing((str(sep_token), sep_token_id), (str(cls_token), cls_token_id)) + tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix) + + parameters = { + "model": "BertWordPiece", + "unk_token": unk_token, + "sep_token": sep_token, + "cls_token": cls_token, + "pad_token": pad_token, + "mask_token": mask_token, + "clean_text": clean_text, + "handle_chinese_chars": handle_chinese_chars, + "strip_accents": strip_accents, + "lowercase": lowercase, + "wordpieces_prefix": wordpieces_prefix, + } + + super().__init__(tokenizer, parameters) + + @staticmethod + def from_file(vocab: str, **kwargs): + vocab = WordPiece.read_file(vocab) + return BertWordPieceTokenizer(vocab, **kwargs) + + def train( + self, + files: Union[str, List[str]], + vocab_size: int = 30000, + min_frequency: int = 2, + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + special_tokens: List[Union[str, AddedToken]] = [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + ], + show_progress: bool = True, + wordpieces_prefix: str = "##", + ): + """Train the model using the given files""" + + trainer = trainers.WordPieceTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + special_tokens=special_tokens, + show_progress=show_progress, + continuing_subword_prefix=wordpieces_prefix, + ) + if isinstance(files, str): + files = [files] + self._tokenizer.train(files, trainer=trainer) + + def train_from_iterator( + self, + iterator: Union[Iterator[str], Iterator[Iterator[str]]], + vocab_size: int = 30000, + min_frequency: int = 2, + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + special_tokens: List[Union[str, AddedToken]] = [ + "[PAD]", + "[UNK]", + "[CLS]", + "[SEP]", + "[MASK]", + ], + show_progress: bool = True, + wordpieces_prefix: str = "##", + length: Optional[int] = None, + ): + """Train the model using the given iterator""" + + trainer = trainers.WordPieceTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + special_tokens=special_tokens, + show_progress=show_progress, + continuing_subword_prefix=wordpieces_prefix, + ) + self._tokenizer.train_from_iterator( + iterator, + trainer=trainer, + length=length, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/byte_level_bpe.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/byte_level_bpe.py new file mode 100644 index 0000000000000000000000000000000000000000..f573c15c5a7e4344f712deea45322733819be120 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/byte_level_bpe.py @@ -0,0 +1,122 @@ +from typing import Dict, Iterator, List, Optional, Tuple, Union + +from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, processors, trainers +from tokenizers.models import BPE +from tokenizers.normalizers import Lowercase, Sequence, unicode_normalizer_from_str + +from .base_tokenizer import BaseTokenizer + + +class ByteLevelBPETokenizer(BaseTokenizer): + """ByteLevelBPETokenizer + + Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model + """ + + def __init__( + self, + vocab: Optional[Union[str, Dict[str, int]]] = None, + merges: Optional[Union[str, List[Tuple[str, str]]]] = None, + add_prefix_space: bool = False, + lowercase: bool = False, + dropout: Optional[float] = None, + unicode_normalizer: Optional[str] = None, + continuing_subword_prefix: Optional[str] = None, + end_of_word_suffix: Optional[str] = None, + trim_offsets: bool = False, + ): + if vocab is not None and merges is not None: + tokenizer = Tokenizer( + BPE( + vocab, + merges, + dropout=dropout, + continuing_subword_prefix=continuing_subword_prefix or "", + end_of_word_suffix=end_of_word_suffix or "", + ) + ) + else: + tokenizer = Tokenizer(BPE()) + + # Check for Unicode normalization first (before everything else) + normalizers = [] + + if unicode_normalizer: + normalizers += [unicode_normalizer_from_str(unicode_normalizer)] + + if lowercase: + normalizers += [Lowercase()] + + # Create the normalizer structure + if len(normalizers) > 0: + if len(normalizers) > 1: + tokenizer.normalizer = Sequence(normalizers) + else: + tokenizer.normalizer = normalizers[0] + + tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=add_prefix_space) + tokenizer.decoder = decoders.ByteLevel() + tokenizer.post_processor = processors.ByteLevel(trim_offsets=trim_offsets) + + parameters = { + "model": "ByteLevelBPE", + "add_prefix_space": add_prefix_space, + "lowercase": lowercase, + "dropout": dropout, + "unicode_normalizer": unicode_normalizer, + "continuing_subword_prefix": continuing_subword_prefix, + "end_of_word_suffix": end_of_word_suffix, + "trim_offsets": trim_offsets, + } + + super().__init__(tokenizer, parameters) + + @staticmethod + def from_file(vocab_filename: str, merges_filename: str, **kwargs): + vocab, merges = BPE.read_file(vocab_filename, merges_filename) + return ByteLevelBPETokenizer(vocab, merges, **kwargs) + + def train( + self, + files: Union[str, List[str]], + vocab_size: int = 30000, + min_frequency: int = 2, + show_progress: bool = True, + special_tokens: List[Union[str, AddedToken]] = [], + ): + """Train the model using the given files""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + show_progress=show_progress, + special_tokens=special_tokens, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), + ) + if isinstance(files, str): + files = [files] + self._tokenizer.train(files, trainer=trainer) + + def train_from_iterator( + self, + iterator: Union[Iterator[str], Iterator[Iterator[str]]], + vocab_size: int = 30000, + min_frequency: int = 2, + show_progress: bool = True, + special_tokens: List[Union[str, AddedToken]] = [], + length: Optional[int] = None, + ): + """Train the model using the given iterator""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + show_progress=show_progress, + special_tokens=special_tokens, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), + ) + self._tokenizer.train_from_iterator( + iterator, + trainer=trainer, + length=length, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/char_level_bpe.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/char_level_bpe.py new file mode 100644 index 0000000000000000000000000000000000000000..1891d0435ed429613c1e09486e86165cf19479be --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/char_level_bpe.py @@ -0,0 +1,150 @@ +from typing import Dict, Iterator, List, Optional, Tuple, Union + +from .. import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers +from ..models import BPE +from ..normalizers import BertNormalizer, Lowercase, Sequence, unicode_normalizer_from_str +from .base_tokenizer import BaseTokenizer + + +class CharBPETokenizer(BaseTokenizer): + """Original BPE Tokenizer + + Represents the BPE algorithm, as introduced by Rico Sennrich + (https://arxiv.org/abs/1508.07909) + + The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original + Sennrich subword-nmt implementation by the following options that you can deactivate: + - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by: + * removing any control characters and replacing all whitespaces by the classic one. + * handle chinese chars by putting spaces around them. + * strip all accents. + - spitting on punctuation in addition to whitespaces (deactivate it with + `split_on_whitespace_only=True`) + """ + + def __init__( + self, + vocab: Optional[Union[str, Dict[str, int]]] = None, + merges: Optional[Union[str, List[Tuple[str, str]]]] = None, + unk_token: Union[str, AddedToken] = "", + suffix: str = "", + dropout: Optional[float] = None, + lowercase: bool = False, + unicode_normalizer: Optional[str] = None, + bert_normalizer: bool = True, + split_on_whitespace_only: bool = False, + ): + if vocab is not None and merges is not None: + tokenizer = Tokenizer( + BPE( + vocab, + merges, + dropout=dropout, + unk_token=str(unk_token), + end_of_word_suffix=suffix, + ) + ) + else: + tokenizer = Tokenizer(BPE(unk_token=str(unk_token), dropout=dropout, end_of_word_suffix=suffix)) + + if tokenizer.token_to_id(str(unk_token)) is not None: + tokenizer.add_special_tokens([str(unk_token)]) + + # Check for Unicode normalization first (before everything else) + normalizers = [] + + if unicode_normalizer: + normalizers += [unicode_normalizer_from_str(unicode_normalizer)] + + if bert_normalizer: + normalizers += [BertNormalizer(lowercase=False)] + + if lowercase: + normalizers += [Lowercase()] + + # Create the normalizer structure + if len(normalizers) > 0: + if len(normalizers) > 1: + tokenizer.normalizer = Sequence(normalizers) + else: + tokenizer.normalizer = normalizers[0] + + if split_on_whitespace_only: + tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + else: + tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer() + + tokenizer.decoder = decoders.BPEDecoder(suffix=suffix) + + parameters = { + "model": "BPE", + "unk_token": unk_token, + "suffix": suffix, + "dropout": dropout, + "lowercase": lowercase, + "unicode_normalizer": unicode_normalizer, + "bert_normalizer": bert_normalizer, + "split_on_whitespace_only": split_on_whitespace_only, + } + + super().__init__(tokenizer, parameters) + + @staticmethod + def from_file(vocab_filename: str, merges_filename: str, **kwargs): + vocab, merges = BPE.read_file(vocab_filename, merges_filename) + return CharBPETokenizer(vocab, merges, **kwargs) + + def train( + self, + files: Union[str, List[str]], + vocab_size: int = 30000, + min_frequency: int = 2, + special_tokens: List[Union[str, AddedToken]] = [""], + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + suffix: Optional[str] = "", + show_progress: bool = True, + ): + """Train the model using the given files""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + special_tokens=special_tokens, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + end_of_word_suffix=suffix, + show_progress=show_progress, + ) + if isinstance(files, str): + files = [files] + self._tokenizer.train(files, trainer=trainer) + + def train_from_iterator( + self, + iterator: Union[Iterator[str], Iterator[Iterator[str]]], + vocab_size: int = 30000, + min_frequency: int = 2, + special_tokens: List[Union[str, AddedToken]] = [""], + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + suffix: Optional[str] = "", + show_progress: bool = True, + length: Optional[int] = None, + ): + """Train the model using the given iterator""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + special_tokens=special_tokens, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + end_of_word_suffix=suffix, + show_progress=show_progress, + ) + self._tokenizer.train_from_iterator( + iterator, + trainer=trainer, + length=length, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_bpe.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_bpe.py new file mode 100644 index 0000000000000000000000000000000000000000..912fa4676930004ebc0a620b5bc7c1cfb61c3be7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_bpe.py @@ -0,0 +1,103 @@ +from typing import Dict, Iterator, List, Optional, Tuple, Union + +from tokenizers import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers +from tokenizers.models import BPE +from tokenizers.normalizers import NFKC + +from .base_tokenizer import BaseTokenizer + + +class SentencePieceBPETokenizer(BaseTokenizer): + """SentencePiece BPE Tokenizer + + Represents the BPE algorithm, with the pretokenization used by SentencePiece + """ + + def __init__( + self, + vocab: Optional[Union[str, Dict[str, int]]] = None, + merges: Optional[Union[str, List[Tuple[str, str]]]] = None, + unk_token: Union[str, AddedToken] = "", + replacement: str = "▁", + add_prefix_space: bool = True, + dropout: Optional[float] = None, + fuse_unk: Optional[bool] = False, + ): + if vocab is not None and merges is not None: + tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)) + else: + tokenizer = Tokenizer(BPE(dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk)) + + if tokenizer.token_to_id(str(unk_token)) is not None: + tokenizer.add_special_tokens([str(unk_token)]) + + tokenizer.normalizer = NFKC() + prepend_scheme = "always" if add_prefix_space else "never" + tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + + parameters = { + "model": "SentencePieceBPE", + "unk_token": unk_token, + "replacement": replacement, + "add_prefix_space": add_prefix_space, + "dropout": dropout, + } + + super().__init__(tokenizer, parameters) + + @staticmethod + def from_file(vocab_filename: str, merges_filename: str, **kwargs): + vocab, merges = BPE.read_file(vocab_filename, merges_filename) + return SentencePieceBPETokenizer(vocab, merges, **kwargs) + + def train( + self, + files: Union[str, List[str]], + vocab_size: int = 30000, + min_frequency: int = 2, + special_tokens: List[Union[str, AddedToken]] = [""], + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + show_progress: bool = True, + ): + """Train the model using the given files""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + special_tokens=special_tokens, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + show_progress=show_progress, + ) + if isinstance(files, str): + files = [files] + self._tokenizer.train(files, trainer=trainer) + + def train_from_iterator( + self, + iterator: Union[Iterator[str], Iterator[Iterator[str]]], + vocab_size: int = 30000, + min_frequency: int = 2, + special_tokens: List[Union[str, AddedToken]] = [""], + limit_alphabet: int = 1000, + initial_alphabet: List[str] = [], + show_progress: bool = True, + length: Optional[int] = None, + ): + """Train the model using the given iterator""" + + trainer = trainers.BpeTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + special_tokens=special_tokens, + limit_alphabet=limit_alphabet, + initial_alphabet=initial_alphabet, + show_progress=show_progress, + ) + self._tokenizer.train_from_iterator( + iterator, + trainer=trainer, + length=length, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_unigram.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_unigram.py new file mode 100644 index 0000000000000000000000000000000000000000..26ce12e7d1eee6202668f125f255d7c5fe470c8a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/implementations/sentencepiece_unigram.py @@ -0,0 +1,196 @@ +import json +import os +from typing import Iterator, List, Optional, Union, Tuple + +from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers +from tokenizers.models import Unigram + +from .base_tokenizer import BaseTokenizer + + +class SentencePieceUnigramTokenizer(BaseTokenizer): + """SentencePiece Unigram Tokenizer + + Represents the Unigram algorithm, with the pretokenization used by SentencePiece + """ + + def __init__( + self, + vocab: Optional[List[Tuple[str, float]]] = None, + replacement: str = "▁", + add_prefix_space: bool = True, + ): + if vocab is not None: + # Let Unigram(..) fail if only one of them is None + tokenizer = Tokenizer(Unigram(vocab)) + else: + tokenizer = Tokenizer(Unigram()) + + tokenizer.normalizer = normalizers.Sequence( + [normalizers.Nmt(), normalizers.NFKC(), normalizers.Replace(Regex(" {2,}"), " ")] + ) + prepend_scheme = "always" if add_prefix_space else "never" + tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + + parameters = { + "model": "SentencePieceUnigram", + "replacement": replacement, + "add_prefix_space": add_prefix_space, + } + + super().__init__(tokenizer, parameters) + + def train( + self, + files: Union[str, List[str]], + vocab_size: int = 8000, + show_progress: bool = True, + special_tokens: Optional[List[Union[str, AddedToken]]] = None, + initial_alphabet: Optional[List[str]] = None, + unk_token: Optional[str] = None, + ): + """ + Train the model using the given files + + Args: + files (:obj:`List[str]`): + A list of path to the files that we should use for training + vocab_size (:obj:`int`): + The size of the final vocabulary, including all tokens and alphabet. + show_progress (:obj:`bool`): + Whether to show progress bars while training. + special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): + A list of special tokens the model should know of. + initial_alphabet (:obj:`List[str]`, `optional`): + A list of characters to include in the initial alphabet, even + if not seen in the training dataset. + If the strings contain more than one character, only the first one + is kept. + unk_token (:obj:`str`, `optional`): + The unknown token to be used by the model. + """ + + if special_tokens is None: + special_tokens = [] + + if initial_alphabet is None: + initial_alphabet = [] + + trainer = trainers.UnigramTrainer( + vocab_size=vocab_size, + special_tokens=special_tokens, + show_progress=show_progress, + initial_alphabet=initial_alphabet, + unk_token=unk_token, + ) + + if isinstance(files, str): + files = [files] + self._tokenizer.train(files, trainer=trainer) + + def train_from_iterator( + self, + iterator: Union[Iterator[str], Iterator[Iterator[str]]], + vocab_size: int = 8000, + show_progress: bool = True, + special_tokens: Optional[List[Union[str, AddedToken]]] = None, + initial_alphabet: Optional[List[str]] = None, + unk_token: Optional[str] = None, + length: Optional[int] = None, + ): + """ + Train the model using the given iterator + + Args: + iterator (:obj:`Union[Iterator[str], Iterator[Iterator[str]]]`): + Any iterator over strings or list of strings + vocab_size (:obj:`int`): + The size of the final vocabulary, including all tokens and alphabet. + show_progress (:obj:`bool`): + Whether to show progress bars while training. + special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): + A list of special tokens the model should know of. + initial_alphabet (:obj:`List[str]`, `optional`): + A list of characters to include in the initial alphabet, even + if not seen in the training dataset. + If the strings contain more than one character, only the first one + is kept. + unk_token (:obj:`str`, `optional`): + The unknown token to be used by the model. + length (:obj:`int`, `optional`): + The total number of sequences in the iterator. This is used to + provide meaningful progress tracking + """ + + if special_tokens is None: + special_tokens = [] + + if initial_alphabet is None: + initial_alphabet = [] + + trainer = trainers.UnigramTrainer( + vocab_size=vocab_size, + special_tokens=special_tokens, + show_progress=show_progress, + initial_alphabet=initial_alphabet, + unk_token=unk_token, + ) + + self._tokenizer.train_from_iterator( + iterator, + trainer=trainer, + length=length, + ) + + @staticmethod + def from_spm(filename: str): + try: + import sys + + sys.path.append(".") + + import sentencepiece_model_pb2 as model # type: ignore[import] + except Exception: + raise Exception( + "You don't seem to have the required protobuf file, in order to use this function you need to run `pip install protobuf` and `wget https://raw.githubusercontent.com/google/sentencepiece/master/python/src/sentencepiece/sentencepiece_model_pb2.py` for us to be able to read the intrinsics of your spm_file. `pip install sentencepiece` is not required." + ) + + m = model.ModelProto() + m.ParseFromString(open(filename, "rb").read()) + + precompiled_charsmap = m.normalizer_spec.precompiled_charsmap + vocab = [(piece.piece, piece.score) for piece in m.pieces] + unk_id = m.trainer_spec.unk_id + model_type = m.trainer_spec.model_type + byte_fallback = m.trainer_spec.byte_fallback + if model_type != 1: + raise Exception( + "You're trying to run a `Unigram` model but you're file was trained with a different algorithm" + ) + + replacement = "▁" + add_prefix_space = True + + tokenizer = Tokenizer(Unigram(vocab, unk_id, byte_fallback)) + + if precompiled_charsmap: + tokenizer.normalizer = normalizers.Sequence( + [ + normalizers.Precompiled(precompiled_charsmap), + normalizers.Replace(Regex(" {2,}"), " "), + ] + ) + else: + tokenizer.normalizer = normalizers.Sequence([normalizers.Replace(Regex(" {2,}"), " ")]) + prepend_scheme = "always" if add_prefix_space else "never" + tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + tokenizer.decoder = decoders.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme) + + parameters = { + "model": "SentencePieceUnigram", + } + + obj = BaseTokenizer.__new__(SentencePieceUnigramTokenizer, tokenizer, parameters) # type: ignore[arg-type] + BaseTokenizer.__init__(obj, tokenizer, parameters) + return obj diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0094328283214245bae20d1f18b2ba3db716ed2c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.py @@ -0,0 +1,8 @@ +# Generated content DO NOT EDIT +from .. import models + +Model = models.Model +BPE = models.BPE +Unigram = models.Unigram +WordLevel = models.WordLevel +WordPiece = models.WordPiece diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..8a0de5e6b9aa80d3272ca943ea83be1f57804da4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__init__.pyi @@ -0,0 +1,744 @@ +# Generated content DO NOT EDIT +class Model: + """ + Base class for all models + + The model represents the actual tokenization algorithm. This is the part that + will contain and manage the learned vocabulary. + + This class cannot be constructed directly. Please use one of the concrete models. + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + def get_trainer(self): + """ + Get the associated :class:`~tokenizers.trainers.Trainer` + + Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this + :class:`~tokenizers.models.Model`. + + Returns: + :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model + """ + pass + + def id_to_token(self, id): + """ + Get the token associated to an ID + + Args: + id (:obj:`int`): + An ID to convert to a token + + Returns: + :obj:`str`: The token associated to the ID + """ + pass + + def save(self, folder, prefix): + """ + Save the current model + + Save the current model in the given folder, using the given prefix for the various + files that will get created. + Any file with the same name that already exists in this folder will be overwritten. + + Args: + folder (:obj:`str`): + The path to the target folder in which to save the various files + + prefix (:obj:`str`, `optional`): + An optional prefix, used to prefix each file name + + Returns: + :obj:`List[str]`: The list of saved files + """ + pass + + def token_to_id(self, tokens): + """ + Get the ID associated to a token + + Args: + token (:obj:`str`): + A token to convert to an ID + + Returns: + :obj:`int`: The ID associated to the token + """ + pass + + def tokenize(self, sequence): + """ + Tokenize a sequence + + Args: + sequence (:obj:`str`): + A sequence to tokenize + + Returns: + A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens + """ + pass + +class BPE(Model): + """ + An implementation of the BPE (Byte-Pair Encoding) algorithm + + Args: + vocab (:obj:`Dict[str, int]`, `optional`): + A dictionary of string keys and their ids :obj:`{"am": 0,...}` + + merges (:obj:`List[Tuple[str, str]]`, `optional`): + A list of pairs of tokens (:obj:`Tuple[str, str]`) :obj:`[("a", "b"),...]` + + cache_capacity (:obj:`int`, `optional`): + The number of words that the BPE cache can contain. The cache allows + to speed-up the process by keeping the result of the merge operations + for a number of words. + + dropout (:obj:`float`, `optional`): + A float between 0 and 1 that represents the BPE dropout to use. + + unk_token (:obj:`str`, `optional`): + The unknown token to be used by the model. + + continuing_subword_prefix (:obj:`str`, `optional`): + The prefix to attach to subword units that don't represent a beginning of word. + + end_of_word_suffix (:obj:`str`, `optional`): + The suffix to attach to subword units that represent an end of word. + + fuse_unk (:obj:`bool`, `optional`): + Whether to fuse any subsequent unknown tokens into a single one + + byte_fallback (:obj:`bool`, `optional`): + Whether to use spm byte-fallback trick (defaults to False) + + ignore_merges (:obj:`bool`, `optional`): + Whether or not to match tokens with the vocab before using merges. + """ + def __init__( + self, + vocab=None, + merges=None, + cache_capacity=None, + dropout=None, + unk_token=None, + continuing_subword_prefix=None, + end_of_word_suffix=None, + fuse_unk=None, + byte_fallback=False, + ignore_merges=False, + ): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def byte_fallback(self): + """ """ + pass + + @byte_fallback.setter + def byte_fallback(self, value): + """ """ + pass + + @property + def continuing_subword_prefix(self): + """ """ + pass + + @continuing_subword_prefix.setter + def continuing_subword_prefix(self, value): + """ """ + pass + + @property + def dropout(self): + """ """ + pass + + @dropout.setter + def dropout(self, value): + """ """ + pass + + @property + def end_of_word_suffix(self): + """ """ + pass + + @end_of_word_suffix.setter + def end_of_word_suffix(self, value): + """ """ + pass + + @staticmethod + def from_file(vocab, merges, **kwargs): + """ + Instantiate a BPE model from the given files. + + This method is roughly equivalent to doing:: + + vocab, merges = BPE.read_file(vocab_filename, merges_filename) + bpe = BPE(vocab, merges) + + If you don't need to keep the :obj:`vocab, merges` values lying around, + this method is more optimized than manually calling + :meth:`~tokenizers.models.BPE.read_file` to initialize a :class:`~tokenizers.models.BPE` + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.json` file + + merges (:obj:`str`): + The path to a :obj:`merges.txt` file + + Returns: + :class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files + """ + pass + + @property + def fuse_unk(self): + """ """ + pass + + @fuse_unk.setter + def fuse_unk(self, value): + """ """ + pass + + def get_trainer(self): + """ + Get the associated :class:`~tokenizers.trainers.Trainer` + + Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this + :class:`~tokenizers.models.Model`. + + Returns: + :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model + """ + pass + + def id_to_token(self, id): + """ + Get the token associated to an ID + + Args: + id (:obj:`int`): + An ID to convert to a token + + Returns: + :obj:`str`: The token associated to the ID + """ + pass + + @property + def ignore_merges(self): + """ """ + pass + + @ignore_merges.setter + def ignore_merges(self, value): + """ """ + pass + + @staticmethod + def read_file(vocab, merges): + """ + Read a :obj:`vocab.json` and a :obj:`merges.txt` files + + This method provides a way to read and parse the content of these files, + returning the relevant data structures. If you want to instantiate some BPE models + from memory, this method gives you the expected input from the standard files. + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.json` file + + merges (:obj:`str`): + The path to a :obj:`merges.txt` file + + Returns: + A :obj:`Tuple` with the vocab and the merges: + The vocabulary and merges loaded into memory + """ + pass + + def save(self, folder, prefix): + """ + Save the current model + + Save the current model in the given folder, using the given prefix for the various + files that will get created. + Any file with the same name that already exists in this folder will be overwritten. + + Args: + folder (:obj:`str`): + The path to the target folder in which to save the various files + + prefix (:obj:`str`, `optional`): + An optional prefix, used to prefix each file name + + Returns: + :obj:`List[str]`: The list of saved files + """ + pass + + def token_to_id(self, tokens): + """ + Get the ID associated to a token + + Args: + token (:obj:`str`): + A token to convert to an ID + + Returns: + :obj:`int`: The ID associated to the token + """ + pass + + def tokenize(self, sequence): + """ + Tokenize a sequence + + Args: + sequence (:obj:`str`): + A sequence to tokenize + + Returns: + A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens + """ + pass + + @property + def unk_token(self): + """ """ + pass + + @unk_token.setter + def unk_token(self, value): + """ """ + pass + +class Unigram(Model): + """ + An implementation of the Unigram algorithm + + Args: + vocab (:obj:`List[Tuple[str, float]]`, `optional`, `optional`): + A list of vocabulary items and their relative score [("am", -0.2442),...] + """ + def __init__(self, vocab=None, unk_id=None, byte_fallback=None): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + def get_trainer(self): + """ + Get the associated :class:`~tokenizers.trainers.Trainer` + + Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this + :class:`~tokenizers.models.Model`. + + Returns: + :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model + """ + pass + + def id_to_token(self, id): + """ + Get the token associated to an ID + + Args: + id (:obj:`int`): + An ID to convert to a token + + Returns: + :obj:`str`: The token associated to the ID + """ + pass + + def save(self, folder, prefix): + """ + Save the current model + + Save the current model in the given folder, using the given prefix for the various + files that will get created. + Any file with the same name that already exists in this folder will be overwritten. + + Args: + folder (:obj:`str`): + The path to the target folder in which to save the various files + + prefix (:obj:`str`, `optional`): + An optional prefix, used to prefix each file name + + Returns: + :obj:`List[str]`: The list of saved files + """ + pass + + def token_to_id(self, tokens): + """ + Get the ID associated to a token + + Args: + token (:obj:`str`): + A token to convert to an ID + + Returns: + :obj:`int`: The ID associated to the token + """ + pass + + def tokenize(self, sequence): + """ + Tokenize a sequence + + Args: + sequence (:obj:`str`): + A sequence to tokenize + + Returns: + A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens + """ + pass + +class WordLevel(Model): + """ + An implementation of the WordLevel algorithm + + Most simple tokenizer model based on mapping tokens to their corresponding id. + + Args: + vocab (:obj:`str`, `optional`): + A dictionary of string keys and their ids :obj:`{"am": 0,...}` + + unk_token (:obj:`str`, `optional`): + The unknown token to be used by the model. + """ + def __init__(self, vocab=None, unk_token=None): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def from_file(vocab, unk_token=None): + """ + Instantiate a WordLevel model from the given file + + This method is roughly equivalent to doing:: + + vocab = WordLevel.read_file(vocab_filename) + wordlevel = WordLevel(vocab) + + If you don't need to keep the :obj:`vocab` values lying around, this method is + more optimized than manually calling :meth:`~tokenizers.models.WordLevel.read_file` to + initialize a :class:`~tokenizers.models.WordLevel` + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.json` file + + Returns: + :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file + """ + pass + + def get_trainer(self): + """ + Get the associated :class:`~tokenizers.trainers.Trainer` + + Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this + :class:`~tokenizers.models.Model`. + + Returns: + :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model + """ + pass + + def id_to_token(self, id): + """ + Get the token associated to an ID + + Args: + id (:obj:`int`): + An ID to convert to a token + + Returns: + :obj:`str`: The token associated to the ID + """ + pass + + @staticmethod + def read_file(vocab): + """ + Read a :obj:`vocab.json` + + This method provides a way to read and parse the content of a vocabulary file, + returning the relevant data structures. If you want to instantiate some WordLevel models + from memory, this method gives you the expected input from the standard files. + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.json` file + + Returns: + :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` + """ + pass + + def save(self, folder, prefix): + """ + Save the current model + + Save the current model in the given folder, using the given prefix for the various + files that will get created. + Any file with the same name that already exists in this folder will be overwritten. + + Args: + folder (:obj:`str`): + The path to the target folder in which to save the various files + + prefix (:obj:`str`, `optional`): + An optional prefix, used to prefix each file name + + Returns: + :obj:`List[str]`: The list of saved files + """ + pass + + def token_to_id(self, tokens): + """ + Get the ID associated to a token + + Args: + token (:obj:`str`): + A token to convert to an ID + + Returns: + :obj:`int`: The ID associated to the token + """ + pass + + def tokenize(self, sequence): + """ + Tokenize a sequence + + Args: + sequence (:obj:`str`): + A sequence to tokenize + + Returns: + A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens + """ + pass + + @property + def unk_token(self): + """ """ + pass + + @unk_token.setter + def unk_token(self, value): + """ """ + pass + +class WordPiece(Model): + """ + An implementation of the WordPiece algorithm + + Args: + vocab (:obj:`Dict[str, int]`, `optional`): + A dictionary of string keys and their ids :obj:`{"am": 0,...}` + + unk_token (:obj:`str`, `optional`): + The unknown token to be used by the model. + + max_input_chars_per_word (:obj:`int`, `optional`): + The maximum number of characters to authorize in a single word. + """ + def __init__(self, vocab=None, unk_token="[UNK]", max_input_chars_per_word=100, continuing_subword_prefix="##"): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def continuing_subword_prefix(self): + """ """ + pass + + @continuing_subword_prefix.setter + def continuing_subword_prefix(self, value): + """ """ + pass + + @staticmethod + def from_file(vocab, **kwargs): + """ + Instantiate a WordPiece model from the given file + + This method is roughly equivalent to doing:: + + vocab = WordPiece.read_file(vocab_filename) + wordpiece = WordPiece(vocab) + + If you don't need to keep the :obj:`vocab` values lying around, this method is + more optimized than manually calling :meth:`~tokenizers.models.WordPiece.read_file` to + initialize a :class:`~tokenizers.models.WordPiece` + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.txt` file + + Returns: + :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file + """ + pass + + def get_trainer(self): + """ + Get the associated :class:`~tokenizers.trainers.Trainer` + + Retrieve the :class:`~tokenizers.trainers.Trainer` associated to this + :class:`~tokenizers.models.Model`. + + Returns: + :class:`~tokenizers.trainers.Trainer`: The Trainer used to train this model + """ + pass + + def id_to_token(self, id): + """ + Get the token associated to an ID + + Args: + id (:obj:`int`): + An ID to convert to a token + + Returns: + :obj:`str`: The token associated to the ID + """ + pass + + @property + def max_input_chars_per_word(self): + """ """ + pass + + @max_input_chars_per_word.setter + def max_input_chars_per_word(self, value): + """ """ + pass + + @staticmethod + def read_file(vocab): + """ + Read a :obj:`vocab.txt` file + + This method provides a way to read and parse the content of a standard `vocab.txt` + file as used by the WordPiece Model, returning the relevant data structures. If you + want to instantiate some WordPiece models from memory, this method gives you the + expected input from the standard files. + + Args: + vocab (:obj:`str`): + The path to a :obj:`vocab.txt` file + + Returns: + :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` + """ + pass + + def save(self, folder, prefix): + """ + Save the current model + + Save the current model in the given folder, using the given prefix for the various + files that will get created. + Any file with the same name that already exists in this folder will be overwritten. + + Args: + folder (:obj:`str`): + The path to the target folder in which to save the various files + + prefix (:obj:`str`, `optional`): + An optional prefix, used to prefix each file name + + Returns: + :obj:`List[str]`: The list of saved files + """ + pass + + def token_to_id(self, tokens): + """ + Get the ID associated to a token + + Args: + token (:obj:`str`): + A token to convert to an ID + + Returns: + :obj:`int`: The ID associated to the token + """ + pass + + def tokenize(self, sequence): + """ + Tokenize a sequence + + Args: + sequence (:obj:`str`): + A sequence to tokenize + + Returns: + A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens + """ + pass + + @property + def unk_token(self): + """ """ + pass + + @unk_token.setter + def unk_token(self, value): + """ """ + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c4f2d71238b51d1facb0c21884e0ecab65c4731 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/models/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e7b7476ef32ee94036ad57c7f88956552018ca7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.py @@ -0,0 +1,29 @@ +from .. import normalizers + + +Normalizer = normalizers.Normalizer +BertNormalizer = normalizers.BertNormalizer +NFD = normalizers.NFD +NFKD = normalizers.NFKD +NFC = normalizers.NFC +NFKC = normalizers.NFKC +Sequence = normalizers.Sequence +Lowercase = normalizers.Lowercase +Prepend = normalizers.Prepend +Strip = normalizers.Strip +StripAccents = normalizers.StripAccents +Nmt = normalizers.Nmt +Precompiled = normalizers.Precompiled +Replace = normalizers.Replace +ByteLevel = normalizers.ByteLevel + +NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD} + + +def unicode_normalizer_from_str(normalizer: str) -> Normalizer: + if normalizer not in NORMALIZERS: + raise ValueError( + "{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys()) + ) + + return NORMALIZERS[normalizer]() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..f2e38f60b092a2e1b8f9481e236370beca9ada91 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__init__.pyi @@ -0,0 +1,946 @@ +# Generated content DO NOT EDIT +class Normalizer: + """ + Base class for all normalizers + + This class is not supposed to be instantiated directly. Instead, any implementation of a + Normalizer will return an instance of this class when instantiated. + """ + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class BertNormalizer(Normalizer): + """ + BertNormalizer + + Takes care of normalizing raw text before giving it to a Bert model. + This includes cleaning the text, handling accents, chinese chars and lowercasing + + Args: + clean_text (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to clean the text, by removing any control characters + and replacing all whitespaces by the classic one. + + handle_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to handle chinese chars by putting spaces around them. + + strip_accents (:obj:`bool`, `optional`): + Whether to strip all accents. If this option is not specified (ie == None), + then it will be determined by the value for `lowercase` (as in the original Bert). + + lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to lowercase. + """ + def __init__(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def clean_text(self): + """ """ + pass + + @clean_text.setter + def clean_text(self, value): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + @property + def handle_chinese_chars(self): + """ """ + pass + + @handle_chinese_chars.setter + def handle_chinese_chars(self, value): + """ """ + pass + + @property + def lowercase(self): + """ """ + pass + + @lowercase.setter + def lowercase(self, value): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + + @property + def strip_accents(self): + """ """ + pass + + @strip_accents.setter + def strip_accents(self, value): + """ """ + pass + +class ByteLevel(Normalizer): + """ + Bytelevel Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class Lowercase(Normalizer): + """ + Lowercase Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class NFC(Normalizer): + """ + NFC Unicode Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class NFD(Normalizer): + """ + NFD Unicode Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class NFKC(Normalizer): + """ + NFKC Unicode Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class NFKD(Normalizer): + """ + NFKD Unicode Normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class Nmt(Normalizer): + """ + Nmt normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class Precompiled(Normalizer): + """ + Precompiled normalizer + Don't use manually it is used for compatibility for SentencePiece. + """ + def __init__(self, precompiled_charsmap): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class Prepend(Normalizer): + """ + Prepend normalizer + """ + def __init__(self, prepend): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + + @property + def prepend(self): + """ """ + pass + + @prepend.setter + def prepend(self, value): + """ """ + pass + +class Replace(Normalizer): + """ + Replace normalizer + """ + def __init__(self, pattern, content): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def content(self): + """ """ + pass + + @content.setter + def content(self, value): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + + @property + def pattern(self): + """ """ + pass + + @pattern.setter + def pattern(self, value): + """ """ + pass + +class Sequence(Normalizer): + """ + Allows concatenating multiple other Normalizer as a Sequence. + All the normalizers run in sequence in the given order + + Args: + normalizers (:obj:`List[Normalizer]`): + A list of Normalizer to be run as a sequence + """ + def __init__(self, normalizers): + pass + + def __getitem__(self, key): + """ + Return self[key]. + """ + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setitem__(self, key, value): + """ + Set self[key] to value. + """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +class Strip(Normalizer): + """ + Strip normalizer + """ + def __init__(self, left=True, right=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + @property + def left(self): + """ """ + pass + + @left.setter + def left(self, value): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + + @property + def right(self): + """ """ + pass + + @right.setter + def right(self, value): + """ """ + pass + +class StripAccents(Normalizer): + """ + StripAccents normalizer + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(normalizer): + """ """ + pass + + def normalize(self, normalized): + """ + Normalize a :class:`~tokenizers.NormalizedString` in-place + + This method allows to modify a :class:`~tokenizers.NormalizedString` to + keep track of the alignment information. If you just want to see the result + of the normalization on a raw string, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize_str` + + Args: + normalized (:class:`~tokenizers.NormalizedString`): + The normalized string on which to apply this + :class:`~tokenizers.normalizers.Normalizer` + """ + pass + + def normalize_str(self, sequence): + """ + Normalize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.normalizers.Normalizer` but it does not keep track of the alignment + information. If you need to get/convert offsets, you can use + :meth:`~tokenizers.normalizers.Normalizer.normalize` + + Args: + sequence (:obj:`str`): + A string to normalize + + Returns: + :obj:`str`: A string after normalization + """ + pass + +from typing import Dict + +NORMALIZERS: Dict[str, Normalizer] + +def unicode_normalizer_from_str(normalizer: str) -> Normalizer: ... diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8060544b02df885ec1718cf3759d27f31c134bfd Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/normalizers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..156a980cae8685c0b2ad2466a47087c5a95d75a5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.py @@ -0,0 +1,16 @@ +# Generated content DO NOT EDIT +from .. import pre_tokenizers + +PreTokenizer = pre_tokenizers.PreTokenizer +BertPreTokenizer = pre_tokenizers.BertPreTokenizer +ByteLevel = pre_tokenizers.ByteLevel +CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit +Digits = pre_tokenizers.Digits +FixedLength = pre_tokenizers.FixedLength +Metaspace = pre_tokenizers.Metaspace +Punctuation = pre_tokenizers.Punctuation +Sequence = pre_tokenizers.Sequence +Split = pre_tokenizers.Split +UnicodeScripts = pre_tokenizers.UnicodeScripts +Whitespace = pre_tokenizers.Whitespace +WhitespaceSplit = pre_tokenizers.WhitespaceSplit diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cd809f853229e713fb0a361513437f2b92ea364d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__init__.pyi @@ -0,0 +1,1015 @@ +# Generated content DO NOT EDIT +class PreTokenizer: + """ + Base class for all pre-tokenizers + + This class is not supposed to be instantiated directly. Instead, any implementation of a + PreTokenizer will return an instance of this class when instantiated. + """ + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class BertPreTokenizer(PreTokenizer): + """ + BertPreTokenizer + + This pre-tokenizer splits tokens on spaces, and also on punctuation. + Each occurrence of a punctuation character will be treated separately. + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class ByteLevel(PreTokenizer): + """ + ByteLevel PreTokenizer + + This pre-tokenizer takes care of replacing all bytes of the given string + with a corresponding representation, as well as splitting into words. + + Args: + add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to add a space to the first word if there isn't already one. This + lets us treat `hello` exactly like `say hello`. + use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`): + Set this to :obj:`False` to prevent this `pre_tokenizer` from using + the GPT2 specific regexp for spliting on whitespace. + """ + def __init__(self, add_prefix_space=True, trim_offsets=True, use_regex=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def add_prefix_space(self): + """ """ + pass + + @add_prefix_space.setter + def add_prefix_space(self, value): + """ """ + pass + + @staticmethod + def alphabet(): + """ + Returns the alphabet used by this PreTokenizer. + + Since the ByteLevel works as its name suggests, at the byte level, it + encodes each byte value to a unique visible character. This means that there is a + total of 256 different characters composing this alphabet. + + Returns: + :obj:`List[str]`: A list of characters that compose the alphabet + """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + + @property + def trim_offsets(self): + """ """ + pass + + @trim_offsets.setter + def trim_offsets(self, value): + """ """ + pass + + @property + def use_regex(self): + """ """ + pass + + @use_regex.setter + def use_regex(self, value): + """ """ + pass + +class CharDelimiterSplit(PreTokenizer): + """ + This pre-tokenizer simply splits on the provided char. Works like `.split(delimiter)` + + Args: + delimiter: str: + The delimiter char that will be used to split input + """ + def __init__(self, delimiter): + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + @property + def delimiter(self): + """ """ + pass + + @delimiter.setter + def delimiter(self, value): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class Digits(PreTokenizer): + """ + This pre-tokenizer simply splits using the digits in separate tokens + + Args: + individual_digits (:obj:`bool`, `optional`, defaults to :obj:`False`): + If set to True, digits will each be separated as follows:: + + "Call 123 please" -> "Call ", "1", "2", "3", " please" + + If set to False, digits will grouped as follows:: + + "Call 123 please" -> "Call ", "123", " please" + """ + def __init__(self, individual_digits=False): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + @property + def individual_digits(self): + """ """ + pass + + @individual_digits.setter + def individual_digits(self, value): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class FixedLength(PreTokenizer): + """ + This pre-tokenizer splits the text into fixed length chunks as used + [here](https://www.biorxiv.org/content/10.1101/2023.01.11.523679v1.full) + + Args: + length (:obj:`int`, `optional`, defaults to :obj:`5`): + The length of the chunks to split the text into. + + Strings are split on the character level rather than the byte level to avoid + splitting unicode characters consisting of multiple bytes. + """ + def __init__(self, length=5): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + @property + def length(self): + """ """ + pass + + @length.setter + def length(self, value): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class Metaspace(PreTokenizer): + """ + Metaspace pre-tokenizer + + This pre-tokenizer replaces any whitespace by the provided replacement character. + It then tries to split on these spaces. + + Args: + replacement (:obj:`str`, `optional`, defaults to :obj:`▁`): + The replacement character. Must be exactly one character. By default we + use the `▁` (U+2581) meta symbol (Same as in SentencePiece). + + prepend_scheme (:obj:`str`, `optional`, defaults to :obj:`"always"`): + Whether to add a space to the first word if there isn't already one. This + lets us treat `hello` exactly like `say hello`. + Choices: "always", "never", "first". First means the space is only added on the first + token (relevant when special tokens are used or other pre_tokenizer are used). + + """ + def __init__(self, replacement="_", prepend_scheme="always", split=True): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + + @property + def prepend_scheme(self): + """ """ + pass + + @prepend_scheme.setter + def prepend_scheme(self, value): + """ """ + pass + + @property + def replacement(self): + """ """ + pass + + @replacement.setter + def replacement(self, value): + """ """ + pass + + @property + def split(self): + """ """ + pass + + @split.setter + def split(self, value): + """ """ + pass + +class Punctuation(PreTokenizer): + """ + This pre-tokenizer simply splits on punctuation as individual characters. + + Args: + behavior (:class:`~tokenizers.SplitDelimiterBehavior`): + The behavior to use when splitting. + Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next", + "contiguous" + """ + def __init__(self, behavior="isolated"): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def behavior(self): + """ """ + pass + + @behavior.setter + def behavior(self, value): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class Sequence(PreTokenizer): + """ + This pre-tokenizer composes other pre_tokenizers and applies them in sequence + """ + def __init__(self, pretokenizers): + pass + + def __getitem__(self, key): + """ + Return self[key]. + """ + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setitem__(self, key, value): + """ + Set self[key] to value. + """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class Split(PreTokenizer): + """ + Split PreTokenizer + + This versatile pre-tokenizer splits using the provided pattern and + according to the provided behavior. The pattern can be inverted by + making use of the invert flag. + + Args: + pattern (:obj:`str` or :class:`~tokenizers.Regex`): + A pattern used to split the string. Usually a string or a regex built with `tokenizers.Regex`. + If you want to use a regex pattern, it has to be wrapped around a `tokenizers.Regex`, + otherwise we consider is as a string pattern. For example `pattern="|"` + means you want to split on `|` (imagine a csv file for example), while + `pattern=tokenizers.Regex("1|2")` means you split on either '1' or '2'. + behavior (:class:`~tokenizers.SplitDelimiterBehavior`): + The behavior to use when splitting. + Choices: "removed", "isolated", "merged_with_previous", "merged_with_next", + "contiguous" + + invert (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to invert the pattern. + """ + def __init__(self, pattern, behavior, invert=False): + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def behavior(self): + """ """ + pass + + @behavior.setter + def behavior(self, value): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + @property + def invert(self): + """ """ + pass + + @invert.setter + def invert(self, value): + """ """ + pass + + @property + def pattern(self): + """ """ + pass + + @pattern.setter + def pattern(self, value): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class UnicodeScripts(PreTokenizer): + """ + This pre-tokenizer splits on characters that belong to different language family + It roughly follows https://github.com/google/sentencepiece/blob/master/data/Scripts.txt + Actually Hiragana and Katakana are fused with Han, and 0x30FC is Han too. + This mimicks SentencePiece Unigram implementation. + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class Whitespace(PreTokenizer): + """ + This pre-tokenizer splits on word boundaries according to the `\w+|[^\w\s]+` + regex pattern. It splits on word characters or characters that aren't words or + whitespaces (punctuation such as hyphens, apostrophes, commas, etc.). + + Example: + Use the `Whitespace` function as shown below:: + + ```python + from tokenizers.pre_tokenizers import Whitespace + + pre_tokenizer = Whitespace() + text = "Hello, world! Let's try the Whitespace pre-tokenizer." + pre_tokenizer.pre_tokenize_str(text) + [('Hello', (0, 5)), + (',', (5, 6)), + ('world', (7, 12)), + ('!', (12, 13)), + ('Let', (14, 17)), + ("'", (17, 18)), + ('s', (18, 19)), + ('try', (20, 23)), + ('the', (24, 27)), + ('Whitespace', (28, 38)), + ('pre', (39, 42)), + ('-', (42, 43)), + ('tokenizer', (43, 52)), + ('.', (52, 53))] + ``` + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass + +class WhitespaceSplit(PreTokenizer): + """ + This pre-tokenizer simply splits on the whitespace. Works like `.split()` + """ + def __init__(self): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @staticmethod + def custom(pretok): + """ """ + pass + + def pre_tokenize(self, pretok): + """ + Pre-tokenize a :class:`~tokenizers.PyPreTokenizedString` in-place + + This method allows to modify a :class:`~tokenizers.PreTokenizedString` to + keep track of the pre-tokenization, and leverage the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you just want to see the result of + the pre-tokenization of a raw string, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize_str` + + Args: + pretok (:class:`~tokenizers.PreTokenizedString): + The pre-tokenized string on which to apply this + :class:`~tokenizers.pre_tokenizers.PreTokenizer` + """ + pass + + def pre_tokenize_str(self, sequence): + """ + Pre tokenize the given string + + This method provides a way to visualize the effect of a + :class:`~tokenizers.pre_tokenizers.PreTokenizer` but it does not keep track of the + alignment, nor does it provide all the capabilities of the + :class:`~tokenizers.PreTokenizedString`. If you need some of these, you can use + :meth:`~tokenizers.pre_tokenizers.PreTokenizer.pre_tokenize` + + Args: + sequence (:obj:`str`): + A string to pre-tokeize + + Returns: + :obj:`List[Tuple[str, Offsets]]`: + A list of tuple with the pre-tokenized parts and their offsets + """ + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0826950b2ca3a6ab68c6e4964fac634d6c534465 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/pre_tokenizers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9c8feef829cd34819b7b872a79004990d3abf2fc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.py @@ -0,0 +1,9 @@ +# Generated content DO NOT EDIT +from .. import processors + +PostProcessor = processors.PostProcessor +BertProcessing = processors.BertProcessing +ByteLevel = processors.ByteLevel +RobertaProcessing = processors.RobertaProcessing +Sequence = processors.Sequence +TemplateProcessing = processors.TemplateProcessing diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..dacac8f189fb6e617b21e155f89d546f12e194fd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__init__.pyi @@ -0,0 +1,519 @@ +# Generated content DO NOT EDIT +class PostProcessor: + """ + Base class for all post-processors + + This class is not supposed to be instantiated directly. Instead, any implementation of + a PostProcessor will return an instance of this class when instantiated. + """ + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + +class BertProcessing(PostProcessor): + """ + This post-processor takes care of adding the special tokens needed by + a Bert model: + + - a SEP token + - a CLS token + + Args: + sep (:obj:`Tuple[str, int]`): + A tuple with the string representation of the SEP token, and its id + + cls (:obj:`Tuple[str, int]`): + A tuple with the string representation of the CLS token, and its id + """ + def __init__(self, sep, cls): + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def cls(self): + """ """ + pass + + @cls.setter + def cls(self, value): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + + @property + def sep(self): + """ """ + pass + + @sep.setter + def sep(self, value): + """ """ + pass + +class ByteLevel(PostProcessor): + """ + This post-processor takes care of trimming the offsets. + + By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't + want the offsets to include these whitespaces, then this PostProcessor must be used. + + Args: + trim_offsets (:obj:`bool`): + Whether to trim the whitespaces from the produced offsets. + + add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): + If :obj:`True`, keeps the first token's offset as is. If :obj:`False`, increments + the start of the first token's offset by 1. Only has an effect if :obj:`trim_offsets` + is set to :obj:`True`. + """ + def __init__(self, add_prefix_space=None, trim_offsets=None, use_regex=None): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def add_prefix_space(self): + """ """ + pass + + @add_prefix_space.setter + def add_prefix_space(self, value): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + + @property + def trim_offsets(self): + """ """ + pass + + @trim_offsets.setter + def trim_offsets(self, value): + """ """ + pass + + @property + def use_regex(self): + """ """ + pass + + @use_regex.setter + def use_regex(self, value): + """ """ + pass + +class RobertaProcessing(PostProcessor): + """ + This post-processor takes care of adding the special tokens needed by + a Roberta model: + + - a SEP token + - a CLS token + + It also takes care of trimming the offsets. + By default, the ByteLevel BPE might include whitespaces in the produced tokens. If you don't + want the offsets to include these whitespaces, then this PostProcessor should be initialized + with :obj:`trim_offsets=True` + + Args: + sep (:obj:`Tuple[str, int]`): + A tuple with the string representation of the SEP token, and its id + + cls (:obj:`Tuple[str, int]`): + A tuple with the string representation of the CLS token, and its id + + trim_offsets (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to trim the whitespaces from the produced offsets. + + add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether the add_prefix_space option was enabled during pre-tokenization. This + is relevant because it defines the way the offsets are trimmed out. + """ + def __init__(self, sep, cls, trim_offsets=True, add_prefix_space=True): + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def add_prefix_space(self): + """ """ + pass + + @add_prefix_space.setter + def add_prefix_space(self, value): + """ """ + pass + + @property + def cls(self): + """ """ + pass + + @cls.setter + def cls(self, value): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + + @property + def sep(self): + """ """ + pass + + @sep.setter + def sep(self, value): + """ """ + pass + + @property + def trim_offsets(self): + """ """ + pass + + @trim_offsets.setter + def trim_offsets(self, value): + """ """ + pass + +class Sequence(PostProcessor): + """ + Sequence Processor + + Args: + processors (:obj:`List[PostProcessor]`) + The processors that need to be chained + """ + def __init__(self, processors): + pass + + def __getitem__(self, key): + """ + Return self[key]. + """ + pass + + def __getnewargs__(self): + """ """ + pass + + def __getstate__(self): + """ """ + pass + + def __setitem__(self, key, value): + """ + Set self[key] to value. + """ + pass + + def __setstate__(self, state): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + +class TemplateProcessing(PostProcessor): + """ + Provides a way to specify templates in order to add the special tokens to each + input sequence as relevant. + + Let's take :obj:`BERT` tokenizer as an example. It uses two special tokens, used to + delimitate each sequence. :obj:`[CLS]` is always used at the beginning of the first + sequence, and :obj:`[SEP]` is added at the end of both the first, and the pair + sequences. The final result looks like this: + + - Single sequence: :obj:`[CLS] Hello there [SEP]` + - Pair sequences: :obj:`[CLS] My name is Anthony [SEP] What is my name? [SEP]` + + With the type ids as following:: + + [CLS] ... [SEP] ... [SEP] + 0 0 0 1 1 + + You can achieve such behavior using a TemplateProcessing:: + + TemplateProcessing( + single="[CLS] $0 [SEP]", + pair="[CLS] $A [SEP] $B:1 [SEP]:1", + special_tokens=[("[CLS]", 1), ("[SEP]", 0)], + ) + + In this example, each input sequence is identified using a ``$`` construct. This identifier + lets us specify each input sequence, and the type_id to use. When nothing is specified, + it uses the default values. Here are the different ways to specify it: + + - Specifying the sequence, with default ``type_id == 0``: ``$A`` or ``$B`` + - Specifying the `type_id` with default ``sequence == A``: ``$0``, ``$1``, ``$2``, ... + - Specifying both: ``$A:0``, ``$B:1``, ... + + The same construct is used for special tokens: ``(:)?``. + + **Warning**: You must ensure that you are giving the correct tokens/ids as these + will be added to the Encoding without any further check. If the given ids correspond + to something totally different in a `Tokenizer` using this `PostProcessor`, it + might lead to unexpected results. + + Args: + single (:obj:`Template`): + The template used for single sequences + + pair (:obj:`Template`): + The template used when both sequences are specified + + special_tokens (:obj:`Tokens`): + The list of special tokens used in each sequences + + Types: + + Template (:obj:`str` or :obj:`List`): + - If a :obj:`str` is provided, the whitespace is used as delimiter between tokens + - If a :obj:`List[str]` is provided, a list of tokens + + Tokens (:obj:`List[Union[Tuple[int, str], Tuple[str, int], dict]]`): + - A :obj:`Tuple` with both a token and its associated ID, in any order + - A :obj:`dict` with the following keys: + - "id": :obj:`str` => The special token id, as specified in the Template + - "ids": :obj:`List[int]` => The associated IDs + - "tokens": :obj:`List[str]` => The associated tokens + + The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have + the same length. + """ + def __init__(self, single=None, pair=None, special_tokens=None): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + def num_special_tokens_to_add(self, is_pair): + """ + Return the number of special tokens that would be added for single/pair sentences. + + Args: + is_pair (:obj:`bool`): + Whether the input would be a pair of sequences + + Returns: + :obj:`int`: The number of tokens to add + """ + pass + + def process(self, encoding, pair=None, add_special_tokens=True): + """ + Post-process the given encodings, generating the final one + + Args: + encoding (:class:`~tokenizers.Encoding`): + The encoding for the first sequence + + pair (:class:`~tokenizers.Encoding`, `optional`): + The encoding for the pair sequence + + add_special_tokens (:obj:`bool`): + Whether to add the special tokens + + Return: + :class:`~tokenizers.Encoding`: The final encoding + """ + pass + + @property + def single(self): + """ """ + pass + + @single.setter + def single(self, value): + """ """ + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fab2abda4b83de28d98c760f798b245572f1fb4c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/processors/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0d166cb9015bd967669e50341c3966297c447735 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__init__.py @@ -0,0 +1 @@ +from .visualizer import Annotation, EncodingVisualizer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c300aaeb62ef37854a245bf08ed19295dde04a4 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/visualizer.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/visualizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3cd4fd68469bfd76e1f041ddbcf891d94bf9e91 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/__pycache__/visualizer.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer-styles.css b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer-styles.css new file mode 100644 index 0000000000000000000000000000000000000000..c108d667b391694d94c15fd583b2491e438ec058 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer-styles.css @@ -0,0 +1,170 @@ +.tokenized-text { + width:100%; + padding:2rem; + max-height: 400px; + overflow-y: auto; + box-sizing:border-box; + line-height:4rem; /* Lots of space between lines */ + font-family: "Roboto Light", "Ubuntu Light", "Ubuntu", monospace; + box-shadow: 2px 2px 2px rgba(0,0,0,0.2); + background-color: rgba(0,0,0,0.01); + letter-spacing:2px; /* Give some extra separation between chars */ +} +.non-token{ + /* White space and other things the tokenizer ignores*/ + white-space: pre; + letter-spacing:4px; + border-top:1px solid #A0A0A0; /* A gentle border on top and bottom makes tabs more ovious*/ + border-bottom:1px solid #A0A0A0; + line-height: 1rem; + height: calc(100% - 2px); +} + +.token { + white-space: pre; + position:relative; + color:black; + letter-spacing:2px; +} + +.annotation{ + white-space:nowrap; /* Important - ensures that annotations appears even if the annotated text wraps a line */ + border-radius:4px; + position:relative; + width:fit-content; +} +.annotation:before { + /*The before holds the text and the after holds the background*/ + z-index:1000; /* Make sure this is above the background */ + content:attr(data-label); /* The annotations label is on a data attribute */ + color:white; + position:absolute; + font-size:1rem; + text-align:center; + font-weight:bold; + + top:1.75rem; + line-height:0; + left:0; + width:100%; + padding:0.5rem 0; + /* These make it so an annotation doesn't stretch beyond the annotated text if the label is longer*/ + overflow: hidden; + white-space: nowrap; + text-overflow:ellipsis; +} + +.annotation:after { + content:attr(data-label); /* The content defines the width of the annotation*/ + position:absolute; + font-size:0.75rem; + text-align:center; + font-weight:bold; + text-overflow:ellipsis; + top:1.75rem; + line-height:0; + overflow: hidden; + white-space: nowrap; + + left:0; + width:100%; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/ + + padding:0.5rem 0; + /* Nast hack below: + We set the annotations color in code because we don't know the colors at css time. + But you can't pass a color as a data attribute to get it into the pseudo element (this thing) + So to get around that, annotations have the color set on them with a style attribute and then we + can get the color with currentColor. + Annotations wrap tokens and tokens set the color back to black + */ + background-color: currentColor; +} +.annotation:hover::after, .annotation:hover::before{ + /* When the user hovers over an annotation expand the label to display in full + */ + min-width: fit-content; +} + +.annotation:hover{ + /* Emphasize the annotation start end with a border on hover*/ + border-color: currentColor; + border: 2px solid; +} +.special-token:not(:empty){ + /* + A none empty special token is like UNK (as opposed to CLS which has no representation in the text ) + */ + position:relative; +} +.special-token:empty::before{ + /* Special tokens that don't have text are displayed as pseudo elements so we dont select them with the mouse*/ + content:attr(data-stok); + background:#202020; + font-size:0.75rem; + color:white; + margin: 0 0.25rem; + padding: 0.25rem; + border-radius:4px +} + +.special-token:not(:empty):before { + /* Special tokens that have text (UNK) are displayed above the actual text*/ + content:attr(data-stok); + position:absolute; + bottom:1.75rem; + min-width:100%; + width:100%; + height:1rem; + line-height:1rem; + font-size:1rem; + text-align:center; + color:white; + font-weight:bold; + background:#202020; + border-radius:10%; +} +/* +We want to alternate the color of tokens, but we can't use nth child because tokens might be broken up by annotations +instead we apply even and odd class at generation time and color them that way + */ +.even-token{ + background:#DCDCDC ; + border: 1px solid #DCDCDC; +} +.odd-token{ + background:#A0A0A0; + border: 1px solid #A0A0A0; +} +.even-token.multi-token,.odd-token.multi-token{ + background: repeating-linear-gradient( + 45deg, + transparent, + transparent 1px, + #ccc 1px, + #ccc 1px + ), + /* on "bottom" */ + linear-gradient( + to bottom, + #FFB6C1, + #999 + ); +} + +.multi-token:hover::after { + content:"This char has more than 1 token"; /* The content defines the width of the annotation*/ + color:white; + background-color: black; + position:absolute; + font-size:0.75rem; + text-align:center; + font-weight:bold; + text-overflow:ellipsis; + top:1.75rem; + line-height:0; + overflow: hidden; + white-space: nowrap; + left:0; + width:fit-content; /* 100% of the parent, which is the annotation whose width is the tokens inside it*/ + padding:0.5rem 0; +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..b03cbefa5b2151d368b328ab83164441fb7276d3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/tools/visualizer.py @@ -0,0 +1,407 @@ +import itertools +import os +import re +from string import Template +from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple + +from tokenizers import Encoding, Tokenizer + + +dirname = os.path.dirname(__file__) +css_filename = os.path.join(dirname, "visualizer-styles.css") +with open(css_filename) as f: + css = f.read() + + +class Annotation: + start: int + end: int + label: str + + def __init__(self, start: int, end: int, label: str): + self.start = start + self.end = end + self.label = label + + +AnnotationList = List[Annotation] +PartialIntList = List[Optional[int]] + + +class CharStateKey(NamedTuple): + token_ix: Optional[int] + anno_ix: Optional[int] + + +class CharState: + char_ix: Optional[int] + + def __init__(self, char_ix): + self.char_ix = char_ix + + self.anno_ix: Optional[int] = None + self.tokens: List[int] = [] + + @property + def token_ix(self): + return self.tokens[0] if len(self.tokens) > 0 else None + + @property + def is_multitoken(self): + """ + BPE tokenizers can output more than one token for a char + """ + return len(self.tokens) > 1 + + def partition_key(self) -> CharStateKey: + return CharStateKey( + token_ix=self.token_ix, + anno_ix=self.anno_ix, + ) + + +class Aligned: + pass + + +class EncodingVisualizer: + """ + Build an EncodingVisualizer + + Args: + + tokenizer (:class:`~tokenizers.Tokenizer`): + A tokenizer instance + + default_to_notebook (:obj:`bool`): + Whether to render html output in a notebook by default + + annotation_converter (:obj:`Callable`, `optional`): + An optional (lambda) function that takes an annotation in any format and returns + an Annotation object + """ + + unk_token_regex = re.compile("(.{1}\b)?(unk|oov)(\b.{1})?", flags=re.IGNORECASE) + + def __init__( + self, + tokenizer: Tokenizer, + default_to_notebook: bool = True, + annotation_converter: Optional[Callable[[Any], Annotation]] = None, + ): + if default_to_notebook: + try: + from IPython.core.display import HTML, display # type: ignore[attr-defined] + except ImportError: + raise Exception( + """We couldn't import IPython utils for html display. + Are you running in a notebook? + You can also pass `default_to_notebook=False` to get back raw HTML + """ + ) + + self.tokenizer = tokenizer + self.default_to_notebook = default_to_notebook + self.annotation_coverter = annotation_converter + pass + + def __call__( + self, + text: str, + annotations: Optional[List[Any]] = None, + default_to_notebook: Optional[bool] = None, + ) -> Optional[str]: + """ + Build a visualization of the given text + + Args: + text (:obj:`str`): + The text to tokenize + + annotations (:obj:`List[Annotation]`, `optional`): + An optional list of annotations of the text. The can either be an annotation class + or anything else if you instantiated the visualizer with a converter function + + default_to_notebook (:obj:`bool`, `optional`, defaults to `False`): + If True, will render the html in a notebook. Otherwise returns an html string. + + Returns: + The HTML string if default_to_notebook is False, otherwise (default) returns None and + renders the HTML in the notebook + + """ + final_default_to_notebook = self.default_to_notebook + if default_to_notebook is not None: + final_default_to_notebook = default_to_notebook + if final_default_to_notebook: + try: + from IPython.core.display import HTML, display # type: ignore[attr-defined] + except ImportError: + raise Exception( + """We couldn't import IPython utils for html display. + Are you running in a notebook?""" + ) + if annotations is None: + annotations = [] + if self.annotation_coverter is not None: + annotations = list(map(self.annotation_coverter, annotations)) + encoding = self.tokenizer.encode(text) + html = EncodingVisualizer.__make_html(text, encoding, annotations) + if final_default_to_notebook: + display(HTML(html)) + else: + return html + + @staticmethod + def calculate_label_colors(annotations: AnnotationList) -> Dict[str, str]: + """ + Generates a color palette for all the labels in a given set of annotations + + Args: + annotations (:obj:`Annotation`): + A list of annotations + + Returns: + :obj:`dict`: A dictionary mapping labels to colors in HSL format + """ + if len(annotations) == 0: + return {} + labels = set(map(lambda x: x.label, annotations)) + num_labels = len(labels) + h_step = int(255 / num_labels) + if h_step < 20: + h_step = 20 + s = 32 + l = 64 # noqa: E741 + h = 10 + colors = {} + + for label in sorted(labels): # sort so we always get the same colors for a given set of labels + colors[label] = f"hsl({h},{s}%,{l}%)" + h += h_step + return colors + + @staticmethod + def consecutive_chars_to_html( + consecutive_chars_list: List[CharState], + text: str, + encoding: Encoding, + ): + """ + Converts a list of "consecutive chars" into a single HTML element. + Chars are consecutive if they fall under the same word, token and annotation. + The CharState class is a named tuple with a "partition_key" method that makes it easy to + compare if two chars are consecutive. + + Args: + consecutive_chars_list (:obj:`List[CharState]`): + A list of CharStates that have been grouped together + + text (:obj:`str`): + The original text being processed + + encoding (:class:`~tokenizers.Encoding`): + The encoding returned from the tokenizer + + Returns: + :obj:`str`: The HTML span for a set of consecutive chars + """ + first = consecutive_chars_list[0] + if first.char_ix is None: + # its a special token + stoken = encoding.tokens[first.token_ix] + # special tokens are represented as empty spans. We use the data attribute and css + # magic to display it + return f'' + # We're not in a special token so this group has a start and end. + last = consecutive_chars_list[-1] + assert first.char_ix is not None + assert last.char_ix is not None + start = first.char_ix + end = last.char_ix + 1 + span_text = text[start:end] + css_classes = [] # What css classes will we apply on the resulting span + data_items = {} # What data attributes will we apply on the result span + if first.token_ix is not None: + # We can either be in a token or not (e.g. in white space) + css_classes.append("token") + if first.is_multitoken: + css_classes.append("multi-token") + if first.token_ix % 2: + # We use this to color alternating tokens. + # A token might be split by an annotation that ends in the middle of it, so this + # lets us visually indicate a consecutive token despite its possible splitting in + # the html markup + css_classes.append("odd-token") + else: + # Like above, but a different color so we can see the tokens alternate + css_classes.append("even-token") + if EncodingVisualizer.unk_token_regex.search(encoding.tokens[first.token_ix]) is not None: + # This is a special token that is in the text. probably UNK + css_classes.append("special-token") + # TODO is this the right name for the data attribute ? + data_items["stok"] = encoding.tokens[first.token_ix] + else: + # In this case we are looking at a group/single char that is not tokenized. + # e.g. white space + css_classes.append("non-token") + css = f'''class="{" ".join(css_classes)}"''' + data = "" + for key, val in data_items.items(): + data += f' data-{key}="{val}"' + return f"{span_text}" + + @staticmethod + def __make_html(text: str, encoding: Encoding, annotations: AnnotationList) -> str: + char_states = EncodingVisualizer.__make_char_states(text, encoding, annotations) + current_consecutive_chars = [char_states[0]] + prev_anno_ix = char_states[0].anno_ix + spans = [] + label_colors_dict = EncodingVisualizer.calculate_label_colors(annotations) + cur_anno_ix = char_states[0].anno_ix + if cur_anno_ix is not None: + # If we started in an annotation make a span for it + anno = annotations[cur_anno_ix] + label = anno.label + color = label_colors_dict[label] + spans.append(f'') + + for cs in char_states[1:]: + cur_anno_ix = cs.anno_ix + if cur_anno_ix != prev_anno_ix: + # If we've transitioned in or out of an annotation + spans.append( + # Create a span from the current consecutive characters + EncodingVisualizer.consecutive_chars_to_html( + current_consecutive_chars, + text=text, + encoding=encoding, + ) + ) + current_consecutive_chars = [cs] + + if prev_anno_ix is not None: + # if we transitioned out of an annotation close it's span + spans.append("") + if cur_anno_ix is not None: + # If we entered a new annotation make a span for it + anno = annotations[cur_anno_ix] + label = anno.label + color = label_colors_dict[label] + spans.append(f'') + prev_anno_ix = cur_anno_ix + + if cs.partition_key() == current_consecutive_chars[0].partition_key(): + # If the current charchter is in the same "group" as the previous one + current_consecutive_chars.append(cs) + else: + # Otherwise we make a span for the previous group + spans.append( + EncodingVisualizer.consecutive_chars_to_html( + current_consecutive_chars, + text=text, + encoding=encoding, + ) + ) + # An reset the consecutive_char_list to form a new group + current_consecutive_chars = [cs] + # All that's left is to fill out the final span + # TODO I think there is an edge case here where an annotation's span might not close + spans.append( + EncodingVisualizer.consecutive_chars_to_html( + current_consecutive_chars, + text=text, + encoding=encoding, + ) + ) + res = HTMLBody(spans) # Send the list of spans to the body of our html + return res + + @staticmethod + def __make_anno_map(text: str, annotations: AnnotationList) -> PartialIntList: + """ + Args: + text (:obj:`str`): + The raw text we want to align to + + annotations (:obj:`AnnotationList`): + A (possibly empty) list of annotations + + Returns: + A list of length len(text) whose entry at index i is None if there is no annotation on + character i or k, the index of the annotation that covers index i where k is with + respect to the list of annotations + """ + annotation_map = [None] * len(text) + for anno_ix, a in enumerate(annotations): + for i in range(a.start, a.end): + annotation_map[i] = anno_ix + return annotation_map + + @staticmethod + def __make_char_states(text: str, encoding: Encoding, annotations: AnnotationList) -> List[CharState]: + """ + For each character in the original text, we emit a tuple representing it's "state": + + * which token_ix it corresponds to + * which word_ix it corresponds to + * which annotation_ix it corresponds to + + Args: + text (:obj:`str`): + The raw text we want to align to + + annotations (:obj:`List[Annotation]`): + A (possibly empty) list of annotations + + encoding: (:class:`~tokenizers.Encoding`): + The encoding returned from the tokenizer + + Returns: + :obj:`List[CharState]`: A list of CharStates, indicating for each char in the text what + it's state is + """ + annotation_map = EncodingVisualizer.__make_anno_map(text, annotations) + # Todo make this a dataclass or named tuple + char_states: List[CharState] = [CharState(char_ix) for char_ix in range(len(text))] + for token_ix, token in enumerate(encoding.tokens): + offsets = encoding.token_to_chars(token_ix) + if offsets is not None: + start, end = offsets + for i in range(start, end): + char_states[i].tokens.append(token_ix) + for char_ix, anno_ix in enumerate(annotation_map): + char_states[char_ix].anno_ix = anno_ix + + return char_states + + +def HTMLBody(children: List[str], css_styles=css) -> str: + """ + Generates the full html with css from a list of html spans + + Args: + children (:obj:`List[str]`): + A list of strings, assumed to be html elements + + css_styles (:obj:`str`, `optional`): + Optional alternative implementation of the css + + Returns: + :obj:`str`: An HTML string with style markup + """ + children_text = "".join(children) + return f""" + + + + + +
+ {children_text} +
+ + + """ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..92d58beee3d4502a7c7df02d74a2cd3c1fa67714 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.py @@ -0,0 +1,8 @@ +# Generated content DO NOT EDIT +from .. import trainers + +Trainer = trainers.Trainer +BpeTrainer = trainers.BpeTrainer +UnigramTrainer = trainers.UnigramTrainer +WordLevelTrainer = trainers.WordLevelTrainer +WordPieceTrainer = trainers.WordPieceTrainer diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.pyi b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..f296988dcaf70cbe6fe5e2d9b99b12d80b8394b1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__init__.pyi @@ -0,0 +1,462 @@ +# Generated content DO NOT EDIT +class Trainer: + """ + Base class for all trainers + + This class is not supposed to be instantiated directly. Instead, any implementation of a + Trainer will return an instance of this class when instantiated. + """ + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + +class BpeTrainer(Trainer): + """ + Trainer capable of training a BPE model + + Args: + vocab_size (:obj:`int`, `optional`): + The size of the final vocabulary, including all tokens and alphabet. + + min_frequency (:obj:`int`, `optional`): + The minimum frequency a pair should have in order to be merged. + + show_progress (:obj:`bool`, `optional`): + Whether to show progress bars while training. + + special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): + A list of special tokens the model should know of. + + limit_alphabet (:obj:`int`, `optional`): + The maximum different characters to keep in the alphabet. + + initial_alphabet (:obj:`List[str]`, `optional`): + A list of characters to include in the initial alphabet, even + if not seen in the training dataset. + If the strings contain more than one character, only the first one + is kept. + + continuing_subword_prefix (:obj:`str`, `optional`): + A prefix to be used for every subword that is not a beginning-of-word. + + end_of_word_suffix (:obj:`str`, `optional`): + A suffix to be used for every subword that is a end-of-word. + + max_token_length (:obj:`int`, `optional`): + Prevents creating tokens longer than the specified size. + This can help with reducing polluting your vocabulary with + highly repetitive tokens like `======` for wikipedia + + """ + def __init__( + self, + vocab_size=30000, + min_frequency=0, + show_progress=True, + special_tokens=[], + limit_alphabet=None, + initial_alphabet=[], + continuing_subword_prefix=None, + end_of_word_suffix=None, + max_token_length=None, + words={}, + ): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def continuing_subword_prefix(self): + """ """ + pass + + @continuing_subword_prefix.setter + def continuing_subword_prefix(self, value): + """ """ + pass + + @property + def end_of_word_suffix(self): + """ """ + pass + + @end_of_word_suffix.setter + def end_of_word_suffix(self, value): + """ """ + pass + + @property + def initial_alphabet(self): + """ """ + pass + + @initial_alphabet.setter + def initial_alphabet(self, value): + """ """ + pass + + @property + def limit_alphabet(self): + """ """ + pass + + @limit_alphabet.setter + def limit_alphabet(self, value): + """ """ + pass + + @property + def max_token_length(self): + """ """ + pass + + @max_token_length.setter + def max_token_length(self, value): + """ """ + pass + + @property + def min_frequency(self): + """ """ + pass + + @min_frequency.setter + def min_frequency(self, value): + """ """ + pass + + @property + def show_progress(self): + """ """ + pass + + @show_progress.setter + def show_progress(self, value): + """ """ + pass + + @property + def special_tokens(self): + """ """ + pass + + @special_tokens.setter + def special_tokens(self, value): + """ """ + pass + + @property + def vocab_size(self): + """ """ + pass + + @vocab_size.setter + def vocab_size(self, value): + """ """ + pass + +class UnigramTrainer(Trainer): + """ + Trainer capable of training a Unigram model + + Args: + vocab_size (:obj:`int`): + The size of the final vocabulary, including all tokens and alphabet. + + show_progress (:obj:`bool`): + Whether to show progress bars while training. + + special_tokens (:obj:`List[Union[str, AddedToken]]`): + A list of special tokens the model should know of. + + initial_alphabet (:obj:`List[str]`): + A list of characters to include in the initial alphabet, even + if not seen in the training dataset. + If the strings contain more than one character, only the first one + is kept. + + shrinking_factor (:obj:`float`): + The shrinking factor used at each step of the training to prune the + vocabulary. + + unk_token (:obj:`str`): + The token used for out-of-vocabulary tokens. + + max_piece_length (:obj:`int`): + The maximum length of a given token. + + n_sub_iterations (:obj:`int`): + The number of iterations of the EM algorithm to perform before + pruning the vocabulary. + """ + def __init__( + self, + vocab_size=8000, + show_progress=True, + special_tokens=[], + initial_alphabet=[], + shrinking_factor=0.75, + unk_token=None, + max_piece_length=16, + n_sub_iterations=2, + ): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def initial_alphabet(self): + """ """ + pass + + @initial_alphabet.setter + def initial_alphabet(self, value): + """ """ + pass + + @property + def show_progress(self): + """ """ + pass + + @show_progress.setter + def show_progress(self, value): + """ """ + pass + + @property + def special_tokens(self): + """ """ + pass + + @special_tokens.setter + def special_tokens(self, value): + """ """ + pass + + @property + def vocab_size(self): + """ """ + pass + + @vocab_size.setter + def vocab_size(self, value): + """ """ + pass + +class WordLevelTrainer(Trainer): + """ + Trainer capable of training a WorldLevel model + + Args: + vocab_size (:obj:`int`, `optional`): + The size of the final vocabulary, including all tokens and alphabet. + + min_frequency (:obj:`int`, `optional`): + The minimum frequency a pair should have in order to be merged. + + show_progress (:obj:`bool`, `optional`): + Whether to show progress bars while training. + + special_tokens (:obj:`List[Union[str, AddedToken]]`): + A list of special tokens the model should know of. + """ + def __init__(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[]): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def min_frequency(self): + """ """ + pass + + @min_frequency.setter + def min_frequency(self, value): + """ """ + pass + + @property + def show_progress(self): + """ """ + pass + + @show_progress.setter + def show_progress(self, value): + """ """ + pass + + @property + def special_tokens(self): + """ """ + pass + + @special_tokens.setter + def special_tokens(self, value): + """ """ + pass + + @property + def vocab_size(self): + """ """ + pass + + @vocab_size.setter + def vocab_size(self, value): + """ """ + pass + +class WordPieceTrainer(Trainer): + """ + Trainer capable of training a WordPiece model + + Args: + vocab_size (:obj:`int`, `optional`): + The size of the final vocabulary, including all tokens and alphabet. + + min_frequency (:obj:`int`, `optional`): + The minimum frequency a pair should have in order to be merged. + + show_progress (:obj:`bool`, `optional`): + Whether to show progress bars while training. + + special_tokens (:obj:`List[Union[str, AddedToken]]`, `optional`): + A list of special tokens the model should know of. + + limit_alphabet (:obj:`int`, `optional`): + The maximum different characters to keep in the alphabet. + + initial_alphabet (:obj:`List[str]`, `optional`): + A list of characters to include in the initial alphabet, even + if not seen in the training dataset. + If the strings contain more than one character, only the first one + is kept. + + continuing_subword_prefix (:obj:`str`, `optional`): + A prefix to be used for every subword that is not a beginning-of-word. + + end_of_word_suffix (:obj:`str`, `optional`): + A suffix to be used for every subword that is a end-of-word. + """ + def __init__( + self, + vocab_size=30000, + min_frequency=0, + show_progress=True, + special_tokens=[], + limit_alphabet=None, + initial_alphabet=[], + continuing_subword_prefix="##", + end_of_word_suffix=None, + ): + pass + + def __getstate__(self): + """ """ + pass + + def __setstate__(self, state): + """ """ + pass + + @property + def continuing_subword_prefix(self): + """ """ + pass + + @continuing_subword_prefix.setter + def continuing_subword_prefix(self, value): + """ """ + pass + + @property + def end_of_word_suffix(self): + """ """ + pass + + @end_of_word_suffix.setter + def end_of_word_suffix(self, value): + """ """ + pass + + @property + def initial_alphabet(self): + """ """ + pass + + @initial_alphabet.setter + def initial_alphabet(self, value): + """ """ + pass + + @property + def limit_alphabet(self): + """ """ + pass + + @limit_alphabet.setter + def limit_alphabet(self, value): + """ """ + pass + + @property + def min_frequency(self): + """ """ + pass + + @min_frequency.setter + def min_frequency(self, value): + """ """ + pass + + @property + def show_progress(self): + """ """ + pass + + @show_progress.setter + def show_progress(self, value): + """ """ + pass + + @property + def special_tokens(self): + """ """ + pass + + @special_tokens.setter + def special_tokens(self, value): + """ """ + pass + + @property + def vocab_size(self): + """ """ + pass + + @vocab_size.setter + def vocab_size(self, value): + """ """ + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__pycache__/__init__.cpython-312.pyc b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2009d77920671a3834abb8f71985b398c5d3b092 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/tokenizers/trainers/__pycache__/__init__.cpython-312.pyc differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..85b32dbbfe04a286ec3935a852422cbf9a76cdbb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/__init__.py @@ -0,0 +1,299 @@ +r""" +This package introduces support for the current :ref:`accelerator` in python. +""" + +from functools import cache +from typing import Any +from typing_extensions import deprecated + +import torch + +from ._utils import _device_t, _get_device_index +from .memory import ( + empty_cache, + get_memory_info, + max_memory_allocated, + max_memory_reserved, + memory_allocated, + memory_reserved, + memory_stats, + reset_accumulated_memory_stats, + reset_peak_memory_stats, +) + + +__all__ = [ + "current_accelerator", + "current_device_idx", # deprecated + "current_device_index", + "get_device_capability", + "current_stream", + "device_count", + "device_index", + "empty_cache", + "get_memory_info", + "is_available", + "max_memory_allocated", + "max_memory_reserved", + "memory_allocated", + "memory_reserved", + "memory_stats", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", + "set_device_idx", # deprecated + "set_device_index", + "set_stream", + "synchronize", +] + + +def device_count() -> int: + r"""Return the number of current :ref:`accelerator` available. + + Returns: + int: the number of the current :ref:`accelerator` available. + If there is no available accelerators, return 0. + + .. note:: This API delegates to the device-specific version of `device_count`. + On CUDA, this API will NOT poison fork if NVML discovery succeeds. + Otherwise, it will. For more details, see :ref:`multiprocessing-poison-fork-note`. + """ + acc = current_accelerator() + if acc is None: + return 0 + + mod = torch.get_device_module(acc) + return mod.device_count() + + +def is_available() -> bool: + r"""Check if the current accelerator is available at runtime: it was build, all the + required drivers are available and at least one device is visible. + See :ref:`accelerator` for details. + + Returns: + bool: A boolean indicating if there is an available :ref:`accelerator`. + + .. note:: This API delegates to the device-specific version of `is_available`. + On CUDA, when the environment variable ``PYTORCH_NVML_BASED_CUDA_CHECK=1`` is set, + this function will NOT poison fork. Otherwise, it will. For more details, see + :ref:`multiprocessing-poison-fork-note`. + + Example:: + + >>> assert torch.accelerator.is_available() "No available accelerators detected." + """ + # Why not just check "device_count() > 0" like other is_available call? + # Because device like CUDA have a python implementation of is_available that is + # non-poisoning and some features like Dataloader rely on it. + # So we are careful to delegate to the Python version of the accelerator here + acc = current_accelerator() + if acc is None: + return False + + mod = torch.get_device_module(acc) + return mod.is_available() + + +def current_accelerator(check_available: bool = False) -> torch.device | None: + r"""Return the device of the accelerator available at compilation time. + If no accelerator were available at compilation time, returns None. + See :ref:`accelerator` for details. + + Args: + check_available (bool, optional): if True, will also do a runtime check to see + if the device :func:`torch.accelerator.is_available` on top of the compile-time + check. + Default: ``False`` + + Returns: + torch.device: return the current accelerator as :class:`torch.device`. + + .. note:: The index of the returned :class:`torch.device` will be ``None``, please use + :func:`torch.accelerator.current_device_index` to know the current index being used. + This API does NOT poison fork. For more details, see :ref:`multiprocessing-poison-fork-note`. + + Example:: + + >>> # xdoctest: + >>> # If an accelerator is available, sent the model to it + >>> model = torch.nn.Linear(2, 2) + >>> if (current_device := current_accelerator(check_available=True)) is not None: + >>> model.to(current_device) + """ + if (acc := torch._C._accelerator_getAccelerator()) is not None: + if (not check_available) or (check_available and is_available()): + return acc + return None + + +def current_device_index() -> int: + r"""Return the index of a currently selected device for the current :ref:`accelerator`. + + Returns: + int: the index of a currently selected device. + """ + return torch._C._accelerator_getDeviceIndex() + + +current_device_idx = deprecated( + "Use `current_device_index` instead.", + category=FutureWarning, +)(current_device_index) + +current_device_idx.__doc__ = r""" + (Deprecated) Return the index of a currently selected device for the current :ref:`accelerator`. + + Returns: + int: the index of a currently selected device. + + .. warning:: + + :func:`torch.accelerator.current_device_idx` is deprecated in favor of :func:`torch.accelerator.current_device_index` + and will be removed in a future PyTorch release. + """ + + +@cache +def get_device_capability(device: _device_t = None, /) -> dict[str, Any]: + r"""Return the capability of the currently selected device. + + Args: + device (:class:`torch.device`, str, int, optional): The device to query capabilities for + :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + Returns: + dict[str, Any]: A dictionary containing device capability information. The dictionary includes: + - ``supported_dtypes`` (set(torch.dtype)): Set of PyTorch data types supported by the device + + Examples: + >>> # xdoctest: +SKIP("requires cuda") + >>> # Query capabilities for current device + >>> capabilities = torch.accelerator.get_device_capability("cuda:0") + >>> print("Supported dtypes:", capabilities["supported_dtypes"]) + """ + device_index = _get_device_index(device, optional=True) + return torch._C._accelerator_getDeviceCapability(device_index) + + +def set_device_index(device: _device_t, /) -> None: + r"""Set the current device index to a given device. + + Args: + device (:class:`torch.device`, str, int): a given device that must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if this device index is negative. + """ + device_index = _get_device_index(device, optional=False) + torch._C._accelerator_setDeviceIndex(device_index) + + +set_device_idx = deprecated( + "Use `set_device_index` instead.", + category=FutureWarning, +)(set_device_index) + +set_device_idx.__doc__ = r""" + (Deprecated) Set the current device index to a given device. + + Args: + device (:class:`torch.device`, str, int): a given device that must match the current + :ref:`accelerator` device type. + + .. warning:: + + :func:`torch.accelerator.set_device_idx` is deprecated in favor of :func:`torch.accelerator.set_device_index` + and will be removed in a future PyTorch release. + """ + + +def current_stream(device: _device_t = None, /) -> torch.Stream: + r"""Return the currently selected stream for a given device. + + Args: + device (:class:`torch.device`, str, int, optional): a given device that must match the current + :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + Returns: + torch.Stream: the currently selected stream for a given device. + """ + device_index = _get_device_index(device, optional=True) + return torch._C._accelerator_getStream(device_index) + + +def set_stream(stream: torch.Stream) -> None: + r"""Set the current stream to a given stream. + + Args: + stream (torch.Stream): a given stream that must match the current :ref:`accelerator` device type. + + .. note:: This function will set the current device index to the device index of the given stream. + """ + torch._C._accelerator_setStream(stream) + + +def synchronize(device: _device_t = None, /) -> None: + r"""Wait for all kernels in all streams on the given device to complete. + + Args: + device (:class:`torch.device`, str, int, optional): device for which to synchronize. It must match + the current :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + .. note:: This function is a no-op if the current :ref:`accelerator` is not initialized. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> assert torch.accelerator.is_available() "No available accelerators detected." + >>> start_event = torch.Event(enable_timing=True) + >>> end_event = torch.Event(enable_timing=True) + >>> start_event.record() + >>> tensor = torch.randn(100, device=torch.accelerator.current_accelerator()) + >>> sum = torch.sum(tensor) + >>> end_event.record() + >>> torch.accelerator.synchronize() + >>> elapsed_time_ms = start_event.elapsed_time(end_event) + """ + device_index = _get_device_index(device, optional=True) + torch._C._accelerator_synchronizeDevice(device_index) + + +class device_index: + r"""Context manager to set the current device index for the current :ref:`accelerator`. + Temporarily changes the current device index to the specified value for the duration + of the context, and automatically restores the previous device index when exiting + the context. + + Args: + device (Optional[int]): a given device index to temporarily set. If None, + no device index switching occurs. + + Examples: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # Set device 0 as the current device temporarily + >>> with torch.accelerator.device_index(0): + ... # Code here runs with device 0 as the current device + ... pass + >>> # Original device is now restored + >>> # No-op when None is passed + >>> with torch.accelerator.device_index(None): + ... # No device switching occurs + ... pass + """ + + def __init__(self, device: int | None, /) -> None: + self.idx = device + self.prev_idx = -1 + + def __enter__(self) -> None: + if self.idx is not None: + self.prev_idx = torch._C._accelerator_exchangeDevice(self.idx) + + def __exit__(self, *exc_info: object) -> None: + if self.idx is not None: + torch._C._accelerator_maybeExchangeDevice(self.prev_idx) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f811b7c29bb362a499417d51da6c92fd6a14884c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/_utils.py @@ -0,0 +1,26 @@ +import torch +from torch.types import Device as _device_t + + +def _get_device_index(device: _device_t, optional: bool = False) -> int: + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + device_index: int | None = None + if isinstance(device, torch.device): + acc = torch.accelerator.current_accelerator() + if acc is None: + raise RuntimeError("Accelerator expected") + if acc.type != device.type: + raise ValueError( + f"{device.type} doesn't match the current accelerator {acc}." + ) + device_index = device.index + if device_index is None: + if not optional: + raise ValueError( + f"Expected a torch.device with a specified index or an integer, but got:{device}" + ) + return torch.accelerator.current_device_index() + return device_index diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/memory.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..de666e39e4d0b847135298097f23048cf3d25e52 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/accelerator/memory.py @@ -0,0 +1,236 @@ +from collections import OrderedDict +from typing import Any + +import torch + +from ._utils import _device_t, _get_device_index + + +__all__ = [ + "empty_cache", + "get_memory_info", + "max_memory_allocated", + "max_memory_reserved", + "memory_allocated", + "memory_reserved", + "memory_stats", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", +] + + +def empty_cache() -> None: + r"""Release all unoccupied cached memory currently held by the caching + allocator so that those can be used in other application. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + if not torch._C._accelerator_isAllocatorInitialized(): + return + torch._C._accelerator_emptyCache() + + +def memory_stats(device_index: _device_t = None, /) -> OrderedDict[str, Any]: + r"""Return a dictionary of accelerator device memory allocator statistics for a given device index. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of reserved segments from device memory allocation. + - ``"reserved_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of reserved memory. + - ``"active.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of active memory blocks. + - ``"active_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of active memory. + - ``"inactive_split.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of inactive, non-releasable memory blocks. + - ``"inactive_split_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of inactive, non-releasable memory. + + For these core statistics, values are broken down as follows. + + Pool type: + + - ``all``: combined statistics across all memory pools. + - ``large_pool``: statistics for the large allocation pool + (as of June 2025, for size >= 1MB allocations). + - ``small_pool``: statistics for the small allocation pool + (as of June 2025, for size < 1MB allocations). + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_alloc_retries"``: number of failed device memory allocation calls that + result in a cache flush and retry. + - ``"num_ooms"``: number of out-of-memory errors thrown. + - ``"num_sync_all_streams"``: number of ``synchronize_and_free_events`` calls. + - ``"num_device_alloc"``: number of device memory allocation calls. + - ``"num_device_free"``: number of device memory free calls. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + OrderedDict[str, Any]: an ordered dictionary mapping statistic names to their values. + """ + if not torch._C._accelerator_isAllocatorInitialized(): + return OrderedDict() + device_index = _get_device_index(device_index, optional=True) + stats = torch._C._accelerator_getDeviceStats(device_index) + flat_stats = [] + + def flatten(prefix: str, value: Any) -> None: + if isinstance(value, dict): + for k, v in value.items(): + nested_prefix = f"{prefix}.{k}" if prefix else k + flatten(nested_prefix, v) + else: + flat_stats.append((prefix, value)) + + flatten("", stats) + flat_stats.sort() + # pyrefly: ignore [no-matching-overload] + return OrderedDict(flat_stats) + + +def memory_allocated(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` device memory occupied by tensors + in bytes for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the current memory occupied by live tensors (in bytes) within the current process. + """ + return memory_stats(device_index).get("allocated_bytes.all.current", 0) + + +def max_memory_allocated(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` maximum device memory occupied by tensors + in bytes for a given device index. + + By default, this returns the peak allocated memory since the beginning of + this program. :func:`~torch.accelerator.reset_peak_memory_stats` can be used to + reset the starting point in tracking this metric. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the peak memory occupied by live tensors (in bytes) within the current process. + """ + return memory_stats(device_index).get("allocated_bytes.all.peak", 0) + + +def memory_reserved(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` device memory managed by the caching allocator + in bytes for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the current memory reserved by PyTorch (in bytes) within the current process. + """ + return memory_stats(device_index).get("reserved_bytes.all.current", 0) + + +def max_memory_reserved(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` maximum device memory managed by the caching allocator + in bytes for a given device index. + + By default, this returns the peak cached memory since the beginning of this + program. :func:`~torch.accelerator.reset_peak_memory_stats` can be used to reset + the starting point in tracking this metric. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the peak memory reserved by PyTorch (in bytes) within the current process. + """ + return memory_stats(device_index).get("reserved_bytes.all.peak", 0) + + +def reset_accumulated_memory_stats(device_index: _device_t = None, /) -> None: + r"""Reset the "accumulated" (historical) stats tracked by the current :ref:`accelerator` + memory allocator for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_resetAccumulatedStats(device_index) + + +def reset_peak_memory_stats(device_index: _device_t = None, /) -> None: + r"""Reset the "peak" stats tracked by the current :ref:`accelerator` + memory allocator for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_resetPeakStats(device_index) + + +def get_memory_info(device_index: _device_t = None, /) -> tuple[int, int]: + r"""Return the current device memory information for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + tuple[int, int]: a tuple of two integers (free_memory, total_memory) in bytes. + The first value is the free memory on the device (available across all processes and applications), + The second value is the device's total hardware memory capacity. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_getMemoryInfo(device_index) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..83fd1cd1484034e19b407f6f665a6132b562dde5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/__init__.py @@ -0,0 +1,9 @@ +from .autocast_mode import ( + _enter_autocast, + _exit_autocast, + autocast, + custom_bwd, + custom_fwd, + is_autocast_available, +) +from .grad_scaler import GradScaler diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/autocast_mode.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/autocast_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..8ec7840ec8cb39a772a87e37c2d80b618180c9a2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/autocast_mode.py @@ -0,0 +1,525 @@ +# mypy: allow-untyped-defs +import collections +import functools +import warnings +from typing import Any, Optional + +import torch +from torch.types import _dtype + + +try: + import numpy as np + + HAS_NUMPY = True +except ModuleNotFoundError: + HAS_NUMPY = False + np = None # type: ignore[assignment] + +__all__ = [ + "autocast_decorator", + "autocast", + "is_autocast_available", + "custom_fwd", + "custom_bwd", +] + + +def is_autocast_available(device_type: str) -> bool: + r""" + Return a bool indicating if autocast is available on :attr:`device_type`. + + Args: + device_type(str): Device type to use. Possible values are: 'cuda', 'cpu', 'mtia', 'maia', 'xpu', and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + """ + return torch._C._is_autocast_available(device_type) + + +def autocast_decorator(autocast_instance, func): + @functools.wraps(func) + def decorate_autocast(*args, **kwargs): + with autocast_instance: + return func(*args, **kwargs) + + decorate_autocast.__script_unsupported = ( # type: ignore[attr-defined] + "@autocast() decorator is not supported in script mode" + ) + return decorate_autocast + + +class autocast: + r""" + Instances of :class:`autocast` serve as context managers or decorators that + allow regions of your script to run in mixed precision. + + In these regions, ops run in an op-specific dtype chosen by autocast + to improve performance while maintaining accuracy. + See the :ref:`Autocast Op Reference` for details. + + When entering an autocast-enabled region, Tensors may be any type. + You should not call ``half()`` or ``bfloat16()`` on your model(s) or inputs when using autocasting. + + :class:`autocast` should wrap only the forward pass(es) of your network, including the loss + computation(s). Backward passes under autocast are not recommended. + Backward ops run in the same type that autocast used for corresponding forward ops. + + Example for CUDA Devices:: + + # Creates model and optimizer in default precision + model = Net().cuda() + optimizer = optim.SGD(model.parameters(), ...) + + for input, target in data: + optimizer.zero_grad() + + # Enables autocasting for the forward pass (model + loss) + with torch.autocast(device_type="cuda"): + output = model(input) + loss = loss_fn(output, target) + + # Exits the context manager before backward() + loss.backward() + optimizer.step() + + See the :ref:`Automatic Mixed Precision examples` for usage (along with gradient scaling) + in more complex scenarios (e.g., gradient penalty, multiple models/losses, custom autograd functions). + + :class:`autocast` can also be used as a decorator, e.g., on the ``forward`` method of your model:: + + class AutocastModel(nn.Module): + ... + + @torch.autocast(device_type="cuda") + def forward(self, input): ... + + Floating-point Tensors produced in an autocast-enabled region may be ``float16``. + After returning to an autocast-disabled region, using them with floating-point + Tensors of different dtypes may cause type mismatch errors. If so, cast the Tensor(s) + produced in the autocast region back to ``float32`` (or other dtype if desired). + If a Tensor from the autocast region is already ``float32``, the cast is a no-op, + and incurs no additional overhead. + CUDA Example:: + + # Creates some tensors in default dtype (here assumed to be float32) + a_float32 = torch.rand((8, 8), device="cuda") + b_float32 = torch.rand((8, 8), device="cuda") + c_float32 = torch.rand((8, 8), device="cuda") + d_float32 = torch.rand((8, 8), device="cuda") + + with torch.autocast(device_type="cuda"): + # torch.mm is on autocast's list of ops that should run in float16. + # Inputs are float32, but the op runs in float16 and produces float16 output. + # No manual casts are required. + e_float16 = torch.mm(a_float32, b_float32) + # Also handles mixed input types + f_float16 = torch.mm(d_float32, e_float16) + + # After exiting autocast, calls f_float16.float() to use with d_float32 + g_float32 = torch.mm(d_float32, f_float16.float()) + + CPU Training Example:: + + # Creates model and optimizer in default precision + model = Net() + optimizer = optim.SGD(model.parameters(), ...) + + for epoch in epochs: + for input, target in data: + optimizer.zero_grad() + + # Runs the forward pass with autocasting. + with torch.autocast(device_type="cpu", dtype=torch.bfloat16): + output = model(input) + loss = loss_fn(output, target) + + loss.backward() + optimizer.step() + + + CPU Inference Example:: + + # Creates model in default precision + model = Net().eval() + + with torch.autocast(device_type="cpu", dtype=torch.bfloat16): + for input in data: + # Runs the forward pass with autocasting. + output = model(input) + + CPU Inference Example with Jit Trace:: + + class TestModel(nn.Module): + def __init__(self, input_size, num_classes): + super().__init__() + self.fc1 = nn.Linear(input_size, num_classes) + + def forward(self, x): + return self.fc1(x) + + + input_size = 2 + num_classes = 2 + model = TestModel(input_size, num_classes).eval() + + # For now, we suggest to disable the Jit Autocast Pass, + # As the issue: https://github.com/pytorch/pytorch/issues/75956 + torch._C._jit_set_autocast_mode(False) + + with torch.cpu.amp.autocast(cache_enabled=False): + model = torch.jit.trace(model, torch.randn(1, input_size)) + model = torch.jit.freeze(model) + # Models Run + for _ in range(3): + model(torch.randn(1, input_size)) + + Type mismatch errors *in* an autocast-enabled region are a bug; if this is what you observe, + please file an issue. + + ``autocast(enabled=False)`` subregions can be nested in autocast-enabled regions. + Locally disabling autocast can be useful, for example, if you want to force a subregion + to run in a particular ``dtype``. Disabling autocast gives you explicit control over + the execution type. In the subregion, inputs from the surrounding region + should be cast to ``dtype`` before use:: + + # Creates some tensors in default dtype (here assumed to be float32) + a_float32 = torch.rand((8, 8), device="cuda") + b_float32 = torch.rand((8, 8), device="cuda") + c_float32 = torch.rand((8, 8), device="cuda") + d_float32 = torch.rand((8, 8), device="cuda") + + with torch.autocast(device_type="cuda"): + e_float16 = torch.mm(a_float32, b_float32) + with torch.autocast(device_type="cuda", enabled=False): + # Calls e_float16.float() to ensure float32 execution + # (necessary because e_float16 was created in an autocasted region) + f_float32 = torch.mm(c_float32, e_float16.float()) + + # No manual casts are required when re-entering the autocast-enabled region. + # torch.mm again runs in float16 and produces float16 output, regardless of input types. + g_float16 = torch.mm(d_float32, f_float32) + + The autocast state is thread-local. If you want it enabled in a new thread, the context manager or decorator + must be invoked in that thread. This affects :class:`torch.nn.DataParallel` and + :class:`torch.nn.parallel.DistributedDataParallel` when used with more than one GPU per process + (see :ref:`Working with Multiple GPUs`). + + Args: + device_type(str, required): Device type to use. Possible values are: 'cuda', 'cpu', 'mtia', 'maia', 'xpu', and 'hpu'. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + enabled(bool, optional): Whether autocasting should be enabled in the region. + Default: ``True`` + dtype(torch_dtype, optional): Data type for ops run in autocast. It uses the default value + (``torch.float16`` for CUDA and ``torch.bfloat16`` for CPU), given by + :func:`~torch.get_autocast_dtype`, if :attr:`dtype` is ``None``. + Default: ``None`` + cache_enabled(bool, optional): Whether the weight cache inside autocast should be enabled. + Default: ``True`` + """ + + def __init__( + self, + device_type: str, + dtype: Optional[_dtype] = None, + enabled: bool = True, + cache_enabled: Optional[bool] = None, + ): + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + self.fast_dtype = ( + torch.get_autocast_dtype(device_type) if dtype is None else dtype + ) + if torch._jit_internal.is_scripting(): + self._enabled = enabled + self.device = device_type + assert self.fast_dtype is not None + return + self.device = device_type + if not is_autocast_available(self.device): + raise RuntimeError( + f"User specified an unsupported autocast device_type '{self.device}'" + ) + + device_supported_dtypes = [torch.bfloat16, torch.float16] + + self.custom_backend_name = torch._C._get_privateuse1_backend_name() + if self.device == self.custom_backend_name: + necessary_funcs = [ + "get_amp_supported_dtype", + ] + message = f"Tried to use AMP with the `{self.custom_backend_name}` backend, but the backend has not " + message += "registered a module or the module miss some necessary funcs. The backend should register " + message += "a module by `torch._register_device_module`, and the module must have these funcs: \n" + message += "`get_amp_supported_dtype() -> List[torch.dtype]`. \n" + + assert hasattr(torch, self.custom_backend_name), message + self.custom_device_mod = getattr(torch, self.custom_backend_name) + for func in necessary_funcs: + assert hasattr(self.custom_device_mod, func), ( + message + f"But the func `{func}` is missing. \n" + ) + device_supported_dtypes = self.custom_device_mod.get_amp_supported_dtype() + + self._cache_enabled = ( + torch.is_autocast_cache_enabled() + if cache_enabled is None + else cache_enabled + ) + + device_name = ( + self.device + if self.device == self.custom_backend_name + else self.device.upper() + ) + if enabled: + # Special case for CUDA AMP and bfloat16 support + if self.device == "cuda": + if torch.cuda.amp.common.amp_definitely_not_available(): + warnings.warn( + "CUDA is not available or torch_xla is imported. Disabling autocast.", + stacklevel=2, + ) + enabled = False + elif ( + self.fast_dtype == torch.bfloat16 + and not torch.cuda.is_bf16_supported() + ): + raise RuntimeError( + "Current CUDA Device does not support bfloat16. Please switch dtype to float16." + ) + elif self.fast_dtype not in device_supported_dtypes: + error_message = ( + f"In {device_name} autocast, but the target dtype is not supported. Disabling autocast.\n" + f"{device_name} Autocast only supports dtypes of " + + ", ".join(map(str, device_supported_dtypes)) + + " currently." + ) + warnings.warn(error_message, stacklevel=2) + enabled = False + # Special case for MPS bfloat16 support on macOS < 14 + if ( + self.device == "mps" + and self.fast_dtype == torch.bfloat16 + and not torch.backends.mps.is_macos_or_newer(14, 0) + ): + error_message = ( + "In MPS autocast, but the target dtype torch.bfloat16 is not supported " + "on macOS versions below 14. Disabling autocast." + ) + warnings.warn(error_message, stacklevel=2) + enabled = False + self._enabled = enabled + + def __enter__(self): + if torch._jit_internal.is_scripting(): + assert self.fast_dtype is not None + return self + + self.prev_cache_enabled = torch.is_autocast_cache_enabled() + self.prev = torch.is_autocast_enabled(self.device) + self.prev_fastdtype = torch.get_autocast_dtype(self.device) + torch.set_autocast_enabled(self.device, self._enabled) + torch.set_autocast_dtype(self.device, self.fast_dtype) # type: ignore[arg-type] + torch.autocast_increment_nesting() + torch.set_autocast_cache_enabled(self._cache_enabled) + + # only dispatch to PreDispatchTorchFunctionMode to avoid exposing this + # API to other functional modes. We only expose to PreDispatchTorchFunctionMode + # for preserving autocast in torch.export.export. + if torch._C._is_torch_function_mode_enabled(): + stacks = torch.overrides._get_current_function_mode_stack() + for mode in stacks: + if isinstance( + mode, + torch.fx.experimental.proxy_tensor.PreDispatchTorchFunctionMode, + ): + args = ( + self.device, + self.fast_dtype, + self._enabled, + self._cache_enabled, + ) + mode.__torch_function__(torch.amp._enter_autocast, (), args) + return self + + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[override] + if torch._jit_internal.is_scripting(): + return + + # Drop the cache when we exit to a nesting level that's outside any instance of autocast. + if torch.autocast_decrement_nesting() == 0: + torch.clear_autocast_cache() + torch.set_autocast_enabled(self.device, self.prev) + torch.set_autocast_dtype(self.device, self.prev_fastdtype) + torch.set_autocast_cache_enabled(self.prev_cache_enabled) + + # only dispatch to PreDispatchTorchFunctionMode to avoid exposing this + # API to other functional modes. We only expose to PreDispatchTorchFunctionMode + # for preserving autocast in torch.export.export. + if torch._C._is_torch_function_mode_enabled(): + stacks = torch.overrides._get_current_function_mode_stack() + for mode in stacks: + if isinstance( + mode, + torch.fx.experimental.proxy_tensor.PreDispatchTorchFunctionMode, + ): + mode.__torch_function__(torch.amp._exit_autocast, (), ()) + # This is very important because the above line actually doesn't + # run exit code so it end up swallowing exceptions. + return False + return False + + def __call__(self, func): + if torch._jit_internal.is_scripting(): + return func + return autocast_decorator(self, func) + + +# These functions aren't meant for public usage. +# They are what we trace into a graph during pre_dispatch tracing +# when we encounter an autocast context manager. +def _enter_autocast(*vals): + # For pre-dispatch tracing, if a TorchFunction mode is active, we'll want to trace this into a graph. + if torch._C._is_torch_function_mode_enabled(): + return torch.overrides.handle_torch_function( + torch.amp._enter_autocast, [], *vals + ) + mode = torch.amp.autocast(*vals) + mode.__enter__() + return mode + + +def _exit_autocast(mode): + if torch._C._is_torch_function_mode_enabled(): + return torch.overrides.handle_torch_function(torch.amp._exit_autocast, [], mode) + mode.__exit__(None, None, None) + + +# Casts Tensors and containers of Tensors. Special-cases passthroughs for strings and np.ndarrays, which +# may be falsely detected as "Iterables." +def _cast(value, device_type: str, dtype: _dtype): + if isinstance(value, torch.Tensor): + is_eligible = ( + value.is_floating_point() + and value.device.type == device_type + and (value.dtype is not torch.float64) + ) + return value.to(dtype) if is_eligible else value + elif isinstance(value, (str, bytes)): + return value + elif HAS_NUMPY and isinstance( + value, + # pyrefly: ignore [missing-attribute] + np.ndarray, + ): + return value + elif isinstance(value, collections.abc.Mapping): + return { + _cast(k, device_type, dtype): _cast(v, device_type, dtype) + for k, v in value.items() + } + elif isinstance(value, collections.abc.Iterable): + iterable = (_cast(v, device_type, dtype) for v in value) + if isinstance(value, (list, tuple)): + return type(value)(iterable) + else: + return iterable + else: + return value + + +def custom_fwd( + fwd=None, + *, + device_type: str, + cast_inputs: Optional[_dtype] = None, +): + """ + Create a helper decorator for ``forward`` methods of custom autograd functions. + + Autograd functions are subclasses of :class:`torch.autograd.Function`. + See the :ref:`example page` for more detail. + + Args: + device_type(str): Device type to use. 'cuda', 'cpu', 'mtia', 'maia', 'xpu' and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + cast_inputs (:class:`torch.dtype` or None, optional, default=None): If not ``None``, + when ``forward`` runs in an autocast-enabled region, casts incoming + floating-point Tensors to the target dtype (non-floating-point Tensors are not affected), + then executes ``forward`` with autocast disabled. + If ``None``, ``forward``'s internal ops execute with the current autocast state. + + .. note:: + If the decorated ``forward`` is called outside an autocast-enabled region, + :func:`custom_fwd` is a no-op and ``cast_inputs`` has no effect. + """ + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + if fwd is None: + return functools.partial( + custom_fwd, device_type=device_type, cast_inputs=cast_inputs + ) + + @functools.wraps(fwd) + def decorate_fwd(*args, **kwargs): + args[0]._dtype = torch.get_autocast_dtype(device_type) + if cast_inputs is None: + args[0]._fwd_used_autocast = torch.is_autocast_enabled(device_type) + return fwd(*args, **kwargs) # pyrefly: ignore [not-callable] + else: + autocast_context = torch.is_autocast_enabled(device_type) + args[0]._fwd_used_autocast = False + if autocast_context: + with autocast(device_type=device_type, enabled=False): + return fwd( # pyrefly: ignore # not-callable + *_cast(args, device_type, cast_inputs), + **_cast(kwargs, device_type, cast_inputs), + ) + else: + return fwd(*args, **kwargs) # pyrefly: ignore [not-callable] + + return decorate_fwd + + +# Autograd ensures incoming gradients are the same type as forward outputs. Allowing a separate +# cast_inputs argument on custom_bwd is unnecessary and could cause errors if it doesn't match +# cast_inputs supplied to custom_fwd. +def custom_bwd(bwd=None, *, device_type: str): + """Create a helper decorator for backward methods of custom autograd functions. + + Autograd functions are subclasses of :class:`torch.autograd.Function`. + Ensures that ``backward`` executes with the same autocast state as ``forward``. + See the :ref:`example page` for more detail. + + Args: + device_type(str): Device type to use. 'cuda', 'cpu', 'mtia', 'maia', 'xpu' and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + """ + + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + if bwd is None: + return functools.partial(custom_bwd, device_type=device_type) + + @functools.wraps(bwd) + def decorate_bwd(*args, **kwargs): + with autocast( + device_type=device_type, + enabled=args[0]._fwd_used_autocast, + dtype=args[0]._dtype, + ): + return bwd(*args, **kwargs) # pyrefly: ignore [not-callable] + + return decorate_bwd diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/grad_scaler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/grad_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..bc009e77869db1186b9c40519b1d55fb143aeb97 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/amp/grad_scaler.py @@ -0,0 +1,693 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import inspect +import warnings +from collections import abc, defaultdict +from enum import Enum +from typing import Any, cast, Optional, overload, TYPE_CHECKING, Union + +import torch + + +if TYPE_CHECKING: + from collections.abc import Iterable + + +__all__ = ["OptState", "GradScaler"] + + +class _MultiDeviceReplicator: + """Lazily serves copies of a tensor to requested devices. + + Copies are cached per-device. + """ + + def __init__(self, master_tensor: torch.Tensor) -> None: + self.master = master_tensor + self._per_device_tensors: dict[torch.device, torch.Tensor] = {} + + def get(self, device: torch.device) -> torch.Tensor: + retval = self._per_device_tensors.get(device, None) + if retval is None: + retval = self.master.to(device=device, non_blocking=True, copy=True) + self._per_device_tensors[device] = retval + return retval + + +# Defines default_factory for GradScaler's _per_optimizer_states defaultdict, +# as well as associated "enum" values. Prefers defining these at top level because +# - Lambdas can't be pickled, so we don't want to supply a lambda as the factory. +# - Defining READY, UNSCALED, STEPPED and _refresh_per_optimizer_state within GradScaler +# causes a circular reference, which we'd rather avoid. +class OptState(Enum): + READY = 0 + UNSCALED = 1 + STEPPED = 2 + + +def _refresh_per_optimizer_state() -> dict[str, Any]: + return {"stage": OptState.READY, "found_inf_per_device": {}} + + +class GradScaler: + """An instance ``scaler`` of :class:`GradScaler`. + + Helps perform the steps of gradient scaling + conveniently. + + * ``scaler.scale(loss)`` multiplies a given loss by ``scaler``'s current scale factor. + * ``scaler.step(optimizer)`` safely unscales gradients and calls ``optimizer.step()``. + * ``scaler.update()`` updates ``scaler``'s scale factor. + + Example:: + + # Creates a GradScaler once at the beginning of training. + scaler = GradScaler() + + for epoch in epochs: + for input, target in data: + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + + # Scales loss. Calls backward() on scaled loss to create scaled gradients. + scaler.scale(loss).backward() + + # scaler.step() first unscales gradients of the optimizer's params. + # If gradients don't contain infs/NaNs, optimizer.step() is then called, + # otherwise, optimizer.step() is skipped. + scaler.step(optimizer) + + # Updates the scale for next iteration. + scaler.update() + + See the :ref:`Automatic Mixed Precision examples` for usage + (along with autocasting) in more complex cases like gradient clipping, gradient accumulation, gradient penalty, + and multiple losses/optimizers. + + ``scaler`` dynamically estimates the scale factor each iteration. To minimize gradient underflow, + a large scale factor should be used. However, ``float16`` values can "overflow" (become inf or NaN) if + the scale factor is too large. Therefore, the optimal scale factor is the largest factor that can be used + without incurring inf or NaN gradient values. + ``scaler`` approximates the optimal scale factor over time by checking the gradients for infs and NaNs during every + ``scaler.step(optimizer)`` (or optional separate ``scaler.unscale_(optimizer)``, see :meth:`unscale_`). + + * If infs/NaNs are found, ``scaler.step(optimizer)`` skips the underlying ``optimizer.step()`` (so the params + themselves remain uncorrupted) and ``update()`` multiplies the scale by ``backoff_factor``. + + * If no infs/NaNs are found, ``scaler.step(optimizer)`` runs the underlying ``optimizer.step()`` as usual. + If ``growth_interval`` unskipped iterations occur consecutively, ``update()`` multiplies the scale by + ``growth_factor``. + + The scale factor often causes infs/NaNs to appear in gradients for the first few iterations as its + value calibrates. ``scaler.step`` will skip the underlying ``optimizer.step()`` for these + iterations. After that, step skipping should occur rarely (once every few hundred or thousand iterations). + + Args: + device (str, optional, default="cuda"): Device type to use. Possible values are: 'cuda' and 'cpu'. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + init_scale (float, optional, default=2.**16): Initial scale factor. + growth_factor (float, optional, default=2.0): Factor by which the scale is multiplied during + :meth:`update` if no inf/NaN gradients occur for ``growth_interval`` consecutive iterations. + backoff_factor (float, optional, default=0.5): Factor by which the scale is multiplied during + :meth:`update` if inf/NaN gradients occur in an iteration. + growth_interval (int, optional, default=2000): Number of consecutive iterations without inf/NaN gradients + that must occur for the scale to be multiplied by ``growth_factor``. + enabled (bool, optional): If ``False``, disables gradient scaling. :meth:`step` simply + invokes the underlying ``optimizer.step()``, and other methods become no-ops. + Default: ``True`` + """ + + def __init__( + self, + device: str = "cuda", + init_scale: float = 2.0**16, + growth_factor: float = 2.0, + backoff_factor: float = 0.5, + growth_interval: int = 2000, + enabled: bool = True, + ) -> None: + self._device = device + self._enabled = enabled + if self._device == "cuda": + if enabled and torch.cuda.amp.common.amp_definitely_not_available(): + warnings.warn( + "torch.cuda.amp.GradScaler is enabled, but CUDA is not available. Disabling.", + stacklevel=2, + ) + self._enabled = False + + if self._enabled: + assert growth_factor > 1.0, "The growth factor must be > 1.0." + assert backoff_factor < 1.0, "The backoff factor must be < 1.0." + + self._init_scale = init_scale + # self._scale will be lazily initialized during the first call to scale() + self._scale: Optional[torch.Tensor] = None + self._growth_factor = growth_factor + self._backoff_factor = backoff_factor + self._growth_interval = growth_interval + self._init_growth_tracker = 0 + # self._growth_tracker will be lazily initialized during the first call to scale() + self._growth_tracker: Optional[torch.Tensor] = None + self._per_optimizer_states: dict[int, dict[str, Any]] = defaultdict( + _refresh_per_optimizer_state + ) + + def _check_scale_growth_tracker( + self, funcname: str + ) -> tuple[torch.Tensor, torch.Tensor]: + fix = "This may indicate your script did not use scaler.scale(loss or outputs) earlier in the iteration." + assert self._scale is not None, ( + f"Attempted {funcname} but _scale is None. " + fix + ) + assert self._growth_tracker is not None, ( + f"Attempted {funcname} but _growth_tracker is None. " + fix + ) + return (self._scale, self._growth_tracker) + + def _lazy_init_scale_growth_tracker(self, dev: torch.device) -> None: + assert self._growth_tracker is None, "_growth_tracker initialized before _scale" + self._scale = torch.full((), self._init_scale, dtype=torch.float32, device=dev) + self._growth_tracker = torch.full( + (), self._init_growth_tracker, dtype=torch.int32, device=dev + ) + + @overload + def scale(self, outputs: torch.Tensor) -> torch.Tensor: ... + + @overload + def scale(self, outputs: list[torch.Tensor]) -> list[torch.Tensor]: ... + + @overload + def scale(self, outputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]: ... + + @overload + def scale(self, outputs: Iterable[torch.Tensor]) -> Iterable[torch.Tensor]: ... + + def scale( + self, + outputs: Union[torch.Tensor, Iterable[torch.Tensor]], + ) -> Union[torch.Tensor, Iterable[torch.Tensor]]: + """ + Multiplies ('scales') a tensor or list of tensors by the scale factor. + + Returns scaled outputs. If this instance of :class:`GradScaler` is not enabled, outputs are returned + unmodified. + + Args: + outputs (Tensor or iterable of Tensors): Outputs to scale. + """ + if not self._enabled: + return outputs + + # Short-circuit for the common case. + if isinstance(outputs, torch.Tensor): + if self._scale is None: + self._lazy_init_scale_growth_tracker(outputs.device) + assert self._scale is not None + return outputs * self._scale.to(device=outputs.device, non_blocking=True) + + # Invoke the more complex machinery only if we're treating multiple outputs. + stash: list[ + _MultiDeviceReplicator + ] = [] # holds a reference that can be overwritten by apply_scale + + def apply_scale(val: Union[torch.Tensor, Iterable[torch.Tensor]]): + if isinstance(val, torch.Tensor): + if len(stash) == 0: + if self._scale is None: + self._lazy_init_scale_growth_tracker(val.device) + assert self._scale is not None + stash.append(_MultiDeviceReplicator(self._scale)) + return val * stash[0].get(val.device) + if isinstance(val, abc.Iterable): + iterable = map(apply_scale, val) + if isinstance(val, (list, tuple)): + return type(val)(iterable) + return iterable + raise ValueError("outputs must be a Tensor or an iterable of Tensors") + + return apply_scale(outputs) + + def _unscale_grads_( + self, + optimizer: torch.optim.Optimizer, + inv_scale: torch.Tensor, + found_inf: torch.Tensor, + allow_fp16: bool, + ) -> dict[torch.device, torch.Tensor]: + per_device_inv_scale = _MultiDeviceReplicator(inv_scale) + per_device_found_inf = _MultiDeviceReplicator(found_inf) + + # To set up _amp_foreach_non_finite_check_and_unscale_, split grads by device and dtype. + # There could be hundreds of grads, so we'd like to iterate through them just once. + # However, we don't know their devices or dtypes in advance. + + # https://stackoverflow.com/questions/5029934/defaultdict-of-defaultdict + # Google says mypy struggles with defaultdicts type annotations. + per_device_and_dtype_grads: dict[ + torch.device, dict[torch.dtype, list[torch.Tensor]] + ] = defaultdict(lambda: defaultdict(list)) + with torch.no_grad(): + for group in optimizer.param_groups: + for param in group["params"]: + assert isinstance(param, torch.Tensor) + if param.grad is None: + continue + if (not allow_fp16) and param.grad.dtype == torch.float16: + raise ValueError("Attempting to unscale FP16 gradients.") + if param.grad.is_sparse: + # is_coalesced() == False means the sparse grad has values with duplicate indices. + # coalesce() deduplicates indices and adds all values that have the same index. + # For scaled fp16 values, there's a good chance coalescing will cause overflow, + # so we should check the coalesced _values(). + if param.grad.dtype is torch.float16: + param.grad = param.grad.coalesce() + to_unscale = param.grad._values() + else: + to_unscale = param.grad + + # TODO: is there a way to split by device and dtype without appending in the inner loop? + per_device_and_dtype_grads[to_unscale.device][ + to_unscale.dtype + ].append(to_unscale) + + for device, per_dtype_grads in per_device_and_dtype_grads.items(): + for grads in per_dtype_grads.values(): + torch._amp_foreach_non_finite_check_and_unscale_( + grads, + per_device_found_inf.get(device), + per_device_inv_scale.get(device), + ) + + return per_device_found_inf._per_device_tensors + + def unscale_(self, optimizer: torch.optim.Optimizer) -> None: + """ + Divides ("unscales") the optimizer's gradient tensors by the scale factor. + + :meth:`unscale_` is optional, serving cases where you need to + :ref:`modify or inspect gradients` + between the backward pass(es) and :meth:`step`. + If :meth:`unscale_` is not called explicitly, gradients will be unscaled automatically during :meth:`step`. + + Simple example, using :meth:`unscale_` to enable clipping of unscaled gradients:: + + ... + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + scaler.step(optimizer) + scaler.update() + + Args: + optimizer (torch.optim.Optimizer): Optimizer that owns the gradients to be unscaled. + + .. note:: + :meth:`unscale_` does not incur a CPU-GPU sync. + + .. warning:: + :meth:`unscale_` should only be called once per optimizer per :meth:`step` call, + and only after all gradients for that optimizer's assigned parameters have been accumulated. + Calling :meth:`unscale_` twice for a given optimizer between each :meth:`step` triggers a RuntimeError. + + .. warning:: + :meth:`unscale_` may unscale sparse gradients out of place, replacing the ``.grad`` attribute. + """ + if not self._enabled: + return + + self._check_scale_growth_tracker("unscale_") + + optimizer_state = self._per_optimizer_states[id(optimizer)] + + if optimizer_state["stage"] is OptState.UNSCALED: + raise RuntimeError( + "unscale_() has already been called on this optimizer since the last update()." + ) + elif optimizer_state["stage"] is OptState.STEPPED: + raise RuntimeError("unscale_() is being called after step().") + + # FP32 division can be imprecise for certain compile options, so we carry out the reciprocal in FP64. + assert self._scale is not None + inv_scale = ( + self._scale.double().reciprocal().float() + if self._scale.device != torch.device("mps:0") + else self._scale.reciprocal() + ) + found_inf = torch.full((), 0.0, dtype=torch.float32, device=self._scale.device) + + optimizer_state["found_inf_per_device"] = self._unscale_grads_( + optimizer, inv_scale, found_inf, False + ) + optimizer_state["stage"] = OptState.UNSCALED + + def _maybe_opt_step( + self, + optimizer: torch.optim.Optimizer, + optimizer_state: dict[str, Any], + *args: Any, + **kwargs: Any, + ) -> Optional[float]: + retval: Optional[float] = None + if not sum(v.item() for v in optimizer_state["found_inf_per_device"].values()): + retval = optimizer.step(*args, **kwargs) + return retval + + def step( + self, optimizer: torch.optim.Optimizer, *args: Any, **kwargs: Any + ) -> Optional[float]: + """Invoke ``unscale_(optimizer)`` followed by parameter update, if gradients are not infs/NaN. + + :meth:`step` carries out the following two operations: + + 1. Internally invokes ``unscale_(optimizer)`` (unless :meth:`unscale_` was explicitly called for ``optimizer`` + earlier in the iteration). As part of the :meth:`unscale_`, gradients are checked for infs/NaNs. + 2. If no inf/NaN gradients are found, invokes ``optimizer.step()`` using the unscaled + gradients. Otherwise, ``optimizer.step()`` is skipped to avoid corrupting the params. + + ``*args`` and ``**kwargs`` are forwarded to ``optimizer.step()``. + + Returns the return value of ``optimizer.step(*args, **kwargs)``. + + Args: + optimizer (torch.optim.Optimizer): Optimizer that applies the gradients. + args: Any arguments. + kwargs: Any keyword arguments. + + .. warning:: + Closure use is not currently supported. + """ + if not self._enabled: + return optimizer.step(*args, **kwargs) + + if "closure" in kwargs: + raise RuntimeError( + "Closure use is not currently supported if GradScaler is enabled." + ) + + self._check_scale_growth_tracker("step") + + optimizer_state = self._per_optimizer_states[id(optimizer)] + + if optimizer_state["stage"] is OptState.STEPPED: + raise RuntimeError( + "step() has already been called since the last update()." + ) + + retval: Optional[float] = None + + if getattr(optimizer, "_step_supports_amp_scaling", False): + # This optimizer has customized scale-handling logic, so we can call optimizer.step() directly. + # The contract with custom optimizers is that their step() should accept an additional, + # optional grad_scaler kwarg. We append self to the kwargs so the custom optimizer has full information: + # it can query its own state, invoke unscale_ on itself, etc + # The contract above is being deprecated to avoid introducing `grad_scaler: GradScaler` argument + # to `Optimizer.step`. The new behavior is going to add two Tensor attributes of `grad_scale` + # and `found_inf` to the passed optimizer so that the optimizer can utilize those + # to skip the parameter updates or unscale gradients before updating parameters in + # the fused kernel, e.g. `FusedAdamMathFunctor`. + # In this behavior, `GradScaler._check_inf_per_device` is called if `OptState.READY`, + # while the method is expected to be called by users side, i.e. their optimizers. + kwargs_ = kwargs + has_grad_scaler_kwarg = ( + "grad_scaler" in inspect.signature(optimizer.step).parameters + ) + if has_grad_scaler_kwarg: + warnings.warn( + "GradScaler is going to stop passing itself as a keyword argument to the passed " + "optimizer. In the near future GradScaler registers `grad_scale: Tensor` and " + "`found_inf: Tensor` to the passed optimizer and let the optimizer use them directly.", + FutureWarning, + stacklevel=2, + ) + kwargs_.update({"grad_scaler": self}) + else: + if optimizer_state["stage"] is OptState.READY: + self._check_inf_per_device(optimizer) + scaler = self._get_scale_async() + assert scaler is not None + found_inf = cast( + torch.Tensor, + sum( + [ # noqa: C419 + t.to(scaler.device, non_blocking=True) + for t in optimizer_state["found_inf_per_device"].values() + ] + ), + ) + # Take the product of the scales, if the user has already set `optimizer.grad_scale`. + optimizer.grad_scale = ( # type: ignore[attr-defined] + getattr(optimizer, "grad_scale", None) + if optimizer_state["stage"] == OptState.UNSCALED + else scaler * getattr(optimizer, "grad_scale", 1) + ) + optimizer.found_inf = found_inf # type: ignore[attr-defined] + retval = optimizer.step(*args, **kwargs_) + optimizer_state["stage"] = OptState.STEPPED + if not has_grad_scaler_kwarg: + del optimizer.grad_scale # type: ignore[attr-defined] + del optimizer.found_inf # type: ignore[attr-defined] + return retval + + if optimizer_state["stage"] is OptState.READY: + self.unscale_(optimizer) + + assert len(optimizer_state["found_inf_per_device"]) > 0, ( + "No inf checks were recorded for this optimizer." + ) + + retval = self._maybe_opt_step(optimizer, optimizer_state, *args, **kwargs) + + optimizer_state["stage"] = OptState.STEPPED + + return retval + + def update(self, new_scale: Optional[Union[float, torch.Tensor]] = None) -> None: + """Update the scale factor. + + If any optimizer steps were skipped the scale is multiplied by ``backoff_factor`` + to reduce it. If ``growth_interval`` unskipped iterations occurred consecutively, + the scale is multiplied by ``growth_factor`` to increase it. + + Passing ``new_scale`` sets the new scale value manually. (``new_scale`` is not + used directly, it's used to fill GradScaler's internal scale tensor. So if + ``new_scale`` was a tensor, later in-place changes to that tensor will not further + affect the scale GradScaler uses internally.) + + Args: + new_scale (float or :class:`torch.Tensor`, optional, default=None): New scale factor. + + .. warning:: + :meth:`update` should only be called at the end of the iteration, after ``scaler.step(optimizer)`` has + been invoked for all optimizers used this iteration. + + .. warning:: + For performance reasons, we do not check the scale factor value to avoid synchronizations, + so the scale factor is not guaranteed to be above 1. If the scale falls below 1 and/or + you are seeing NaNs in your gradients or loss, something is likely wrong. For example, + bf16-pretrained models are often incompatible with AMP/fp16 due to differing dynamic ranges. + """ + if not self._enabled: + return + + _scale, _growth_tracker = self._check_scale_growth_tracker("update") + + if new_scale is not None: + assert self._scale is not None + # Accept a new user-defined scale. + if isinstance(new_scale, float): + self._scale.fill_(new_scale) + else: + reason = ( + "new_scale should be a float or a 1-element torch.cuda.FloatTensor or " + "torch.FloatTensor with requires_grad=False." + ) + assert new_scale.device.type == self._device, reason + assert new_scale.numel() == 1, reason + assert new_scale.requires_grad is False, reason + self._scale.copy_(new_scale) + else: + # Consume shared inf/nan data collected from optimizers to update the scale. + # If all found_inf tensors are on the same device as self._scale, this operation is asynchronous. + found_infs = [ + found_inf.to(device=_scale.device, non_blocking=True) + for state in self._per_optimizer_states.values() + for found_inf in state["found_inf_per_device"].values() + ] + + assert len(found_infs) > 0, "No inf checks were recorded prior to update." + + found_inf_combined = found_infs[0] + if len(found_infs) > 1: + for i in range(1, len(found_infs)): + found_inf_combined += found_infs[i] + + torch._amp_update_scale_( + _scale, + _growth_tracker, + found_inf_combined, + self._growth_factor, + self._backoff_factor, + self._growth_interval, + ) + + # To prepare for next iteration, clear the data collected from optimizers this iteration. + self._per_optimizer_states = defaultdict(_refresh_per_optimizer_state) + + def _get_scale_async(self) -> Optional[torch.Tensor]: + return self._scale + + def get_scale(self) -> float: + """Return a Python float containing the current scale, or 1.0 if scaling is disabled. + + .. warning:: + :meth:`get_scale` incurs a CPU-GPU sync. + """ + if self._enabled: + return ( + self._init_scale + if (scale := self._get_scale_async()) is None + else cast(float, scale.item()) + ) + return 1.0 + + def get_growth_factor(self) -> float: + r"""Return a Python float containing the scale growth factor.""" + return self._growth_factor + + def set_growth_factor(self, new_factor: float) -> None: + r"""Set a new scale growth factor. + + Args: + new_scale (float): Value to use as the new scale growth factor. + """ + self._growth_factor = new_factor + + def get_backoff_factor(self) -> float: + r"""Return a Python float containing the scale backoff factor.""" + return self._backoff_factor + + def set_backoff_factor(self, new_factor: float) -> None: + r"""Set a new scale backoff factor. + + Args: + new_scale (float): Value to use as the new scale backoff factor. + """ + self._backoff_factor = new_factor + + def get_growth_interval(self) -> int: + r"""Return a Python int containing the growth interval.""" + return self._growth_interval + + def set_growth_interval(self, new_interval: int) -> None: + r"""Set a new growth interval. + + Args: + new_interval (int): Value to use as the new growth interval. + """ + self._growth_interval = new_interval + + def _get_growth_tracker(self) -> int: + if self._enabled: + return ( + self._init_growth_tracker + if self._growth_tracker is None + else cast(int, self._growth_tracker.item()) + ) + return 0 + + def is_enabled(self) -> bool: + r"""Return a bool indicating whether this instance is enabled.""" + return self._enabled + + def state_dict(self) -> dict[str, Any]: + r"""Return the state of the scaler as a :class:`dict`. + + It contains five entries: + + * ``"scale"`` - a Python float containing the current scale + * ``"growth_factor"`` - a Python float containing the current growth factor + * ``"backoff_factor"`` - a Python float containing the current backoff factor + * ``"growth_interval"`` - a Python int containing the current growth interval + * ``"_growth_tracker"`` - a Python int containing the number of recent consecutive unskipped steps. + + If this instance is not enabled, returns an empty dict. + + .. note:: + If you wish to checkpoint the scaler's state after a particular iteration, :meth:`state_dict` + should be called after :meth:`update`. + """ + if self._enabled: + return { + "scale": self.get_scale(), + "growth_factor": self._growth_factor, + "backoff_factor": self._backoff_factor, + "growth_interval": self._growth_interval, + "_growth_tracker": self._get_growth_tracker(), + } + return {} + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + r"""Load the scaler state. + + If this instance is disabled, :meth:`load_state_dict` is a no-op. + + Args: + state_dict(dict): scaler state. Should be an object returned from a call to :meth:`state_dict`. + """ + if not self._enabled: + return + + if len(state_dict) == 0: + raise RuntimeError( + "The source state dict is empty, possibly because it was saved " + "from a disabled instance of GradScaler." + ) + + self._init_scale = cast(float, state_dict["scale"]) + if self._scale is not None: + self._scale.fill_(state_dict["scale"]) + self._growth_factor = cast(float, state_dict["growth_factor"]) + self._backoff_factor = cast(float, state_dict["backoff_factor"]) + self._growth_interval = cast(int, state_dict["growth_interval"]) + self._init_growth_tracker = cast(int, state_dict["_growth_tracker"]) + if self._growth_tracker is not None: + self._growth_tracker.fill_(state_dict["_growth_tracker"]) + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + if self._enabled: + assert len(self._per_optimizer_states) == 0, ( + "A GradScaler instance may only be pickled at the beginning " + "of an iteration, or at the end after scaler.update()." + ) + # Pickling _scale and _growth_tracker Tensors directly triggers + # "warnings.warn("pickle support for Storage will be removed in 1.5..." + # so instead, we set the unpickled instance up to reinitialize them lazily. + state["_init_scale"] = self.get_scale() + state["_init_growth_tracker"] = self._get_growth_tracker() + state["_scale"] = None + state["_growth_tracker"] = None + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + + def _check_inf_per_device(self, optimizer: torch.optim.Optimizer) -> dict[str, Any]: + _scale, _ = self._check_scale_growth_tracker("_check_inf_per_device") + + dummy_inv_scale = torch.full((), 1.0, dtype=torch.float32, device=_scale.device) + found_inf = torch.full((), 0.0, dtype=torch.float32, device=_scale.device) + + self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] = ( + self._unscale_grads_(optimizer, dummy_inv_scale, found_inf, True) + ) + + return self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] + + def _found_inf_per_device(self, optimizer: torch.optim.Optimizer) -> dict[str, Any]: + return self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/ao/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/ao/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..63e5f3868deb7e9596c81ae8e5fdd59853bba61a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/ao/__init__.py @@ -0,0 +1,31 @@ +# torch.ao is a package with a lot of interdependencies. +# We will use lazy import to avoid cyclic dependencies here. + +from typing import TYPE_CHECKING as _TYPE_CHECKING + + +if _TYPE_CHECKING: + from types import ModuleType + + from torch.ao import ( # noqa: TC004 + nn as nn, + ns as ns, + pruning as pruning, + quantization as quantization, + ) + + +__all__ = [ + "nn", + "ns", + "pruning", + "quantization", +] + + +def __getattr__(name: str) -> "ModuleType": + if name in __all__: + import importlib + + return importlib.import_module("." + name, __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a0ff7da4c42f77e225fe019145f5a20dae96e55 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/__init__.py @@ -0,0 +1,618 @@ +# mypy: allow-untyped-defs +""" +``torch.autograd`` provides classes and functions implementing automatic differentiation of arbitrary scalar valued functions. + +It requires minimal changes to the existing code - you only need to declare :class:`Tensor` s +for which gradients should be computed with the ``requires_grad=True`` keyword. +As of now, we only support autograd for floating point :class:`Tensor` types ( +half, float, double and bfloat16) and complex :class:`Tensor` types (cfloat, cdouble). +""" + +import warnings +from collections.abc import Sequence +from typing import cast, Optional, Union + +import torch +from torch import _vmap_internals +from torch.overrides import handle_torch_function, has_torch_function, is_tensor_like +from torch.types import _size, _TensorOrTensors, _TensorOrTensorsOrGradEdge + +from . import forward_ad, functional, graph +from .anomaly_mode import detect_anomaly, set_detect_anomaly +from .function import Function, NestedIOFunction +from .grad_mode import ( + _force_original_view_tracking, + _unsafe_preserve_version_counter, + enable_grad, + inference_mode, + no_grad, + set_grad_enabled, + set_multithreading_enabled, +) +from .gradcheck import gradcheck, gradgradcheck +from .graph import _engine_run_backward +from .variable import Variable + + +__all__ = [ + "Variable", + "Function", + "backward", + "grad_mode", + "NestedIOFunction", + "detect_anomaly", + "enable_grad", + "grad", + "gradcheck", + "gradgradcheck", + "inference_mode", + "no_grad", + "set_detect_anomaly", + "set_grad_enabled", + "set_multithreading_enabled", + "variable", +] + +_OptionalTensor = Optional[torch.Tensor] +_ShapeorNestedShape = Union[_size, Sequence[_size], torch.Tensor] + + +def _calculate_shape( + output: Union[torch.Tensor, graph.GradientEdge], + grad: torch.Tensor, + is_grads_batched: bool, +) -> tuple[_ShapeorNestedShape, _ShapeorNestedShape]: + # is_same_size ensures that both tensors are either nested or non nested + # circular import + from torch.nested._internal.nested_tensor import NestedTensor + + if isinstance(output, graph.GradientEdge): + # We have already checked that we are not a C++ NestedTensor + if is_grads_batched: + raise RuntimeError("Batched grads are not supported with GradientEdge") + out_metadata = output.node._input_metadata[output.output_nr] + return torch.Size(out_metadata.shape), grad.shape + + if output.is_nested and not isinstance(output, NestedTensor): + if is_grads_batched: + raise RuntimeError("Batched grads are not supported with Nested Tensor.") + out_shape = output._nested_tensor_size() + grad_shape = grad._nested_tensor_size() + + return out_shape, grad_shape + + reg_out_shape = output.shape + reg_grad_shape = grad.shape if not is_grads_batched else grad.shape[1:] + return reg_out_shape, reg_grad_shape + + +def _make_grads( + outputs: Union[Sequence[torch.Tensor], Sequence[graph.GradientEdge]], + grads: Sequence[_OptionalTensor], + is_grads_batched: bool, +) -> tuple[_OptionalTensor, ...]: + new_grads: list[_OptionalTensor] = [] + + for out, grad in zip(outputs, grads): + out = cast(Union[torch.Tensor, graph.GradientEdge], out) + out_size = None + out_device = None + + if isinstance(out, graph.GradientEdge): + out_metadata = out.node._input_metadata[out.output_nr] + out_size = torch.Size(out_metadata.shape) + out_dtype = out_metadata.dtype + out_device = out_metadata.device + out_is_nested = out_metadata.is_nested_tensor + if out_metadata.is_cpp_nested_tensor: + raise RuntimeError( + "C++ NestedTensor are not supported with GradientEdge" + ) + out_is_cpp_nested = False + else: + # circular import + from torch.nested._internal.nested_tensor import NestedTensor + + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + out_dtype = out.dtype + out_is_nested = out.is_nested + out_is_cpp_nested = out_is_nested and not isinstance(out, NestedTensor) + if not out_is_cpp_nested: + out_size = out.shape + + if isinstance(grad, torch.Tensor): + from torch.fx.experimental.symbolic_shapes import expect_true, sym_eq + + first_grad = grad if not is_grads_batched else grad[0] + + # TODO: We can remove this conditional once we uniformly use + # singleton int to represent jagged dimension, so that size() call + # on nested tensor works. + if out_is_cpp_nested: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor.") + shape_matches = torch.is_same_size(out, first_grad) + else: + # We need to do a regular size check, without going through + # the operator, to be able to handle unbacked symints + # (expect_true ensures we can deal with unbacked) + if out_size is None: + raise AssertionError("Expected out_size to be set.") + shape_matches = expect_true(sym_eq(out_size, first_grad.size())) + + if not shape_matches: + out = cast(Union[torch.Tensor, graph.GradientEdge], out) # type: ignore[redundant-cast] + out_shape, grad_shape = _calculate_shape( + out, first_grad, is_grads_batched + ) + if is_grads_batched: + raise RuntimeError( + "If `is_grads_batched=True`, we interpret the first " + "dimension of each grad_output as the batch dimension. " + "The sizes of the remaining dimensions are expected to match " + "the shape of corresponding output, but a mismatch " + "was detected: grad_output[" + + str(grads.index(grad)) + + "] has a shape of " + + str(grad_shape) + + " and output[" + + str(outputs.index(out)) + + "] has a shape of " + + str(out_shape) + + ". " + "If you only want some tensors in `grad_output` to be considered " + "batched, consider using vmap." + ) + else: + raise RuntimeError( + "Mismatch in shape: grad_output[" + + str(grads.index(grad)) + + "] has a shape of " + + str(grad_shape) + + " and output[" + + str(outputs.index(out)) + + "] has a shape of " + + str(out_shape) + + "." + ) + if out_dtype.is_complex != grad.dtype.is_complex: + raise RuntimeError( + "For complex Tensors, both grad_output and output" + " are required to have the same dtype." + " Mismatch in dtype: grad_output[" + + str(grads.index(grad)) + + "] has a dtype of " + + str(grad.dtype) + + " and output[" + + str(outputs.index(out)) + + "] has a dtype of " + + str(out_dtype) + + "." + ) + new_grads.append(grad) + elif grad is None: + if isinstance(out, graph.GradientEdge) or out.requires_grad: # type: ignore[attr-defined] + if isinstance(out, graph.GradientEdge): + if out_size is None: + raise AssertionError("Expected out_size to be set.") + out_numel_is_1 = all(o == 1 for o in out_size) + else: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + out_numel_is_1 = out.numel() == 1 + if not out_numel_is_1: + raise RuntimeError( + "grad can be implicitly created only for scalar outputs" + ) + if not out_dtype.is_floating_point: + msg = ( + "grad can be implicitly created only for real scalar outputs" + f" but got {out_dtype}" + ) + raise RuntimeError(msg) + if isinstance(out, graph.GradientEdge): + if out_size is None: + raise AssertionError("Expected out_size to be set.") + if out_device is None: + raise AssertionError("Expected out_device to be set.") + new_grads.append( + torch.ones( + out_size, + dtype=out_dtype, + device=out_device, + ) + ) + else: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + new_grads.append( + torch.ones_like(out, memory_format=torch.preserve_format) + ) + else: + new_grads.append(None) + else: + raise TypeError( + "gradients can be either Tensors or None, but got " + + type(grad).__name__ + ) + return tuple(new_grads) + + +def _tensor_or_tensors_to_tuple( + tensors: Optional[_TensorOrTensors], length: int +) -> tuple[_OptionalTensor, ...]: + if tensors is None: + return (None,) * length + if isinstance(tensors, torch.Tensor): + return (tensors,) + return tuple(tensors) + + +def backward( + tensors: _TensorOrTensorsOrGradEdge, + grad_tensors: Optional[_TensorOrTensors] = None, + retain_graph: Optional[bool] = None, + create_graph: bool = False, + grad_variables: Optional[_TensorOrTensors] = None, + inputs: Optional[_TensorOrTensorsOrGradEdge] = None, +) -> None: + r"""Compute the sum of gradients of given tensors with respect to graph leaves. + + The graph is differentiated using the chain rule. If any of ``tensors`` + are non-scalar (i.e. their data has more than one element) and require + gradient, then the Jacobian-vector product would be computed, in this + case the function additionally requires specifying ``grad_tensors``. + It should be a sequence of matching length, that contains the "vector" + in the Jacobian-vector product, usually the gradient of the differentiated + function w.r.t. corresponding tensors (``None`` is an acceptable value for + all tensors that don't need gradient tensors). + + This function accumulates gradients in the leaves - you might need to zero + ``.grad`` attributes or set them to ``None`` before calling it. + See :ref:`Default gradient layouts` + for details on the memory layout of accumulated gradients. + + .. note:: + Using this method with ``create_graph=True`` will create a reference cycle + between the parameter and its gradient which can cause a memory leak. + We recommend using ``autograd.grad`` when creating the graph to avoid this. + If you have to use this function, make sure to reset the ``.grad`` fields of your + parameters to ``None`` after use to break the cycle and avoid the leak. + + .. note:: + + If you run any forward ops, create ``grad_tensors``, and/or call ``backward`` + in a user-specified CUDA stream context, see + :ref:`Stream semantics of backward passes`. + + .. note:: + + When ``inputs`` are provided and a given input is not a leaf, + the current implementation will call its grad_fn (even though it is not strictly needed to get this gradients). + It is an implementation detail on which the user should not rely. + See https://github.com/pytorch/pytorch/pull/60521#issuecomment-867061780 for more details. + + Args: + tensors (Sequence[Tensor] or Tensor or Sequence[GradientEdge] or GradientEdge): Tensors of which + the derivative will be computed. + grad_tensors (Sequence[Tensor or None] or Tensor, optional): The "vector" in + the Jacobian-vector product, usually gradients w.r.t. each element of + corresponding tensors. None values can be specified for scalar Tensors or + ones that don't require grad. If a None value would be acceptable for all + grad_tensors, then this argument is optional. + retain_graph (bool, optional): If ``False``, the graph used to compute the grad + will be freed. Note that in nearly all cases setting this option to ``True`` + is not needed and often can be worked around in a much more efficient + way. Defaults to the value of ``create_graph``. + create_graph (bool, optional): If ``True``, graph of the derivative will + be constructed, allowing to compute higher order derivative products. + Defaults to ``False``. + inputs (Sequence[Tensor] or Tensor or Sequence[GradientEdge], optional): Inputs w.r.t. which the gradient + be will accumulated into ``.grad``. All other Tensors will be ignored. If + not provided, the gradient is accumulated into all the leaf Tensors that + were used to compute the :attr:`tensors`. + """ + if torch._C._are_functorch_transforms_active(): + raise RuntimeError( + "backward() called inside a functorch transform. This is not " + "supported, please use functorch.grad or functorch.vjp instead " + "or call backward() outside of functorch transforms." + ) + + if grad_variables is not None: + warnings.warn( + "`grad_variables` is deprecated. Use `grad_tensors` instead.", + FutureWarning, + stacklevel=2, + ) + if grad_tensors is None: + grad_tensors = grad_variables + else: + raise RuntimeError( + "`grad_tensors` and `grad_variables` (deprecated) " + "arguments both passed to `backward()`. Please only " + "use `grad_tensors`." + ) + + inputs_tuple: tuple[Union[torch.Tensor, graph.GradientEdge], ...] + if inputs is None: + inputs_tuple = () + elif isinstance(inputs, (torch.Tensor, graph.GradientEdge)): + inputs_tuple = (inputs,) + else: + inputs_tuple = tuple(inputs) + if len(inputs_tuple) == 0: + raise RuntimeError("`inputs` argument to `backward()` cannot be empty.") + + if is_tensor_like(tensors) or isinstance(tensors, graph.GradientEdge): + tensors = cast( + Union[tuple[torch.Tensor], tuple[graph.GradientEdge]], (tensors,) + ) + else: + # pyrefly: ignore [bad-argument-type] + tensors = tuple(tensors) + + grad_tensors_ = _tensor_or_tensors_to_tuple(grad_tensors, len(tensors)) + grad_tensors_ = _make_grads(tensors, grad_tensors_, is_grads_batched=False) + if retain_graph is None: + retain_graph = create_graph + + # The reason we repeat the same comment below is that + # some Python versions print out the first line of a multi-line function + # calls in the traceback and some print out the last line + _engine_run_backward( + tensors, + grad_tensors_, + retain_graph, + create_graph, + inputs_tuple, + allow_unreachable=True, + accumulate_grad=True, + ) + + +def grad( + outputs: _TensorOrTensorsOrGradEdge, + inputs: _TensorOrTensorsOrGradEdge, + grad_outputs: Optional[_TensorOrTensors] = None, + retain_graph: Optional[bool] = None, + create_graph: bool = False, + only_inputs: bool = True, + allow_unused: Optional[bool] = None, + is_grads_batched: bool = False, + materialize_grads: bool = False, +) -> tuple[torch.Tensor, ...]: + r"""Compute and return the sum of gradients of outputs with respect to the inputs. + + ``grad_outputs`` should be a sequence of length matching ``output`` + containing the "vector" in vector-Jacobian product, usually the pre-computed + gradients w.r.t. each of the outputs. If an output doesn't require_grad, + then the gradient can be ``None``). + + .. note:: + + If you run any forward ops, create ``grad_outputs``, and/or call ``grad`` + in a user-specified CUDA stream context, see + :ref:`Stream semantics of backward passes`. + + .. note:: + + ``only_inputs`` argument is deprecated and is ignored now (defaults to ``True``). + To accumulate gradient for other parts of the graph, please use + ``torch.autograd.backward``. + + Args: + outputs (sequence of Tensor or GradientEdge): outputs of the differentiated function. + inputs (sequence of Tensor or GradientEdge): Inputs w.r.t. which the gradient will be + returned (and not accumulated into ``.grad``). + grad_outputs (sequence of Tensor): The "vector" in the vector-Jacobian product. + Usually gradients w.r.t. each output. None values can be specified for scalar + Tensors or ones that don't require grad. If a None value would be acceptable + for all grad_tensors, then this argument is optional. Default: None. + retain_graph (bool, optional): If ``False``, the graph used to compute the grad + will be freed. Note that in nearly all cases setting this option to ``True`` + is not needed and often can be worked around in a much more efficient + way. Defaults to the value of ``create_graph``. + create_graph (bool, optional): If ``True``, graph of the derivative will + be constructed, allowing to compute higher order derivative products. + Default: ``False``. + allow_unused (Optional[bool], optional): If ``False``, specifying inputs + that were not used when computing outputs (and therefore their grad is + always zero) is an error. Defaults to the value of ``materialize_grads``. + is_grads_batched (bool, optional): If ``True``, the first dimension of each + tensor in ``grad_outputs`` will be interpreted as the batch dimension. + Instead of computing a single vector-Jacobian product, we compute a + batch of vector-Jacobian products for each "vector" in the batch. + We use the vmap prototype feature as the backend to vectorize calls + to the autograd engine so that this computation can be performed in a + single call. This should lead to performance improvements when compared + to manually looping and performing backward multiple times. Note that + due to this feature being experimental, there may be performance + cliffs. Please use ``torch._C._debug_only_display_vmap_fallback_warnings(True)`` + to show any performance warnings and file an issue on github if warnings exist + for your use case. Defaults to ``False``. + materialize_grads (bool, optional): If ``True``, set the gradient for unused inputs + to zero instead of None. This is useful when computing higher-order derivatives. + If ``materialize_grads`` is ``True`` and ``allow_unused`` is ``False``, an error + will be raised. Defaults to ``False``. + + """ + if materialize_grads and allow_unused is False: + raise ValueError( + "Expected allow_unused to be True or not passed when materialize_grads=True, " + "but got: allow_unused=False." + ) + if allow_unused is None: + allow_unused = materialize_grads + if is_tensor_like(outputs) or isinstance(outputs, graph.GradientEdge): + outputs = cast( + Union[Sequence[torch.Tensor], Sequence[graph.GradientEdge]], (outputs,) + ) + else: + # pyrefly: ignore [bad-argument-type] + outputs = tuple(outputs) + if is_tensor_like(inputs) or isinstance(inputs, graph.GradientEdge): + inputs = cast(_TensorOrTensorsOrGradEdge, (inputs,)) + else: + # pyrefly: ignore [bad-argument-type] + inputs = tuple(inputs) + t_outputs = tuple(i for i in outputs if is_tensor_like(i)) + t_inputs = tuple(i for i in inputs if is_tensor_like(i)) + overridable_args = t_outputs + t_inputs + if has_torch_function(overridable_args): + return handle_torch_function( + grad, + overridable_args, + outputs, + inputs, + grad_outputs=grad_outputs, + retain_graph=retain_graph, + create_graph=create_graph, + only_inputs=only_inputs, + allow_unused=allow_unused, + is_grads_batched=is_grads_batched, + materialize_grads=materialize_grads, + ) + + if not only_inputs: + warnings.warn( + "only_inputs argument is deprecated and is ignored now " + "(defaults to True). To accumulate gradient for other " + "parts of the graph, please use torch.autograd.backward.", + FutureWarning, + stacklevel=2, + ) + + grad_outputs_ = _tensor_or_tensors_to_tuple(grad_outputs, len(outputs)) + grad_outputs_ = _make_grads( + outputs, grad_outputs_, is_grads_batched=is_grads_batched + ) + + if retain_graph is None: + retain_graph = create_graph + + # The reason we repeat the same comment several times below is because + # some Python versions print out the first line of multi-line function + # calls in the traceback and some print out the last line + if is_grads_batched: + + def vjp(gO): + return _engine_run_backward( + outputs, + gO, + retain_graph, + create_graph, + inputs, + allow_unused, + accumulate_grad=False, + ) + + result = _vmap_internals._vmap(vjp, 0, 0, allow_none_pass_through=True)( + grad_outputs_ + ) + else: + result = _engine_run_backward( + outputs, + grad_outputs_, + retain_graph, + create_graph, + inputs, + allow_unused, + accumulate_grad=False, + ) + if materialize_grads: + if any( + result[i] is None and not is_tensor_like(inputs[i]) + for i in range(len(inputs)) + ): + raise RuntimeError( + "materialize_grads cannot be used when the given input is a GradientEdge" + ) + result = tuple( + output + if output is not None + else torch.zeros_like(input, requires_grad=create_graph) + for (output, input) in zip(result, inputs) + ) + return result + + +# This function applies in case of gradient checkpointing for memory +# optimization. Currently, gradient checkpointing is supported only if the +# execution engine is invoked through torch.autograd.backward() and its +# inputs argument is not passed. It is not supported for torch.autograd.grad(). +# This is because if inputs are specified, the gradient won't be calculated for +# anything else e.g. model parameters like weights, bias etc. +# +# This function returns whether the checkpointing is valid i.e. torch.autograd.backward +# or not i.e. torch.autograd.grad. The implementation works by maintaining a thread +# local variable in torch/csrc/autograd/engine.cpp which looks at the NodeTask +# in the stack and before a NodeTask is executed in evaluate_function, it +# checks for whether reentrant backwards is imperative or not. +# See https://github.com/pytorch/pytorch/pull/4594 for more discussion/context +def _is_checkpoint_valid(): + return Variable._execution_engine.is_checkpoint_valid() + + +def variable(*args, **kwargs): # noqa: D103 + raise RuntimeError( + "torch.autograd.variable(...) is deprecated, use torch.tensor(...) instead" + ) + + +# Monkey patching variable.Variable to fix FX codegen. FX generates a call by roughly doing +# f"{fn.__module__}.{fn.__name__}(...). This yields torch.autograd.variable.Variable(...) in the +# output of an FX graph. Unfortunately the module name torch.autograd.variable is shadowed by the +# deprecated function - variable(...). +variable.Variable = Variable # type: ignore[attr-defined] + +if not torch._C._autograd_init(): + raise RuntimeError("autograd initialization failed") + +# Import all native method/classes +from torch._C._autograd import ( + _add_metadata_json, + _disable_profiler, + _disable_profiler_legacy, + _enable_profiler, + _enable_profiler_legacy, + _enable_record_function, + _get_sequence_nr, + _kineto_step, + _KinetoEvent, + _pop_saved_tensors_default_hooks, + _prepare_profiler, + _profiler_enabled, + _ProfilerResult, + _push_saved_tensors_default_hooks, + _record_function_with_args_enter, + _record_function_with_args_exit, + _set_empty_test_observer, + _supported_activities, + _toggle_collection_dynamic, + DeviceType, + kineto_available, + ProfilerEvent, + SavedTensor, +) +from torch._C._profiler import ProfilerActivity, ProfilerConfig, ProfilerState + +from . import profiler + + +def _register_py_tensor_class_for_device(device, cls): + if not isinstance(cls, type): + raise RuntimeError("cls isn't a typeinfo object") + torch._C._register_py_class_for_device(device, cls) + + +is_multithreading_enabled = torch._C._is_multithreading_enabled +torch._C._add_docstr( + is_multithreading_enabled, "Returns True if multithreading is currently enabled." +) + +is_view_replay_enabled = torch._C._is_view_replay_enabled +torch._C._add_docstr( + is_view_replay_enabled, "Returns True if view-replay is currently enabled." +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/anomaly_mode.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/anomaly_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..6036245c4856f0cffc9471dd456d877971619998 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/anomaly_mode.py @@ -0,0 +1,123 @@ +# mypy: allow-untyped-defs +r"""Autograd anomaly mode.""" + +import warnings + +import torch + + +__all__ = ["detect_anomaly", "set_detect_anomaly"] + + +class detect_anomaly: + r"""Context-manager that enable anomaly detection for the autograd engine. + + This does two things: + + - Running the forward pass with detection enabled will allow the backward + pass to print the traceback of the forward operation that created the failing + backward function. + - If ``check_nan`` is ``True``, any backward computation that generate "nan" + value will raise an error. Default ``True``. + + .. warning:: + This mode should be enabled only for debugging as the different tests + will slow down your program execution. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ANOMALY) + >>> import torch + >>> from torch import autograd + >>> class MyFunc(autograd.Function): + ... @staticmethod + ... def forward(ctx, inp): + ... return inp.clone() + ... + ... @staticmethod + ... def backward(ctx, gO): + ... # Error during the backward pass + ... raise RuntimeError("Some error in backward") + ... return gO.clone() + >>> def run_fn(a): + ... out = MyFunc.apply(a) + ... return out.sum() + >>> inp = torch.rand(10, 10, requires_grad=True) + >>> out = run_fn(inp) + >>> out.backward() + Traceback (most recent call last): + File "", line 1, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + >>> with autograd.detect_anomaly(): + ... inp = torch.rand(10, 10, requires_grad=True) + ... out = run_fn(inp) + ... out.backward() + Traceback of forward call that caused the error: + File "tmp.py", line 53, in + out = run_fn(inp) + File "tmp.py", line 44, in run_fn + out = MyFunc.apply(a) + Traceback (most recent call last): + File "", line 4, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + + """ + + def __init__(self, check_nan=True) -> None: # noqa: D107 + self.prev = torch.is_anomaly_enabled() + self.check_nan = check_nan + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + warnings.warn( + "Anomaly Detection has been enabled. " + "This mode will increase the runtime " + "and should only be enabled for debugging.", + stacklevel=2, + ) + + def __enter__(self) -> None: # noqa: D105 + torch.set_anomaly_enabled(True, self.check_nan) + + def __exit__(self, *args: object) -> None: # noqa: D105 + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) + + +class set_detect_anomaly: + r"""Context-manager that sets the anomaly detection for the autograd engine on or off. + + ``set_detect_anomaly`` will enable or disable the autograd anomaly detection + based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + See ``detect_anomaly`` above for details of the anomaly detection behaviour. + + Args: + mode (bool): Flag whether to enable anomaly detection (``True``), + or disable (``False``). + check_nan (bool): Flag whether to raise an error when the backward + generate "nan" + + """ + + def __init__(self, mode: bool, check_nan: bool = True) -> None: # noqa: D107 + self.prev = torch.is_anomaly_enabled() + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + torch.set_anomaly_enabled(mode, check_nan) + + def __enter__(self) -> None: # noqa: D105 + pass + + def __exit__(self, *args: object) -> None: # noqa: D105 + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/forward_ad.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/forward_ad.py new file mode 100644 index 0000000000000000000000000000000000000000..36ef0da38864bb15c561cc67aebd77018515a53a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/forward_ad.py @@ -0,0 +1,229 @@ +# mypy: allow-untyped-defs +import os +from typing import Any, NamedTuple, Optional + +import torch + +from .grad_mode import _DecoratorContextManager + + +__all__ = [ + "UnpackedDualTensor", + "enter_dual_level", + "exit_dual_level", + "make_dual", + "unpack_dual", + "dual_level", +] + +# Global variable used to make the python API simpler to use +_current_level = -1 + + +def enter_dual_level(): + r"""Enter a new forward grad level. + + This level can be used to make and unpack dual Tensors to compute + forward gradients. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + global _current_level + new_level = torch._C._enter_dual_level() + if new_level != _current_level + 1: + raise RuntimeError( + "Entering a new forward AD level but the current level " + "is not valid. Make sure you did not modified it directly." + ) + _current_level = new_level + return new_level + + +def exit_dual_level(*, level=None): + r"""Exit a forward grad level. + + This function deletes all the gradients associated with this + level. Only deleting the latest entered level is allowed. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + global _current_level + if level is None: + level = _current_level + if level != _current_level: + raise RuntimeError( + "Trying to exit a forward AD level that was not the last one " + "that was created. This is not supported." + ) + torch._C._exit_dual_level(level=level) + _current_level = level - 1 + + +def _maybe_load_decompositions(): + if os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__: + from torch._decomp import decompositions_for_jvp # noqa: F401 + + +def make_dual(tensor, tangent, *, level=None): + r"""Associate a tensor value with its tangent to create a "dual tensor" for forward AD gradient computation. + + The result is a new tensor aliased to :attr:`tensor` with :attr:`tangent` embedded + as an attribute as-is if it has the same storage layout or copied otherwise. + The tangent attribute can be recovered with :func:`unpack_dual`. + + This function is backward differentiable. + + Given a function `f` whose jacobian is `J`, it allows one to compute the Jacobian-vector product (`jvp`) + between `J` and a given vector `v` as follows. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, v) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + + """ + # See NOTE: [forward-mode AD decompositions mechanism] + # + # Import from torch._decomp import decompositions_for_jvp to register + # decompositions for jvp to the jit registry + # + # FIXME: We specify that __debug__ must be True because + # if python is run with -OO or -O flags (i.e., __debug__ is False), we encounter the + # following error: + # + # Return value was annotated as having type Tuple[NoneType, NoneType] but is actually of + # type Tuple[Tensor, Tensor]: + # File ".../torch/_decomp/__init__.py", line 1585 + # else: + # buffer = z + # return min - torch.log1p(z), buffer + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + _maybe_load_decompositions() + + if level is None: + level = _current_level + + if level < 0: + raise RuntimeError( + "Trying to create a dual Tensor for forward AD but no level " + "exists, make sure to enter_dual_level() first." + ) + if not (tensor.is_floating_point() or tensor.is_complex()): + raise ValueError( + f"Expected primal to be floating point or complex, but got: {tensor.dtype}" + ) + if not (tangent.is_floating_point() or tangent.is_complex()): + raise ValueError( + f"Expected tangent to be floating point or complex, but got: {tangent.dtype}" + ) + + return torch._VF._make_dual(tensor, tangent, level=level) + + +class UnpackedDualTensor(NamedTuple): + r"""Namedtuple returned by :func:`unpack_dual` containing the primal and tangent components of the dual tensor. + + See :func:`unpack_dual` for more details. + """ + + primal: torch.Tensor + tangent: Optional[torch.Tensor] + + +def unpack_dual(tensor, *, level=None): + r"""Unpack a "dual tensor" to get both its Tensor value and its forward AD gradient. + + The result is a namedtuple ``(primal, tangent)`` where ``primal`` is a view of + :attr:`tensor`'s primal and ``tangent`` is :attr:`tensor`'s tangent as-is. + Neither of these tensors can be dual tensor of level :attr:`level`. + + This function is backward differentiable. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + ... jvp = unpack_dual(out).tangent + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + if level is None: + level = _current_level + + if level < 0: + return UnpackedDualTensor(tensor, None) + + primal, dual = torch._VF._unpack_dual(tensor, level=level) + + return UnpackedDualTensor(primal, dual) + + +class dual_level(_DecoratorContextManager): + r"""Context-manager for forward AD, where all forward AD computation must occur within the ``dual_level`` context. + + .. Note:: + + The ``dual_level`` context appropriately enters and exit the dual level to + controls the current forward AD level, which is used by default by the other + functions in this API. + + We currently don't plan to support nested ``dual_level`` contexts, however, so + only a single forward AD level is supported. To compute higher-order + forward grads, one can use :func:`torch.func.jvp`. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> x = torch.tensor([1]) + >>> x_t = torch.tensor([1]) + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... # Do computations with inp + ... out = your_fn(inp) + ... _, grad = unpack_dual(out) + >>> grad is None + False + >>> # After exiting the level, the grad is deleted + >>> _, grad_after = unpack_dual(out) + >>> grad is None + True + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + + def __enter__(self): + return enter_dual_level() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + exit_dual_level() + + +# Private helper functions +_is_fwd_grad_enabled = torch._C._is_fwd_grad_enabled + + +# Private helper function to enable or disable fwd grad. +# If you're a user and want to use this, please file an issue to discuss the use case. +class _set_fwd_grad_enabled(_DecoratorContextManager): + def __init__(self, mode: bool) -> None: + self.prev = _is_fwd_grad_enabled() + torch._C._set_fwd_grad_enabled(mode) + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_fwd_grad_enabled(self.prev) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/function.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/function.py new file mode 100644 index 0000000000000000000000000000000000000000..771a51de7e8c99e61cb39643ea1161b3cc388dae --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/function.py @@ -0,0 +1,856 @@ +# mypy: allow-untyped-defs +import functools +import inspect +import itertools +import warnings +from collections import OrderedDict +from collections.abc import Callable +from typing import Any, Concatenate, Optional, TypeVar +from typing_extensions import deprecated, ParamSpec + +import torch +import torch._C as _C +import torch._functorch as _functorch +import torch.utils.hooks as hooks +from torch._C import _functions +from torch._functorch.autograd_function import custom_function_call + + +__all__ = [ + "FunctionCtx", + "BackwardCFunction", + "FunctionMeta", + "Function", + "once_differentiable", + "InplaceFunction", + "NestedIOFunction", +] + +# Unique id provider for each class inheriting from Function +# This is incremented in FunctionMeta during class definition +AUTOGRAD_FUNCTION_COUNTER = itertools.count() + +_T = TypeVar("_T") +_R = TypeVar("_R") +_P = ParamSpec("_P") + + +# Formerly known as: _ContextMethodMixin +class FunctionCtx: + def save_for_backward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.backward`. + + ``save_for_backward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and only with tensors. + + All tensors intended to be used in the backward pass should be saved + with ``save_for_backward`` (as opposed to directly on ``ctx``) to prevent + incorrect gradients and memory leaks, and enable the application of saved + tensor hooks. See :class:`torch.autograd.graph.saved_tensors_hooks`. + See :ref:`extending-autograd` for more details. + + Note that if intermediary tensors, tensors that are neither inputs + nor outputs of :func:`forward`, are saved for backward, your custom Function + may not support double backward. + Custom Functions that do not support double backward should decorate their + :func:`backward` method with ``@once_differentiable`` so that performing + double backward raises an error. If you'd like to support double backward, + you can either recompute intermediaries based on the inputs during backward + or return the intermediaries as the outputs of the custom Function. See the + `double backward tutorial `_ + for more details. + + In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors` + attribute. Before returning them to the user, a check is made to ensure + they weren't used in any in-place operation that modified their content. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> w = x * z + >>> out = x * y + y * z + w * y + >>> ctx.save_for_backward(x, y, w, out) + >>> ctx.z = z # z is not a tensor + >>> return out + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_out): + >>> x, y, w, out = ctx.saved_tensors + >>> z = ctx.z + >>> gx = grad_out * (y + y * z) + >>> gy = grad_out * (x + z + w) + >>> gz = None + >>> return gx, gy, gz + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> d = Func.apply(a, b, c) + + """ + self.to_save = tensors + + def save_for_forward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.jvp`. + + ``save_for_forward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and all arguments + should be tensors. + + In :func:`jvp`, saved objects can be accessed through the :attr:`saved_tensors` + attribute. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +SKIP + >>> class Func(torch.autograd.Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> ctx.save_for_backward(x, y) + >>> ctx.save_for_forward(x, y) + >>> ctx.z = z + >>> return x * y * z + >>> + >>> @staticmethod + >>> def jvp(ctx, x_t, y_t, _): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * (y * x_t + x * y_t) + >>> + >>> @staticmethod + >>> def vjp(ctx, grad_out): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * grad_out * y, z * grad_out * x, None + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> t = torch.tensor(1., dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> + >>> with fwAD.dual_level(): + >>> a_dual = fwAD.make_dual(a, t) + >>> d = Func.apply(a_dual, b, c) + + """ + for tensor in tensors: + if not (isinstance(tensor, torch.Tensor) or tensor is None): + raise AssertionError( + "save_for_forward expects all arguments to be tensors; you should " + "save non-tensors as attributes on ctx." + ) + + self.saved_for_forward = tensors + + def mark_dirty(self, *args: torch.Tensor): + r"""Mark given tensors as modified in an in-place operation. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be inputs. + + Every tensor that's been modified in-place in a call to :func:`forward` + should be given to this function, to ensure correctness of our checks. + It doesn't matter whether the function is called before or after + modification. + + Examples:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Inplace(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> x_npy = x.numpy() # x_npy shares storage with x + >>> x_npy += 1 + >>> ctx.mark_dirty(x) + >>> return x + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_output): + >>> return grad_output + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone() + >>> b = a * a + >>> Inplace.apply(a) # This would lead to wrong gradients! + >>> # but the engine would not know unless we mark_dirty + >>> # xdoctest: +SKIP + >>> b.backward() # RuntimeError: one of the variables needed for gradient + >>> # computation has been modified by an inplace operation + + """ + self.dirty_tensors = args + + @deprecated( + "`mark_shared_storage` is deprecated. " + "Tensors with shared storages are automatically tracked. " + "Note that calls to `set_()` are not tracked", + category=FutureWarning, + ) + def mark_shared_storage(self, *pairs): + pass + + def mark_non_differentiable(self, *args: torch.Tensor): + r"""Mark outputs as non-differentiable. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be tensor outputs. + + This will mark outputs as not requiring gradients, increasing the + efficiency of backward computation. You still need to accept a gradient + for each output in :meth:`~Function.backward`, but it's always going to + be a zero tensor with the same shape as the shape of a corresponding + output. + + This is used e.g. for indices returned from a sort. See example:: + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> sorted, idx = x.sort() + >>> ctx.mark_non_differentiable(idx) + >>> ctx.save_for_backward(x, idx) + >>> return sorted, idx + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): # still need to accept g2 + >>> x, idx = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> grad_input.index_add_(0, idx, g1) + >>> return grad_input + + """ + self.non_differentiable = args + + def set_materialize_grads(self, value: bool): + r"""Set whether to materialize grad tensors. Default is ``True``. + + This should be called only from either the :func:`setup_context` or + :func:`forward` methods. + + If ``True``, undefined grad tensors will be expanded to tensors full of zeros + prior to calling the :func:`backward` and :func:`jvp` methods. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class SimpleFunc(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> return g1 + g2 # No check for None necessary + >>> + >>> # We modify SimpleFunc to handle non-materialized grad outputs + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> ctx.set_materialize_grads(False) + >>> ctx.save_for_backward(x) + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> x, = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> if g1 is not None: # We must check for None now + >>> grad_input += g1 + >>> if g2 is not None: + >>> grad_input += g2 + >>> return grad_input + >>> + >>> a = torch.tensor(1., requires_grad=True) + >>> b, _ = Func.apply(a) # induces g2 to be undefined + + """ + self.materialize_grads = value + + +# DO NOT USE: This is only defined to be able to load old serialized models +_ContextMethodMixin = FunctionCtx + + +class _HookMixin: + @staticmethod + def _register_hook(backward_hooks, hook): + if backward_hooks is None: + backward_hooks = OrderedDict() + handle = hooks.RemovableHandle(backward_hooks) + backward_hooks[handle.id] = hook + return backward_hooks, handle + + +class BackwardCFunction(_C._FunctionBase, FunctionCtx, _HookMixin): + r""" + This class is used for internal autograd work. Do not use. + """ + + def apply(self, *args): + r""" + Apply method used when executing this Node during the backward + """ + # _forward_cls is defined by derived class + # The user should define either backward or vjp but never both. + backward_fn = self._forward_cls.backward # type: ignore[attr-defined] + vjp_fn = self._forward_cls.vjp # type: ignore[attr-defined] + if backward_fn is not Function.backward and vjp_fn is not Function.vjp: + raise RuntimeError( + "Implementing both 'backward' and 'vjp' for a custom " + "Function is not allowed. You should only implement one " + "of them." + ) + user_fn = vjp_fn if vjp_fn is not Function.vjp else backward_fn + return user_fn(self, *args) + + def apply_jvp(self, *args): + r""" + Apply method used when executing forward mode AD during the forward + """ + # _forward_cls is defined by derived class + return self._forward_cls.jvp(self, *args) # type: ignore[attr-defined] + + def _compiled_autograd_key(self): + return self._forward_cls._compiled_autograd_key(self) # type: ignore[attr-defined] + + +class FunctionMeta(type): + """Function metaclass. + + This metaclass sets up the following properties: + _backward_cls: The Function class corresponding to the differentiated + version of this function (which is generated on the fly by this + metaclass). + """ + + def __init__(cls, name, bases, attrs): + backward_fn = type( + name + "Backward", (BackwardCFunction,), {"_forward_cls": cls} + ) + backward_fn._autograd_function_id = next(AUTOGRAD_FUNCTION_COUNTER) # type: ignore[attr-defined] + cls._backward_cls = backward_fn + + super().__init__(name, bases, attrs) + + +class _SingleLevelFunction( + _C._FunctionBase, FunctionCtx, _HookMixin, metaclass=FunctionMeta +): + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + r"""Define the forward of the custom autograd Function. + + This function is to be overridden by all subclasses. + There are two ways to define forward: + + Usage 1 (Combined forward and ctx):: + + @staticmethod + def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any: + pass + + - It must accept a context ctx as the first argument, followed by any + number of arguments (tensors or other types). + - See :ref:`combining-forward-context` for more details + + Usage 2 (Separate forward and ctx):: + + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + pass + + + @staticmethod + def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None: + pass + + - The forward no longer accepts a ctx argument. + - Instead, you must also override the :meth:`torch.autograd.Function.setup_context` + staticmethod to handle setting up the ``ctx`` object. + ``output`` is the output of the forward, ``inputs`` are a Tuple of inputs + to the forward. + - See :ref:`extending-autograd` for more details + + The context can be used to store arbitrary data that can be then + retrieved during the backward pass. Tensors should not be stored + directly on `ctx` (though this is not currently enforced for + backward compatibility). Instead, tensors should be saved either with + :func:`ctx.save_for_backward` if they are intended to be used in + ``backward`` (equivalently, ``vjp``) or :func:`ctx.save_for_forward` + if they are intended to be used for in ``jvp``. + """ + raise NotImplementedError( + "You must implement the forward function for custom autograd.Function." + ) + + @staticmethod + def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> Any: + r"""There are two ways to define the forward pass of an autograd.Function. + + Either: + + 1. Override forward with the signature ``forward(ctx, *args, **kwargs)``. + ``setup_context`` is not overridden. Setting up the ctx for backward + happens inside the ``forward``. + 2. Override forward with the signature ``forward(*args, **kwargs)`` and + override ``setup_context``. Setting up the ctx for backward happens + inside ``setup_context`` (as opposed to inside the ``forward``) + + See :meth:`torch.autograd.Function.forward` and :ref:`extending-autograd` for more details. + """ + raise NotImplementedError("setup_context is not implemented.") + + @staticmethod + def backward(ctx: Any, *grad_outputs: Any) -> Any: + r"""Define a formula for differentiating the operation with backward mode automatic differentiation. + + This function is to be overridden by all subclasses. + (Defining this function is equivalent to defining the ``vjp`` function.) + + It must accept a context :attr:`ctx` as the first argument, followed by + as many outputs as the :func:`forward` returned (None will be passed in + for non tensor outputs of the forward function), + and it should return as many tensors, as there were inputs to + :func:`forward`. Each argument is the gradient w.r.t the given output, + and each returned value should be the gradient w.r.t. the + corresponding input. If an input is not a Tensor or is a Tensor not + requiring grads, you can just pass None as a gradient for that input. + + The context can be used to retrieve tensors saved during the forward + pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple + of booleans representing whether each input needs gradient. E.g., + :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the + first input to :func:`forward` needs gradient computed w.r.t. the + output. + """ + raise NotImplementedError( + "You must implement either the backward or vjp method for " + "your custom autograd.Function to use it with backward " + "mode AD." + ) + + # vjp and backward are alias of each other + vjp = backward + + @staticmethod + def jvp(ctx: Any, *grad_inputs: Any) -> Any: + r"""Define a formula for differentiating the operation with forward mode automatic differentiation. + + This function is to be overridden by all subclasses. + It must accept a context :attr:`ctx` as the first argument, followed by + as many inputs as the :func:`forward` got (None will be passed in + for non tensor inputs of the forward function), + and it should return as many tensors as there were outputs to + :func:`forward`. Each argument is the gradient w.r.t the given input, + and each returned value should be the gradient w.r.t. the + corresponding output. If an output is not a Tensor or the function is not + differentiable with respect to that output, you can just pass None as a + gradient for that input. + + You can use the :attr:`ctx` object to pass any value from the forward to this + functions. + """ + raise NotImplementedError( + "You must implement the jvp function for custom " + "autograd.Function to use it with forward mode AD." + ) + + +class Function(_SingleLevelFunction): + r"""Base class to create custom `autograd.Function`. + + To create a custom `autograd.Function`, subclass this class and implement + the :meth:`forward` and :meth:`backward` static methods. Then, to use your custom + op in the forward pass, call the class method ``apply``. Do not call + :meth:`forward` directly. + + To ensure correctness and best performance, make sure you are calling the + correct methods on ``ctx`` and validating your backward function using + :func:`torch.autograd.gradcheck`. + + See :ref:`extending-autograd` for more details on how to use this class. + + Examples:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Exp(Function): + >>> @staticmethod + >>> def forward(ctx, i): + >>> result = i.exp() + >>> ctx.save_for_backward(result) + >>> return result + >>> + >>> @staticmethod + >>> def backward(ctx, grad_output): + >>> result, = ctx.saved_tensors + >>> return grad_output * result + >>> + >>> # Use it by calling the apply method: + >>> # xdoctest: +SKIP + >>> output = Exp.apply(input) + """ + + def __init__(self, *args, **kwargs): + warnings.warn( + f"{self.__class__} should not be instantiated. Methods on autograd functions " + "are all static, so you should invoke them on the class itself. " + "Instantiating an autograd function will raise an " + "error in a future version of PyTorch.", + DeprecationWarning, + stacklevel=2, + ) + + def __call__(self, *args, **kwargs): + raise RuntimeError( + "Legacy autograd function with non-static forward method is deprecated. " + "Please use new-style autograd function with static forward method. " + "(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)" + ) + + """ + Bool that specifies if PyTorch should attempt to autogenerate + :func:`torch.vmap` support for this autograd.Function. You may set this to + True only if this autograd.Function's forward, backward, and jvp (if they + exist) are written using PyTorch operations; otherwise, please override + :meth:`torch.autograd.Function.vmap` to add support for :func:`torch.vmap`. + + Please see :ref:`func-autograd-function` for more details. + """ + generate_vmap_rule = False + + @staticmethod + def vmap(info, in_dims, *args): + r"""Define the behavior for this autograd.Function underneath :func:`torch.vmap`. + + For a :func:`torch.autograd.Function` to support + :func:`torch.vmap`, you must either override this static method, or set + ``generate_vmap_rule`` to ``True`` (you may not do both). + + If you choose to override this staticmethod: it must accept + + - an ``info`` object as the first argument. ``info.batch_size`` + specifies the size of the dimension being vmapped over, + while ``info.randomness`` is the randomness option passed to + :func:`torch.vmap`. + - an ``in_dims`` tuple as the second argument. + For each arg in ``args``, ``in_dims`` has a corresponding + ``Optional[int]``. It is ``None`` if the arg is not a Tensor or if + the arg is not being vmapped over, otherwise, it is an integer + specifying what dimension of the Tensor is being vmapped over. + - ``*args``, which is the same as the args to :meth:`~Function.forward`. + + The return of the vmap staticmethod is a tuple of ``(output, out_dims)``. + Similar to ``in_dims``, ``out_dims`` should be of the same structure as + ``output`` and contain one ``out_dim`` per output that specifies if the + output has the vmapped dimension and what index it is in. + + Please see :ref:`func-autograd-function` for more details. + """ + raise NotImplementedError( + "To use autograd.Function with vmap, you must either override the " + "vmap staticmethod or set generate_vmap_rule=True." + ) + + @classmethod + def apply(cls, *args, **kwargs): + def bind_default_args(func, *args, **kwargs): + signature = inspect.signature(func) + bound_args = signature.bind(*args, **kwargs) + bound_args.apply_defaults() + + return bound_args.args + + is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context) + if is_setup_ctx_defined: + args = bind_default_args(cls.forward, *args, **kwargs) + + if not torch._C._are_functorch_transforms_active(): + # See NOTE: [functorch vjp and autograd interaction] + args = _functorch.utils.unwrap_dead_wrappers(args) + return super().apply(*args, **kwargs) # type: ignore[misc] + + if not is_setup_ctx_defined: + raise RuntimeError( + "In order to use an autograd.Function with functorch transforms " + "(vmap, grad, jvp, jacrev, ...), it must override the setup_context " + "staticmethod. For more details, please see " + "https://pytorch.org/docs/main/notes/extending.func.html" + ) + + return custom_function_call(cls, *args, **kwargs) + + @staticmethod + def _compiled_autograd_key(ctx): + return (ctx._autograd_function_id,) + + +def _is_setup_context_defined(fn): + return fn != _SingleLevelFunction.setup_context + + +def once_differentiable( + fn: Callable[Concatenate[_T, _P], _R], +) -> Callable[Concatenate[_T, _P], _R]: + @functools.wraps(fn) + def wrapper(ctx: _T, *args: _P.args, **kwargs: _P.kwargs) -> _R: + with torch.no_grad(): + outputs = fn(ctx, *args, **kwargs) + + if not torch.is_grad_enabled(): + return outputs + + # If any of the inputs have requires_grad=True, we force the outputs + # to have requires_grad=True but point to a grad_fn which throws an + # error message during (double) back-propagation. + # XXX: this is only an approximation of requires_grad - there's no way + # to figure out if fn didn't use ctx.saved_tensors and as a result + # some Tensors might require grad, even if no args do. + # Unfortunately, this leads to unexpected error messages ("no nodes + # require computing gradients"), but I don't have a better idea. + # These functions would raise an error in backward anyway. + requires_grad = any( + isinstance(arg, torch.Tensor) and arg.requires_grad for arg in args + ) + if not requires_grad: + return outputs + + if not isinstance(outputs, tuple): + outputs_ = (outputs,) + else: + outputs_ = outputs + + err_fn = _functions.DelayedError( + b"trying to differentiate twice a function that was marked " + b"with @once_differentiable", + len(outputs_), + ) + + # Create aliases of each output that has requires_grad=True. We need + # at least one of the inputs to err_fn to require grad so that the + # output will have a grad_fn. + def fake_requires_grad(var): + if var is not None: + var = var.detach() + var.requires_grad = True + return var + + return err_fn(*[fake_requires_grad(v) for v in outputs_]) # type: ignore[return-value] + + return wrapper + + +class InplaceFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + def __init__(self, inplace=False): + super().__init__() + self.inplace = inplace + + +def _nested_map(condition, fn, condition_msg=None): + def _map(obj): + if condition(obj): + return fn(obj) + elif obj is None: + return None + elif isinstance(obj, (list, tuple)): + mapped = (_map(x) for x in obj) + if hasattr(obj, "_fields"): + # obj is namedtuple + return type(obj)(*mapped) + return type(obj)(mapped) + elif isinstance(obj, dict): + return {x: _map(obj[x]) for x in obj} + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _map + + +def _jit_unwrap_structured(obj): + if hasattr(obj, "_jit_unwrap"): + return obj._jit_unwrap() + return obj + + +def _iter_filter(condition, allow_unknown=False, condition_msg=None, conversion=None): + def _iter(obj): + if conversion is not None: + obj = conversion(obj) + if condition(obj): + yield obj + elif obj is None: + return + elif isinstance(obj, (list, tuple)): + for o in obj: + yield from _iter(o) + elif isinstance(obj, dict): + # We only accept primitive key types, so we needn't inspect them + for o in obj.values(): + yield from _iter(o) + elif allow_unknown: + yield obj + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _iter + + +def _unflatten(input, proto): + # unflatten a list or tuple input into a nested list/tuple structure + # specified by proto + def unflatten_helper(input, proto): + res: list[Optional[torch.Tensor]] = [] + if hasattr(proto, "_jit_wrap"): + return proto._jit_wrap(input) + if not isinstance(proto, (list, tuple)): + return input[0], input[1:] + for e in proto: + if e is None: + res.append(e) + else: + res_e, input = unflatten_helper(input, e) + res.append(res_e) + return type(proto)(res), input + + return unflatten_helper(input, proto)[0] + + +_iter_jit_values = _iter_filter( + lambda o: o is None or isinstance(o, torch._C.Value), + condition_msg="jit's Values or None", +) +_iter_tensors = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + condition_msg="Tensors", + conversion=_jit_unwrap_structured, +) +_iter_tensors_permissive = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + allow_unknown=True, + condition_msg="Tensors (permissive)", +) +_iter_None_tensors = _iter_filter( + lambda o: o is None or isinstance(o, torch.Tensor), condition_msg="Tensors or None" +) +_map_tensor_data = _nested_map( + lambda x: isinstance(x, torch.Tensor), lambda o: o.data, condition_msg="Tensors" +) + + +class NestedIOFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + # The 'type: ignore' statements are needed here because these functions are declared as '@staticmethod' in the + # superclass (Function) but are instance methods here, which mypy reports as incompatible. + + def _do_forward(self, *input): + self._nested_input = input + flat_input = tuple(_iter_tensors(input)) + flat_output = super()._do_forward(*flat_input) # type: ignore[misc] + nested_tensors = _unflatten(flat_output, self._nested_output) + return nested_tensors + + def _do_backward(self, gradients, retain_variables): + self.retain_variables = retain_variables + result = super()._do_backward(gradients, retain_variables) # type: ignore[misc] + if not retain_variables: + del self._nested_output + del self._to_save_nested + return result + + def backward(self, *gradients: Any) -> Any: # type: ignore[override] + r""" + Shared backward utility. + """ + nested_gradients = _unflatten(gradients, self._nested_output) + result = self.backward_extended(*nested_gradients) # type: ignore[func-returns-value] + return tuple(_iter_None_tensors(result)) + + __call__ = _do_forward + + def forward(self, *args: Any) -> Any: # type: ignore[override] + r""" + Shared forward utility. + """ + nested_tensors = _map_tensor_data(self._nested_input) + result = self.forward_extended(*nested_tensors) # type: ignore[func-returns-value] + del self._nested_input + self._nested_output = result + return tuple(_iter_tensors(result)) + + def save_for_backward(self, *args: Any) -> None: + r""" + See :meth:`Function.save_for_backward`. + """ + self.to_save = tuple(_iter_tensors(args)) + self._to_save_nested = args + + @property + def saved_tensors(self): # type: ignore[override] + r""" + See :meth:`Function.saved_tensors`. + """ + flat_tensors = super().saved_tensors # type: ignore[misc] + return _unflatten(flat_tensors, self._to_save_nested) + + def mark_dirty(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_dirty`. + """ + self.dirty_tensors = tuple(_iter_tensors((args, kwargs))) + + def mark_non_differentiable(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_non_differentiable`. + """ + self.non_differentiable = tuple(_iter_tensors((args, kwargs))) + + def forward_extended(self, *input: Any) -> None: + r""" + User defined forward. + """ + raise NotImplementedError + + def backward_extended(self, *grad_output: Any) -> None: + r""" + User defined backward. + """ + raise NotImplementedError diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/functional.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..020b39b178c234e4cdca9a8e5f36f9e30b39efb6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/functional.py @@ -0,0 +1,1210 @@ +# mypy: allow-untyped-defs + +import torch +from torch._vmap_internals import _vmap + +from . import forward_ad as fwAD + + +__all__ = ["vjp", "jvp", "jacobian", "hessian", "hvp", "vhp"] + +# Utility functions + + +def _as_tuple_nocheck(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _as_tuple(inp, arg_name=None, fn_name=None): + # Ensures that inp is a tuple of Tensors + # Returns whether or not the original inp was a tuple and the tupled version of the input + if arg_name is None and fn_name is None: + return _as_tuple_nocheck(inp) + + is_inp_tuple = True + if not isinstance(inp, tuple): + inp = (inp,) + is_inp_tuple = False + + for i, el in enumerate(inp): + if not isinstance(el, torch.Tensor): + if is_inp_tuple: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" value at index {i} has type {type(el)}." + ) + else: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" given {arg_name} has type {type(el)}." + ) + + return is_inp_tuple, inp + + +def _tuple_postprocess(res, to_unpack): + # Unpacks a potentially nested tuple of Tensors + # to_unpack should be a single boolean or a tuple of two booleans. + # It is used to: + # - invert _as_tuple when res should match the inp given to _as_tuple + # - optionally remove nesting of two tuples created by multiple calls to _as_tuple + if isinstance(to_unpack, tuple): + if len(to_unpack) != 2: + raise AssertionError("Expected to_unpack tuple to have exactly 2 elements") + if not to_unpack[1]: + res = tuple(el[0] for el in res) + if not to_unpack[0]: + res = res[0] + else: + if not to_unpack: + res = res[0] + return res + + +def _grad_preprocess(inputs, create_graph, need_graph): + # Preprocess the inputs to make sure they require gradient + # inputs is a tuple of Tensors to preprocess + # create_graph specifies if the user wants gradients to flow back to the Tensors in inputs + # need_graph specifies if we internally want gradients to flow back to the Tensors in res + # Note that we *always* create a new Tensor object to be able to see the difference between + # inputs given as arguments and the same Tensors automatically captured by the user function. + # Check this issue for more details on how that can happen: https://github.com/pytorch/pytorch/issues/32576 + res = [] + for inp in inputs: + if create_graph and inp.requires_grad: + # Create at least a new Tensor object in a differentiable way + if not inp.is_sparse: + # Use .view_as() to get a shallow copy + res.append(inp.view_as(inp)) + else: + # We cannot use view for sparse Tensors so we clone + res.append(inp.clone()) + else: + res.append(inp.detach().requires_grad_(need_graph)) + return tuple(res) + + +def _grad_postprocess(inputs, create_graph): + # Postprocess the generated Tensors to avoid returning Tensors with history when the user did not + # request it. + if isinstance(inputs[0], torch.Tensor): + if not create_graph: + return tuple(inp.detach() for inp in inputs) + else: + return inputs + else: + return tuple(_grad_postprocess(inp, create_graph) for inp in inputs) + + +def _validate_v(v, other, is_other_tuple): + # This assumes that other is the correct shape, and v should match + # Both are assumed to be tuples of Tensors + if len(other) != len(v): + if is_other_tuple: + raise RuntimeError( + f"v is a tuple of invalid length: should be {len(other)} but got {len(v)}." + ) + else: + raise RuntimeError("The given v should contain a single Tensor.") + + for idx, (el_v, el_other) in enumerate(zip(v, other)): + if el_v.size() != el_other.size(): + prepend = "" + if is_other_tuple: + prepend = f"Entry {idx} in " + raise RuntimeError( + f"{prepend}v has invalid size: should be {el_other.size()} but got {el_v.size()}." + ) + + +def _check_requires_grad(inputs, input_type, strict): + # Used to make all the necessary checks to raise nice errors in strict mode. + if not strict: + return + + if input_type not in ["outputs", "grad_inputs", "jacobian", "hessian"]: + raise RuntimeError("Invalid input_type to _check_requires_grad") + for i, inp in enumerate(inputs): + if inp is None: + # This can only be reached for grad_inputs. + raise RuntimeError( + f"The output of the user-provided function is independent of input {i}." + " This is not allowed in strict mode." + ) + if not inp.requires_grad: + if input_type == "hessian": + raise RuntimeError( + f"The hessian of the user-provided function with respect to input {i}" + " is independent of the input. This is not allowed in strict mode." + " You should ensure that your function is thrice differentiable and that" + " the hessian depends on the inputs." + ) + elif input_type == "jacobian": + raise RuntimeError( + "While computing the hessian, found that the jacobian of the user-provided" + f" function with respect to input {i} is independent of the input. This is not" + " allowed in strict mode. You should ensure that your function is twice" + " differentiable and that the jacobian depends on the inputs (this would be" + " violated by a linear function for example)." + ) + elif input_type == "grad_inputs": + raise RuntimeError( + f"The gradient with respect to input {i} is independent of the inputs of the" + " user-provided function. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + f"Output {i} of the user-provided function does not require gradients." + " The outputs must be computed in a differentiable manner from the input" + " when running in strict mode." + ) + + +def _autograd_grad( + outputs, + inputs, + grad_outputs=None, + create_graph=False, + retain_graph=None, + is_grads_batched=False, +): + # Version of autograd.grad that accepts `None` in outputs and do not compute gradients for them. + # This has the extra constraint that inputs has to be a tuple + if not isinstance(outputs, tuple): + raise AssertionError("Expected outputs to be a tuple") + if grad_outputs is None: + grad_outputs = (None,) * len(outputs) + if not isinstance(grad_outputs, tuple): + raise AssertionError("Expected grad_outputs to be a tuple") + if len(outputs) != len(grad_outputs): + raise AssertionError( + f"Expected outputs and grad_outputs to have the same length, " + f"but got {len(outputs)} and {len(grad_outputs)}" + ) + + new_outputs: tuple[torch.Tensor, ...] = () + new_grad_outputs: tuple[torch.Tensor, ...] = () + for out, grad_out in zip(outputs, grad_outputs): + if out is not None and out.requires_grad: + new_outputs += (out,) + new_grad_outputs += (grad_out,) + + if len(new_outputs) == 0: + # No differentiable output, we don't need to call the autograd engine + return (None,) * len(inputs) + else: + return torch.autograd.grad( + new_outputs, + inputs, + new_grad_outputs, + allow_unused=True, + create_graph=create_graph, + retain_graph=retain_graph, + is_grads_batched=is_grads_batched, + ) + + +def _fill_in_zeros(grads, refs, strict, create_graph, stage): + # Used to detect None in the grads and depending on the flags, either replace them + # with Tensors full of 0s of the appropriate size based on the refs or raise an error. + # strict and create graph allow us to detect when it is appropriate to raise an error + # stage gives us information of which backward call we consider to give good error message + if stage not in ["back", "back_trick", "double_back", "double_back_trick"]: + raise RuntimeError(f"Invalid stage argument '{stage}' to _fill_in_zeros") + + res: tuple[torch.Tensor, ...] = () + for i, grads_i in enumerate(grads): + if grads_i is None: + if strict: + if stage == "back": + raise RuntimeError( + "The output of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + elif stage == "back_trick": + raise RuntimeError( + f"The gradient with respect to the input is independent of entry {i}" + " in the grad_outputs when using the double backward trick to compute" + " forward mode gradients. This is not allowed in strict mode." + ) + elif stage == "double_back": + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"entry {i} in the grad_jacobian. This is not allowed in strict " + "mode as it prevents from using the double backward trick to " + "replace forward mode AD." + ) + + grads_i = torch.zeros_like(refs[i]) + else: + if strict and create_graph and not grads_i.requires_grad: + if "double" not in stage: + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + + res += (grads_i,) + + return res + + +# Public API + + +def vjp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between a vector ``v`` and the Jacobian of the given function at the point given by the inputs. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector + Jacobian product is computed. Must be the same size as the output + of ``func``. This argument is optional when the output of ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vjp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vjp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vjp(exp_reducer, inputs, v) + (tensor([5.7817, 7.2458, 5.7830, 6.7782]), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]])) + + >>> vjp(exp_reducer, inputs, v, create_graph=True) + (tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = torch.ones(2) + >>> vjp(adder, inputs, v) + (tensor([2.4225, 2.3340]), + (tensor([2., 2.]), tensor([3., 3.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vjp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vjp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if v is not None: + _, v = _as_tuple(v, "v", "vjp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, outputs, is_outputs_tuple) + else: + if len(outputs) != 1 or outputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the " + "user-provided function returns " + "a single Tensor with a single element." + ) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(outputs, inputs, v, create_graph=create_graph) + vjp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "back") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + vjp = _grad_postprocess(vjp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vjp, is_inputs_tuple + ) + + +def jvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the Jacobian of the given function at the point given by the inputs and a vector ``v``. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Jacobian + vector product is computed. Must be the same size as the input of + ``func``. This argument is optional when the input to ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + jvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the output. + + Note: + ``autograd.functional.jvp`` computes the jvp by using the backward of + the backward (sometimes called the double backwards trick). This is not + the most performant way of computing the jvp. Please consider using + :func:`torch.func.jvp` or the + :ref:`low-level forward-mode AD API ` instead. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4, 4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jvp(exp_reducer, inputs, v) + (tensor([6.3090, 4.6742, 7.9114, 8.2106]), + tensor([6.3090, 4.6742, 7.9114, 8.2106])) + + >>> jvp(exp_reducer, inputs, v, create_graph=True) + (tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=), + tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.ones(2), torch.ones(2)) + >>> jvp(adder, inputs, v) + (tensor([2.2399, 2.5005]), + tensor([5., 5.])) + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "jvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to " + "the user-provided function is a single Tensor " + "with a single element." + ) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + # The backward is linear so the value of grad_outputs is not important as + # it won't appear in the double backward graph. We only need to ensure that + # it does not contain inf or nan. + grad_outputs = tuple( + torch.zeros_like(out, requires_grad=True) for out in outputs + ) + + grad_inputs = _autograd_grad(outputs, inputs, grad_outputs, create_graph=True) + _check_requires_grad(grad_inputs, "grad_inputs", strict=strict) + + if create_graph: + with torch.enable_grad(): + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + else: + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + jvp = _grad_postprocess(jvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + jvp, is_outputs_tuple + ) + + +def _construct_standard_basis_for( + tensors: tuple[torch.Tensor, ...], tensor_numels: tuple[int, ...] +) -> tuple[torch.Tensor, ...]: + # This function: + # - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix. + # - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`. + # - Each chunk corresponds to one tensor. The chunk has the same dtype and + # device as the tensor + # + # For example, with tensor_numels = [1, 2, 1], this function returns: + # ( tensor([[1], tensor([[0, 0], tensor([[0], + # [0], [1, 0], [0], + # [0], [0, 1], [0], + # [0]]) , [0, 0]]) , [1]]) ) + # + # Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors) + # Precondition: tensors always has at least one element. + # + # See NOTE: [Computing jacobian with vmap and grad for multiple tensors] + # for context behind this function. All the pre-conditions are guarded for + # in torch.autograd.functional.jacobian. + if len(tensors) != len(tensor_numels): + raise AssertionError( + f"Expected tensors and tensor_numels to have the same length, " + f"but got {len(tensors)} and {len(tensor_numels)}" + ) + if len(tensors) == 0: + raise AssertionError("Expected at least one tensor") + total_numel = sum(tensor_numels) + chunks = tuple( + tensor.new_zeros(total_numel, tensor_numel) + for tensor, tensor_numel in zip(tensors, tensor_numels) + ) + diag_start_idx = 0 + for chunk, numel in zip(chunks, tensor_numels): + chunk.diagonal(diag_start_idx).fill_(1) + diag_start_idx -= numel + return chunks + + +def _jacfwd(func, inputs, strict=False, vectorize=False): + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `strict=False` or " + '`strategy="reverse-mode"`.' + ) + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + output_info = [] + + if vectorize: + # See NOTE: [Computing jacobian with vmap and grad for multiple outputs] + input_numels = tuple(input.numel() for input in inputs) + + # Step 1: Prepare tangents + tangents = _construct_standard_basis_for(inputs, input_numels) + + # Step 2: Compute vmap over computation with dual tensors + def jvp(tangents): + with fwAD.dual_level(): + dual_inputs = tuple( + fwAD.make_dual(input, tangent.view_as(input)) + for input, tangent in zip(inputs, tangents) + ) + _is_outputs_tuple, dual_outputs = _as_tuple( + func(*dual_inputs), "outputs" + ) + output_info.append(_is_outputs_tuple) + jv = [] + primal_outs = [] + for dual_out in dual_outputs: + primal, tangent = fwAD.unpack_dual(dual_out) + primal_outs.append(primal) + if tangent is not None: + jv.append(tangent) + else: + jv.append(torch.zeros_like(primal)) + output_info.append(primal_outs) + return tuple(jv) + + outputs_before_split = _vmap(jvp)(tangents) + is_outputs_tuple, outputs = output_info + # Step 3: for each of the output tangents, split along dim 0 + jacobian_input_output = [] + for jac_output_i, output_i in zip(outputs_before_split, outputs): + jacobian_output_i_output = [] + for jac, input_j in zip(jac_output_i.split(input_numels, dim=0), inputs): + # We need to transpose the Jacobian because in forward AD, the + # batch dimension represents that of the inputs + jacobian_input_i_output_j = jac.permute(*range(1, jac.ndim), 0).reshape( + (*output_i.shape, *input_j.shape) + ) # noqa: C409 + + jacobian_output_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_output_i_output) + + # Omit [Step 4] because everything is already transposed w/ forward AD + return _tuple_postprocess( + jacobian_input_output, (is_outputs_tuple, is_inputs_tuple) + ) + else: + raise NotImplementedError( + "Computing Jacobian using forward-AD or forward-over-reverse Hessian is" + "only implemented for `vectorize=True`." + ) + + +def jacobian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + strategy="reverse-mode", +): + r"""Compute the Jacobian of a given function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Jacobian will be + computed in a differentiable manner. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jacobian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.jacrev` or + :func:`torch.func.jacfwd` instead if you are looking for something + less experimental and more performant. + When computing the jacobian, usually we invoke + ``autograd.grad`` once per row of the jacobian. If this flag is + ``True``, we perform only a single ``autograd.grad`` call with + ``batched_grad=True`` which uses the vmap prototype feature. + Though this should lead to performance improvements in many cases, + because this feature is still experimental, there may be performance + cliffs. See :func:`torch.autograd.grad`'s ``batched_grad`` parameter for + more information. + strategy (str, optional): Set to ``"forward-mode"`` or ``"reverse-mode"`` to + determine whether the Jacobian will be computed with forward or reverse + mode AD. Currently, ``"forward-mode"`` requires ``vectorized=True``. + Defaults to ``"reverse-mode"``. If ``func`` has more outputs than + inputs, ``"forward-mode"`` tends to be more performant. Otherwise, + prefer to use ``"reverse-mode"``. + + Returns: + Jacobian (Tensor or nested tuple of Tensors): if there is a single + input and output, this will be a single Tensor containing the + Jacobian for the linearized inputs and output. If one of the two is + a tuple, then the Jacobian will be a tuple of Tensors. If both of + them are tuples, then the Jacobian will be a tuple of tuple of + Tensors where ``Jacobian[i][j]`` will contain the Jacobian of the + ``i``\th output and ``j``\th input and will have as size the + concatenation of the sizes of the corresponding output and the + corresponding input and will have same dtype and device as the + corresponding input. If strategy is ``forward-mode``, the dtype will be + that of the output; otherwise, the input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jacobian(exp_reducer, inputs) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]]) + + >>> jacobian(exp_reducer, inputs, create_graph=True) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]], grad_fn=) + + >>> def exp_adder(x, y): + ... return 2 * x.exp() + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> jacobian(exp_adder, inputs) + (tensor([[2.8052, 0.0000], + [0.0000, 3.3963]]), + tensor([[3., 0.], + [0., 3.]])) + + >>> def linear_model(x): + ... W = torch.tensor([[2.0, -1.0], [0.0, 1.0]]) + ... b = torch.tensor([1.0, 0.5]) + ... return x @ W.T + b + + >>> x = torch.randn(4, 2, requires_grad=True) + >>> jac = jacobian(linear_model, x, vectorize=True) + >>> jac.shape + torch.Size([4, 2, 4, 2]) + """ + if strategy not in ("forward-mode", "reverse-mode"): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode". Hint: If your ' + 'function has more outputs than inputs, "forward-mode" tends to be more performant. ' + 'Otherwise, prefer to use "reverse-mode".' + ) + if strategy == "forward-mode": + if create_graph: + raise NotImplementedError( + "torch.autograd.functional.jacobian: `create_graph=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `create_graph=False` or " + '`strategy="reverse-mode"`.' + ) + return _jacfwd(func, inputs, strict, vectorize) + + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jacobian" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if vectorize: + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + "and `vectorized=True` are not supported together. " + "Please either set `strict=False` or " + "`vectorize=False`." + ) + # NOTE: [Computing jacobian with vmap and grad for multiple outputs] + # + # Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3). + # It turns out we can compute the jacobian of this function with a single + # call to autograd.grad by using vmap over the correct grad_outputs. + # + # Firstly, one way to compute the jacobian is to stack x**2 and x.sum() + # into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()]) + # + # To get the first row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0])) + # To get the 2nd row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0])) + # and so on. + # + # Using vmap, we can vectorize all 4 of these computations into one by + # passing the standard basis for R^4 as the grad_output. + # vmap(partial(autograd.grad, g(x), x))(torch.eye(4)). + # + # Now, how do we compute the jacobian *without stacking the output*? + # We can just split the standard basis across the outputs. So to + # compute the jacobian of f(x), we'd use + # >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...)) + # The grad_outputs looks like the following: + # ( torch.tensor([[1, 0, 0], + # [0, 1, 0], + # [0, 0, 1], + # [0, 0, 0]]), + # torch.tensor([[0], + # [0], + # [0], + # [1]]) ) + # + # But we're not done yet! + # >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...))) + # returns a Tensor of shape [4, 3]. We have to remember to split the + # jacobian of shape [4, 3] into two: + # - one of shape [3, 3] for the first output + # - one of shape [ 3] for the second output + + # Step 1: Construct grad_outputs by splitting the standard basis + output_numels = tuple(output.numel() for output in outputs) + grad_outputs = _construct_standard_basis_for(outputs, output_numels) + flat_outputs = tuple(output.reshape(-1) for output in outputs) + + # Step 2: Call vmap + autograd.grad + def vjp(grad_output): + vj = list( + _autograd_grad( + flat_outputs, + inputs, + grad_output, + create_graph=create_graph, + is_grads_batched=True, + ) + ) + for el_idx, vj_el in enumerate(vj): + if vj_el is not None: + continue + vj[el_idx] = torch.zeros_like(inputs[el_idx]).expand( + (sum(output_numels),) + inputs[el_idx].shape + ) + return tuple(vj) + + jacobians_of_flat_output = vjp(grad_outputs) + + # Step 3: The returned jacobian is one big tensor per input. In this step, + # we split each Tensor by output. + jacobian_input_output = [] + for jac_input_i, input_i in zip(jacobians_of_flat_output, inputs): + jacobian_input_i_output = [] + for jac, output_j in zip( + jac_input_i.split(output_numels, dim=0), outputs + ): + jacobian_input_i_output_j = jac.view(output_j.shape + input_i.shape) + jacobian_input_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_input_i_output) + + # Step 4: Right now, `jacobian` is a List[List[Tensor]]. + # The outer List corresponds to the number of inputs, + # the inner List corresponds to the number of outputs. + # We need to exchange the order of these and convert to tuples + # before returning. + jacobian_output_input = tuple(zip(*jacobian_input_output)) + + jacobian_output_input = _grad_postprocess( + jacobian_output_input, create_graph + ) + return _tuple_postprocess( + jacobian_output_input, (is_outputs_tuple, is_inputs_tuple) + ) + + jacobian: tuple[torch.Tensor, ...] = () + + for i, out in enumerate(outputs): + # mypy complains that expression and variable have different types due to the empty list + jac_i: tuple[list[torch.Tensor]] = tuple([] for _ in range(len(inputs))) # type: ignore[assignment] + for j in range(out.nelement()): + vj = _autograd_grad( + (out.reshape(-1)[j],), + inputs, + retain_graph=True, + create_graph=create_graph, + ) + + for el_idx, (jac_i_el, vj_el, inp_el) in enumerate( + zip(jac_i, vj, inputs) + ): + if vj_el is not None: + if strict and create_graph and not vj_el.requires_grad: + msg = ( + "The jacobian of the user-provided function is " + f"independent of input {i}. This is not allowed in " + "strict mode when create_graph=True." + ) + raise RuntimeError(msg) + jac_i_el.append(vj_el) + else: + if strict: + msg = ( + f"Output {i} of the user-provided function is " + f"independent of input {el_idx}. This is not allowed in " + "strict mode." + ) + raise RuntimeError(msg) + jac_i_el.append(torch.zeros_like(inp_el)) + + jacobian += ( + tuple( + torch.stack(jac_i_el, dim=0).view( + out.size() + inputs[el_idx].size() # type: ignore[operator] + ) + for (el_idx, jac_i_el) in enumerate(jac_i) + ), + ) + + jacobian = _grad_postprocess(jacobian, create_graph) + + return _tuple_postprocess(jacobian, (is_outputs_tuple, is_inputs_tuple)) + + +def hessian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + outer_jacobian_strategy="reverse-mode", +): + r"""Compute the Hessian of a given scalar function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Hessian will be computed in + a differentiable manner. Note that when ``strict`` is ``False``, the result can not + require gradients or be disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we detect that there exists an input + such that all the outputs are independent of it. If ``False``, we return a Tensor of zeros as the + hessian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.hessian` + instead if you are looking for something less experimental and more performant. + When computing the hessian, usually we invoke + ``autograd.grad`` once per row of the hessian. If this flag is + ``True``, we use the vmap prototype feature as the backend to + vectorize calls to ``autograd.grad`` so we only invoke it once + instead of once per row. This should lead to performance + improvements in many use cases, however, due to this feature + being incomplete, there may be performance cliffs. Please + use `torch._C._debug_only_display_vmap_fallback_warnings(True)` + to show any performance warnings and file us issues if + warnings exist for your use case. Defaults to ``False``. + outer_jacobian_strategy (str, optional): The Hessian is computed by + computing the Jacobian of a Jacobian. The inner Jacobian is always + computed in reverse-mode AD. Setting strategy to ``"forward-mode"`` + or ``"reverse-mode"`` determines whether the outer Jacobian will be + computed with forward or reverse mode AD. Currently, computing the outer + Jacobian in ``"forward-mode"`` requires ``vectorized=True``. Defaults + to ``"reverse-mode"``. + + Returns: + Hessian (Tensor or a tuple of tuple of Tensors): if there is a single input, + this will be a single Tensor containing the Hessian for the input. + If it is a tuple, then the Hessian will be a tuple of tuples where + ``Hessian[i][j]`` will contain the Hessian of the ``i``\th input + and ``j``\th input with size the sum of the size of the ``i``\th input plus + the size of the ``j``\th input. ``Hessian[i][j]`` will have the same + dtype and device as the corresponding ``i``\th input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hessian(pow_reducer, inputs) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]]) + + >>> hessian(pow_reducer, inputs, create_graph=True) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]], grad_fn=) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> hessian(pow_adder_reducer, inputs) + ((tensor([[4., 0.], + [0., 4.]]), + tensor([[0., 0.], + [0., 0.]])), + (tensor([[0., 0.], + [0., 0.]]), + tensor([[6., 0.], + [0., 6.]]))) + """ + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hessian") + if outer_jacobian_strategy not in ( + "forward-mode", + "reverse-mode", + ): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode".' + ) + + def ensure_single_output_function(*inp): + out = func(*inp) + is_out_tuple, t_out = _as_tuple( + out, "outputs of the user-provided function", "hessian" + ) + _check_requires_grad(t_out, "outputs", strict=strict) + + if is_out_tuple or not isinstance(out, torch.Tensor): + raise RuntimeError( + "The function given to hessian should return a single Tensor" + ) + + if out.nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hessian should contain a single element" + ) + + return out.squeeze() + + def jac_func(*inp): + if outer_jacobian_strategy == "forward-mode": + # _grad_preprocess requires create_graph=True and input to require_grad + # or else the input will be detached + inp = tuple(t.requires_grad_(True) for t in inp) + jac = jacobian(ensure_single_output_function, inp, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + return jac + + res = jacobian( + jac_func, + inputs, + create_graph=create_graph, + strict=strict, + vectorize=vectorize, + strategy=outer_jacobian_strategy, + ) + return _tuple_postprocess(res, (is_inputs_tuple, is_inputs_tuple)) + + +def vhp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between vector ``v`` and Hessian of a given scalar function at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector Hessian + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vhp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vhp (tuple of Tensors or Tensor): result of the dot product with the + same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vhp(pow_reducer, inputs, v) + (tensor(0.5591), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]])) + >>> vhp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.5591, grad_fn=), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]], grad_fn=)) + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> vhp(pow_adder_reducer, inputs, v) + (tensor(4.8053), + (tensor([0., 0.]), + tensor([6., 6.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vhp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "vhp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vhp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to vhp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to vhp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(jac, inputs, v, create_graph=create_graph) + vhp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "double_back") + + outputs = _grad_postprocess(outputs, create_graph) + vhp = _grad_postprocess(vhp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vhp, is_inputs_tuple + ) + + +def hvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the scalar function's Hessian and a vector ``v`` at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Hessian vector + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result will be + computed in a differentiable way. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + hvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + hvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hvp(pow_reducer, inputs, v) + (tensor(0.1448), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]])) + + >>> hvp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.1448, grad_fn=), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]], grad_fn=)) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> hvp(pow_adder_reducer, inputs, v) + (tensor(2.3030), + (tensor([0., 0.]), + tensor([6., 6.]))) + + Note: + + This function is significantly slower than `vhp` due to backward mode AD constraints. + If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you + know that your function satisfies this condition, you should use vhp instead that is + much faster with the current implementation. + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "hvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "hvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to hvp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hvp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + grad_jac = tuple(torch.zeros_like(inp, requires_grad=True) for inp in inputs) + + double_back = _autograd_grad(jac, inputs, grad_jac, create_graph=True) + _check_requires_grad(jac, "hessian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(double_back, grad_jac, v, create_graph=create_graph) + hvp = _fill_in_zeros( + grad_res, inputs, strict, create_graph, "double_back_trick" + ) + + outputs = _grad_postprocess(outputs, create_graph) + hvp = _grad_postprocess(hvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + hvp, is_inputs_tuple + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/grad_mode.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/grad_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..d6716a6b1a1a220ec71c7205d7f444675d0f6461 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/grad_mode.py @@ -0,0 +1,420 @@ +# mypy: allow-untyped-defs +from typing import Any, Union + +import torch +from torch.utils._contextlib import ( + _DecoratorContextManager, + _NoParamDecoratorContextManager, + F, +) + + +__all__ = [ + "no_grad", + "enable_grad", + "set_grad_enabled", + "inference_mode", + "set_multithreading_enabled", +] + + +class no_grad(_NoParamDecoratorContextManager): + r"""Context-manager that disables gradient calculation. + + Disabling gradient calculation is useful for inference, when you are sure + that you will not call :meth:`Tensor.backward()`. It will reduce memory + consumption for computations that would otherwise have `requires_grad=True`. + + In this mode, the result of every computation will have + `requires_grad=False`, even when the inputs have `requires_grad=True`. + There is an exception! All factory functions, or functions that create + a new Tensor and take a requires_grad kwarg, will NOT be affected by + this mode. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + No-grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + If you want to disable forward AD for a computation, you can unpack + your dual tensors. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... y = x * 2 + >>> y.requires_grad + False + >>> @torch.no_grad() + ... def doubler(x): + ... return x * 2 + >>> z = doubler(x) + >>> z.requires_grad + False + >>> @torch.no_grad() + ... def tripler(x): + ... return x * 3 + >>> z = tripler(x) + >>> z.requires_grad + False + >>> # factory function exception + >>> with torch.no_grad(): + ... a = torch.nn.Parameter(torch.rand(10)) + >>> a.requires_grad + True + """ + + def __init__(self) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.prev = False + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch.set_grad_enabled(False) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch.set_grad_enabled(self.prev) + + +class enable_grad(_NoParamDecoratorContextManager): + r"""Context-manager that enables gradient calculation. + + Enables gradient calculation, if it has been disabled via :class:`~no_grad` + or :class:`~set_grad_enabled`. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + enable_grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... with torch.enable_grad(): + ... y = x * 2 + >>> y.requires_grad + True + >>> y.backward() + >>> x.grad + tensor([2.]) + >>> @torch.enable_grad() + ... def doubler(x): + ... return x * 2 + >>> with torch.no_grad(): + ... z = doubler(x) + >>> z.requires_grad + True + >>> @torch.enable_grad() + ... def tripler(x): + ... return x * 3 + >>> with torch.no_grad(): + ... z = tripler(x) + >>> z.requires_grad + True + + """ + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch._C._set_grad_enabled(True) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + +class set_grad_enabled(_DecoratorContextManager): + r"""Context-manager that sets gradient calculation on or off. + + ``set_grad_enabled`` will enable or disable grads based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + Args: + mode (bool): Flag whether to enable grad (``True``), or disable + (``False``). This can be used to conditionally enable + gradients. + + .. note:: + set_grad_enabled is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> is_train = False + >>> with torch.set_grad_enabled(is_train): + ... y = x * 2 + >>> y.requires_grad + False + >>> _ = torch.set_grad_enabled(True) + >>> y = x * 2 + >>> y.requires_grad + True + >>> _ = torch.set_grad_enabled(False) + >>> y = x * 2 + >>> y.requires_grad + False + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch.is_grad_enabled() + self.mode = mode + torch._C._set_grad_enabled(mode) + + def __call__(self, orig_func: F) -> F: + torch._C._set_grad_enabled(self.prev) + return super().__call__(orig_func) + + def __enter__(self) -> None: + torch._C._set_grad_enabled(self.mode) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + def __str__(self) -> str: + return f"{torch.typename(self)}(mode={self.mode})" + + def __repr__(self) -> str: + return str(self) + + def clone(self) -> "set_grad_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class inference_mode(_DecoratorContextManager): + r"""Context manager that enables or disables inference mode. + + InferenceMode is analogous to :class:`~no_grad` and should be used + when you are certain your operations will not interact with autograd + (e.g., during data loading or model evaluation). Compared to + :class:`~no_grad`, it removes additional overhead by disabling view + tracking and version counter bumps. It is also more restrictive, in + that tensors created in this mode cannot be used in computations + recorded by autograd. + + This context manager is thread-local; it does not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + Inference mode is one of several mechanisms that can locally enable + or disable gradients. See :ref:`locally-disable-grad-doc` for a + comparison. If avoiding the use of tensors created in inference mode + in autograd-tracked regions is difficult, consider benchmarking your + code with and without inference mode to weigh the performance benefits + against the trade-offs. You can always use :class:`~no_grad` instead. + + .. note:: + Unlike some other mechanisms that locally enable or disable grad, + entering inference_mode also disables :ref:`forward-mode AD `. + + .. warning:: + `inference_mode` does NOT automatically set the model to evaluation mode. + For proper inference behavior (e.g., disabling dropout, using running statistics + in batch normalization), you must explicitly set your model to evaluation mode using + `model.eval()` in addition to using this context manager. + + Args: + mode (bool or function): Either a boolean flag to enable or disable + inference mode, or a Python function to decorate with inference + mode enabled. + + Example:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> import torch + >>> x = torch.ones(1, 2, 3, requires_grad=True) + >>> with torch.inference_mode(): + ... y = x * x + >>> y.requires_grad + False + >>> # xdoctest: +SKIP("want string isn't quite right") + >>> y._version + Traceback (most recent call last): + File "", line 1, in + RuntimeError: Inference tensors do not track version counter. + >>> @torch.inference_mode() + ... def func(x): + ... return x * x + >>> out = func(x) + >>> out.requires_grad + False + >>> @torch.inference_mode() + ... def doubler(x): + ... return x * 2 + >>> out = doubler(x) + >>> out.requires_grad + False + + """ + + def __init__(self, mode: bool = True) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.mode = mode + + def __new__(cls, mode=True): + if isinstance(mode, bool): + return super().__new__(cls) + return cls()(mode) + + def __enter__(self) -> None: + self._inference_mode_context = torch._C._InferenceMode(self.mode) + self._inference_mode_context.__enter__() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self._inference_mode_context.__exit__(exc_type, exc_value, traceback) + + def clone(self) -> "inference_mode": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +def _enter_inference_mode(mode): + mode_context = torch._C._InferenceMode(mode) + mode_context.__enter__() + return mode_context + + +def _exit_inference_mode(mode): + mode.__exit__(None, None, None) + + +class set_multithreading_enabled(_DecoratorContextManager): + r"""Context-manager that sets multithreaded backwards on or off. + + ``set_multithreading_enabled`` will enable or disable multithreaded backwards based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + Args: + mode (bool): Flag whether to enable multithreaded backwards (``True``), or disable + (``False``). + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_multithreading_enabled() + torch._C._set_multithreading_enabled(mode) + self.mode = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_multithreading_enabled(self.prev) + + def clone(self) -> "set_multithreading_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class _force_original_view_tracking(_DecoratorContextManager): + r"""Context-manager that sets whether or not to always enable view-replay in autograd. + + ``set_view_replay_enabled`` will enable or disable view-replay based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + When a tensor view is mutated, the autograd engine needs to decide whether or not + to regenerate the "updated view" by either replaying the chain of views from the updated base, + or with a single call to as_strided. + + If set_view_replay_enabled is set to True, then autograd will always use view replay. + Otherwise, it will fall back to its existing logic. + + Args: + mode (bool): Flag whether to enable view-replay (``True``), or disable + (``False``). + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_view_replay_enabled() + torch._C._set_view_replay_enabled(mode) + self.mode = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_view_replay_enabled(self.prev) + + def clone(self): + return self.__class__(self.mode) + + +class _unsafe_preserve_version_counter(_DecoratorContextManager): + r"""DO NOT USE THIS UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING. + + This context manager can lead to arbitrary silent-correctness issues in any other part of your code + (even the ones not touched directly by the context manager)! + + Ordinarily, autograd will track mutations to tensors by incrementing it's `._version` attribute. + This is generally important for correctness, as for example, mutating a tensor that autograd has saved + for the backwards pass can result in incorrect gradients, and autograd uses the version counter to detect + and error out in this situation. + + However, there are rare instances where it might be useful to hide mutations from autograd. For example: + if a tensor is very large, and you'd like to free its memory by storing it elsewhere, and re-populate + the tensor right before it is needed by autograd. + + Args: + tensor (torch.Tensor): the tensor in question, that you would like to preserve the version counter of. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + """ + + def __init__(self, tensors: Union[torch.Tensor, tuple[torch.Tensor, ...]]) -> None: + self.tensors = (tensors,) if isinstance(tensors, torch.Tensor) else tensors + if not isinstance(self.tensors, tuple): + raise AssertionError("Expected tensors to be a tuple") + self.prev_versions = tuple(t._version for t in self.tensors) + + def __enter__(self) -> None: + pass + + # pyrefly: ignore [bad-override] + def __exit__(self, *args) -> None: + torch._C._autograd._unsafe_set_version_counter(self.tensors, self.prev_versions) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/gradcheck.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/gradcheck.py new file mode 100644 index 0000000000000000000000000000000000000000..50f9ba78e383e615c830d2e2ed75d68bfa523b63 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/gradcheck.py @@ -0,0 +1,2306 @@ +# mypy: allow-untyped-defs +import collections +import functools +import warnings +from collections.abc import Callable, Iterable +from itertools import product +from typing import Optional, Union +from typing_extensions import deprecated + +import torch +import torch.testing + +# pyrefly: ignore [deprecated] +from torch._vmap_internals import _vmap, vmap +from torch.overrides import is_tensor_like +from torch.types import _TensorOrTensors + + +# Note: `get_*_jacobian` functions are added here even though we didn't intend to make them public +# since they have been exposed from before we added `__all__` and we already maintain BC for them +# We should eventually deprecate them and remove them from `__all__` +__all__ = [ + "gradcheck", + "gradgradcheck", + "GradcheckError", + "get_numerical_jacobian", + "get_analytical_jacobian", + "get_numerical_jacobian_wrt_specific_input", +] + + +class GradcheckError(RuntimeError): + r"""Error raised by :func:`gradcheck` and :func:`gradgradcheck`.""" + + +def _is_sparse_compressed_tensor(obj: torch.Tensor): + return obj.layout in { + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + } + + +def _is_sparse_any_tensor(obj: torch.Tensor): + return _is_sparse_compressed_tensor(obj) or obj.layout is torch.sparse_coo + + +def _is_float_or_complex_tensor(obj): + return is_tensor_like(obj) and (obj.is_floating_point() or obj.is_complex()) + + +def _allocate_jacobians_with_inputs( + input_tensors: tuple, numel_output +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from inputs. If `numel_output` is not None, for + # each tensor in `input_tensors`, returns a new zero-filled tensor with height + # of `t.numel` and width of `numel_output`. Otherwise, for each tensor, returns + # a 1-d tensor with size `(t.numel,)`. Each new tensor will be strided and have + # the same dtype and device as those of the corresponding input. + out: list[torch.Tensor] = [ + t.new_zeros((t.numel(), numel_output), layout=torch.strided) + for t in input_tensors + if _is_float_or_complex_tensor(t) and t.requires_grad + ] + return tuple(out) + + +def _allocate_jacobians_with_outputs( + output_tensors: tuple, numel_input, dtype=None, device=None +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from outputs. If `dim` is not None, for each tensor + # in `output_tensors`, returns a new zero-filled tensor with height of `dim` and + # width of `t.numel`. Otherwise, for each tensor, returns a 1-d tensor with size + # (t.numel,). + options = {"dtype": dtype, "device": device, "layout": torch.strided} + out: list[torch.Tensor] = [ + t.new_zeros((numel_input, t.numel()), **options) + for t in output_tensors + if _is_float_or_complex_tensor(t) + ] + return tuple(out) + + +def _iter_tensors( + x: Union[torch.Tensor, Iterable[torch.Tensor]], only_requiring_grad: bool = False +) -> Iterable[torch.Tensor]: + if is_tensor_like(x): + # mypy doesn't narrow type of `x` to torch.Tensor + if x.requires_grad or not only_requiring_grad: # type: ignore[union-attr] + yield x # type: ignore[misc] + elif isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + for elem in x: + yield from _iter_tensors(elem, only_requiring_grad) + + +def _densify(x): + # return a copy of sparse x with all unspecified elements + # "replaced" with zero-valued elements + if isinstance(x, (list, tuple)): + return type(x)(map(_densify, x)) + elif not is_tensor_like(x) or x.layout in {torch.strided, torch._mkldnn}: # type: ignore[attr-defined] # no attr _mkldnn + return x + elif x.layout is torch.sparse_coo: + device = x.device + indices_dtype = x._indices().dtype + tmp = torch.ones(x.shape[: x.sparse_dim()], dtype=torch.int8, device=device) + indices = tmp.nonzero().t().to(dtype=indices_dtype) + values = torch.zeros( + (tmp.numel(), *x.shape[x.sparse_dim() :]), dtype=x.dtype, device=device + ) + x_coalesced = x.detach().coalesce() + if x_coalesced.numel() > 0: + stride = tmp.stride() + flat_indices = ( + x_coalesced.indices() + .mul( + torch.tensor(stride, dtype=indices_dtype, device=device).unsqueeze( + 1 + ) + ) + .sum(0) + ) + values[flat_indices] = x_coalesced.values() + return ( + torch.sparse_coo_tensor(indices, values, x.shape) + ._coalesced_(True) + .requires_grad_(x.requires_grad) + ) + elif _is_sparse_compressed_tensor(x): + blocksize = ( + x.values().shape[1:3] + if x.layout in {torch.sparse_bsr, torch.sparse_bsc} + else None + ) + compressed_indices = ( + x.crow_indices() + if x.layout in {torch.sparse_csr, torch.sparse_bsr} + else x.ccol_indices() + ) + # We'll use intermediate sparse COO for simplicity + r = _densify(x.detach().to_sparse(layout=torch.sparse_coo)).to_sparse( + layout=x.layout, blocksize=blocksize + ) + # Check that all elements are specified also after `to_sparse` op: + dense_numel = r.values().numel() // max(1, r.values().shape[0]) + batch_numel = compressed_indices.numel() // compressed_indices.shape[-1] + sparse_numel = r.numel() // max(1, dense_numel * batch_numel) + if sparse_numel != r._nnz(): + raise AssertionError( + f"{x.layout} densify failed: expected nnz={sparse_numel} but got {r._nnz()}" + ) + return r.requires_grad_(x.requires_grad) + elif _is_sparse_any_tensor(x): + raise NotImplementedError(x.layout) + return x + + +def _iter_tensor(x_tensor): + # (Only used for slow gradcheck) Returns a generator that yields the following + # elements at each iteration: + # 1) a tensor: the same tensor is returned across all iterations. The tensor + # is not the same as the original x_tensor as given as input - it is + # prepared so that it can be modified in-place. Depending on whether the + # input tensor is strided, sparse, or dense, the returned tensor may or may + # not share storage with x_tensor. + # 2) a tuple of indices that can be used with advanced indexing (yielded in + # dictionary order) + # 3) flattened index that will be used to index into the Jacobian tensor + # + # For a tensor t with size (2, 2), _iter_tensor yields: + # `x, (0, 0), 0`, `x, (0, 1), 1`, `x, (1, 0), 2`, `x, (1, 1), 3` + # + # where x is the t.data of the original tensor. Perturbing the entry of x + # at index (1, 1) yields the 3rd column of the overall Jacobian matrix. + if _is_sparse_any_tensor(x_tensor): + + def get_stride(size): + dim = len(size) + tmp = 1 + stride = [0] * dim + for i in reversed(range(dim)): + stride[i] = tmp + tmp *= size[i] + return stride + + x_nnz = x_tensor._nnz() + x_size = list(x_tensor.size()) + if x_tensor.layout is torch.sparse_coo: + x_indices = x_tensor._indices().t() + x_values = x_tensor._values() + elif x_tensor.layout is torch.sparse_csr: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_csc: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_bsr: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + elif x_tensor.layout is torch.sparse_bsc: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + else: + raise NotImplementedError(f"_iter_tensor for {x_tensor.layout} input") + x_stride = get_stride(x_size) + # Use .data here to get around the version check + x_values = x_values.data + for i in range(x_nnz): + x_value = x_values[i] + for x_idx in product(*[range(m) for m in x_values.size()[1:]]): + indices = x_indices[i].tolist() + list(x_idx) + d_idx = sum(indices[k] * x_stride[k] for k in range(len(x_size))) + yield x_value, x_idx, d_idx + elif x_tensor.layout == torch._mkldnn: # type: ignore[attr-defined] + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + # this is really inefficient, but without indexing implemented, there's + # not really a better way than converting back and forth + x_tensor_dense = x_tensor.to_dense() + yield x_tensor_dense, x_idx, d_idx + else: + # Use .data here to get around the version check + x_tensor = x_tensor.data + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + yield x_tensor, x_idx, d_idx + + +def _get_numerical_jacobian( + fn, inputs, outputs=None, target=None, eps=1e-3, is_forward_ad=False +) -> list[tuple[torch.Tensor, ...]]: + """Compute the numerical Jacobian of `fn(inputs)` with respect to `target`. + + If not specified, targets are the input. Returns M * N Jacobians where N is the + number of tensors in target that require grad and M is the number of non-integral + outputs. + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + target: the Tensors wrt whom Jacobians are calculated (default=`inputs`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + is_forward_ad: if this numerical jacobian is computed to be checked wrt + forward AD gradients (this is used for error checking only) + + Returns: + A list of M N-tuples of tensors + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + jacobians: list[tuple[torch.Tensor, ...]] = [] + if outputs is None: + outputs = _as_tuple(fn(*_as_tuple(inputs))) + if not is_forward_ad and any(o.is_complex() for o in outputs): + raise ValueError( + "Expected output to be non-complex. get_numerical_jacobian no " + "longer supports functions that return complex outputs." + ) + if target is None: + target = inputs + inp_indices = [ + i for i, a in enumerate(target) if is_tensor_like(a) and a.requires_grad + ] + for inp, inp_idx in zip(_iter_tensors(target, True), inp_indices): + jacobians += [ + get_numerical_jacobian_wrt_specific_input( + fn, + inp_idx, + inputs, + outputs, + eps, + input=inp, + is_forward_ad=is_forward_ad, + ) + ] + return jacobians + + +@deprecated( + "`get_numerical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_numerical_jacobian(fn, inputs, target=None, eps=1e-3, grad_out=1.0): + """Compute the numerical Jacobian for a given fn and its inputs. + + This is a Deprecated API. + + Args: + fn: the function to compute the Jacobian for (must take inputs as a tuple) + inputs: input to `fn` + target: the Tensors wrt whom Jacobians are calculated (default=`input`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + grad_out: defaults to 1.0. + + Returns: + A list of Jacobians of `fn` (restricted to its first output) with respect to + each input or target, if provided. + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_numerical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + + def fn_pack_inps(*inps): + return fn(inps) + + jacobians = _get_numerical_jacobian(fn_pack_inps, inputs, None, target, eps) + + return tuple(jacobian_for_each_output[0] for jacobian_for_each_output in jacobians) + + +def _compute_numerical_gradient(fn, entry, v, norm_v, nbhd_checks_fn): + # Computes numerical directional derivative as finite difference + # of function `fn` at input `entry`, perturbed by vector `v`. + if _is_sparse_compressed_tensor(entry): + # sparse compressed tensors don't implement sub/add/copy_ + # yet. However, in non-masked semantics context entry and v + # have the same sparse indices ... + if entry.layout != v.layout: + raise AssertionError( + f"Expected entry and v to have the same layout, but got {entry.layout} and {v.layout}" + ) + if entry._nnz() != v._nnz(): + raise AssertionError( + f"Expected entry and v to have the same nnz, but got {entry._nnz()} and {v._nnz()} " + f"with entry shape {entry.shape}" + ) + # ... the finite differencing can be performed on values only: + entry = entry.values() + v = v.values() + # we'll detach to avoid backward computations that sparse + # tensors have limited support for. + entry = entry.detach() + + orig = entry.clone() + entry.copy_(orig - v) + outa = fn() + entry.copy_(orig + v) + outb = fn() + entry.copy_(orig) + + def compute(a, b): + nbhd_checks_fn(a, b) + ret = (b - a) / (2 * norm_v) # use central difference approx + return ret.detach().reshape(-1) + + return tuple(compute(a, b) for (a, b) in zip(outa, outb)) + + +def _compute_numerical_jvps_wrt_specific_input( + jvp_fn, delta, input_is_complex, is_forward_ad=False +) -> list[torch.Tensor]: + # Computing the jacobian only works for real delta + # For details on the algorithm used here, refer: + # Section 3.5.3 https://arxiv.org/pdf/1701.00392.pdf + # s = fn(z) where z = x for real valued input + # and z = x + yj for complex valued input + jvps: list[torch.Tensor] = [] + ds_dx_tup = jvp_fn(delta[0] if isinstance(delta, tuple) else delta) + + if input_is_complex: # C -> R + ds_dy_tup = ( + jvp_fn(delta[1] * 1j) if isinstance(delta, tuple) else jvp_fn(delta * 1j) + ) + for ds_dx, ds_dy in zip(ds_dx_tup, ds_dy_tup): + if ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex") + # conjugate wirtinger derivative + conj_w_d = ds_dx + ds_dy * 1j + jvps.append(conj_w_d) + else: + for ds_dx in ds_dx_tup: # R -> R or (R -> C for the forward AD case) + if not is_forward_ad and ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex.") + jvps.append(ds_dx) + return jvps + + +def _combine_jacobian_cols( + jacobians_cols: dict[int, list[torch.Tensor]], outputs, input, numel +) -> tuple[torch.Tensor, ...]: + # jacobian_cols maps column_idx -> output_idx -> single column of jacobian Tensor + # we return a list that maps output_idx -> full jacobian Tensor + jacobians = _allocate_jacobians_with_outputs( + outputs, numel, dtype=input.dtype if input.dtype.is_complex else None + ) + for i, jacobian in enumerate(jacobians): + for k, v in jacobians_cols.items(): + jacobian[k] = v[i] + return jacobians + + +def _prepare_input( + input: torch.Tensor, maybe_perturbed_input: Optional[torch.Tensor], fast_mode=False +) -> torch.Tensor: + # Prepares the inputs to be passed into the function while including the new + # modified input. + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert back to mkldnn + if maybe_perturbed_input is not None: + return maybe_perturbed_input.to_mkldnn() + else: + return input + elif _is_sparse_any_tensor(input): + if fast_mode and maybe_perturbed_input is not None: + # entry is already a "cloned" version of the original tensor + # thus changes to entry are not reflected in the input + return maybe_perturbed_input + else: + return input + else: + # We cannot use entry (input.data) if we want gradgrad to work because + # fn (in the gradgrad case) needs to compute grad wrt input + return input + + +def _check_outputs_same_dtype_and_shape(output1, output2, eps, idx=None) -> None: + # Check that the returned outputs don't have different dtype or shape when you + # perturb the input + on_index = f"on index {idx} " if idx is not None else "" + if output1.shape != output2.shape: + raise AssertionError( + f"Expected `func` to return outputs with the same shape" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" shapes {output1.shape} and {output2.shape}." + ) + if output1.dtype != output2.dtype: + raise AssertionError( + f"Expected `func` to return outputs with the same dtype" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" dtypes {output1.dtype} and {output2.dtype}." + ) + + +def get_numerical_jacobian_wrt_specific_input( + fn, input_idx, inputs, outputs, eps, input=None, is_forward_ad=False +) -> tuple[torch.Tensor, ...]: + # Computes the numerical jacobians wrt to a single input. Returns N jacobian + # tensors, where N is the number of outputs. We use a dictionary for + # jacobian_cols because indices aren't necessarily consecutive for sparse inputs + # When we perturb only a single element of the input tensor at a time, the jvp + # is equivalent to a single col of the Jacobian matrix of fn. + jacobian_cols: dict[int, list[torch.Tensor]] = {} + input = inputs[input_idx] if input is None else input + if not input.requires_grad: + raise AssertionError("Expected input to have requires_grad=True") + for x, idx, d_idx in _iter_tensor(input): + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, x) + input_to_perturb = x[idx] + nbhd_checks_fn = functools.partial( + _check_outputs_same_dtype_and_shape, idx=idx, eps=eps + ) + jvp_fn = _get_numerical_jvp_fn( + wrapped_fn, input_to_perturb, eps, nbhd_checks_fn + ) + jacobian_cols[d_idx] = _compute_numerical_jvps_wrt_specific_input( + jvp_fn, eps, x.is_complex(), is_forward_ad + ) + return _combine_jacobian_cols(jacobian_cols, outputs, input, input.numel()) + + +def _get_analytical_jacobian_forward_ad( + fn, inputs, outputs, *, check_grad_dtypes=False, all_u=None +) -> tuple[tuple[torch.Tensor, ...], ...]: + """Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`. + + Return N * M Jacobians where N is the number of tensors in target that require grad and + M is the number of non-integral outputs. + Contrary to other functions here, this function requires "inputs" to actually be used by the function. + The computed value is expected to be wrong if the function captures the inputs by side effect instead of + using the passed ones (many torch.nn tests do this). + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + check_grad_dtypes: if True, will check that the gradient dtype are valid + all_u (optional): if provided, the Jacobian will be right multiplied with this vector + + Returns: + A tuple of M N-tuples of tensors + """ + # To avoid early import issues + fwAD = torch.autograd.forward_ad + + tensor_inputs = tuple(i for i in inputs if is_tensor_like(i) and i.requires_grad) + + if any(i.is_complex() for i in tensor_inputs): + raise ValueError( + "Expected inputs to be non-complex for _get_analytical_jacobian_forward_ad." + ) + + if all_u: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, 1) for i in tensor_inputs + ) + else: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, i.numel()) for i in tensor_inputs + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + for inp in inputs: + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + dual_inputs.append(inp) + + if all_u: + # Do the full reduction in one pass + # To be consistent with numerical evaluation, we actually compute one reduction per input + for i, (fw_grad, u) in enumerate(zip(fw_grads, all_u)): + fw_grad.copy_(u.view_as(fw_grad)) + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError("Forward AD gradient has dtype mismatch.") + + # Remove extra dimension of size 1 corresponding to the reduced input + jacobians[i][index_o].squeeze_(0) + if res is None: + jacobians[i][index_o].zero_() + else: + jacobians[i][index_o].copy_(res.reshape(-1)) + fw_grad.zero_() + else: + # Reconstruct the full Jacobian column by column + for i, fw_grad in enumerate(fw_grads): + for lin_idx, grad_idx in enumerate( + product(*[range(m) for m in fw_grad.size()]) + ): + fw_grad[grad_idx] = 1.0 + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError( + "Forward AD gradient has dtype mismatch." + ) + + if res is None: + jacobians[i][index_o][lin_idx].zero_() + else: + jacobians[i][index_o][lin_idx].copy_(res.reshape(-1)) + fw_grad[grad_idx] = 0.0 + + return jacobians + + +def _get_input_to_perturb(input): + # Prepare the input so that it can be modified in-place and do certain + # operations that require the tensor to have strides. If fast_mode=False, + # _iter_tensor would handle the below cases: + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert to dense so we can perform operations that require strided tensors + input_to_perturb = input.to_dense() + elif _is_sparse_any_tensor(input): + # Clone because input may require grad, and copy_ calls resize_, + # which is not allowed for .data + input_to_perturb = input.clone() + else: + input_to_perturb = input.data + return input_to_perturb + + +def _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, fast_mode=False): + # Wraps `fn` so that its inputs are already supplied + def wrapped_fn(): + inp = tuple( + _prepare_input(a, input_to_perturb if i == input_idx else None, fast_mode) + if is_tensor_like(a) + else a + for i, a in enumerate(_as_tuple(inputs)) + ) + return tuple(a.clone() for a in _as_tuple(fn(*inp))) + + return wrapped_fn + + +def _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn): + # Wraps jvp_fn so that certain arguments are already supplied + def jvp_fn(delta): + return _compute_numerical_gradient( + wrapped_fn, input_to_perturb, delta, eps, nbhd_checks_fn + ) + + return jvp_fn + + +def _reshape_tensor_or_tuple(u, shape): + # We don't need to reshape when input corresponding to u is sparse + if isinstance(u, tuple): + if not _is_sparse_any_tensor(u[0]): + return (u[0].reshape(shape), u[1].reshape(shape)) + else: + if not _is_sparse_any_tensor(u): + return u.reshape(shape) + return u + + +def _mul_tensor_or_tuple(u, k): + if isinstance(u, tuple): + return (k * u[0], k * u[1]) + else: + return k * u + + +def _get_numerical_jvp_wrt_specific_input( + fn, input_idx, inputs, u, eps, is_forward_ad=False +) -> list[torch.Tensor]: + input = inputs[input_idx] + input_to_perturb = _get_input_to_perturb(input) + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, True) + nbhd_checks_fn = functools.partial(_check_outputs_same_dtype_and_shape, eps=eps) + jvp_fn = _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn) + u = _reshape_tensor_or_tuple(u, input_to_perturb.shape) + u = _mul_tensor_or_tuple(u, eps) + return _compute_numerical_jvps_wrt_specific_input( + jvp_fn, u, input.is_complex(), is_forward_ad + ) + + +def _get_numerical_vJu( + fn, inputs, inp_indices, func_out, all_u, all_v, eps, is_forward_ad +): + # Note that all_v can also be None, in that case, this function only computes Ju. + reduced_jacobians: list[list[torch.Tensor]] = [] + for inp_idx, u in zip(inp_indices, all_u): + all_Ju = _get_numerical_jvp_wrt_specific_input( + fn, inp_idx, inputs, u, eps, is_forward_ad + ) + # Filter out the Ju for non floating point outputs + filtered_Ju = [] + func_out = _as_tuple(func_out) + if len(all_Ju) != len(func_out): + raise AssertionError( + f"Expected all_Ju and func_out to have the same length, " + f"but got {len(all_Ju)} and {len(func_out)}" + ) + for Ju, output in zip(all_Ju, func_out): + if _is_float_or_complex_tensor(output): + filtered_Ju.append(Ju) + else: + # TODO: handle the other Ju + pass + if all_v is not None: + jacobian_scalars: list[torch.Tensor] = [] + for v, Ju in zip(all_v, filtered_Ju): + jacobian_scalars.append(_dot_with_type_promotion(v, Ju)) + reduced_jacobians.append(jacobian_scalars) + else: + reduced_jacobians.append(filtered_Ju) + return reduced_jacobians + + +def _check_jacobians_equal(j1, j2, atol): + # Check whether the max difference between two Jacobian tensors are within some + # tolerance `atol`. + for j1_x, j2_x in zip(j1, j2): + if j1_x.numel() != 0 and (j1_x - j2_x).abs().max() > atol: + return False + return True + + +def _stack_and_check_tensors( + list_of_list_of_tensors, inputs, numel_outputs +) -> tuple[tuple[torch.Tensor, ...], bool, bool]: + # For the ith tensor in the inner list checks whether it has the same size and + # dtype as the ith differentiable input. + out_jacobians = _allocate_jacobians_with_inputs(inputs, numel_outputs) + diff_input_list = list(_iter_tensors(inputs, True)) + correct_grad_sizes = True + correct_grad_types = True + for i, tensor_list in enumerate(list_of_list_of_tensors): + inp = diff_input_list[i] + out_jacobian = out_jacobians[i] + for j, tensor in enumerate(tensor_list): + if tensor is not None and tensor.size() != inp.size(): + correct_grad_sizes = False + elif tensor is not None and tensor.dtype != inp.dtype: + correct_grad_types = False + if tensor is None: + out_jacobian[:, j].zero_() + else: + dense = tensor.to_dense() if tensor.layout != torch.strided else tensor + if out_jacobian[:, j].numel() != dense.numel(): + raise AssertionError( + f"Expected out_jacobian column to have {dense.numel()} elements, " + f"but got {out_jacobian[:, j].numel()}" + ) + out_jacobian[:, j] = dense.reshape(-1) + return out_jacobians, correct_grad_sizes, correct_grad_types + + +FAILED_NONDET_MSG = """\n +NOTE: If your op relies on non-deterministic operations i.e., it is listed here: +https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html +this failure might be expected. + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `nondet_tol=` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_nondet_tol=`. +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_nondet_tol=` +""" + + +def _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=False, v=None +) -> tuple[torch.Tensor, ...]: + # This is used by both fast and slow mode: + # - For slow mode, vjps[i][j] is the jth row of the Jacobian wrt the ith + # input. + # - For fast mode, vjps[i][0] is a linear combination of the rows + # of the Jacobian wrt the ith input + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + if fast_mode: + vjps1 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + vjps2 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + else: + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() if not fast_mode else 1 + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + if not types_ok and check_grad_dtypes: + raise GradcheckError("Gradient has dtype mismatch") + if not sizes_ok: + raise GradcheckError("Analytical gradient has incorrect size") + if not reentrant: + raise GradcheckError( + "Backward is not reentrant, i.e., running backward with " + "same input and grad_output multiple times gives different values, " + "although analytical gradient matches numerical gradient." + f"The tolerance for nondeterminism was {nondet_tol}." + FAILED_NONDET_MSG + ) + return jacobians1 + + +def _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u +): + reduced_jacobians: list[list[torch.Tensor]] = [] + for output, v in zip(outputs, all_v): + all_vJ = _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=True, v=v + ) + jacobian_scalars: list[torch.Tensor] = [] + for vJ, u in zip(all_vJ, all_u): + # Why do we need squeeze here? vJ is a 2-d tensor so that we can reuse + # the error checking logic from slow mode + vJ = vJ.T.squeeze(0) + if vJ.is_complex(): # C -> R + tv = torch.view_as_real(vJ.resolve_conj()) + tr = tv.select(-1, 0) + ti = tv.select(-1, 1) + jacobian_scalars.append(tr.dot(u[0]) + 1j * ti.dot(u[1])) + else: # R -> R + jacobian_scalars.append(vJ.dot(u)) + reduced_jacobians.append(jacobian_scalars) + return reduced_jacobians + + +@deprecated( + "`get_analytical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_analytical_jacobian(inputs, output, nondet_tol=0.0, grad_out=1.0): + # Replicates the behavior of the old get_analytical_jacobian before the refactor + # This shares much of its code with _check_analytical_jacobian_attributes + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_analytical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + if output.is_complex(): + raise ValueError( + "Expected output to be non-complex. get_analytical_jacobian no " + "longer supports functions that return complex outputs." + ) + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + return jacobians1, reentrant, sizes_ok, types_ok + + +def _get_analytical_jacobian(inputs, outputs, input_idx, output_idx): + # Computes the analytical Jacobian in slow mode for a single input-output pair. + # Forgoes performing checks on dtype, shape, and reentrancy. + jacobians = _check_analytical_jacobian_attributes( + inputs, outputs[output_idx], nondet_tol=float("inf"), check_grad_dtypes=False + ) + return jacobians[input_idx] + + +def _compute_analytical_jacobian_rows( + vjp_fn, sample_output +) -> list[list[Optional[torch.Tensor]]]: + # Computes Jacobian row-by-row by projecting `vjp_fn` = v^T J on standard basis + # vectors: vjp_fn(e) = e^T J is a corresponding row of the Jacobian. + # NB: this function does not assume vjp_fn(v) to return tensors with the same + # number of elements for different v. This is checked when we later combine the + # rows into a single tensor. + grad_out_base = torch.zeros_like( + sample_output, memory_format=torch.legacy_contiguous_format + ) + flat_grad_out = grad_out_base.view(-1) + # jacobians_rows[i][j] is the Jacobian jth row for the ith input + jacobians_rows: list[list[Optional[torch.Tensor]]] = [] + for j in range(flat_grad_out.numel()): + flat_grad_out.zero_() + flat_grad_out[j] = 1.0 # projection for jth row of Jacobian + grad_inputs = vjp_fn(grad_out_base) + for i, d_x in enumerate(grad_inputs): + if j == 0: + jacobians_rows.append([]) + jacobians_rows[i] += [ + d_x.clone() if isinstance(d_x, torch.Tensor) else None + ] + return jacobians_rows + + +def _get_analytical_vjps_wrt_specific_output( + vjp_fn, sample_output, v +) -> list[list[Optional[torch.Tensor]]]: + grad_inputs = vjp_fn(v.reshape(sample_output.shape)) + vjps: list[list[Optional[torch.Tensor]]] = [ + [vjp.clone() if isinstance(vjp, torch.Tensor) else None] for vjp in grad_inputs + ] + return vjps + + +def _check_inputs(tupled_inputs) -> bool: + # Make sure that gradients are saved for at least one input + any_input_requiring_grad = False + for idx, inp in enumerate(tupled_inputs): + if is_tensor_like(inp) and inp.requires_grad: + if not (inp.dtype == torch.float64 or inp.dtype == torch.complex128): + warnings.warn( + f"Input #{idx} requires gradient and " + "is not a double precision floating point or complex. " + "This check will likely fail if all the inputs are " + "not of double precision floating point or complex. ", + stacklevel=2, + ) + if inp.is_sparse: + content = inp._values() + elif _is_sparse_compressed_tensor(inp): + content = inp.values() + else: + content = inp + # TODO: To cover more problematic cases, replace stride = 0 check with + # "any overlap in memory" once we have a proper function to check it. + if content.layout is not torch._mkldnn: # type: ignore[attr-defined] + if not all( + st > 0 or sz <= 1 + for st, sz in zip(content.stride(), content.size()) + ): + raise RuntimeError( + f"The {idx}th input has a dimension with stride 0. gradcheck only " + "supports inputs that are non-overlapping to be able to " + "compute the numerical gradients correctly. You should call " + ".contiguous on the input before passing it to gradcheck." + ) + any_input_requiring_grad = True + + if not any_input_requiring_grad: + raise ValueError( + "gradcheck expects at least one input tensor to require gradient, " + "but none of the them have requires_grad=True." + ) + return True + + +def _check_outputs(outputs) -> None: + if any(_is_sparse_any_tensor(t) for t in outputs if isinstance(t, torch.Tensor)): + # it is easier to call to_dense() on the sparse output than + # to modify analytical jacobian + raise ValueError( + "Sparse output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + if any(t.layout == torch._mkldnn for t in outputs if isinstance(t, torch.Tensor)): # type: ignore[attr-defined] + raise ValueError( + "MKLDNN output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + + +def _check_no_differentiable_outputs( + func, inputs, func_out, eps, *, is_forward_ad +) -> bool: + # When there are no differentiable outputs, numerical gradient for a function is + # expected to be zero. + jacobians_all_inputs_outputs = _get_numerical_jacobian( + func, inputs, func_out, eps=eps, is_forward_ad=is_forward_ad + ) + for jacobians_all_outputs_and_fixed_input in jacobians_all_inputs_outputs: + for jacobian in jacobians_all_outputs_and_fixed_input: + if torch.ne(jacobian, 0).sum() > 0: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +def _check_no_differentiable_outputs_fast( + func, func_out, all_inputs, inputs_indices, all_u, eps, nondet_tol +): + for inp_idx, u in zip(inputs_indices, all_u): + jvps = _get_numerical_jvp_wrt_specific_input(func, inp_idx, all_inputs, u, eps) + for jvp in jvps: + if jvp.numel() == 0: + continue + if (jvp - torch.zeros_like(jvp)).abs().max() > nondet_tol: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +FAILED_BATCHED_GRAD_MSG = """ +gradcheck or gradgradcheck failed while testing batched gradient computation. +This could have been invoked in a number of ways (via a test that calls +gradcheck/gradgradcheck directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`. + +If you're modifying an existing operator that supports batched grad computation, +or wish to make a new operator work with batched grad computation, please read +the following. + +To compute batched grads (e.g., jacobians, hessians), we vmap over the backward +computation. The most common failure case is if there is a 'vmap-incompatible +operation' in the backward pass. Please see +NOTE: [How to write vmap-compatible backward formulas] +in the codebase for an explanation of how to fix this. +""".strip() + +FAILED_BATCHED_GRAD_MSG_FWD_AD = """ +gradcheck failed while testing batched gradient computation with forward-mode AD. +This test is enabled automatically when both `check_batched_grad=True` +and `check_forward_ad=True`, but can be disabled in the following ways +dependong on how the test was invoked (via a test that calls gradcheck +directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_forward_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_forward_grad=False` +""" + + +def _get_failed_batched_grad_test_msg( + output_idx, input_idx, res, exp, is_forward_ad=False +): + return f""" +For output {output_idx} and input {input_idx}: + +{FAILED_BATCHED_GRAD_MSG_FWD_AD if is_forward_ad else FAILED_BATCHED_GRAD_MSG} + +Got: +{res} + +Expected: +{exp} +""".strip() + + +def _test_batched_grad_forward_ad(func, inputs) -> bool: + fwAD = torch.autograd.forward_ad # To avoid early import issues (do we need this?) + if not isinstance(inputs, tuple): + raise AssertionError("Expected inputs to be a tuple") + + for input_idx, current_input in enumerate(inputs): + if not (is_tensor_like(current_input) and current_input.requires_grad): + continue + + def jvp(tangent: torch.Tensor): + with fwAD.dual_level(): + dual = fwAD.make_dual(current_input.detach(), tangent) + inputs_with_dual = tuple( + dual + if idx == input_idx + else (inp.detach() if is_tensor_like(inp) else inp) + for idx, inp in enumerate(inputs) + ) + dual_outputs = _as_tuple(func(*inputs_with_dual)) + ret = [] + for dual_output in dual_outputs: + if dual_output is None: + continue + primal_out, tangent_out = fwAD.unpack_dual(dual_output) + if tangent_out is not None: + ret.append(tangent_out) + else: + ret.append( + torch.zeros( + [], dtype=primal_out.dtype, device=primal_out.device + ).expand(primal_out.shape) + ) + return tuple(ret) + + if not _is_float_or_complex_tensor(current_input): + continue + + tangents = [torch.randn_like(current_input) for _ in range(2)] + expected = [jvp(t) for t in tangents] + expected = [torch.stack(shards) for shards in zip(*expected)] + + try: + result = _vmap(jvp)(torch.stack(tangents)) + except RuntimeError as ex: + # Rethrow to provide a better error message + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG_FWD_AD}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg( + input_idx, input_idx, res, exp, is_forward_ad=True + ) + ) + return True + + +def _test_batched_grad(input, output, output_idx) -> bool: + # NB: _test_batched_grad compares two autograd.grad invocations with a single + # vmap(autograd.grad) invocation. It's not exactly a "gradcheck" in the + # sense that we're not comparing an analytical jacobian with a numeric one, + # but it is morally similar (we could have computed a full analytic jac + # via vmap, but that is potentially slow) + diff_input_list = list(_iter_tensors(input, True)) + grad = functools.partial( + torch.autograd.grad, + output, + diff_input_list, + retain_graph=True, + allow_unused=True, + ) + + def vjp(v): + results = grad(v) + results = tuple( + grad + if grad is not None + else torch.zeros([], dtype=inp.dtype, device=inp.device).expand(inp.shape) + for grad, inp in zip(results, diff_input_list) + ) + return results + + grad_outputs = [torch.randn_like(output) for _ in range(2)] + + expected = [vjp(gO) for gO in grad_outputs] + expected = [torch.stack(shards) for shards in zip(*expected)] + + # Squash warnings since these are expected to happen in most cases + # NB: this doesn't work for CUDA tests: https://github.com/pytorch/pytorch/issues/50209 + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="There is a performance drop") + warnings.filterwarnings("ignore", message="Please use `torch.vmap`") + try: + result = vmap(vjp)(torch.stack(grad_outputs)) + except RuntimeError as ex: + # It's OK that we're not raising the error at the correct callsite. + # That's because the callsite is always going to inside the Python + # autograd.grad instead of the C++ traceback of what line in the + # backward formula + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg(output_idx, input_idx, res, exp) + ) + return True + + +def _test_backward_mul_by_grad_output(outputs, inputs, masked) -> bool: + # Tests that backward is multiplied by grad_output + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + grads_input = torch.autograd.grad( + outputs, + diff_input_list, + [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in outputs + ], + allow_unused=True, + ) + for gi, di in zip(grads_input, diff_input_list): + if gi is None: + continue + if isinstance(gi, torch.Tensor) and gi.layout != torch.strided: + if gi.layout != di.layout: + raise GradcheckError( + "grad is incorrect layout (" + + str(gi.layout) + + " is not " + + str(di.layout) + + ")" + ) + if _is_sparse_any_tensor(gi): + sparse_kind = str(gi.layout).replace("torch.", "").replace("_coo", "") + if gi.sparse_dim() != di.sparse_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect sparse_dim" + f" {gi.sparse_dim()}, expected {di.sparse_dim()}" + ) + if gi.dense_dim() != di.dense_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect dense_dim" + f" {gi.dense_dim()}, expected {di.dense_dim()}" + ) + gi = gi.to_dense() + di = di.to_dense() + if masked: + if not torch.allclose(gi, torch.zeros_like(gi)): + raise GradcheckError("backward not multiplied by grad_output") + elif not gi.eq(0).all(): + raise GradcheckError("backward not multiplied by grad_output") + if gi.dtype != di.dtype: + raise GradcheckError("grad is incorrect type") + if gi.device != di.device: + raise GradcheckError("grad is incorrect device") + if gi.size() != di.size(): + raise GradcheckError("grad is incorrect size") + return True + + +def _test_undefined_forward_mode(func, outputs, inputs): + fwAD = torch.autograd.forward_ad + + _inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + _all_v, all_u, _all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=True + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + tensor_indices = set() + for i, inp in enumerate(inputs): + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + tensor_indices.add(i) + dual_inputs.append(inp) + + for fw_grad, u in zip(fw_grads, all_u): + fw_grad.copy_(u.view_as(fw_grad)) + + for idx, inp in enumerate(inputs): + if idx not in tensor_indices: + continue + dual_inp_obj = dual_inputs[idx] + + # case 1 (Materialized Zero Tensor Tangent) + dual_inputs[idx] = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs1 = filter(_is_float_or_complex_tensor, raw_outputs) + + # case 2 (Efficient Zero Tensor Tangent since we don't make a dual object and pass a regular tensor) + dual_inputs[idx] = inp.detach() + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs2 = filter(_is_float_or_complex_tensor, raw_outputs) + + # reset + dual_inputs[idx] = dual_inp_obj + + for index_o, (d_o1, d_o2) in enumerate(zip(dual_outputs1, dual_outputs2)): + _val1, res1 = fwAD.unpack_dual(d_o1) + _val2, res2 = fwAD.unpack_dual(d_o2) + + if not (res1 is None or res2 is None): + if not torch.allclose(res1, res2): + raise GradcheckError( + "Mismatch in tangent values for output with index: ", + index_o, + " when input: ", + inp, + " has an undefined tangent value. ", + " Got: ", + res1, + " but expected: ", + res2, + ) + return True + + +def _test_undefined_backward_mode(func, outputs, inputs) -> bool: + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + + def warn_bc_breaking(): + warnings.warn( + "Backwards compatibility: New undefined gradient support checking " + "feature is enabled by default, but it may break existing callers " + "of this function. If this is true for you, you can call this " + 'function with "check_undefined_grad=False" to disable the feature', + stacklevel=2, + ) + + def check_undefined_grad_support(output_to_check): + grads_output = [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in output_to_check + ] + try: + grads_input = torch.autograd.grad( + output_to_check, diff_input_list, grads_output, allow_unused=True + ) + except RuntimeError as e: + warn_bc_breaking() + raise GradcheckError( + "Expected backward function to handle undefined output grads. " + 'Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) from e + + for gi in grads_input: + if (gi is not None) and (not gi.eq(0).all()): + warn_bc_breaking() + raise GradcheckError( + "Expected all input grads to be undefined or zero when all output grads are undefined " + 'or zero. Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) + return True + + # All backward functions must work properly if all output grads are undefined + outputs_to_check = [ + [ + torch._C._functions.UndefinedGrad()(o) + for o in _differentiable_outputs(func(*inputs)) + # This check filters out Tensor-likes that aren't instances of Tensor. + if isinstance(o, torch.Tensor) + ] + ] + + # If there are multiple output grads, we should be able to undef one at a time without error + if len(outputs_to_check[0]) > 1: + for undef_grad_idx in range(len(outputs)): + output_to_check = _differentiable_outputs(func(*inputs)) + outputs_to_check.append( + [ + torch._C._functions.UndefinedGrad()(o) + if idx == undef_grad_idx + else o + for idx, o in enumerate(output_to_check) + ] + ) + + return all(check_undefined_grad_support(output) for output in outputs_to_check) + + +def _as_tuple(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _differentiable_outputs(x): + return tuple(o for o in _as_tuple(x) if o.requires_grad) + + +def _get_notallclose_msg( + analytical, + numerical, + output_idx, + input_idx, + complex_indices, + test_imag=False, + is_forward_ad=False, +) -> str: + out_is_complex = ( + (not is_forward_ad) and complex_indices and output_idx in complex_indices + ) + inp_is_complex = is_forward_ad and complex_indices and input_idx in complex_indices + part = "imaginary" if test_imag else "real" + element = "inputs" if is_forward_ad else "outputs" + prefix = ( + "" + if not (out_is_complex or inp_is_complex) + else f"While considering the {part} part of complex {element} only, " + ) + mode = "computed with forward mode " if is_forward_ad else "" + return ( + prefix + + f"Jacobian {mode}mismatch for output {output_idx:d} with respect to input {input_idx:d},\n" + f"numerical:{numerical}\nanalytical:{analytical}\n" + ) + + +def _transpose(matrix_of_tensors): + # returns list of tuples + return list(zip(*matrix_of_tensors)) + + +def _real_and_imag_output(fn): + # returns new functions real(fn), and imag(fn) where real(fn) and imag(fn) behave the same as + # the original fn, except torch.real or torch.imag are applied to the complex outputs + def apply_to_c_outs(fn, fn_to_apply): + def wrapped_fn(*inputs): + outs = _as_tuple(fn(*inputs)) + return tuple(fn_to_apply(o) if o.is_complex() else o for o in outs) + + return wrapped_fn + + return apply_to_c_outs(fn, torch.real), apply_to_c_outs(fn, torch.imag) + + +def _real_and_imag_input(fn, complex_inp_indices, tupled_inputs): + # returns new functions that take real inputs instead of complex inputs as + # (x, y) -> fn(x + y * 1j). And it computes: inp -> fn(inp + y * 1j) and inp -> fn(x + inp * 1j). + # In each case, the other part is considered constant. + # We do not use 0 for the constant here to make sure we always call the user function with a valid input. + def apply_to_c_inps(fn, fn_to_apply): + def wrapped_fn(*inputs): + new_inputs = list(inputs) + for should_be_complex in complex_inp_indices: + new_inputs[should_be_complex] = fn_to_apply( + new_inputs[should_be_complex], tupled_inputs[should_be_complex] + ) + return _as_tuple(fn(*new_inputs)) + + return wrapped_fn + + real_fn = apply_to_c_inps(fn, lambda inp, orig: inp + orig.imag * 1j) + imag_fn = apply_to_c_inps(fn, lambda inp, orig: orig.real + inp * 1j) + return real_fn, imag_fn + + +def _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad, + check_backward_ad, + nondet_tol, + check_undefined_grad, +): + complex_out_indices = [i for i, o in enumerate(outputs) if o.is_complex()] + has_any_complex_output = any(o.is_complex() for o in _as_tuple(func_out)) + if check_backward_ad: + if has_any_complex_output: + real_fn, imag_fn = _real_and_imag_output(func) + + imag_func_out = imag_fn(*tupled_inputs) + imag_outputs = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + tupled_inputs, + imag_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + test_imag=True, + ) + + real_func_out = real_fn(*tupled_inputs) + real_outputs = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + tupled_inputs, + real_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + ) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + ) + + if check_forward_ad: + complex_inp_indices = [ + i + for i, inp in enumerate(tupled_inputs) + if is_tensor_like(inp) and inp.is_complex() + ] + if complex_inp_indices: + real_fn, imag_fn = _real_and_imag_input( + func, complex_inp_indices, tupled_inputs + ) + + imag_inputs = [ + inp.imag if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + imag_func_out = imag_fn(*imag_inputs) + diff_imag_func_out = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + imag_inputs, + diff_imag_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + test_imag=True, + use_forward_ad=True, + ) + + real_inputs = [ + inp.real if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + real_func_out = real_fn(*real_inputs) + diff_real_func_out = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + real_inputs, + diff_real_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(imag_fn, imag_func_out, imag_inputs) + _test_undefined_forward_mode(real_fn, real_func_out, real_inputs) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(func, outputs, tupled_inputs) + + +def _slow_gradcheck( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + func_out = _as_tuple(func_out) + if not outputs: + return _check_no_differentiable_outputs( + func, tupled_inputs, func_out, eps=eps, is_forward_ad=use_forward_ad + ) + tupled_inputs_numerical = tupled_inputs if masked else _densify(tupled_inputs) + + numerical = _transpose( + _get_numerical_jacobian( + func, + tupled_inputs_numerical, + func_out, + eps=eps, + is_forward_ad=use_forward_ad, + ) + ) + # Note: [numerical vs analytical output length] + # The numerical path returns jacobian quantity for all outputs, even if requires_grad of that + # output is False. This behavior is necessary for _check_no_differentiable_outputs to work. + numerical = [nj for o, nj in zip(func_out, numerical) if o.requires_grad] + if use_forward_ad: + analytical_forward = _get_analytical_jacobian_forward_ad( + func, tupled_inputs, func_out, check_grad_dtypes=check_grad_dtypes + ) + + for i, n_per_out in enumerate(numerical): + for j, n in enumerate(n_per_out): + a = analytical_forward[j][i] + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg( + a, n, i, j, complex_indices, test_imag, is_forward_ad=True + ) + ) + else: + for i, o in enumerate(outputs): + analytical = _check_analytical_jacobian_attributes( + tupled_inputs, o, nondet_tol, check_grad_dtypes + ) + + for j, (a, n) in enumerate(zip(analytical, numerical[i])): + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg(a, n, i, j, complex_indices, test_imag) + ) + + return True + + +def _dot_with_type_promotion(u, v): + if u.dim() != 1 or v.dim() != 1: + raise AssertionError( + f"Expected u and v to be 1D tensors, but got dims {u.dim()} and {v.dim()}" + ) + return (u * v).sum() + + +def _allclose_with_type_promotion(a, b, rtol, atol): + promoted_type = torch.promote_types(a.dtype, b.dtype) + a = a.to(dtype=promoted_type) + b = b.to(dtype=promoted_type) + return torch.allclose(a, b, rtol, atol) + + +def _to_real_dtype(dtype): + if dtype == torch.complex128: + return torch.float64 + elif dtype == torch.complex64: + return torch.float32 + else: + return dtype + + +def _vec_from_tensor(x, generator, downcast_complex=False): + # Create a random vector with the same number of elements as x and the same + # dtype/device. If x is complex and downcast_complex is False, we create a + # complex tensor with only real component. + if x.layout == torch.sparse_coo: + # For sparse, create a random sparse vec with random values in the same + # indices. Make sure size is set so that it isn't inferred to be smaller. + x_values = x._values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_coo_tensor(x._indices(), values, x.size(), device=x.device) + elif _is_sparse_compressed_tensor(x): + if x.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices, plain_indices = x.crow_indices(), x.col_indices() + else: + compressed_indices, plain_indices = x.ccol_indices(), x.row_indices() + x_values = x.values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_compressed_tensor( + compressed_indices, + plain_indices, + values, + x.size(), + layout=x.layout, + device=x.device, + ) + else: + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + vec = torch.rand(x.numel(), generator=generator).to( + dtype=dtype, device=x.device + ) + vec /= vec.norm() + return vec + + +def _get_inp_tensors(tupled_inputs): + inp_idx_tup = [ + (i, t) + for i, t in enumerate(tupled_inputs) + if is_tensor_like(t) and t.requires_grad + ] + return [tup[0] for tup in inp_idx_tup], [tup[1] for tup in inp_idx_tup] + + +def _adjusted_atol(atol, u, v): + # In slow gradcheck, we compare A and B element-wise, i.e., for some a, b we + # allow: |a - b| < atol + rtol * b. But since we now compare q1 = v^T A u and + # q2 = v^T B u, we must allow |q1 - q2| < v^T E u + rtol * v^T B u, where E is + # the correctly sized matrix in which each entry is atol. + # + # We see that atol needs to be scaled by v^T M u (where M is an all-ones M x N + # matrix): v^T M u = \sum_{i} \sum_{j} u_i * v_j = (\sum_{i} u_i)(\sum_{i} v_i) + # TODO: properly handle case when u is tuple instead of only taking first element + u = u[0] if isinstance(u, tuple) else u + sum_u = u.sum() + sum_v = 1.0 if v is None else v.sum() + return atol * float(sum_u) * float(sum_v) + + +FAST_FAIL_SLOW_OK_MSG = """ +Fast gradcheck failed but element-wise differences are small. This means that the +test might've passed in slow_mode! + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck: + +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `fast_mode=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_fast_mode=False` +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_fast_mode=False` +""".strip() + + +def _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, input_idx, output_idx, rtol, atol, eps, is_forward_ad +): + # Compute jacobians in slow mode for better error message + slow_numerical = _get_numerical_jacobian( + func, tupled_inputs, outputs, eps=eps, is_forward_ad=is_forward_ad + )[input_idx][output_idx] + if is_forward_ad: + + def new_fn(inp): + new_inputs = list(tupled_inputs) + new_inputs[input_idx] = inp + return _as_tuple(func(*new_inputs))[output_idx] + + slow_analytical = _get_analytical_jacobian_forward_ad( + new_fn, (tupled_inputs[input_idx],), (outputs[output_idx],) + )[0][0] + else: + slow_analytical = _get_analytical_jacobian( + tupled_inputs, outputs, input_idx, output_idx + ) + + # Assume jacobians are non-empty and have the same shape + slow_max_diff = (slow_numerical - slow_analytical).abs().max() + + slow_allclose = torch.allclose(slow_analytical, slow_numerical, rtol, atol) + msg = ( + "\nThe above quantities relating the numerical and analytical jacobians are computed \n" + "in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background \n" + "about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:\n\n" + f"Numerical:\n {slow_numerical}\n" + f"Analytical:\n{slow_analytical}\n\n" + f"The max per-element difference (slow mode) is: {slow_max_diff}.\n" + ) + if slow_allclose: + # Slow gradcheck would've passed! + msg += FAST_FAIL_SLOW_OK_MSG + return msg + + +def _to_flat_dense_if_sparse(tensor): + if _is_sparse_any_tensor(tensor): + return tensor.to_dense().reshape(-1) + else: + return tensor + + +def _make_vectors(inp_tensors, outputs, *, use_forward_ad): + # Use our own generator to avoid messing with the user's RNG state + g_cpu = torch.Generator() + + def _vec_from_tensor_cpu(*args): + # Default allocate all tensors on CPU, so they are on the same device as the generator + # even if the user specified a default device + with torch.device("cpu"): + return _vec_from_tensor(*args) + + all_u = [] + all_u_dense = [] + for inp in inp_tensors: + ur = _vec_from_tensor_cpu(inp, g_cpu, True) + ur_dense = _to_flat_dense_if_sparse(ur) + if inp.is_complex(): + ui = _vec_from_tensor_cpu(inp, g_cpu, True) + all_u.append((ur, ui)) + ui_dense = _to_flat_dense_if_sparse(ui) + all_u_dense.append((ur_dense, ui_dense)) + else: + all_u.append(ur) + all_u_dense.append(ur_dense) + all_v = ( + None + if use_forward_ad + else [_vec_from_tensor_cpu(out, g_cpu) for out in outputs] + ) + return all_v, all_u, all_u_dense + + +def _check_analytical_numerical_equal( + all_analytical, + all_numerical, + complex_indices, + tupled_inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + *, + is_forward_ad=False, +): + for i, all_numerical_for_input_i in enumerate(all_numerical): + for j, n in enumerate(all_numerical_for_input_i): + # Forward AD generates the transpose of what this function expects + if is_forward_ad: + a = all_analytical[i][j] + else: + a = all_analytical[j][i] + n = n.to(device=a.device) + updated_atol = _adjusted_atol(atol, all_u[i], all_v[j] if all_v else None) + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, updated_atol): + jacobians_str = _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, i, j, rtol, atol, eps, is_forward_ad + ) + raise GradcheckError( + _get_notallclose_msg( + a, n, j, i, complex_indices, test_imag, is_forward_ad + ) + + jacobians_str + ) + + +def _fast_gradcheck( + func, + func_out, + inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + # See https://github.com/pytorch/pytorch/issues/53876 for details + inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + # Backward mode computes v^T * J (VJP) + # Since we computed J * u (JVP) through finite difference method, we perform an equality check + # between VJP * u, v * JVP + # ---- + # Forward mode computes J * u (JVP) + # Since we already compute JVP through finite difference method, + # we don't need v for correctness check here as asserted below + all_v, all_u, all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=use_forward_ad + ) + + inputs_numerical, all_u_numerical, all_v_numerical = ( + (inputs, all_u, all_v) if masked else _densify((inputs, all_u, all_v)) + ) + + numerical_vJu = _get_numerical_vJu( + func, + inputs_numerical, + inp_tensors_idx, + func_out, + all_u_numerical, + all_v_numerical, + eps, + is_forward_ad=use_forward_ad, + ) + # TODO: replicate https://github.com/pytorch/pytorch/pull/77743 for fast gradcheck as well + if use_forward_ad: + if all_v is not None: + raise AssertionError("Expected all_v to be None.") + analytical_vJu = _get_analytical_jacobian_forward_ad( + func, + inputs, + _as_tuple(func_out), + all_u=all_u, + check_grad_dtypes=check_grad_dtypes, + ) + else: + if not outputs: + _check_no_differentiable_outputs_fast( + func, func_out, inputs, inp_tensors_idx, all_u, eps, nondet_tol + ) + + analytical_vJu = _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u_dense + ) + + _check_analytical_numerical_equal( + analytical_vJu, + numerical_vJu, + complex_indices, + inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + is_forward_ad=use_forward_ad, + ) + + return True + + +# Note [VarArg of Tensors] +# ~~~~~~~~~~~~~~~~~~~~~~~~ +# 'func' accepts a vararg of tensors, which isn't expressible in the type system at the moment. +# If https://mypy.readthedocs.io/en/latest/additional_features.html?highlight=callable#extended-callable-types is accepted, +# the '...' first argument of Callable can be replaced with VarArg(Tensor). +# For now, we permit any input. +def gradcheck( + func: Callable[..., Union[_TensorOrTensors]], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_batched_forward_grad: bool = False, + check_forward_ad: bool = False, + check_backward_ad: bool = True, + fast_mode: bool = False, + masked: Optional[bool] = None, +) -> bool: # noqa: D400,D205 + r"""Check gradients computed via small finite differences against analytical + gradients wrt tensors in :attr:`inputs` that are of floating point or complex type + and with ``requires_grad=True``. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + For most of the complex functions we consider for optimization purposes, no notion of + Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of + the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient + computation is done under the assumption that the overall function has a real-valued + output, we treat functions with complex output in a special way. For these functions, + gradcheck is applied to two real-valued functions corresponding to taking the real + components of the complex outputs for the first, and taking the imaginary components + of the complex outputs for the second. For more details, check out + :ref:`complex_autograd-doc`. + + .. note:: + The default values are designed for :attr:`input` of double precision. + This check will likely fail if :attr:`input` is of less precision, e.g., + ``FloatTensor``. + + .. note:: + Gradcheck may fail when evaluated on non-differentiable points + because the numerically computed gradients via finite differencing may differ + those computed analytically (not necessarily because either is incorrect). + For more context, see :ref:`non-differentiable-func-grad`. + + .. warning:: + If any checked tensor in :attr:`input` has overlapping memory, i.e., + different indices pointing to the same memory address (e.g., from + :func:`torch.Tensor.expand`), this check will likely fail because the numerical + gradients computed by point perturbation at such indices will change + values at all other indices that share the same memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. + check_undefined_grad (bool, optional): if ``True``, check if undefined output grads + are supported and treated as zeros, for ``Tensor`` outputs. + check_batched_grad (bool, optional): if ``True``, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute + batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``. + check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward + mode AD match the numerical ones. Defaults to ``False``. + check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on + backward mode AD to be implemented. Defaults to ``True``. + fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only + implemented for R to R functions. If none of the inputs and outputs are complex + a faster implementation of gradcheck that no longer computes the entire jacobian + is run; otherwise, we fall back to the slow implementation. + masked (bool, optional): if ``True``, the gradients of unspecified elements of + sparse tensors are ignored. Defaults to ``False``. + Returns: + ``True`` if all differences satisfy allclose condition + + """ + if not (check_forward_ad or check_backward_ad): + raise AssertionError( + "Expected at least one of check_forward_ad or check_backward_ad to be True" + ) + if check_batched_grad and not check_backward_ad: + raise AssertionError( + "Setting check_batched_grad=True requires check_backward_ad to be True" + ) + if check_batched_forward_grad and not check_forward_ad: + raise AssertionError( + "Setting check_batched_forward_grad=True requires check_forward_ad to be True" + ) + args = locals().copy() + args.pop("raise_exception") + if not raise_exception: + try: + return _gradcheck_helper(**args) + except GradcheckError: + return False + else: + return _gradcheck_helper(**args) + + +def _gradcheck_helper( + func, + inputs, + eps, + atol, + rtol, + nondet_tol, + check_undefined_grad, + check_grad_dtypes, + check_batched_grad, + check_batched_forward_grad, + check_forward_ad, + check_backward_ad, + fast_mode, + masked, +): + tupled_inputs = _as_tuple(inputs) + _check_inputs(tupled_inputs) + + func_out = func(*tupled_inputs) + outputs = _differentiable_outputs(func_out) + _check_outputs(outputs) + + gradcheck_fn = functools.partial( + _fast_gradcheck if fast_mode else _slow_gradcheck, masked=masked + ) + _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad=check_forward_ad, + check_backward_ad=check_backward_ad, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + ) + + if check_batched_forward_grad: + _test_batched_grad_forward_ad(func, tupled_inputs) + + # Short circuit because remaining tests rely on backward AD to be implemented + if not check_backward_ad: + return True + + for i, o in enumerate(outputs): + if check_batched_grad: + _test_batched_grad(tupled_inputs, o, i) + + _test_backward_mul_by_grad_output(outputs, tupled_inputs, masked) + + if check_undefined_grad and check_backward_ad: + _test_undefined_backward_mode(func, outputs, tupled_inputs) + return True + + +def gradgradcheck( + func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + grad_outputs: Optional[_TensorOrTensors] = None, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + gen_non_contig_grad_outputs: bool = False, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_fwd_over_rev: bool = False, + check_rev_over_rev: bool = True, + fast_mode: bool = False, + masked: bool = False, +) -> bool: # noqa: D400,D205 + r"""Check gradients of gradients computed via small finite differences + against analytical gradients wrt tensors in :attr:`inputs` and + :attr:`grad_outputs` that are of floating point or complex type and with + ``requires_grad=True``. + + This function checks that backpropagating through the gradients computed + to the given :attr:`grad_outputs` are correct. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + .. note:: + The default values are designed for :attr:`input` and + :attr:`grad_outputs` of double precision. This check will likely fail if + they are of less precision, e.g., ``FloatTensor``. + + .. warning:: + If any checked tensor in :attr:`input` and :attr:`grad_outputs` has + overlapping memory, i.e., different indices pointing to the same memory + address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail + because the numerical gradients computed by point perturbation at such + indices will change values at all other indices that share the same + memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + grad_outputs (tuple of Tensor or Tensor, optional): The gradients with + respect to the function's outputs. + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is + ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the + randomly generated gradient outputs are made to be noncontiguous + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. Note that a small amount + of nondeterminism in the gradient will lead to larger inaccuracies in + the second derivative. + check_undefined_grad (bool, optional): if True, check if undefined output grads + are supported and treated as zeros + check_batched_grad (bool, optional): if True, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that + no longer computes the entire jacobian. + masked (bool, optional): if True, the gradients of unspecified elements of + sparse tensors are ignored (default, False). + Returns: + True if all differences satisfy allclose condition + """ + if not (check_fwd_over_rev or check_rev_over_rev): + raise AssertionError( + "Expected at least one of check_fwd_over_rev or check_rev_over_rev to be True" + ) + if check_undefined_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_undefined_grad=True requires check_rev_over_rev to be True" + ) + if check_batched_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_batched_grad=True requires check_rev_over_rev to be True" + ) + # TODO: do we want to test this too? + # assert not (check_batched_forward_grad and not check_fwd_over_rev), ( + # "Setting check_batched_forward_grad=True requires check_fwd_over_rev to be True") + tupled_inputs = _as_tuple(inputs) + + if grad_outputs is None: + # If grad_outputs is not specified, create random Tensors of the same shape, type, and device as the outputs + + outputs = _differentiable_outputs(func(*tupled_inputs)) + tupled_grad_outputs = tuple( + torch.testing.make_tensor( + x.shape, + dtype=x.dtype + if x.is_floating_point() or x.is_complex() + else torch.double, + device=x.device, + low=-1, + high=1, + requires_grad=True, + noncontiguous=gen_non_contig_grad_outputs, + ) + for x in outputs + ) + else: + tupled_grad_outputs = _as_tuple(grad_outputs) + + num_outputs = len(tupled_grad_outputs) + + # NB: We need to save the requires_grad information about the inputs here because gradcheck detaches inputs + # before running forward mode AD + diff_input_args_indices = { + i for i, x in enumerate(tupled_inputs) if is_tensor_like(x) and x.requires_grad + } + diff_grad_output_indices = { + i for i, x in enumerate(tupled_grad_outputs) if x.requires_grad + } + + def new_func(*args): + # Restore the requires_grad information + input_args = tuple( + x.requires_grad_() if i in diff_input_args_indices else x + for i, x in enumerate(args[:-num_outputs]) + ) + outputs = _differentiable_outputs(func(*input_args)) + grad_outputs = tuple( + x.requires_grad_() if i in diff_grad_output_indices else x + for i, x in enumerate(args[-num_outputs:]) + ) + diff_input_args = tuple( + x for i, x in enumerate(input_args) if i in diff_input_args_indices + ) + grad_inputs = torch.autograd.grad( + outputs, diff_input_args, grad_outputs, create_graph=True, allow_unused=True + ) + grad_inputs = tuple(g for g in grad_inputs if g is not None) + return grad_inputs + + return gradcheck( + new_func, + tupled_inputs + tupled_grad_outputs, + eps=eps, + atol=atol, + rtol=rtol, + raise_exception=raise_exception, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + check_grad_dtypes=check_grad_dtypes, + check_batched_grad=check_batched_grad, + fast_mode=fast_mode, + check_forward_ad=check_fwd_over_rev, + check_backward_ad=check_rev_over_rev, + masked=masked, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/graph.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..1baef937880ef5498331aa982d4052a3d7ec9567 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/graph.py @@ -0,0 +1,870 @@ +import abc +import contextlib +import functools +import logging +import threading +from collections import defaultdict, deque +from collections.abc import ( + Callable, + Generator, + Iterable, + Iterator, + MutableMapping, + Sequence, +) +from typing import ( + Any, + cast, + Literal, + NamedTuple, + Optional, + TYPE_CHECKING, + TypeAlias, + Union, +) +from weakref import WeakKeyDictionary, WeakValueDictionary + +import torch +from torch.autograd.variable import Variable +from torch.utils._python_dispatch import TorchDispatchMode +from torch.utils.hooks import RemovableHandle + + +if TYPE_CHECKING: + from torch._ops import OpOverload + + +__all__ = [ + "saved_tensors_hooks", + "save_on_cpu", + "disable_saved_tensors_hooks", + "register_multi_grad_hook", + "allow_mutation_on_saved_tensors", + "Node", + "GradientEdge", + "get_gradient_edge", + "increment_version", + "set_warn_on_accumulate_grad_stream_mismatch", +] + + +log = logging.getLogger(__name__) + + +class Node(abc.ABC): + @abc.abstractmethod + def name(self) -> str: + r"""Return the name. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> print(b.grad_fn.name()) + CloneBackward0 + """ + raise NotImplementedError + + @property + @abc.abstractmethod + def next_functions(self) -> tuple[tuple[Optional["Node"], int], ...]: + raise NotImplementedError + + @abc.abstractmethod + def metadata(self) -> dict: + r"""Return the metadata.""" + raise NotImplementedError + + @property + @abc.abstractmethod + def _input_metadata(self) -> list[Any]: + raise NotImplementedError + + @abc.abstractmethod + def _register_hook_dict(self, tensor: torch.Tensor) -> None: + raise NotImplementedError + + @abc.abstractmethod + def register_hook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_inputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + .. note:: + In the rare case where the hook is registered while the Node has already + begun execution, there is no longer any guarantee on :attr:`grad_outputs` + content (it might be as usual or empty depending on other factors). The + hook can still optionally return a new gradient to be used in place of + :attr:`grad_inputs` independent of :attr:`grad_outputs`. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() # Removes the hook + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @abc.abstractmethod + def register_prehook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward pre-hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_outputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, subclass: type) -> bool: + if cls is Node and ( + ( + subclass is not None + and subclass is getattr(torch._C._functions, subclass.__name__, None) + ) + or issubclass(subclass, torch.autograd.function.BackwardCFunction) + ): + return True + return NotImplemented + + +def _get_grad_fn_or_grad_acc(t: Union[torch.Tensor, "GradientEdge"]) -> Node: + if isinstance(t, GradientEdge): + return t.node + if t.requires_grad and t.grad_fn is None: + with torch.enable_grad(): + node = t.view_as(t).grad_fn.next_functions[0][0] # type: ignore[union-attr] + else: + node = t.grad_fn + if node is None: + raise AssertionError("Expected gradient function to be set") + return node + + +class GradientEdge(NamedTuple): + """Object representing a given gradient edge within the autograd graph. + + To get the gradient edge where a given Tensor gradient will be computed, + you can do ``edge = autograd.graph.get_gradient_edge(tensor)``. + """ + + node: Node + output_nr: int + # This token can be used to ensure the graph stays alive when it cannot be + # done via the node field + ownership_token: Optional[Node] = None + + +def get_gradient_edge(tensor: torch.Tensor) -> GradientEdge: + """Get the gradient edge for computing the gradient of the given Tensor. + + In particular, it is equivalent to call + ``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``. + """ + if not tensor.requires_grad: + raise RuntimeError( + "It is not possible to get the gradient edge for a Tensor " + "that does not require gradients", + ) + grad_fn = _get_grad_fn_or_grad_acc(tensor) + + # Python-based Node are owned by the C++ side meaning the python grad_fn + # object we hold here does NOT keep the C++ graph alive. + # Create an ownership token by creating a new C++ node that own the graph + # we care about here. + token = None + if isinstance(grad_fn, torch._C._FunctionBase): + with torch.enable_grad(): + token = tensor.view_as(tensor).grad_fn + + # Note that output_nr default to 0 which is the right value + # for the AccumulateGrad node. + # pyrefly: ignore [bad-argument-type] + return GradientEdge(grad_fn, tensor.output_nr, ownership_token=token) + + +def increment_version(tensor: Union[torch.Tensor, Iterable[torch.Tensor]]) -> None: + """Update autograd metadata tracking whether the given Tensor was modified in place. + + This is to enable more accurate error checking within the autograd engine. + It is already done automatically by PyTorch functions and within custom Function + when mark_dirty() is called appropriately so you only need to call this explicitly + if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't + know about. For example a custom kernel that reads the Tensor data_ptr and modifies + the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors. + + Note that incrementing the version counter multiple times for a single inplace operation + is not problematic. + + Note that if you pass in tensor constructed under torch.inference_mode(), + we will not bump its version counter (because your tensor does not have one). + """ + if isinstance(tensor, torch.Tensor): + tensor = (tensor,) + torch._C._increment_version(tensor) + + +class saved_tensors_hooks: + """Context-manager that sets a pair of pack / unpack hooks for saved tensors. + + Use this context-manager to define how intermediary results of an operation + should be packed before saving, and unpacked on retrieval. + + In that context, the ``pack_hook`` function will be called every time an + operation saves a tensor for backward (this includes intermediary results + saved using + :func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but + also those recorded by a PyTorch-defined operation). The output of + ``pack_hook`` is then stored in the computation graph instead of the + original tensor. + + The ``unpack_hook`` is called when the saved tensor needs to be accessed, + namely when executing :func:`torch.Tensor.backward()` or + :func:`torch.autograd.grad()`. It takes as argument the *packed* object + returned by ``pack_hook`` and should return a tensor which has the same + content as the original tensor (passed as input to the corresponding + ``pack_hook``). + + The hooks should have the following signatures: + + pack_hook(tensor: Tensor) -> Any + + unpack_hook(Any) -> Tensor + + where the return value of ``pack_hook`` is a valid input to ``unpack_hook``. + + In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms + of value, size, dtype and device. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pack_hook(x): + ... print("Packing", x) + ... return x.detach() + >>> + >>> def unpack_hook(x): + ... print("Unpacking", x) + ... return x + >>> + >>> a = torch.ones(5, requires_grad=True) + >>> b = torch.ones(5, requires_grad=True) * 2 + >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook): + ... y = a * b + Packing tensor([1., 1., 1., 1., 1.], requires_grad=True) + Packing tensor([2., 2., 2., 2., 2.], grad_fn=) + >>> y.sum().backward() + Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True) + Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=) + + .. warning :: + Performing an inplace operation on the input to either hooks may lead + to undefined behavior. + + .. warning :: + Only one pair of hooks is allowed at a time. When recursively nesting this + context-manager, only the inner-most pair of hooks will be applied. + + .. warning :: + To avoid reference cycle, the return value of ``pack_hook`` cannot hold a + reference to the input tensor. For example, use `lambda x: x.detach()` + instead of `lambda x: x` as the pack hook. + """ + + def __init__( + self, + pack_hook: Callable[[torch.Tensor], Any], + unpack_hook: Callable[[Any], torch.Tensor], + ) -> None: + self.pack_hook = pack_hook + self.unpack_hook = unpack_hook + + def __enter__(self) -> None: + torch._C._autograd._push_saved_tensors_default_hooks( + self.pack_hook, self.unpack_hook + ) + + def __exit__(self, *args: object) -> None: + torch._C._autograd._pop_saved_tensors_default_hooks() + + +class save_on_cpu(saved_tensors_hooks): + """Context manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward. + + When performing operations within this context manager, intermediary + results saved in the graph during the forward pass will be moved to CPU, + then copied back to the original device when needed for the backward pass. + If the graph was already on CPU, no tensor copy is performed. + + Use this context-manager to trade compute for GPU memory usage (e.g. + when your model doesn't fit in GPU memory during training). + + Args: + pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory + during packing and copied to GPU asynchronously during unpacking. + Defaults to ``False``. + Also see :ref:`cuda-memory-pinning`. + + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> a = torch.randn(5, requires_grad=True, device="cuda") + >>> b = torch.randn(5, requires_grad=True, device="cuda") + >>> c = torch.randn(5, requires_grad=True, device="cuda") + >>> + >>> def f(a, b, c): + ... prod_1 = a * b # a and b are saved on GPU + ... with torch.autograd.graph.save_on_cpu(): + ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU + ... y = prod_2 * a # prod_2 and a are saved on GPU + ... return y + >>> + >>> y = f(a, b, c) + >>> del a, b, c # for illustration only + >>> # the content of a, b, and prod_2 are still alive on GPU + >>> # the content of prod_1 and c only live on CPU + >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward + >>> # all intermediary tensors are released (deleted) after the call to backward + """ + + def __init__(self, pin_memory: bool = False, device_type: str = "cuda") -> None: + device_module = getattr(torch, device_type, torch.cuda) + + def pack_to_cpu(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]: + if not pin_memory: + return (tensor.device, tensor.cpu()) + packed = torch.empty( + tensor.size(), + dtype=tensor.dtype, + layout=tensor.layout, + pin_memory=(device_module.is_available() and not tensor.is_sparse), + ) + packed.copy_(tensor) + return (tensor.device, packed) + + def unpack_from_cpu(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor: + device, tensor = packed + return tensor.to(device, non_blocking=pin_memory) + + super().__init__(pack_to_cpu, unpack_from_cpu) + + +@contextlib.contextmanager +def disable_saved_tensors_hooks(error_message: str) -> Generator[None, None, None]: + """Context-manager that disables the saved tensors default hooks feature. + + Useful for if you are creating a feature that does not work with saved + tensors default hooks. + + Args: + error_message (str): When saved tensors default hooks are used when they + have been are disabled, a RuntimeError with this + error message gets raised. + + Example:: + + >>> # xdoctest: +SKIP(failing) + >>> message = "saved tensors default hooks are disabled" + >>> with torch.autograd.graph.disable_saved_tensors_hooks(message): + ... # Raises RuntimeError: saved tensors default hooks are disabled + ... with torch.autograd.graph.save_on_cpu(): + ... pass + """ + maybe_prev_message = None + try: + maybe_prev_message = ( + torch._C._autograd._saved_tensors_hooks_get_disabled_error_message() + ) + torch._C._autograd._saved_tensors_hooks_disable(error_message) + yield + finally: + # See NOTE: [disabled_error_message invariant] + if maybe_prev_message is None: + torch._C._autograd._saved_tensors_hooks_enable() + else: + torch._C._autograd._saved_tensors_hooks_disable(maybe_prev_message) + + +def set_warn_on_accumulate_grad_stream_mismatch(enabled: bool) -> None: + """Whether to warn when the AccumulateGrad node's stream does not match the stream + of the node that produced the incoming gradient. + """ + return torch._C._set_warn_on_accumulate_grad_stream_mismatch(enabled) + + +class _MultiHandle(RemovableHandle): + handles: tuple[RemovableHandle, ...] + + def __init__(self, handles: tuple[RemovableHandle, ...]) -> None: + self.handles = handles + + def remove(self) -> None: + for handle in self.handles: + handle.remove() + + def __getstate__(self) -> tuple[RemovableHandle, ...]: + return self.handles + + def __setstate__(self, state: tuple[RemovableHandle, ...]) -> None: + self.handles = state + + +def register_multi_grad_hook( + tensors: Sequence[torch.Tensor], + fn: Union[ + Callable[[Sequence[Optional[torch.Tensor]]], None], + Callable[[torch.Tensor], None], + ], + *, + mode: Literal["all", "any"] = "all", +) -> RemovableHandle: + r"""Register a multi-grad backward hook. + + There are two supported modes: ``"all"`` and ``"any"``. + + Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in + :attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but + is not part of the graph, or if a tensor is not needed to compute the gradients + for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call, + this tensor will be ignored and the hook will not wait for its gradient to be + computed. + + After every non-ignored tensor's gradient has been computed, :attr:`fn` will be + called with those gradients. ``None`` will be passed for tensors that did not + have their gradients computed. + + Under the ``"any"`` mode, the hook will be called after the first gradient + with respect to a tensor in :attr:`tensors` has been computed. The hook + will be called with that gradient as its argument. + + The hook should not modify its arguments. + + This function returns a handle with a method ``handle.remove()`` that removes the hook. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> import torch + >>> + >>> a = torch.rand(2, 3, requires_grad=True) + >>> b = torch.rand(2, 3, requires_grad=True) + >>> c = a * b + >>> d = a * b + >>> + >>> def fn(grads): + ... print([g is not None for g in grads]) + ... + >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn) + >>> + >>> c.sum().backward(retain_graph=True) + [True, True, True, False] + >>> c.sum().backward(inputs=(a,), retain_graph=True) + [True, False, True, False] + >>> + """ + supported_modes = ("all", "any") + lock = threading.Lock() + + if mode not in supported_modes: + raise ValueError(f"Expects mode to be one of {supported_modes} but got {mode}") + + if mode == "all": + count: dict[int, int] = {} + nb_calls = None + buffer: dict[int, list[Optional[torch.Tensor]]] = {} + + grad_fns = list(map(_get_grad_fn_or_grad_acc, tensors)) + len_tensors = len(tensors) + + def get_inner_hook(idx: int) -> Callable[[torch.Tensor], None]: + def inner_hook(grad: torch.Tensor) -> None: + nonlocal count, nb_calls, buffer, fn + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + count[id] = count.get(id, 0) + # pyrefly: ignore [unsupported-operation] + buffer[id] = buffer.get(id, [None] * len_tensors) + + with lock: + curr_count, count[id] = count[id], count[id] + 1 + + if curr_count == 0: + # On the first call, compute the actual nb_calls and buffer + nb_calls = sum( + map(torch._C._will_engine_execute_node, grad_fns) + ) + + buffer[id][idx] = grad + + if nb_calls is None: + raise AssertionError("Expected nb_calls to be set") + if curr_count == nb_calls - 1: + fn = cast(Callable[[Sequence[Optional[torch.Tensor]]], None], fn) + fn(buffer[id]) + del count[id] + del buffer[id] + + return inner_hook + + handles = tuple( + t.register_hook(get_inner_hook(i)) for i, t in enumerate(tensors) + ) + elif mode == "any": + fn = cast(Callable[[torch.Tensor], None], fn) + ran_hook: dict[int, bool] = defaultdict(bool) + + @functools.wraps(fn) + def wrapped_fn(grad: torch.Tensor) -> None: + nonlocal ran_hook + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + with lock: + prev, ran_hook[id] = ran_hook[id], True + if prev: + return + fn(grad) + + handles = tuple( + tensor.register_hook(wrapped_fn) + for tensor in tensors + if tensor.requires_grad + ) + + return _MultiHandle(handles) # type: ignore[possibly-undefined] + + +# NOTE [Allow mutation on tensors saved for backward] +# +# 1. Tensor gets saved for backward +# - remember the python object id and the version of the tensor +# - remember aliasing information (data_ptr of base + version) +# - save the original so we control its lifetime +# 2. Any time a tensor gets in-placed +# - for each tensor aliased to it: +# - check using its object id and version to see if it has been saved +# - if it has been saved, clone it +# - delete the reference to the original +# 3. during backward +# - if the clone exists, the tensor must've been modified in-place +_allow_mutation_on_saved_tensors_enabled: bool = False + + +_TID: TypeAlias = tuple[int, int, int] +_SID: TypeAlias = tuple[int, int] + + +def _get_tid(tensor: torch.Tensor) -> _TID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (id(tensor), data_ptr, tensor._version) + + +def _get_sid(tensor: torch.Tensor) -> _SID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (data_ptr, tensor._version) + + +class _Handle: + pass + + +class _swap_with_cloned(saved_tensors_hooks): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + def pack_hook(tensor: torch.Tensor) -> _Handle: + tid = _get_tid(tensor) + sid = _get_sid(tensor) + # Tensors saved for backward have an entry in _tid_to_weakhandle + handle: Optional[_Handle] = None + + # Save aliasing information + ctx.sid_to_tid[sid].add(tid) + + # NB: The same tensor (of the same version) can be saved multiple times + if tid not in ctx.tid_to_weakhandle: + handle = _Handle() + ctx.tid_to_weakhandle[tid] = handle + ctx.original[handle] = tensor + else: + # Store an additional strong reference to the handle + handle = ctx.tid_to_weakhandle[tid] + return handle + + def unpack_hook(handle: _Handle) -> torch.Tensor: + error_msg = ( + "Trying to backward outside of the 'allow_mutation_on_saved_tensors' context" + "in which the graph was originally recorded." + ) + if not _allow_mutation_on_saved_tensors_enabled: + raise AssertionError(error_msg) + if handle in ctx.cloned: + res = ctx.cloned[handle] + else: + if handle not in ctx.original: + raise AssertionError(error_msg) + res = ctx.original[handle] + return res + + super().__init__(pack_hook, unpack_hook) + + +class _CloneArgBeforeMutateMode(TorchDispatchMode): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + self.ctx = ctx + + def __torch_dispatch__( + self, + func: "OpOverload", + types: Iterable[type], + args: tuple[Any, ...] = (), + kwargs: Optional[dict[Any, Any]] = None, + ) -> Any: + kwargs = kwargs or {} + + def maybe_clone(t: torch.Tensor) -> None: + tid = _get_tid(t) + sid = _get_sid(t) + ctx = self.ctx + if sid in ctx.sid_to_tid: + for tid in ctx.sid_to_tid[sid]: + if tid not in ctx.tid_to_weakhandle: + # We know that if tid is in sid_to_tid, then it must also be in + # tid_to_weakhandle. However, it is possible for the tensor to be + # saved at one point, but cleared by backward before it is modified + # in-place. Consider the following example: + # + # >>> a = torch.randn(2, 3, requires_grad=True).clone() + # >>> out = (a**2).sum() + # >>> out.backward() + # >>> a.sin_() + continue + handle = ctx.tid_to_weakhandle[tid] + if handle in ctx.cloned: + # The same exact tensor has been cloned already + continue + ctx.cloned[handle] = ctx.original[handle].clone() + del ctx.original[handle] + + for idx, arg in enumerate(func._schema.arguments): + if arg.alias_info is not None and arg.alias_info.is_write: + if arg.is_out: + maybe_clone(kwargs["out"]) + elif isinstance(args[idx], list): + # Foreach case. (Possible optimization: if most of the + # tensors need to be cloned, use a for each clone?) + for t in args[idx]: + maybe_clone(t) + else: + maybe_clone(args[idx]) + + return func(*args, **kwargs) + + +class _AllowMutationOnSavedContext: + def __init__(self) -> None: + self.cloned: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.original: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.tid_to_weakhandle: MutableMapping[_TID, _Handle] = WeakValueDictionary() + self.sid_to_tid: dict[_SID, set[_TID]] = defaultdict(set) + + def clear(self) -> None: + self.cloned.clear() + self.original.clear() + self.tid_to_weakhandle.clear() + self.sid_to_tid.clear() + + +@contextlib.contextmanager +def allow_mutation_on_saved_tensors() -> Generator[ + _AllowMutationOnSavedContext, None, None +]: + """Context manager under which mutating tensors saved for backward is allowed. + + Under this context manager, tensors saved for backward are cloned on mutation, + so the original version can still be used during backward. Normally, mutating a tensor + saved for backward will result in an error raised when it's used during backward. + + To ensure the correct behavior, both the forward and backward should be run under + the same context manager. + + Returns: + An _AllowMutationOnSavedContext object storing the state managed by this + context manager. This object can be useful for debugging purposes. The state + managed by the context manager is automatically cleared upon exiting. + + Example:: + + >>> import torch + >>> with torch.autograd.graph.allow_mutation_on_saved_tensors(): + ... # forward + ... a = torch.ones(2, 3, requires_grad=True) + ... b = a.clone() + ... out = (b**2).sum() + ... b.sin_() + ... # backward + ... out.sum().backward() + ... + tensor([[0.8415, 0.8415, 0.8415], + [0.8415, 0.8415, 0.8415]], grad_fn=) + """ + global _allow_mutation_on_saved_tensors_enabled + + ctx = _AllowMutationOnSavedContext() + + with _swap_with_cloned(ctx), _CloneArgBeforeMutateMode(ctx): + try: + if _allow_mutation_on_saved_tensors_enabled: + raise RuntimeError( + "allow_mutation_on_saved_tensors contexts cannot be nested" + ) + _allow_mutation_on_saved_tensors_enabled = True + yield ctx + finally: + ctx.clear() + _allow_mutation_on_saved_tensors_enabled = False + + +def _register_logging_hooks_on_whole_graph( + t_outputs: Sequence[Union[torch.Tensor, GradientEdge]], +) -> Callable[[], None]: + grad_fns = list(map(_get_grad_fn_or_grad_acc, t_outputs)) + + def iter_graph(roots: list[Node]) -> Iterator[Node]: + if not roots: + return + seen: set[Node] = set() + q: deque[Node] = deque() + for node in roots: + if node is not None: + seen.add(node) + q.append(node) + + while q: + node = q.popleft() + for fn, _ in node.next_functions: + if fn in seen or fn is None: + continue + seen.add(fn) + q.append(fn) + + yield node + + def fmt(t: Optional[torch.Tensor]) -> str: + # Avoid circular import + from torch.utils._dtype_abbrs import dtype_abbrs + + if t is None: + return "None" + return f"{dtype_abbrs[t.dtype]}[{', '.join(map(str, t.shape))}]" + + def prehook(grad_outputs: Sequence[Optional[torch.Tensor]]) -> None: + node = torch._C._current_autograd_node() + grad_outputs_str = f"[{','.join(fmt(t) for t in grad_outputs)}]" + log_str = f"Executing: {node} with grad_outputs: {grad_outputs_str}" + log.debug(log_str) + + handles = [node.register_prehook(prehook) for node in iter_graph(grad_fns)] + + def unregister_hooks() -> None: + for handle in handles: + handle.remove() + + return unregister_hooks + + +def _engine_run_backward( + t_outputs: Sequence[Union[torch.Tensor, GradientEdge]], + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor, ...]: + attach_logging_hooks = log.getEffectiveLevel() <= logging.DEBUG + if attach_logging_hooks: + unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs) + try: + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass + t_outputs, *args, **kwargs + ) # Calls into the C++ engine to run the backward pass + finally: + if attach_logging_hooks: + unregister_hooks() # type: ignore[possibly-undefined] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..7728dfcb2501accb88d9e45a431ff85ac076446a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler.py @@ -0,0 +1,1242 @@ +# mypy: allow-untyped-defs +import uuid +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from time import perf_counter_ns +from typing import Any, Optional +from warnings import warn + +import torch +import torch.cuda +from torch._C import _get_privateuse1_backend_name +from torch._C._profiler import _ExperimentalConfig +from torch.autograd import ( + _disable_profiler, + _enable_profiler, + _kineto_step, + _prepare_profiler, + _ProfilerResult, + _supported_activities, + _toggle_collection_dynamic, + DeviceType, + kineto_available, + ProfilerActivity, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, + MemRecordsAcc, + OUT_OF_MEMORY_EVENT_NAME, +) +from torch.futures import Future + + +__all__ = [ + "profile", + "record_function", + "emit_itt", + "emit_nvtx", + "load_nvprof", + "EnforceUnique", + "parse_nvprof_trace", + "KinetoStepTracker", + "EventList", + "FunctionEvent", + "MemRecordsAcc", +] + +try: + # Available in Python >= 3.2 + from contextlib import ContextDecorator as _ContextDecorator +except ImportError: + import functools + + class _ContextDecorator: # type: ignore[no-redef] + def __enter__(self): + raise NotImplementedError + + def __exit__(self, exc_type, exc_val, exc_tb): + raise NotImplementedError + + def __call__(self, func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + with self: + return func(*args, **kwargs) + + return wrapped + + +# global python state - whether profiler is currently enabled +# useful for fast python checks to reduce latency +_is_profiler_enabled: bool = False + + +def _set_is_profiler_enabled(enable: bool): + global _is_profiler_enabled + _is_profiler_enabled = enable + + +def _run_on_profiler_start(): + _set_is_profiler_enabled(True) + + +def _run_on_profiler_stop(): + _set_is_profiler_enabled(False) + + +@dataclass +class _ProfilerStats: + "Profiler timing and stats used by developers to catch issues/regressions" + + profiling_window_duration_sec: float = 0 + number_of_events: int = 0 + profiler_prepare_call_duration_us: int = 0 + profiler_enable_call_duration_us: int = 0 + profiler_disable_call_duration_us: int = 0 + parse_kineto_call_duration_us: int = 0 + function_events_build_tree_call_duration_us: int = 0 + + +class profile: + """Context manager that manages autograd profiler state and holds a summary of results. + + .. note:: + This is the backend, most people should use :mod:`torch.profiler` instead. + + Under the hood it just records events of functions being executed in C++ and + exposes those events to Python. You can wrap any code into it and it will + only report runtime of PyTorch functions. + Note: profiler is thread local and is automatically propagated into the async tasks + + Args: + enabled (bool, optional): Setting this to False makes this context manager a no-op. + + use_cuda (bool, optional): Enables timing of CUDA events as well + using the cudaEvent API. (will be deprecated) + + use_device (str, optional): Enables timing of device events. + Adds approximately 4us of overhead to each tensor operation when use cuda. + The valid devices options are 'cuda', 'xpu', 'mtia' and 'privateuseone'. + + record_shapes (bool, optional): If shapes recording is set, information + about input dimensions will be collected. This allows one to see which + dimensions have been used under the hood and further group by them + using prof.key_averages(group_by_input_shape=True). Please note that + shape recording might skew your profiling data. It is recommended to + use separate runs with and without shape recording to validate the timing. + Most likely the skew will be negligible for bottom most events (in a case + of nested function calls). But for higher level functions the total + self cpu time might be artificially increased because of the shape + collection. + + with_flops (bool, optional): If with_flops is set, the profiler will estimate + the FLOPs (floating point operations) value using the operator's input shape. + This allows one to estimate the hardware performance. Currently, + this option only works for the matrix multiplication and 2D convolution operators. + + profile_memory (bool, optional): track tensor memory allocation/deallocation. + + with_stack (bool, optional): record source information (file and line number) for the ops. + + with_modules (bool): record module hierarchy (including function names) + corresponding to the callstack of the op. e.g. If module A's forward call's + module B's forward which contains an aten::add op, + then aten::add's module hierarchy is A.B + Note that this support exist, at the moment, only for TorchScript models + and not eager mode models. + + use_kineto (bool, optional): experimental, enable profiling with Kineto profiler. + + use_cpu (bool, optional): profile CPU events; setting to ``False`` requires + ``use_kineto=True`` and can be used to lower the overhead for GPU-only profiling. + + experimental_config (_ExperimentalConfig) : A set of experimental options + used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed. + + acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles + + + .. warning:: + Enabling memory profiling or source attribution incurs additional profiler + overhead + + .. warning:: + This context managers should not be called recursively, i.e. no nested + instances are allowed + + .. warning:: + Due to some CUDA multiprocessing limitations (see :ref:`multiprocessing-cuda-note`), + one cannot use the profiler with ``use_device = 'cuda'`` to benchmark + DataLoaders with ``num_workers > 0``. If you wish to benchmark data loading, + please use ``use_device = None`` or ``num_workers = 0``. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + >>> for _ in range(100): # any normal python code, really! + >>> y = x ** 2 + >>> y.backward() + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + mul 32.048ms 32.048ms 200 + pow 27.041ms 27.041ms 200 + PowBackward0 9.727ms 55.483ms 100 + torch::autograd::AccumulateGrad 9.148ms 9.148ms 100 + torch::autograd::GraphRoot 691.816us 691.816us 100 + ----------------------------------- --------------- --------------- --------------- + + """ + + def __init__( + self, + enabled=True, + *, + use_cuda=False, # Deprecated + use_device=None, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + use_kineto=False, + use_cpu=True, + experimental_config=None, + acc_events=False, + custom_trace_id_callback=None, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + if self.use_cuda: + warn( + "The attribute `use_cuda` will be deprecated soon, " + "please use ``use_device = 'cuda'`` instead.", + FutureWarning, + stacklevel=2, + ) + self.use_device: Optional[str] = "cuda" + else: + self.use_device = use_device + # TODO Consider changing _function_events into data structure with size cap + self._function_events: Optional[EventList] = None + self._old_function_events: Optional[EventList] = None + # Function event processing is done lazily + self._needs_processing = False + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + self.use_cpu = use_cpu + self.acc_events = acc_events + if experimental_config is None: + experimental_config = _ExperimentalConfig() + self.experimental_config = experimental_config + self.kineto_results: Optional[_ProfilerResult] = None + self.profiling_start_time_ns = 0 + self.profiling_end_time_ns = 0 + self._stats = _ProfilerStats() + self.custom_trace_id_callback = custom_trace_id_callback + self.trace_id = "" + if not self.use_cpu: + if not use_kineto: + raise AssertionError( + "Device-only events supported only with Kineto (use_kineto=True)" + ) + + if self.use_device is not None: + VALID_DEVICE_OPTIONS = ["cuda", "xpu", "mtia", "hpu"] + if _get_privateuse1_backend_name() != "privateuseone": + VALID_DEVICE_OPTIONS.append(_get_privateuse1_backend_name()) + if self.use_device not in VALID_DEVICE_OPTIONS: + warn( + f"The {self.use_device} is not a valid device option.", stacklevel=2 + ) + self.use_device = None + + if self.use_device == "cuda" and not torch.cuda.is_available(): + warn("CUDA is not available, disabling CUDA profiling", stacklevel=2) + self.use_cuda = False + self.use_device = None + + if self.use_device == "xpu" and not torch.xpu.is_available(): + warn("XPU is not available, disabling XPU profiling", stacklevel=2) + self.use_device = None + + if self.use_device == "hpu" and not ( + hasattr(torch, "hpu") and torch.hpu.is_available() + ): + warn("HPU is not available, disabling HPU profiling", stacklevel=2) + self.use_device = None + + self.kineto_activities = set() + if self.use_cpu: + self.kineto_activities.add(ProfilerActivity.CPU) + + self.profiler_kind = ProfilerState.KINETO + if self.use_device == "cuda": + if not use_kineto or ProfilerActivity.CUDA not in _supported_activities(): + if not self.use_cpu: + raise AssertionError("Legacy CUDA profiling requires use_cpu=True") + self.profiler_kind = ProfilerState.KINETO_GPU_FALLBACK + else: + self.kineto_activities.add(ProfilerActivity.CUDA) + elif self.use_device == "xpu": + if not (use_kineto and ProfilerActivity.XPU in _supported_activities()): + raise AssertionError( + "Legacy XPU profiling is not supported. Requires use_kineto=True on XPU devices." + ) + self.kineto_activities.add(ProfilerActivity.XPU) + elif self.use_device == "mtia": + if not (use_kineto and ProfilerActivity.MTIA in _supported_activities()): + raise AssertionError( + "Legacy MTIA profiling is not supported. Requires use_kineto=True on MTIA devices." + ) + self.kineto_activities.add(ProfilerActivity.MTIA) + elif self.use_device == "hpu": + if not (use_kineto and ProfilerActivity.HPU in _supported_activities()): + raise AssertionError( + "Legacy HPU profiling is not supported. Requires use_kineto=True on HPU devices." + ) + self.kineto_activities.add(ProfilerActivity.HPU) + elif self.use_device is not None and self.use_device != "privateuseone": + if ( + not use_kineto + or ProfilerActivity.PrivateUse1 not in _supported_activities() + ): + if not self.use_cpu: + raise AssertionError( + "Legacy custombackend profiling requires use_cpu=True" + ) + self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1_FALLBACK + else: + self.kineto_activities.add(ProfilerActivity.PrivateUse1) + + if len(self.kineto_activities) == 0: + raise AssertionError("No activities specified for the profiler") + + def default_trace_id(self): + # Generate a UUID + uuid_raw = uuid.uuid4() + + return f"{uuid_raw.int:032X}" + + def create_trace_id(self): + if self.custom_trace_id_callback: + return self.custom_trace_id_callback() + return self.default_trace_id() + + def config(self, create_trace_id=False): + # only need to generate new trace id upon prepare trace not start trace + if create_trace_id: + trace_id = self.create_trace_id() + self.trace_id = trace_id + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + self.experimental_config, + self.trace_id, + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self._prepare_trace() + self._start_trace() + return self + + def _prepare_trace(self): + self.entered = True + t0 = perf_counter_ns() + _prepare_profiler(self.config(create_trace_id=True), self.kineto_activities) + t1 = perf_counter_ns() + self._stats.profiler_prepare_call_duration_us = int((t1 - t0) / 1000) + + def _start_trace(self): + self.entered = True + _run_on_profiler_start() + t0 = perf_counter_ns() + _enable_profiler(self.config(create_trace_id=False), self.kineto_activities) + t1 = perf_counter_ns() + self._stats.profiler_enable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_start_time_ns = t1 + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_device and hasattr(torch, self.use_device): + device_module = getattr(torch, self.use_device) + if hasattr(device_module, "synchronize"): + device_module.synchronize() + + if self._function_events and self.acc_events: + self._old_function_events = self._function_events + self._function_events = None + self._needs_processing = True + + t0 = perf_counter_ns() + + self.kineto_results = _disable_profiler() + t1 = perf_counter_ns() + self._stats.profiler_disable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_end_time_ns = t0 + + _run_on_profiler_stop() + + self._stats.profiling_window_duration_sec = ( + (self.profiling_end_time_ns - self.profiling_start_time_ns) * 1.0 / 1e9 + ) + + # If we plan to accumulate events we should post process the function events + # right away to retain the state across multiple start/stop calls + if self.acc_events: + self._ensure_function_events() + return False + + def __repr__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return repr(self._function_events) + + def __str__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return str(self._function_events) + + def _ensure_function_events(self): + """Process function events lazily if required""" + if self._function_events is not None: + return + self._needs_processing = False + + t0 = perf_counter_ns() + parsed_results = [] + if self.kineto_results: + parsed_results = self._parse_kineto_results(self.kineto_results) + t1 = perf_counter_ns() + self._stats.parse_kineto_call_duration_us = int((t1 - t0) / 1000) + + self._function_events = EventList( + parsed_results, + use_device=self.use_device, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + t0 = perf_counter_ns() + self._function_events._build_tree() + t1 = perf_counter_ns() + self._stats.function_events_build_tree_call_duration_us = int((t1 - t0) / 1000) + self._stats.number_of_events = len(self._function_events) + + if self._old_function_events and self.acc_events: + for evt in self._old_function_events: + self._function_events.append(evt) + self._old_function_events = None + + if self._function_events is None: + raise RuntimeError("Profiler didn't finish running") + + @property + def function_events(self): + if self._function_events is None or self._needs_processing: + self._ensure_function_events() + return self._function_events + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def export_chrome_trace(self, path): + """ + Exports the collected trace in Chrome JSON format. If kineto is enabled, only + last cycle in schedule is exported. + """ + if kineto_available(): + self.kineto_results.save(path) # type: ignore[union-attr] + else: + self._ensure_function_events() + return self._function_events.export_chrome_trace(path) # type: ignore[union-attr] + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self._function_events.export_stacks(path, metric) + + def toggle_collection_dynamic( + self, enabled: bool, activities: Iterable[ProfilerActivity] + ): + """ + Toggles the collection of activities for the current profiler instance. + """ + return _toggle_collection_dynamic(enabled, set(activities)) + + def key_averages( + self, + group_by_input_shape=False, + group_by_stack_n=0, + group_by_overload_name=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.key_averages( + group_by_input_shape, group_by_stack_n, group_by_overload_name + ) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Returns total time spent on CPU. + + The total time is a sum of all self times across all the events. + """ + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.self_cpu_time_total + + def _parse_kineto_results(self, result: _ProfilerResult): + # result.events() has most of the events - PyTorch op-level and device-level events + + trace_start_ns = result.trace_start_ns() + mem_records = [ + [evt, False] for evt in result.events() if evt.name() == MEMORY_EVENT_NAME + ] + oom_records = [ + evt for evt in result.events() if evt.name() == OUT_OF_MEMORY_EVENT_NAME + ] + mem_records_acc = MemRecordsAcc(mem_records) + + def _cpu_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [DeviceType.CPU, DeviceType.MKLDNN, DeviceType.IDEEP] + else 0 + ) + + def _device_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.HIP, + DeviceType.XPU, + ] + else 0 + ) + + # Create and return FunctionEvent list, which contains all function events + # Here 2 function events are created: + # all_function_events contains all events associated with each kineto event from result + all_function_events = [] + # frontend_function_events contains the events in aten or torch frontend level, + # whose correlation id is 0 + frontend_function_events = [] + device_corr_map: dict[int, list[FunctionEvent]] = {} + max_evt_id = 0 + for kineto_event in result.events(): + if ( + _filter_name(kineto_event.name()) + or getattr(kineto_event, "is_hidden_event", lambda: False)() + ): + continue + rel_start_ns = kineto_event.start_ns() - trace_start_ns + rel_end_ns = kineto_event.end_ns() - trace_start_ns + abs_end_ns = kineto_event.end_ns() + + cpu_memory_usage = 0 + device_memory_usage = 0 + if kineto_event.device_type() == DeviceType.CPU: + # find the corresponding memory allocation events + for mem_record in mem_records_acc.in_interval( + kineto_event.start_ns() / 1000, abs_end_ns / 1000 + ): + cpu_memory_usage += _cpu_memory_usage(mem_record[0]) + device_memory_usage += _device_memory_usage(mem_record[0]) + mem_record[1] = True + + is_async = kineto_event.is_async() or ( + kineto_event.start_thread_id() != kineto_event.end_thread_id() + ) + + fe = FunctionEvent( + id=kineto_event.correlation_id(), + name=_rewrite_name(name=kineto_event.name(), with_wildcard=True), + overload_name=kineto_event.overload_name(), + trace_name=_rewrite_name(name=kineto_event.name(), with_wildcard=False), + thread=kineto_event.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_end_ns / 1000, + fwd_thread=kineto_event.fwd_thread_id(), + input_shapes=kineto_event.shapes(), + concrete_inputs=kineto_event.concrete_inputs(), + kwinputs=kineto_event.kwinputs(), + stack=[ + entry + for entry in kineto_event.stack() + if _filter_stack_entry(entry) + ], + scope=kineto_event.scope(), + use_device=self.use_device, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=device_memory_usage, + is_async=is_async, + sequence_nr=kineto_event.sequence_nr(), + device_type=kineto_event.device_type(), + device_index=kineto_event.device_index(), + device_resource_id=kineto_event.device_resource_id(), + flops=kineto_event.flops(), + is_user_annotation=kineto_event.is_user_annotation(), + metadata_json=kineto_event.metadata_json(), + ) + max_evt_id = max(max_evt_id, fe.id) + if fe.device_type == DeviceType.CPU and not fe.is_async: + if self.use_device == _get_privateuse1_backend_name(): + privateuse1_time = kineto_event.privateuse1_elapsed_us() + if privateuse1_time > 0: + fe.append_kernel(fe.name, fe.device_index, privateuse1_time) + fe.is_legacy = True + elif self.use_device == "cuda": + # Check if we have CUDA time as a fallback + cuda_time = kineto_event.cuda_elapsed_us() + if cuda_time > 0: + fe.append_kernel(fe.name, fe.device_index, cuda_time) + fe.is_legacy = True + all_function_events.append(fe) + corr_id = kineto_event.linked_correlation_id() + if corr_id > 0: + if corr_id not in device_corr_map: + device_corr_map[corr_id] = [] + device_corr_map[corr_id].append(fe) + elif corr_id == 0: + frontend_function_events.append(fe) + else: + raise RuntimeError( + f"Got negative correlation id {corr_id} in profiler post processing" + ) + + # associate device kernels and device runtime (CPU) with CPU events + for fe in frontend_function_events: + if ( + fe.device_type == DeviceType.CPU + and not fe.is_async + and fe.id in device_corr_map + ): + for f_evt in device_corr_map[fe.id]: + if ( + f_evt.device_type == DeviceType.CUDA + or f_evt.device_type == DeviceType.PrivateUse1 + ): + fe.append_kernel( + f_evt.name, + f_evt.device_index, + f_evt.time_range.end - f_evt.time_range.start, + ) + elif f_evt.device_type == DeviceType.CPU: + # make sure that 'thread' of a CPU Kineto (e.g. Device Runtime) event is associated + # with the 'thread' of the corresponding linked PyTorch event to properly track + # parents and children + f_evt.thread = fe.thread + + def createFunctionEventForMemoryEvents(evt): + rel_start_ns = evt.start_ns() - trace_start_ns + fe = FunctionEvent( + id=max_evt_id, + name=evt.name(), + overload_name="", + trace_name=None, # not outputting in the trace + thread=evt.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_start_ns / 1000, # no duration + fwd_thread=evt.start_thread_id(), + input_shapes=[], + stack=[], + scope=0, # RecordScope::FUNCTION + use_device=self.use_device, + cpu_memory_usage=_cpu_memory_usage(evt), + device_memory_usage=_device_memory_usage(evt), + is_async=False, + sequence_nr=-1, + device_type=DeviceType.CPU, + device_index=0, + ) + return fe + + # output top-level memory events + for mem_record in mem_records: + if not mem_record[1]: + max_evt_id += 1 + fe = createFunctionEventForMemoryEvents(mem_record[0]) + all_function_events.append(fe) + + for oom_record in oom_records: + max_evt_id += 1 + fe = createFunctionEventForMemoryEvents(oom_record) + all_function_events.append(fe) + + all_function_events.sort( + key=lambda evt: [evt.time_range.start, -evt.time_range.end] + ) + return all_function_events + + +# pyrefly: ignore [invalid-inheritance] +class record_function(_ContextDecorator): + """Context manager/function decorator that adds a label to a code block/function when running autograd profiler. + Label will only appear if CPU activity tracing is enabled. + + It is useful when tracing the code profile. + + Args: + name (str): Label assigned to the block of code. + node_id (int): ID of node, for distributed profiling. Unset in + non-distributed cases. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + ... y = x**2 + ... with torch.autograd.profiler.record_function( + ... "label-z" + ... ): # label the block + ... z = y**3 + ... y.backward() + >>> # xdoctest: +IGNORE_WANT + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total % CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + pow 60.77% 47.470us 3 + mul 21.73% 25.465us 2 + PowBackward0 12.03% 121.891us 1 + torch::autograd::AccumulateGrad 2.70% 6.324us 1 + label-z 2.13% 12.421us 1 + torch::autograd::GraphRoot 0.64% 1.503us 1 + ----------------------------------- --------------- --------------- --------------- + Self CPU time total: 234.344us + CUDA time total: 0.000us + + """ + + def __init__(self, name: str, args: Optional[str] = None): + self.name: str = name + self.args: Optional[str] = args + # Whether or not we should run record function's end callbacks when exiting. + self.run_callbacks_on_exit: bool = True + # TODO: TorchScript ignores standard type annotation here + # self.record: Optional["torch.classes.profiler._RecordFunction"] = None + self.record = torch.jit.annotate( + # pyrefly: ignore [not-a-type] + Optional["torch.classes.profiler._RecordFunction"], + None, + ) + + def __enter__(self): + self.record = torch.ops.profiler._record_function_enter_new( + self.name, self.args + ) + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any): + if not self.run_callbacks_on_exit: + return + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + torch.ops.profiler._record_function_exit._RecordFunction(record) + else: + torch.ops.profiler._record_function_exit(record) + + def _call_end_callbacks_on_future(self, fut: Future[Any]) -> Future[Any]: + """Use for profiling async calls that return a future. + + Calling this function will extend recording beyond this scope, until the future is + satisfied. It is useful for profiling the end to end time of asynchronous calls. + This function should only be called once to attach the callback onto the future, and + will throw if called multiple times. + + Args: + fut: (torch._C.Future): future for which to schedule + callback for. + + Returns: + A future that completes with the value of the passed in future when + the profiling callbacks have ran. + + """ + # Throw if we have already attached a callback onto the future. + if not self.run_callbacks_on_exit: + raise RuntimeError("_call_end_callbacks_on_future can only be called once.") + + # We are scheduling to run this RecordFunction's end callbacks when the + # passed in future completes, so don't run end callbacks on exit. + self.run_callbacks_on_exit = False + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + profiled_future = ( + torch.ops.profiler._call_end_callbacks_on_jit_fut._RecordFunction( + record, fut + ) + ) + else: + profiled_future = torch.ops.profiler._call_end_callbacks_on_jit_fut( + record, fut + ) + return profiled_future + + +class emit_itt: + """Context manager that makes every autograd operation emit an ITT range. + + It is useful when running the program under Intel(R) VTune Profiler:: + + vtune <--vtune-flags> + + The Instrumentation and Tracing Technology (ITT) API enables your application to generate and + control the collection of trace data during its execution across different Intel tools. + This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager, + you will be able to see labeled ranges in Intel(R) VTune Profiler GUI. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the itt range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of itt range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.autograd.profiler.emit_itt(): + ... model(x) + + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("ITT annotation context manager is not reentrant") + self.entered = True + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.ITT, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + _disable_profiler() + _run_on_profiler_stop() + return False + + +class emit_nvtx: + """Context manager that makes every autograd operation emit an NVTX range. + + It is useful when running the program under nvprof:: + + nvprof --profile-from-start off -o trace_name.prof -- + + Unfortunately, there's no way to force nvprof to flush the data it collected + to disk, so for CUDA profiling one has to use this context manager to annotate + nvprof traces and wait for the process to exit before inspecting them. + Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or + :func:`torch.autograd.profiler.load_nvprof` can load the results for inspection + e.g. in Python REPL. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the nvtx range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of nvtx range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.cuda.profiler.profile(): + ... model(x) # Warmup CUDA memory allocator and profiler + ... with torch.autograd.profiler.emit_nvtx(): + ... model(x) + + **Forward-backward correlation** + + When viewing a profile created using :class:`emit_nvtx` in the Nvidia Visual Profiler, + correlating each backward-pass op with the corresponding forward-pass op can be difficult. + To ease this task, :class:`emit_nvtx` appends sequence number information to the ranges it + generates. + + During the forward pass, each function range is decorated with ``seq=``. ``seq`` is a running + counter, incremented each time a new backward Function object is created and stashed for backward. + Thus, the ``seq=`` annotation associated with each forward function range tells you that + if a backward Function object is created by this forward function, + the backward object will receive sequence number N. + During the backward pass, the top-level range wrapping each C++ backward Function's + ``apply()`` call is decorated with ``stashed seq=``. ``M`` is the sequence number that + the backward object was created with. By comparing ``stashed seq`` numbers in backward with ``seq`` + numbers in forward, you can track down which forward op created each backward Function. + + Any functions executed during the backward pass are also decorated with ``seq=``. During + default backward (with ``create_graph=False``) this information is irrelevant, and in fact, + ``N`` may simply be 0 for all such functions. Only the top-level ranges associated with + backward Function objects' ``apply()`` methods are useful, as a way to correlate these Function + objects with the earlier forward pass. + + **Double-backward** + + If, on the other hand, a backward pass with ``create_graph=True`` is underway (in other words, + if you are setting up for a double-backward), each function's execution during backward + is given a nonzero, useful ``seq=``. Those functions may themselves create Function objects + to be executed later during double-backward, just as the original functions in the forward pass did. + The relationship between backward and double-backward is conceptually the same as the relationship + between forward and backward: The functions still emit current-sequence-number-tagged ranges, + the Function objects they create still stash those sequence numbers, and during the eventual + double-backward, the Function objects' ``apply()`` ranges are still tagged with ``stashed seq`` + numbers, which can be compared to `seq` numbers from the backward pass. + + .. warning: + The sequence number is thread-local, and some forward functions don't create an associated + backward Function object (instead delegating that to sub-functions further down the call chain). + For these reasons, the correspondence of stashed sequence numbers in + backward Function ``apply()`` ranges with `seq` numbers in forward-pass ranges is + not guaranteed to be 1 to 1. The sequence numbers alone may not be enough to fully + disambiguate which forward function created which + backward Function object. You may need to make a judgment based on analytic knowledge of what + the expected correspondence should be. + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("NVTX annotation context manager is not reentrant") + self.entered = True + torch.cuda.synchronize() + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.NVTX, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + torch.cuda.synchronize() + _disable_profiler() + _run_on_profiler_stop() + return False + + +def load_nvprof(path): + """Open an nvprof trace file and parses autograd annotations. + + Args: + path (str): path to nvprof trace + """ + return EventList(parse_nvprof_trace(path)) + + +class EnforceUnique: + """Raises an error if a key is seen more than once.""" + + def __init__(self): + self.seen = set() + + def see(self, *key): + r""" + Observe a key and raise an error if it is seen multiple times. + """ + if key in self.seen: + raise RuntimeError("duplicate key: " + str(key)) + self.seen.add(key) + + +def parse_nvprof_trace(path): + import sqlite3 + + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + + # Parse strings table + strings = {} + for r in conn.execute("SELECT _id_ as id, value FROM StringTable"): + strings[r["id"]] = torch._C._demangle(r["value"]) + + # First, find all functions and create FunctionEvents for them + marker_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp AS start_time, end.timestamp AS end_time + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + WHERE + start.name != 0 AND end.name = 0 + """ + functions = [] + functions_map = {} + unique = EnforceUnique() + for row in conn.execute(marker_query): + unique.see(row["marker_id"]) + evt = FunctionEvent( + id=row["marker_id"], + node_id=0, # missing a node_id when calling FunctionEvent. This is just to ensure + # that pytorch doesn't crash when creating a FunctionEvent() object + name=strings[row["name"]], + start_us=row["start_time"], + end_us=row["end_time"], + thread=0, + ) # TODO: find in sqlite database + functions.append(evt) + functions_map[evt.id] = evt + + # Now, correlate all kernels with FunctionEvents + kernel_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp, end.timestamp, + runtime._id_ AS runtime_id, runtime.cbid, runtime.start AS runtime_start, runtime.end AS runtime_end, + kernel.start AS kernel_start, kernel.end AS kernel_end, kernel.name AS kernel_name + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start + INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + INNER JOIN CUPTI_ACTIVITY_KIND_RUNTIME as runtime + ON (start.timestamp < runtime.start AND runtime.end < end.timestamp) + INNER JOIN CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL AS kernel + ON kernel.correlationId = runtime.correlationId + """ + unique = EnforceUnique() + for row in conn.execute(kernel_query): + unique.see(row["marker_id"], row["runtime_id"]) + # 211 is cudaKernelLaunch for cuda >= 9.2 + if row["cbid"] != 211: + raise AssertionError(f"Expected cbid to be 211, but got {row['cbid']}") + evt = functions_map[row["marker_id"]] + evt.append_kernel( + row["kernel_name"], 0, row["kernel_end"] - row["kernel_start"] + ) + + functions.sort(key=lambda evt: evt.time_range.start) + return functions + + +class KinetoStepTracker: + """Provides an abstraction for incrementing the step count globally. + + Previously, we only had one place to mark that a step() has occurred + in the program via pytorch profiler step(). We will now add step hooks + in the Optimizer class https://github.com/pytorch/pytorch/issues/88446 + + - This could mean programs that already call profiler.step() every + iteration can end up double incrementing step count. + - If a model uses multiple optimizers we can also have double or more + counting of the step. + + We fix this by adding a layer of abstraction before calling step() + to the kineto library. The idea is to maintain steps per requester in a dict: + + .. code-block:: + + { + "ProfilerStep": 100, # triggered by profiler step() call + "Optimizer1Step": 100, # Optimizer 1 or 2 are just examples, could be SGD, Adam etc + "Optimizer2Step": 100, + } + + To figure out the global step count just take the max of dict values (100). + + If one of the count increments the max will go up. + + .. code-block:: + + { + "ProfilerStep": 100, + "Optimizer1Step": 101, # Optimizer1 got incremented first say + "Optimizer2Step": 100, + } + + Then global step count is 101 + We only call the kineto step() function when global count increments. + + NOTE: Please do not use the KinetoStepTracker in modules beside the Optimizer + for now. The result could be incorrect increments of the step count. + """ + + _current_step = 0 + _step_dict: dict[str, int] = defaultdict(int) + + @classmethod + def init_step_count(cls, requester: str): + r""" + Initialize for a given requester. + """ + cls._step_dict[requester] = cls._current_step + + @classmethod + def erase_step_count(cls, requester: str) -> bool: + r""" + Remove a given requester. + """ + return cls._step_dict.pop(requester, None) is not None + + @classmethod + def increment_step(cls, requester: str) -> int: + """Increments the step count for the requester. + + Additionally if the max over all step counts has incremented then + trigger the _kineto_step() returns global step count + """ + if requester not in cls._step_dict: + cls.init_step_count(requester) + cls._step_dict[requester] += 1 + + new_step = max(cls._step_dict.values()) + if new_step > cls._current_step: + delta = new_step - cls._current_step + if delta > 1: + warn( + "Profiler step count has increased more than 1 - " + f"current_step = {cls._current_step} step dict = {cls._step_dict}", + stacklevel=2, + ) + for _ in range(delta): + _kineto_step() + cls._current_step = new_step + return cls._current_step + + @classmethod + def current_step(cls) -> int: + r""" + Get the latest step for any requester + """ + return cls._current_step diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_legacy.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_legacy.py new file mode 100644 index 0000000000000000000000000000000000000000..04585860ce26c7320bab3dc6520e311e87f7eedf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_legacy.py @@ -0,0 +1,327 @@ +# mypy: allow-untyped-defs +import itertools +import warnings +from typing_extensions import deprecated + +import torch +import torch.cuda +from torch.autograd import ( + _disable_profiler_legacy, + _enable_profiler_legacy, + DeviceType, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, +) + + +__all__ = ["profile"] + + +@deprecated( + "`torch.autograd.profiler_legacy.profile` is deprecated and will be removed in a future release. " + "Please use `torch.profiler` instead.", + category=None, # TODO: change to `FutureWarning` +) +class profile: + """DEPRECATED: use torch.profiler instead.""" + + def __init__( + self, + enabled=True, + *, + use_cuda=False, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + self.function_events = None + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + + if self.use_cuda and not torch.cuda.is_available(): + warnings.warn( + "CUDA is not available, disabling CUDA profiling", + stacklevel=2, + ) + self.use_cuda = False + + if self.use_cuda: + self.profiler_kind = ProfilerState.CUDA + else: + self.profiler_kind = ProfilerState.CPU + + def config(self): + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + # avoid exposing _ExperimentalConfig this in legacy public API + torch._C._profiler._ExperimentalConfig(), + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self.entered = True + self._start_trace() + return self + + def _start_trace(self): + _enable_profiler_legacy(self.config()) + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_cuda: + torch.cuda.synchronize() + + records = _disable_profiler_legacy() + parsed_results = _parse_legacy_records(records) + # pyrefly: ignore [bad-assignment] + self.function_events = EventList( + parsed_results, + use_device="cuda" if self.use_cuda else None, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + # pyrefly: ignore [missing-attribute] + self.function_events._build_tree() + return False + + def __repr__(self): + if self.function_events is None: + return "" + return repr(self.function_events) + + def __str__(self): + if self.function_events is None: + return "" + return str(self.function_events) + + def _check_finish(self): + if self.function_events is None: + raise RuntimeError("Profiler didn't finish running") + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def export_chrome_trace(self, path): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.export_chrome_trace(path) + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self.function_events.export_stacks(path, metric) + + def key_averages(self, group_by_input_shape=False, group_by_stack_n=0): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.key_averages(group_by_input_shape, group_by_stack_n) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Return CPU time as the sum of self times across all events.""" + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.self_cpu_time_total + + +def _parse_legacy_records(thread_records): + def _get_record_key(record): + """Return a tuple for correlating start and end records in `_parse_legacy_records`.""" + return (record.handle(), record.node_id()) + + start_record = None + functions = [] + + # '__start_profile' is not guaranteed to be first, so we must find it here + for record in itertools.chain.from_iterable(thread_records): + name = record.name() + if start_record is None and name == "__start_profile": + start_record = record + + if start_record is None or start_record.is_remote(): + raise AssertionError("Expected a valid local start_record") + + for thread_record_list in thread_records: + # accumulated memory allocations per handle + cpu_memory_allocs = {} + cuda_memory_allocs = {} + # ranges per handle + range_starts = {} + + filtered_handles = set() + prev_record = None + for record in thread_record_list: + record_key = _get_record_key(record) + if _filter_name(record.name()) or record_key in filtered_handles: + filtered_handles.add(record_key) + continue + + if record.kind() == "push": + # workaround to reduce double logging from operator + # wrappers and redispatch + if prev_record is not None: + duplicate = ( + prev_record.name() == record.name() + and prev_record.kind() == record.kind() + and prev_record.node_id() == record.node_id() + ) + if duplicate: + filtered_handles.add(record_key) + continue + + range_starts[record_key] = record + cpu_memory_allocs[record_key] = 0 + cuda_memory_allocs[record_key] = 0 + elif record.kind() == "pop": + if record_key not in range_starts: + raise AssertionError( + f"Expected record with key {record_key} to exist in range_starts. " + "This means that the pop event did not have a corresponding push." + ) + + start = range_starts[record_key] + + cpu_memory_usage = cpu_memory_allocs[record_key] + cuda_memory_usage = cuda_memory_allocs[record_key] + is_async = start.is_async() or (start.thread_id() != record.thread_id()) + is_remote_event = record.is_remote() + start_flops = start.flops() + + fe = FunctionEvent( + id=record.handle(), + node_id=record.node_id(), + name=_rewrite_name(name=start.name(), with_wildcard=True), + trace_name=_rewrite_name(name=start.name(), with_wildcard=False), + thread=start.thread_id(), + start_us=start_record.cpu_elapsed_us(start), + end_us=start_record.cpu_elapsed_us(record), + fwd_thread=start.fwd_thread_id(), + input_shapes=start.shapes(), + stack=[ + entry for entry in start.stack() if _filter_stack_entry(entry) + ], + scope=start.scope(), + use_device="cuda" if start.has_cuda() else None, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=cuda_memory_usage, + is_async=is_async, + is_remote=is_remote_event, + sequence_nr=start.sequence_nr(), + device_type=DeviceType.CPU, + is_legacy=True, + flops=start_flops, + ) + # note: async events have only cpu total time + if not is_async and start.has_cuda(): + duration = start.cuda_elapsed_us(record) + if duration > 0: + fe.append_kernel(start.name(), start.device(), duration) + functions.append(fe) + del range_starts[record_key] + del cpu_memory_allocs[record_key] + del cuda_memory_allocs[record_key] + elif record.kind() == "memory_alloc": + num_open_handles_cpu = len(cpu_memory_allocs) + num_open_handles_cuda = len(cuda_memory_allocs) + if num_open_handles_cpu != num_open_handles_cuda: + raise AssertionError( + f"Expected CPU and CUDA memory allocation handles to match, " + f"but got {num_open_handles_cpu} CPU and {num_open_handles_cuda} CUDA" + ) + for handle in cpu_memory_allocs: + cpu_memory_allocs[handle] += record.cpu_memory_usage() + for handle in cuda_memory_allocs: + cuda_memory_allocs[handle] += record.cuda_memory_usage() + if num_open_handles_cpu == 0: + # output event as a top-level memory event + fe = FunctionEvent( + id=0, + name=MEMORY_EVENT_NAME, + trace_name=None, + thread=0, + start_us=0, + end_us=0, + stack=[], + cpu_memory_usage=record.cpu_memory_usage(), + device_memory_usage=record.cuda_memory_usage(), + is_legacy=True, + ) + functions.append(fe) + prev_record = record + + # Sort functions by start time then by end time ascending. + # This ensures that--in the case of nested events which + # have the same start time (which may happen due to the + # granularity of the given clock tick)--we always show + # the outermost nested call first. This adds stability + # in how FunctionEvents appear + functions.sort(key=lambda evt: [evt.time_range.start, -evt.time_range.end]) + return functions diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_util.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_util.py new file mode 100644 index 0000000000000000000000000000000000000000..6b19815d03ae374e336a19325a83760590b55346 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/profiler_util.py @@ -0,0 +1,1445 @@ +# mypy: allow-untyped-defs +import bisect +import itertools +import math +from collections import defaultdict, namedtuple +from operator import attrgetter +from typing import Any, Optional +from typing_extensions import deprecated + +import torch +from torch.autograd import DeviceType + + +__all__ = [ + "EventList", + "FormattedTimesMixin", + "Interval", + "Kernel", + "FunctionEvent", + "FunctionEventAvg", + "StringTable", + "MemRecordsAcc", +] + + +class EventList(list): + """A list of profiling events with helper methods for analysis and visualization. + + EventList extends the standard Python list to provide specialized methods for + working with profiling events (FunctionEvent or FunctionEventAvg objects). + It includes utilities for aggregating statistics, formatting output tables, + and exporting profiling data. + + This class is typically returned by profiler methods and should not be + instantiated directly by users. + + Args: + *args: Standard list arguments. + use_device (str, optional): Device type for profiling ("cuda", "xpu", etc.). + profile_memory (bool, optional): Whether memory profiling was enabled. Default: False. + with_flops (bool, optional): Whether to include FLOP counts. Default: False. + + Attributes: + _use_device (str): Device type being profiled. + _profile_memory (bool): Whether memory profiling is enabled. + _with_flops (bool): Whether FLOP counting is enabled. + _tree_built (bool): Whether the event tree structure has been built. + + Key Methods: + table(...): Format events as a table string for display. + export_chrome_trace(path): Export to Chrome tracing format. + export_stacks(path, metric): Export stack traces with metrics. + key_averages(...): Compute averaged statistics grouped by operation name. + total_average(): Compute aggregate totals across all events (sums, not averages). + + Properties: + self_cpu_time_total: Sum of self CPU time across all events. + + Example:: + + import torch + from torch.profiler import profile, ProfilerActivity + + with profile(activities=[ProfilerActivity.CPU]) as prof: + x = torch.randn(100, 100) + y = torch.matmul(x, x) + + # EventList is returned by prof.events() + events = prof.events() + + # Display as formatted table + print( + events.table( + sort_by="cpu_time_total", row_limit=20, top_level_events_only=False + ) + ) + + # Export to Chrome tracing format + events.export_chrome_trace("trace.json") + + # Get averaged statistics + avg_events = events.key_averages() + print(avg_events.table()) + + # Export stack traces + events.export_stacks("stacks.txt", "self_cpu_time_total") + + See Also: + - :class:`FunctionEvent`: Individual profiling event + - :class:`FunctionEventAvg`: Averaged profiling statistics + - :meth:`table`: Format events as a readable table + - :meth:`key_averages`: Aggregate events by operation name + """ + + def __init__(self, *args, **kwargs): + use_device = kwargs.pop("use_device", None) + profile_memory = kwargs.pop("profile_memory", False) + with_flops = kwargs.pop("with_flops", False) + # pyrefly: ignore [not-iterable] + super().__init__(*args, **kwargs) + self._use_device = use_device + self._profile_memory = profile_memory + self._tree_built = False + self._with_flops = with_flops + + def _build_tree(self): + self._populate_cpu_children() + self._remove_dup_nodes() + self._set_backward_stacktraces() + self._tree_built = True + + def __str__(self): + return self.table() + + def _remove_dup_nodes(self): + while True: + to_delete = set() + + for idx in range(len(self)): + if ( + self[idx].cpu_parent is not None + and self[idx].cpu_parent.name == self[idx].name + and len(self[idx].cpu_parent.cpu_children) == 1 + ): + self[idx].cpu_parent.cpu_children = self[idx].cpu_children + self[idx].cpu_parent.kernels = self[idx].kernels # lift kernels up + for ch in self[idx].cpu_children: + ch.cpu_parent = self[idx].cpu_parent + to_delete.add(idx) + if len(to_delete) == 0: + break + + new_evts = [ev for ind, ev in enumerate(self) if ind not in to_delete] + + self.clear() + + self.extend(new_evts) + + def _populate_cpu_children(self): + """Populate child events into each underlying FunctionEvent object. + + One event is a child of another if [s1, e1) is inside [s2, e2). Where + s1 and e1 would be start and end of the child event's interval. And + s2 and e2 start and end of the parent event's interval + + Example: In event list [[0, 10], [1, 3], [3, 4]] would have make [0, 10] + be a parent of two other intervals. + + If for any reason two intervals intersect only partially, this function + will not record a parent child relationship between then. + """ + # Some events can be async (i.e. start and end on different threads), + # since it's generally undefined how to attribute children ranges to + # async ranges, we do not use them when calculating nested ranges and stats + sync_events = [ + evt + for evt in self + if not evt.is_async and evt.device_type == DeviceType.CPU + ] + events = sorted( + sync_events, + key=attrgetter("thread"), + ) + # Group by both thread and node_id, so that events that happen to have + # the same thread_id but are from different nodes aren't incorrectly + # grouped together. + threads = itertools.groupby( + events, key=lambda event: (event.thread, event.node_id) + ) + + # For each thread we keep a stack of current nested parents. + # We maintain the invariant that each interval is a subset of all other + # intervals lower in the stack. + # + # First we sort the intervals by their start time. Then we iterate over them. + # Every time we see a new interval we remove several parents from + # the top until we restore the invariant. Then parent child relationship + # if recorded if the stack is not empty. + # Finally we add new interval to the list + # + # Algorithm has O(N * log(N)) complexity where N is number of + # intervals + for _thread_id, thread_events in threads: + thread_events_ = sorted( + thread_events, + key=lambda event: [event.time_range.start, -event.time_range.end], + ) + current_events: list[FunctionEvent] = [] + for event in thread_events_: + while len(current_events) > 0: + parent = current_events[-1] + if ( + event.time_range.start >= parent.time_range.end + or event.time_range.end > parent.time_range.end + ): + # this can't be a parent + current_events.pop() + else: + parent.append_cpu_child(event) + if event.cpu_parent is not None: + raise AssertionError( + f"There is already a CPU parent event for {event.key}" + ) + event.set_cpu_parent(parent) + break + + current_events.append(event) + + def _set_backward_stacktraces(self): + def bw_parent(evt): + if evt is None: + return None + elif evt.scope == 1: # BACKWARD_FUNCTION + return evt + else: + return bw_parent(evt.cpu_parent) + + fwd_stacks = {} + for evt in self: + if bw_parent(evt) is None and evt.stack is not None: + t = (evt.sequence_nr, evt.thread) + if t not in fwd_stacks: + fwd_stacks[t] = evt.stack + + for evt in self: + p = bw_parent(evt) + if p is not None: + if p.fwd_thread is None: + raise AssertionError( + "Expected fwd_thread to be set for backward parent" + ) + t = (p.sequence_nr, p.fwd_thread) + evt.stack = fwd_stacks.get(t, []) + + @property + def self_cpu_time_total(self): + return sum(event.self_cpu_time_total for event in self) + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + time_unit=None, + ): + """Print an EventList as a nicely formatted table. + + Args: + sort_by (str, optional): Attribute used to sort entries. By default + they are printed in the same order as they were registered. + Valid keys include: ``cpu_time``, ``cuda_time``, ``xpu_time``, + ``cpu_time_total``, ``cuda_time_total``, ``xpu_time_total``, + ``cpu_memory_usage``, ``cuda_memory_usage``, ``xpu_memory_usage``, + ``self_cpu_memory_usage``, ``self_cuda_memory_usage``, + ``self_xpu_memory_usage``, ``count``. + top_level_events_only(bool, optional): Boolean flag to determine the + selection of events to display. If true, the profiler will only + display events at top level like top-level invocation of python + `lstm`, python `add` or other functions, nested events like low-level + cpu/cuda/xpu ops events are omitted for profiler result readability. + time_unit(str, optional): A time unit to be used for all values in the + table. Valid options are: ``s``, ``ms`` and ``us``. + + Returns: + A string containing the table. + """ + return _build_table( + self, + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + top_level_events_only=top_level_events_only, + time_unit=time_unit, + ) + + def export_chrome_trace(self, path): + """Export an EventList as a Chrome tracing tools file. + + The checkpoint can be later loaded and inspected under ``chrome://tracing`` URL. + + Args: + path (str): Path where the trace will be written. + """ + import os + + device_name = "cuda" if not self._use_device else self._use_device + with open(path, "w") as f: + next_id = 0 + # Use file IO over using json.dump since JSON dumping is very slow and + # this technique is proven to give a 4x speedup. + f.write("[") + for evt in self: + if evt.trace_name is None: + continue + f.write( + '{{"name": "{}", ' + '"ph": "X", ' + '"ts": {}, ' + '"dur": {}, ' + '"tid": {}, ' + '"pid": "CPU functions", ' + '"args": {{}}}}, '.format( + evt.trace_name, + evt.time_range.start, + evt.time_range.elapsed_us(), + evt.thread + if not evt.is_remote + else f'" node_id:{evt.node_id}, thread_id:{evt.thread} "', + ) + ) + for _ in evt.kernels: + # 's' and 'f' draw Flow arrows from + # the CPU launch to the GPU kernel + f.write( + f'{{"name": "{evt.trace_name}", ' + '"ph": "s", ' + f'"ts": {evt.time_range.start}, ' + f'"tid": {evt.thread}, ' + '"pid": "CPU functions", ' + f'"id": {next_id}, ' + f'"cat": "cpu_to_{device_name}", ' + '"args": {}}, ' + ) + # Note: use torch.profiler to get device kernel trace + next_id += 1 + if len(self) > 0: + # remove trailing whitespace and comma + f.seek(f.tell() - 2, os.SEEK_SET) + f.truncate() + f.write("]") + + def supported_export_stacks_metrics(self): + return [ + "self_cpu_time_total", + "self_cuda_time_total", + "self_xpu_time_total", + "self_privateuse1_time_total", + ] + + def export_stacks(self, path: str, metric: str): + if metric not in self.supported_export_stacks_metrics(): + raise ValueError( + "metric should be one of: " + + str(self.supported_export_stacks_metrics()) + ) + translate_table = str.maketrans(" ;\t\n", "____") + with open(path, "w") as f: + for evt in self: + if evt.stack and len(evt.stack) > 0: + metric_value = getattr( + evt, + metric.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ) + if int(metric_value) > 0: + stack_str = "" + for entry in reversed(evt.stack): + stack_str += entry.translate(translate_table) + stack_str += ";" + stack_str = stack_str[:-1] + " " + str(int(metric_value)) + f.write(stack_str + "\n") + + def key_averages( + self, + group_by_input_shapes=False, + group_by_stack_n=0, + group_by_overload_name=False, + ): + """Averages all function events over their keys. + + Args: + group_by_input_shapes: group entries by + (event name, input shapes) rather than just event name. + This is useful to see which input shapes contribute to the runtime + the most and may help with size-specific optimizations or + choosing the best candidates for quantization (aka fitting a roof line) + + group_by_stack_n: group by top n stack trace entries + + group_by_overload_name: Differentiate operators by their overload name e.g. aten::add.Tensor + and aten::add.out will be aggregated separately + + Returns: + An EventList containing FunctionEventAvg objects. + """ + if not self._tree_built: + raise AssertionError( + "Expected tree to be built before calling key_averages" + ) + stats: dict[tuple[str, ...], FunctionEventAvg] = defaultdict(FunctionEventAvg) + + def get_key( + event, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) -> tuple[str, ...]: + key = [ + str(event.key), + str(event.node_id), + str(event.device_type), + str(event.is_legacy), + str(event.is_user_annotation), + ] + if group_by_overload_name: + key.append(evt.overload_name) + if group_by_input_shapes: + key.append(str(event.input_shapes)) + if group_by_stack_n > 0: + key += event.stack[:group_by_stack_n] + return tuple(key) + + for evt in self: + stats[ + get_key( + evt, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) + ].add(evt) + + avg_list = EventList( + stats.values(), + use_device=self._use_device, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + ) + for evt in avg_list: + evt.stack = evt.stack[:group_by_stack_n] + if not group_by_input_shapes: + evt.input_shapes = "" + if not group_by_overload_name: + evt.overload_name = "" + return avg_list + + def total_average(self): + """Compute aggregate statistics across all events. + + Accumulates statistics from all events into a single FunctionEventAvg object. + This is primarily useful for computing total metrics (total CPU time, total + memory usage, etc.) across the entire profiling session, regardless of + operation type. + + Note: + This sums up times and counts across ALL different operations, so the + "average" metrics (like cpu_time) represent the average time per operation + call across the entire session, mixing all operation types together. + For per-operation averages, use :meth:`key_averages` instead. + + Returns: + FunctionEventAvg: A single aggregate object with key="Total" containing + accumulated statistics. + + """ + total_stat = FunctionEventAvg() + for evt in self: + total_stat += evt + total_stat.key = None + total_stat.key = "Total" + return total_stat + + +def _format_time(time_us): + """Define how to format time in FunctionEvent.""" + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_us >= US_IN_SECOND: + return f"{time_us / US_IN_SECOND:.3f}s" + if time_us >= US_IN_MS: + return f"{time_us / US_IN_MS:.3f}ms" + return f"{time_us:.3f}us" + + +def _format_time_share(time_us, total_time_us): + """Define how to format time in FunctionEvent.""" + if total_time_us == 0: + if time_us != 0: + raise AssertionError(f"Expected time_us == 0 but got {time_us}") + return "NaN" + return f"{time_us * 100.0 / total_time_us:.2f}%" + + +def _format_memory(nbytes): + """Return a formatted memory size string.""" + KB = 1024 + MB = 1024 * KB + GB = 1024 * MB + if abs(nbytes) >= GB: + return f"{nbytes * 1.0 / GB:.2f} GB" + elif abs(nbytes) >= MB: + return f"{nbytes * 1.0 / MB:.2f} MB" + elif abs(nbytes) >= KB: + return f"{nbytes * 1.0 / KB:.2f} KB" + else: + return str(nbytes) + " B" + + +def _attr_formatter(name): + return property(lambda self: _format_time(getattr(self, name))) + + +class FormattedTimesMixin: + """Helpers for FunctionEvent and FunctionEventAvg. + + The subclass should define `*_time_total` and `count` attributes. + """ + + cpu_time_str = _attr_formatter("cpu_time") + device_time_str = _attr_formatter("device_time") + cpu_time_total_str = _attr_formatter("cpu_time_total") + device_time_total_str = _attr_formatter("device_time_total") + self_cpu_time_total_str = _attr_formatter("self_cpu_time_total") + self_device_time_total_str = _attr_formatter("self_device_time_total") + + @property + def cpu_time(self): + return 0.0 if self.count == 0 else 1.0 * self.cpu_time_total / self.count # type: ignore[attr-defined] + + @property + def device_time(self): + return 0.0 if self.count == 0 else 1.0 * self.device_time_total / self.count # type: ignore[attr-defined] + + @property + @deprecated( + "`cuda_time` is deprecated, please use `device_time` instead.", + category=FutureWarning, + ) + def cuda_time(self): # To be deprecated + return self.device_time + + +class Interval: + def __init__(self, start, end): + self.start = start + self.end = end + + def elapsed_us(self): + r""" + Returns the length of the interval + """ + return self.end - self.start + + +Kernel = namedtuple("Kernel", ["name", "device", "duration"]) + + +class FunctionEvent(FormattedTimesMixin): + """Profiling information about a single function. + + FunctionEvent records the execution of a single operation during profiling. + These events are obtained from the profiler/kineto and contain detailed + timing and memory usage information. + + .. note:: + FunctionEvent objects are typically created by the profiler/kineto and should not + be instantiated directly by users. Access them through the profiler's output. + + Attributes: + id (int): Unique identifier for this event. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + name (str): Name of the profiled function/operator. + overload_name (str): Overload name for the operator (requires _ExperimentalConfig(capture_overload_names=True) set). + trace_name (str): Same as name, just changes ProfilerStep* to ProfilerStep# + time_range (Interval): Time interval containing start and end timestamps in microseconds. + thread (int): Thread ID where the operation started. + fwd_thread (int): Thread ID of the corresponding forward operation. + kernels (List[Kernel]): List of device kernels launched by this operation. + count (int): Number of times this event was called (usually 1). + cpu_children (List[FunctionEvent]): Direct CPU child operations. + cpu_parent (FunctionEvent): Direct CPU parent operation. + input_shapes (Tuple[int, ...]): Shapes of input tensors (requires record_shapes=true). + concrete_inputs (List[Any]): Concrete input values (requires record_shapes=true). + kwinputs (Dict[str, Any]): Keyword arguments (requires record_shapes=true). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_memory_usage (int): CPU memory allocated in bytes. + device_memory_usage (int): Device memory allocated in bytes. + is_async (bool): Whether this is an asynchronous operation. + is_remote (bool): Whether this operation occurred on a remote node. + sequence_nr (int): Sequence number for autograd operations. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + device_index (int): Index of the device (e.g., GPU 0, 1, 2). + device_resource_id (int): Resource ID on the device (ie. stream ID). + is_legacy (bool): Whether this is from the legacy profiler. + flops (int): Estimated floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + metadata_json (str): Additional metadata in JSON format. + + Properties: + cpu_time_total (float): Total CPU time in microseconds. + device_time_total (float): Total device (CUDA/XPU/etc) time in microseconds. + self_cpu_time_total (float): CPU time excluding child operations. + self_device_time_total (float): Device time excluding child operations. + self_cpu_memory_usage (int): CPU memory usage excluding child operations. + self_device_memory_usage (int): Device memory usage excluding child operations. + cpu_time (float): Average CPU time per call. + device_time (float): Average device time per call. + key (str): Key used for grouping events (usually same as name). + + See Also: + - :class:`torch.profiler.profile`: Context manager for profiling + - :class:`EventList`: List container for FunctionEvent objects with helper methods + - :class:`FunctionEventAvg`: Averaged statistics over multiple FunctionEvent objects + """ + + def __init__( + self, + id, + name, + thread, + start_us, + end_us, + overload_name=None, + fwd_thread=None, + input_shapes=None, + stack=None, + scope=0, + use_device=None, + cpu_memory_usage=0, + device_memory_usage=0, + is_async=False, + is_remote=False, + sequence_nr=-1, + node_id=-1, + device_type=DeviceType.CPU, + device_index=0, + device_resource_id=None, + is_legacy=False, + flops=None, + trace_name=None, + concrete_inputs=None, + kwinputs=None, + is_user_annotation=False, + metadata_json=None, + ): + self.id: int = id + self.node_id: int = node_id + self.name: str = name + # pyrefly: ignore [bad-assignment] + self.overload_name: str = overload_name + # pyrefly: ignore [bad-assignment] + self.trace_name: str = trace_name + self.time_range: Interval = Interval(start_us, end_us) + self.thread: int = thread + self.fwd_thread: Optional[int] = fwd_thread + self.kernels: list[Kernel] = [] + self.count: int = 1 + self.cpu_children: list[FunctionEvent] = [] + self.cpu_parent: Optional[FunctionEvent] = None + # pyrefly: ignore [bad-assignment] + self.input_shapes: tuple[int, ...] = input_shapes + # pyrefly: ignore [bad-assignment] + self.concrete_inputs: list[Any] = concrete_inputs + # pyrefly: ignore [bad-assignment] + self.kwinputs: dict[str, Any] = kwinputs + # pyrefly: ignore [bad-assignment] + self.stack: list = stack + self.scope: int = scope + self.use_device: Optional[str] = use_device + self.cpu_memory_usage: int = cpu_memory_usage + self.device_memory_usage: int = device_memory_usage + self.is_async: bool = is_async + self.is_remote: bool = is_remote + self.sequence_nr: int = sequence_nr + self.device_type: DeviceType = device_type + self.device_index: int = device_index + self.device_resource_id: int = ( + thread if device_resource_id is None else device_resource_id + ) + self.is_legacy: bool = is_legacy + self.flops: Optional[int] = flops + self.is_user_annotation: Optional[bool] = is_user_annotation + self.self_cpu_percent = -1 + self.total_cpu_percent = -1 + self.total_device_percent = -1 + self.metadata_json = metadata_json + + def append_kernel(self, name, device, duration): + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + self.kernels.append(Kernel(name, device, duration)) + + def append_cpu_child(self, child): + """Append a CPU child of type FunctionEvent. + + One is supposed to append only direct children to the event to have + correct self cpu time being reported. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(child, FunctionEvent): + raise AssertionError("Expected child to be a FunctionEvent") + if child.device_type != DeviceType.CPU: + raise AssertionError("Expected child device_type to be CPU") + self.cpu_children.append(child) + + def set_cpu_parent(self, parent): + """Set the immediate CPU parent of type FunctionEvent. + + One profiling FunctionEvent should have only one CPU parent such that + the child's range interval is completely inside the parent's. We use + this connection to determine the event is from top-level op or not. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(parent, FunctionEvent): + raise AssertionError("Expected parent to be a FunctionEvent") + if parent.device_type != DeviceType.CPU: + raise AssertionError("Expected parent device_type to be CPU") + self.cpu_parent = parent + + # Note: async events don't have children, are not used when computing 'self' + # metrics of other events, have only total cpu time + @property + def self_cpu_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_memory_usage - sum( + child.cpu_memory_usage for child in self.cpu_children + ) + + @property + def self_device_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.device_memory_usage - sum( + child.device_memory_usage for child in self.cpu_children + ) + + @property + @deprecated( + "`self_cuda_memory_usage` is deprecated. Use `self_device_memory_usage` instead.", + category=FutureWarning, + ) + def self_cuda_memory_usage(self): # To be deprecated + return self.self_device_memory_usage + + @property + def cpu_time_total(self): + if self.device_type == DeviceType.CPU: + return self.time_range.elapsed_us() + else: + return 0 + + @property + def self_cpu_time_total(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_time_total - sum( + child.cpu_time_total for child in self.cpu_children + ) + + @property + def device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + if not self.is_legacy: + # account for the kernels in the children ops + return sum(kinfo.duration for kinfo in self.kernels) + sum( + ch.device_time_total for ch in self.cpu_children + ) + else: + # each legacy cpu events has a single (fake) kernel + return sum(kinfo.duration for kinfo in self.kernels) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, or HPU, but got {self.device_type}" + ) + return self.time_range.elapsed_us() + + @property + @deprecated( + "`cuda_time_total` is deprecated. Use `device_time_total` instead.", + category=FutureWarning, + ) + def cuda_time_total(self): # To be deprecated + return self.device_time_total + + @property + def self_device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + return self.device_time_total - sum( + child.device_time_total for child in self.cpu_children + ) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, or HPU, but got {self.device_type}" + ) + return self.device_time_total + + @property + @deprecated( + "`self_cuda_time_total` is deprecated. Use `self_device_time_total` instead.", + category=FutureWarning, + ) + def self_cuda_time_total(self): # To be deprecated + return self.self_device_time_total + + @property + def key(self): + return self.name + + def __repr__(self): + device_name = self.use_device + device_time = self.device_time_str + device_memory_usage = self.device_memory_usage + return ( + f"" + ) + + +class FunctionEventAvg(FormattedTimesMixin): + """Averaged profiling statistics over multiple FunctionEvent objects. + + FunctionEventAvg aggregates statistics from multiple FunctionEvent objects + with the same key (typically same operation name). This is useful for getting + average performance metrics across multiple invocations of the same operation. + + This class is typically created by calling :meth:`EventList.key_averages()` on + a profiler's event list. + + Attributes: + key (str): Grouping key for the events (typically operation name). + count (int): Total number of events aggregated. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + is_async (bool): Whether the operations are asynchronous. + is_remote (bool): Whether the operations occurred on a remote node. + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_time_total (int): Accumulated total CPU time in microseconds. + device_time_total (int): Accumulated total device time in microseconds. + self_cpu_time_total (int): Accumulated self CPU time (excluding children) in microseconds. + self_device_time_total (int): Accumulated self device time (excluding children) in microseconds. + input_shapes (List[List[int]]): Input tensor shapes (requires record_shapes=true). + overload_name (str): Operator overload name (requires _ExperimentalConfig(capture_overload_names=True) set). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + cpu_memory_usage (int): Accumulated CPU memory usage in bytes. + device_memory_usage (int): Accumulated device memory usage in bytes. + self_cpu_memory_usage (int): Accumulated self CPU memory usage in bytes. + self_device_memory_usage (int): Accumulated self device memory usage in bytes. + cpu_children (List[FunctionEvent]): CPU child events. + cpu_parent (FunctionEvent): CPU parent event. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + is_legacy (bool): Whether from legacy profiler. + flops (int): Total floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + + Properties: + cpu_time (float): Average CPU time per invocation. + device_time (float): Average device time per invocation. + + See Also: + - :class:`EventList.key_averages`: Method that creates FunctionEventAvg objects + - :class:`FunctionEvent`: Individual profiling event + - :class:`EventList`: Container for profiling events + """ + + def __init__(self) -> None: + self.key: Optional[str] = None + self.count: int = 0 + self.node_id: int = 0 + self.is_async: bool = False + self.is_remote: bool = False + self.use_device: Optional[str] = None + self.cpu_time_total: int = 0 + self.device_time_total: int = 0 + self.self_cpu_time_total: int = 0 + self.self_device_time_total: int = 0 + self.input_shapes: Optional[list[list[int]]] = None + self.overload_name: Optional[str] = None + self.stack: Optional[list] = None + self.scope: Optional[int] = None + self.cpu_memory_usage: int = 0 + self.device_memory_usage: int = 0 + self.self_cpu_memory_usage: int = 0 + self.self_device_memory_usage: int = 0 + self.cpu_children: Optional[list[FunctionEvent]] = None + self.cpu_parent: Optional[FunctionEvent] = None + self.device_type: DeviceType = DeviceType.CPU + self.is_legacy: bool = False + self.flops: int = 0 + + def add(self, other): + if self.key is None: + # First function being recorded as part of FunctionEventAvg, propagate + # fields. + self.key = other.key + self.node_id = other.node_id + self.is_async = other.is_async + self.is_remote = other.is_remote + self.cpu_parent = other.cpu_parent + self.cpu_children = other.cpu_children + + self.overload_name = other.overload_name + self.input_shapes = other.input_shapes + self.stack = other.stack + self.scope = other.scope + self.device_type = other.device_type + self.is_legacy = other.is_legacy + self.use_device = other.use_device + self.is_user_annotation = other.is_user_annotation + + if not isinstance(other, (FunctionEvent, FunctionEventAvg)): + raise AssertionError( + "Expected other to be a FunctionEvent or FunctionEventAvg" + ) + if other.key != self.key: + raise AssertionError( + f"Expected keys to match, but got {other.key} vs {self.key}" + ) + + self.cpu_time_total += other.cpu_time_total + self.device_time_total += other.device_time_total + self.self_cpu_time_total += other.self_cpu_time_total + self.self_device_time_total += other.self_device_time_total + self.cpu_memory_usage += other.cpu_memory_usage + self.device_memory_usage += other.device_memory_usage + self.self_cpu_memory_usage += other.self_cpu_memory_usage + self.self_device_memory_usage += other.self_device_memory_usage + self.count += other.count + if self.flops is None: + # pyrefly: ignore [bad-assignment] + self.flops = other.flops + elif other.flops is not None: + self.flops += other.flops + return self + + def __iadd__(self, other): + return self.add(other) + + def __repr__(self): + device_name = "cuda" if not self.use_device else self.use_device + self_device_time = self.self_device_time_total_str + device_time = self.device_time_str + device_memory = self.device_memory_usage + return ( + f"" + ) + + +class StringTable(defaultdict): + def __missing__(self, key): + # manage cases like 't' (demangled to 'unsigned short') separately, + # for now simply check the length to avoid unexpected results for + # the short sequences + self[key] = torch._C._demangle(key) if len(key) > 1 else key + return self[key] + + +class MemRecordsAcc: + """Acceleration structure for accessing mem_records in interval.""" + + def __init__(self, mem_records): + self._mem_records = mem_records + self._start_nses: list[int] = [] + self._indices: list[int] = [] + if len(mem_records) > 0: + tmp = sorted([(r[0].start_ns(), i) for i, r in enumerate(mem_records)]) + self._start_nses, self._indices = zip(*tmp) # type: ignore[assignment] + + def in_interval(self, start_us, end_us): + r""" + Return all records in the given interval + To maintain backward compatibility, convert us to ns in function + """ + start_idx = bisect.bisect_left(self._start_nses, start_us * 1000) + end_idx = bisect.bisect_right(self._start_nses, end_us * 1000) + for i in range(start_idx, end_idx): + yield self._mem_records[self._indices[i]] + + +def _filter_stack_entry(entry): + filtered_entries = [ + ("autograd/__init__", "_make_grads"), + ("autograd/__init__", "backward"), + ("torch/tensor", "backward"), + ("_internal/common_utils", "prof_callable"), + ("_internal/common_utils", "prof_func_call"), + ("_internal/common_utils", "prof_meth_call"), + ] + return all(not (f[0] in entry and f[1] in entry) for f in filtered_entries) + + +MEMORY_EVENT_NAME = "[memory]" +OUT_OF_MEMORY_EVENT_NAME = "[OutOfMemory]" + + +def _filter_name(name): + # ignoring the following utility ops + filtered_out_names = [ + MEMORY_EVENT_NAME, # used only for the top-level memory events + OUT_OF_MEMORY_EVENT_NAME, + "profiler::_record_function_enter", + "profiler::_record_function_enter_new", + "profiler::_record_function_exit", + "aten::is_leaf", + "aten::output_nr", + "aten::_version", + ] + return name in filtered_out_names + + +# Demangles and optionally rewrites the provided event name, +# with_wildcard - whether to replace certain numbered event names +# with a wildcard name to aggregate them together in the profiler table +# output +def _rewrite_name(name, with_wildcard=False): + string_table = StringTable() + name = string_table[name] + if with_wildcard: + if name.startswith("ProfilerStep#"): + name = "ProfilerStep*" + return name + + +def _build_table( + events, + sort_by=None, + header=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + with_flops=False, + profile_memory=False, + top_level_events_only=False, + time_unit=None, +): + """Print a summary of events (which can be a list of FunctionEvent or FunctionEventAvg).""" + if len(events) == 0: + return "" + + has_device_time = any(event.self_device_time_total > 0 for event in events) + has_device_mem = any(event.self_device_memory_usage > 0 for event in events) + use_device = events[0].use_device + # Running on PrivateUse1 device with profiler but not enable + # ProfilerActivity.PrivateUse1 can also catch privateuse1 memory usage. + # Here only need to check has_privateuse1_time if not use_device. + if not use_device and has_device_time: + raise RuntimeError("use_device is None, but there is device performance data.") + + has_input_shapes = any( + (event.input_shapes is not None and len(event.input_shapes) > 0) + for event in events + ) + + has_overload_names = any( + (event.overload_name is not None and len(event.overload_name) > 0) + for event in events + ) + + if sort_by is not None: + events = EventList( + sorted( + events, + key=lambda evt: getattr( + evt, + sort_by.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ), + reverse=True, + ), + use_device=use_device, + profile_memory=profile_memory, + with_flops=with_flops, + ) + + name_column_width = max(len(evt.key) for evt in events) + 4 + if max_name_column_width is not None: + name_column_width = min(name_column_width, max_name_column_width) + + shapes_column_width = max(len(str(evt.input_shapes)) for evt in events) + 4 + if max_shapes_column_width is not None: + shapes_column_width = min(shapes_column_width, max_shapes_column_width) + + DEFAULT_COLUMN_WIDTH = 12 + flops_column_width = DEFAULT_COLUMN_WIDTH + + src_column_width = None + stacks = [ + evt.stack for evt in events if evt.stack is not None and len(evt.stack) > 0 + ] + has_stack = len(stacks) > 0 + if has_stack: + src_column_width = ( + max(max(len(entry) for entry in stack) for stack in stacks) + 4 + ) + if max_src_column_width is not None: + src_column_width = min(src_column_width, max_src_column_width) + + headers = ["Name"] + if has_overload_names: + headers.append("Overload Name") + headers += [ + "Self CPU %", + "Self CPU", + "CPU total %", + "CPU total", + "CPU time avg", + ] + + device_name = use_device.upper() if use_device is not None else "None" + if has_device_time: + headers.extend( + [ + f"Self {device_name}", + f"Self {device_name} %", + f"{device_name} total", + f"{device_name} time avg", + ] + ) + if profile_memory: + headers.extend( + [ + "CPU Mem", + "Self CPU Mem", + ] + ) + if use_device and has_device_mem: + headers.extend( + [ + f"{device_name} Mem", + f"Self {device_name} Mem", + ] + ) + headers.append("# of Calls") + # Only append Node ID if any event has a valid (>= 0) Node ID + append_node_id = any(evt.node_id != -1 for evt in events) + if append_node_id: + headers.append("Node ID") + + # Have to use a list because nonlocal is Py3 only... + SPACING_SIZE = 2 + row_format_lst = [""] + header_sep_lst = [""] + line_length_lst = [-SPACING_SIZE] + + def add_column(padding, text_dir=">"): + row_format_lst[0] += ( + "{: " + text_dir + str(padding) + "}" + (" " * SPACING_SIZE) + ) + header_sep_lst[0] += "-" * padding + (" " * SPACING_SIZE) + line_length_lst[0] += padding + SPACING_SIZE + + def auto_scale_flops(flops): + flop_headers = [ + "FLOPs", + "KFLOPs", + "MFLOPs", + "GFLOPs", + "TFLOPs", + "PFLOPs", + ] + if flops <= 0: + raise AssertionError(f"Expected flops to be positive, but got {flops}") + # pyrefly: ignore [no-matching-overload] + log_flops = max(0, min(math.log10(flops) / 3, float(len(flop_headers) - 1))) + if not (log_flops >= 0 and log_flops < len(flop_headers)): + raise AssertionError( + f"Expected log_flops to be in range [0, {len(flop_headers)}), but got {log_flops}" + ) + return (pow(10, (math.floor(log_flops) * -3.0)), flop_headers[int(log_flops)]) + + add_column(name_column_width) + if has_overload_names: + add_column(name_column_width) + for _ in headers[1 + has_overload_names :]: + add_column(DEFAULT_COLUMN_WIDTH) + + if has_input_shapes: + headers.append("Input Shapes") + add_column(shapes_column_width) + + if has_stack: + headers.append("Source Location") + add_column(src_column_width, text_dir="<") + + if with_flops: + # Auto-scaling of flops header + raw_flops = [evt.flops for evt in events if evt.flops > 0] + if len(raw_flops) != 0: + (flops_scale, flops_header) = auto_scale_flops(min(raw_flops)) + headers.append(f"Total {flops_header}") + add_column(flops_column_width) + else: + with_flops = False # can't find any valid flops + + row_format = row_format_lst[0] + header_sep = header_sep_lst[0] + line_length = line_length_lst[0] + add_column = None # type: ignore[assignment] + + # Have to use a list because nonlocal is Py3 only... + result = [] + + def append(s): + result.append(s) + result.append("\n") # Yes, newline after the end as well + + sum_self_cpu_time_total = 0 + sum_self_device_time_total = 0 + for evt in events: + sum_self_cpu_time_total += evt.self_cpu_time_total + if evt.device_type == DeviceType.CPU and evt.is_legacy: + # in legacy profiler, kernel info is stored in cpu events + sum_self_device_time_total += evt.self_device_time_total + elif ( + evt.device_type + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + ] + and not evt.is_user_annotation + ): + # in kineto profiler, there're events with the correct device type (e.g. CUDA) + sum_self_device_time_total += evt.self_device_time_total + + # Actual printing + if header is not None: + append("=" * line_length) + append(header) + if top_level_events_only: + append("=" * line_length) + append("This report only display top-level ops statistics") + append(header_sep) + append(row_format.format(*headers)) + + append(header_sep) + + def trim_path(path, src_column_width): + if len(path) > src_column_width: + offset = len(path) - src_column_width + path = path[offset:] + if len(path) > 3: + path = "..." + path[3:] + return path + + def override_time_unit(time_us, default_str, time_unit): + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_unit == "s": + return f"{time_us / US_IN_SECOND:.3f}s" + elif time_unit == "ms": + return f"{time_us / US_IN_MS:.3f}ms" + elif time_unit == "us": + return f"{time_us:.3f}us" + else: + return default_str + + event_limit = 0 + for evt in events: + if event_limit == row_limit: + break + if top_level_events_only and evt.cpu_parent is not None: + continue + else: + event_limit += 1 + name = evt.key + if max_name_column_width is not None and len(name) >= max_name_column_width - 3: + name = name[: (max_name_column_width - 3)] + "..." + + evt.self_cpu_percent = _format_time_share( + evt.self_cpu_time_total, sum_self_cpu_time_total + ) + evt.total_cpu_percent = ( + _format_time_share(evt.cpu_time_total, sum_self_cpu_time_total) + if not evt.is_async + else 0 + ) + + row_values = [name] + if has_overload_names: + overload_name = evt.overload_name + if ( + max_name_column_width is not None + and len(overload_name) >= max_name_column_width - 3 + ): + overload_name = overload_name[: (max_name_column_width - 3)] + "..." + row_values += [overload_name] + row_values += [ + # Self CPU total %, 0 for async events. + evt.self_cpu_percent, + override_time_unit( + evt.self_cpu_time_total, evt.self_cpu_time_total_str, time_unit + ), # Self CPU total + # CPU total %, 0 for async events. + evt.total_cpu_percent, + override_time_unit( + evt.cpu_time_total, evt.cpu_time_total_str, time_unit + ), # CPU total + override_time_unit( + evt.cpu_time, evt.cpu_time_str, time_unit + ), # CPU time avg + ] + if has_device_time: + evt.total_device_percent = _format_time_share( + evt.self_device_time_total, sum_self_device_time_total + ) + row_values.extend( + [ + override_time_unit( + evt.self_device_time_total, + evt.self_device_time_total_str, + time_unit, + ), + # device time total % + evt.total_device_percent, + override_time_unit( + evt.device_time_total, evt.device_time_total_str, time_unit + ), + override_time_unit( + evt.device_time, evt.device_time_str, time_unit + ), # device time avg + ] + ) + if profile_memory: + row_values.extend( + [ + # CPU Mem Total + _format_memory(evt.cpu_memory_usage), + # Self CPU Mem Total + _format_memory(evt.self_cpu_memory_usage), + ] + ) + if use_device and has_device_mem: + row_values.extend( + [ + # Device Mem Total + _format_memory(evt.device_memory_usage), + # Self Device Mem Total + _format_memory(evt.self_device_memory_usage), + ] + ) + row_values.append( + evt.count, # Number of calls + ) + + if append_node_id: + row_values.append(evt.node_id) + if has_input_shapes: + row_values.append(str(evt.input_shapes)[:shapes_column_width]) + if with_flops: + if evt.flops <= 0: + row_values.append("--") + else: + row_values.append(f"{evt.flops * flops_scale:8.3f}") # type: ignore[possibly-undefined] + if has_stack: + src_field = "" + if len(evt.stack) > 0: + src_field = trim_path(evt.stack[0], src_column_width) + row_values.append(src_field) + append(row_format.format(*row_values)) + + if has_stack: + empty_headers = [""] * (len(headers) - 1) + for entry in evt.stack[1:]: + append( + row_format.format( + *(empty_headers + [trim_path(entry, src_column_width)]) + ) + ) + empty_headers.append("") + append(row_format.format(*empty_headers)) + + append(header_sep) + append( + f"Self CPU time total: {override_time_unit(sum_self_cpu_time_total, _format_time(sum_self_cpu_time_total), time_unit)}" + ) + if has_device_time: + append( + f"Self {use_device.upper() if use_device is not None else 'None'} " + f"time total: {override_time_unit(sum_self_device_time_total, _format_time(sum_self_device_time_total), time_unit)}" + ) + return "".join(result) + + +# Collect all events with stack traces and format them canonically +def _canonicalize_profiler_events(events): + """ + Extract and format all events with stack traces in a canonical way + for deterministic testing. + """ + events_with_traces = [] + + for event in events: + # Extract relevant fields + event_name = event.get("name", "") + node_name = event["args"].get("node_name", "") + stack_trace = event["args"].get("stack_trace", "") + + # Get the last non-empty line of the stack trace + lines = [s.strip() for s in stack_trace.split("\n") if s.strip()] + stack_trace = lines[-1] if lines else "" + + events_with_traces.append( + { + "event_name": event_name[:20], + "node_name": node_name, + "stack_trace": stack_trace, + "start_time": event.get("ts", 0), + } + ) + + # Sort by node_name for deterministic ordering + events_with_traces.sort(key=lambda x: x["start_time"]) + + # Format as a string + lines: list[str] = [] + for evt in events_with_traces: + lines.append( + f"event={evt['event_name']} node={evt['node_name']} stack_trace={evt['stack_trace']}" + ) + + return "\n".join(lines) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/variable.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/variable.py new file mode 100644 index 0000000000000000000000000000000000000000..a81d0af68fd6d692b7f14dd8d0b46e515197131f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/autograd/variable.py @@ -0,0 +1,15 @@ +# mypy: allow-untyped-defs +import torch +from torch._C import _ImperativeEngine as ImperativeEngine + + +__all__ = ["VariableMeta", "Variable"] + + +class VariableMeta(type): + def __instancecheck__(cls, other): + return isinstance(other, torch.Tensor) + + +class Variable(torch._C._LegacyVariableBase, metaclass=VariableMeta): # type: ignore[misc] + _execution_engine = ImperativeEngine() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/backends/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95789be9dda7dd72047a46e28c8843eb4582c7a0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/backends/__init__.py @@ -0,0 +1,139 @@ +# mypy: allow-untyped-defs +import sys +import types +from contextlib import contextmanager + +import torch + + +# The idea for this parameter is that we forbid bare assignment +# to torch.backends..enabled and friends when running our +# test suite, where it's very easy to forget to undo the change +# later. +__allow_nonbracketed_mutation_flag = True + + +def disable_global_flags(): + global __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = False + + +def flags_frozen(): + return not __allow_nonbracketed_mutation_flag + + +@contextmanager +def __allow_nonbracketed_mutation(): + global __allow_nonbracketed_mutation_flag + old = __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = True + try: + yield + finally: + __allow_nonbracketed_mutation_flag = old + + +class ContextProp: + def __init__(self, getter, setter): + self.getter = getter + self.setter = setter + + def __get__(self, obj, objtype): + return self.getter() + + def __set__(self, obj, val): + if not flags_frozen(): + self.setter(val) + else: + raise RuntimeError( + f"not allowed to set {obj.__name__} flags " + "after disable_global_flags; please use flags() context manager instead" + ) + + +class PropModule(types.ModuleType): + def __init__(self, m, name): + super().__init__(name) + self.m = m + + def __getattr__(self, attr): + return self.m.__getattribute__(attr) + + +class _FP32Precision: + def __init__(self, backend, op): + self.backend = backend + self.op = op + + def __setattr__(self, name, value): + if name == "fp32_precision": + torch._C._set_fp32_precision_setter(self.backend, self.op, value) + elif name in ("backend", "op"): + super().__setattr__(name, value) + else: + raise AttributeError("Unknown attribute " + name) + + def __getattr__(self, name): + if name == "fp32_precision": + return torch._C._get_fp32_precision_getter(self.backend, self.op) + else: + raise AttributeError("Unknown attribute " + name) + + +def set_flags(_fp32_precision="none"): + orig_flags = (torch._C._get_fp32_precision_getter("generic", "all"),) + if _fp32_precision is not None: + torch._C._set_fp32_precision_setter("generic", "all", _fp32_precision) + return orig_flags + + +@contextmanager +def flags(fp32_precision="none"): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(fp32_precision) + try: + yield + finally: + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +def _get_fp32_precision_getter(backend, op): + def inner(): + return torch._C._get_fp32_precision_getter(backend, op) + + return inner + + +def _set_fp32_precision_setter(backend, op): + def inner(precision): + return torch._C._set_fp32_precision_setter(backend, op, precision) + + return inner + + +class GenericModule(PropModule): + fp32_precision = ContextProp( + _get_fp32_precision_getter("generic", "all"), + _set_fp32_precision_setter("generic", "all"), + ) + + +sys.modules[__name__] = GenericModule(sys.modules[__name__], __name__) + +from torch.backends import ( + cpu as cpu, + cuda as cuda, + cudnn as cudnn, + cusparselt as cusparselt, + kleidiai as kleidiai, + mha as mha, + miopen as miopen, + mkl as mkl, + mkldnn as mkldnn, + mps as mps, + nnpack as nnpack, + openmp as openmp, + opt_einsum as opt_einsum, + quantized as quantized, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c0df7f2cdefada00d9e56a1bb7ec5a19cdf885b4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/__init__.py @@ -0,0 +1,691 @@ +# mypy: allow-untyped-defs +import io +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, TypeVar, Union +from typing_extensions import ParamSpec + +import torch + +from . import config + + +if TYPE_CHECKING: + from ._cache import CacheInfo + + +__all__ = [ + "compile", + "config", + "assume_constant_result", + "reset", + "allow_in_graph", + "substitute_in_graph", + "list_backends", + "disable", + "set_stance", + "set_enable_guard_collectives", + "cudagraph_mark_step_begin", + "load_compiled_function", + "wrap_numpy", + "is_compiling", + "is_dynamo_compiling", + "is_exporting", + "save_cache_artifacts", + "load_cache_artifacts", + "skip_guard_on_inbuilt_nn_modules_unsafe", + "skip_guard_on_all_nn_modules_unsafe", + "keep_tensor_guards_unsafe", + "skip_guard_on_globals_unsafe", + "skip_all_guards_unsafe", + "nested_compile_region", +] + + +_P = ParamSpec("_P") +_R = TypeVar("_R") +FuncType = Callable[..., Any] +F = TypeVar("F", bound=FuncType) + + +def compile(*args, **kwargs): + """ + See :func:`torch.compile` for details on the arguments for this function. + """ + # pyrefly: ignore [not-iterable] + return torch.compile(*args, **kwargs) + + +def reset() -> None: + """ + This function clears all compilation caches and restores the system to its initial state. + It is recommended to call this function, especially after using operations like `torch.compile(...)` + to ensure a clean state before another unrelated compilation + """ + import torch._dynamo + + torch._dynamo.reset() + + +def allow_in_graph(fn): + """ + Tells the compiler frontend (Dynamo) to skip symbolic introspection of the function + and instead directly write it to the graph when encountered. + + If you are using :func:`torch.compile` (with backend="inductor" (the default)), or + :func:`torch.export.export`, and trying to black-box a Python function throughout + all tracing, do not use this API. + Instead, please create a custom operator (see `PyTorch Custom Operators Landing Page + `_) + + .. warning:: + + If you're a typical torch.compile user (e.g. you're applying torch.compile to + a model to make it run faster), you probably don't want to use this function. + :func:`allow_in_graph` is a footgun because it skips the compiler frontend + (Dynamo) that is responsible for doing safety checks (graph breaks, handling + closures, etc). Incorrect usage will lead to difficult-to-debug silent + incorrectness issues. + + Given a Python function with no allow_in_graph decorator, regular execution + of torch.compile traces through the function. :func:`allow_in_graph` changes + it so that the frontend does not trace inside the function, but the compiler + backend still traces through it. Compare this to custom operators, which + treats a function as a black box throughout the torch.compile stack. The following + table compares these mechanisms. + + +------------------------+-----------------------+--------------------------------+ + | Mechanism | Frontend (Dynamo) | Backend (AOTAutograd+Inductor) | + +========================+=======================+================================+ + | no decorator | trace inside | trace inside | + +------------------------+-----------------------+--------------------------------+ + | allow_in_graph | opaque callable | trace inside | + +------------------------+-----------------------+--------------------------------+ + | custom op | opaque callable | opaque callable | + +------------------------+-----------------------+--------------------------------+ + + One common use case for :func:`allow_in_graph()` is as an escape hatch for the compiler + frontend: if you know the function works w.r.t. to the downstream components of the + compilation stack (AOTAutograd and Inductor) but there is a Dynamo bug that prevents it from + symbolically introspecting the function properly (or if your code is in C/C++ and + therefore cannot be introspected with Dynamo), then one can decorate said function + with :func:`allow_in_graph` to bypass Dynamo. + + We require that ``fn`` adhere to the following restrictions. Failure to adhere + results in undefined behavior: + + - The inputs to ``fn`` must be Proxy-able types in the FX graph. Valid types include: + Tensor/int/bool/float/None/List[Tensor?]/List[int?]/List[float?] + Tuple[Tensor?, ...]/Tuple[int?, ...]/Tuple[float?, ...]/torch.dtype/torch.device + - The outputs to ``fn`` must be Proxy-able types in the FX graph (see previous bullet) + - all Tensors used inside of ``fn`` must be passed directly as inputs to ``fn`` + (as opposed to being captured variables). + + Args: + fn: A callable representing the function to be included in the graph. + If ``fn`` is a list or tuple of callables it recursively applies + :func:`allow_in_graph()` to each function and returns a new list or + tuple containing the modified functions. + + Example:: + + torch.compiler.allow_in_graph(my_custom_function) + + + @torch.compile(...) + def fn(x): + x = torch.add(x, 1) + x = my_custom_function(x) + x = torch.add(x, 1) + return x + + + fn(...) + + Will capture a single graph containing ``my_custom_function()``. + + """ + import torch._dynamo + + return torch._dynamo.allow_in_graph(fn) + + +def substitute_in_graph( + original_fn: Callable[_P, _R], + *, + can_constant_fold_through: bool = False, + skip_signature_check: bool = False, +) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: + """ + Register a polyfill handler for a function, usually a C function from the C extension, to be + used in place of the original function when inlining the original function in the graph. + + .. note:: + + The polyfill handler is only used when inlining the original function. It is not used when + the original function is called directly. In the eager mode, the decorated function calls + the performant C function rather than the polyfill handler. + + The polyfill handler is a function that will be called in place of the original function when + inlining the original function. The polyfill handler should have the same signature and the same + behavior as the original function. + + Args: + original_fn (callable): The original function, usually a C function, to register a polyfill + handler for. + can_constant_fold_through (bool, optional): Whether the polyfill handler can be constant + folded through. That is, if the polyfill handler is a pure function and its arguments + are constant, the result of the polyfill handler can be constant folded during the + compilation. Defaults to ``False``. + skip_signature_check (bool, optional): Whether to skip the signature check between the + original function and the polyfill handler. Defaults to ``False``. + + Returns: + A decorator that registers the polyfill handler for the original function. + + Example:: + + >>> import operator + >>> operator.indexOf([1, 2, 3, 4, 5], 3) + 2 + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + ... # xdoctest: +SKIP("Long tracebacks") + Traceback (most recent call last): + ... + torch._dynamo.exc.Unsupported: ... + + >>> @torch.compiler.substitute_in_graph(operator.indexOf) + ... def indexOf(a, b, /): + ... for i, item in enumerate(a): + ... if item is b or item == b: + ... return i + ... raise ValueError("sequence.index(x): x not in sequence") + >>> + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + 2 + """ + import torch._dynamo + + return torch._dynamo.substitute_in_graph( + original_fn, + can_constant_fold_through=can_constant_fold_through, + skip_signature_check=skip_signature_check, + ) + + +def list_backends(exclude_tags=("debug", "experimental")) -> list[str]: + """ + Return valid strings that can be passed to `torch.compile(..., backend="name")`. + + Args: + exclude_tags(optional): A tuple of strings representing tags to exclude. + """ + import torch._dynamo + + return torch._dynamo.list_backends(exclude_tags) + + +def assume_constant_result(fn): + """ + This function is used to mark a function `fn` as having a constant result. + This allows the compiler to optimize away your function. + Returns The same function `fn` + + Args: + fn: The function to be marked as having a constant result. + + .. warning:: + `assume_constant_result` can if invalid cause safety and soundness issues, :func:`torch.compile` + will not attempt to validate whether the constant assumption is true or not + + """ + import torch._dynamo + + return torch._dynamo.assume_constant_result(fn) + + +def disable(fn=None, recursive=True, *, reason=None): + """ + This function provides a decorator to disable compilation on a function. + It also provides the option of recursively disabling called functions. + + Args: + fn (optional): The function to disable + recursive (optional): A boolean value indicating whether the disabling should be recursive. + reason (optional): A string value indicating the reason for disabling the function. + """ + import torch._dynamo + + return torch._dynamo.disable(fn, recursive, reason=reason) + + +def set_stance( + stance: str = "default", + *, + skip_guard_eval_unsafe: bool = False, + force_backend: Union[str, Callable[..., Any], None] = None, +): + """ + Set the current stance of the compiler. + Can be used as a function, context manager, or decorator. + Do not use this function inside a `torch.compile` region - an error will be raised otherwise. + + .. code-block:: python + + @torch.compile + def foo(x): ... + + + @torch.compiler.set_stance("force_eager") + def bar(): + # will not be compiled + foo(...) + + + bar() + + with torch.compiler.set_stance("force_eager"): + # will also not be compiled + foo(...) + + torch.compiler.set_stance("force_eager") + # will also not be compiled + foo(...) + torch.compiler.set_stance("default") + + # will be compiled + foo(...) + + Args: + stance: The stance to set the compiler to. Valid values are: + + - "default": The default stance, used for normal compilation. + - "force_eager": Ignore all `torch.compile` directives. + - "eager_on_recompile": Run code eagerly when a recompile is necessary. + If there is cached compiled code valid for the input, it will still be used. + - "fail_on_recompile": Raise an error when recompiling a function. + - "eager_then_compile": Run the first invocation in eager mode, then compile on + subsequent calls. This is beneficial for dynamic shapes as it allows inferring + dynamism from the first two invocations instead of wasting a static compile on + the first invocation. + - "aot_eager_then_compile": Run the first invocation with AOT eager to get memory + benefits from activation checkpointing, then compile on subsequent calls. Like + eager_then_compile, this improves handling of dynamic shapes by avoiding an + initial static compile. + + + skip_guard_eval_unsafe: A flag to run only differentiating guards. + CAUTION - This flag is unsafe and should only be used if your setup + meets the following conditions. + + torch.compile uses a guard system to support recompilations and + choose which compiled artifact to run at runtime. These guards, + though efficient, add some overhead, which may impact performance in + scenarios where you need to optimize for minimal guard processing + time. This API enables you to disable guard evaluation, assuming + that you have warmed up the compiled model with a sufficient variety + of inputs. This assumption means that, after the warmup phase, no + further recompilations will be necessary. If this assumption fails, + there is a risk of silently producing incorrect results (hence the + term "unsafe" in the API name). + + force_backend: If `stance` is "default", this argument can be used to force `torch.compile` + to use a specific backend. Otherwise, an error is raised. + """ + import torch._dynamo + + return torch._dynamo.set_stance( + stance, + skip_guard_eval_unsafe=skip_guard_eval_unsafe, + force_backend=force_backend, + ) + + +# forbid in graph +set_stance._dynamo_forbidden = True # type: ignore[attr-defined] + + +def set_enable_guard_collectives(enabled: bool): + """ + Enables use of collectives *during* guard evaluation to synchronize behavior + across ranks. This is expensive: we have to issue a collective every time + we enter a compiled code region, even if no rank actually would need to + compile. This can help prevent NCCL hangs by ensuring that we never have a + situation where one rank starts recompiling while other ranks don't compile; + it is especially useful in conjunction with enable_compiler_collectives + where such a situation would immediately cause a hang (as it is necessary + for all ranks to compile at the same time to run compiler collectives). Like + compiler collectives, you can only run this on SPMD programs; you will hang + otherwise. Note that a guard collective is only issued if there is any + compiled code to guard on; if this the first time we encounter a frame or + the frame is skipped, we don't issue collectives. + + Returns the previous setting of enabled. + """ + from torch._C._dynamo.eval_frame import set_guard_complete_hook # noqa: F401 + from torch._dynamo.eval_frame import guard_collectives_hook + + if enabled: + return set_guard_complete_hook(guard_collectives_hook) is not None # type: ignore[arg-type] + else: + return set_guard_complete_hook(None) is not None + + +set_enable_guard_collectives._dynamo_forbidden = True # type: ignore[attr-defined] + + +def cudagraph_mark_step_begin(): + """ + Indicates that a new iteration of inference or training is about to begin. + + CUDA Graphs will free tensors of a prior iteration. A new iteration is started on each invocation of + torch.compile, so long as there is not a pending backward that has not been called. + + If that heuristic is wrong, such as in the following example, manually mark it with this api. + + .. code-block:: python + + @torch.compile(mode="reduce-overhead") + def rand_foo(): + return torch.rand([4], device="cuda") + + + for _ in range(5): + torch.compiler.cudagraph_mark_step_begin() + rand_foo() + rand_foo() + + For more details, see `torch.compiler_cudagraph_trees `__ + """ + from torch._inductor import cudagraph_trees + + cudagraph_trees.mark_step_begin() + + +def wrap_numpy(fn): + r"""Decorator that turns a function from ``np.ndarray``s to ``np.ndarray``s into a function + from ``torch.Tensor``s to ``torch.Tensor``s. + + It is designed to be used with :func:`torch.compile` with ``fullgraph=True``. It allows to + compile a NumPy function as if it were a PyTorch function. This allows you to run NumPy code + on CUDA or compute its gradients. + + .. note:: + + This decorator does not work without :func:`torch.compile`. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # Compile a NumPy function as a Tensor -> Tensor function + >>> @torch.compile(fullgraph=True) + >>> @torch.compiler.wrap_numpy + >>> def fn(a: np.ndarray): + >>> return np.sum(a * a) + >>> # Execute the NumPy function using Tensors on CUDA and compute the gradients + >>> x = torch.arange(6, dtype=torch.float32, device="cuda", requires_grad=True) + >>> out = fn(x) + >>> out.backward() + >>> print(x.grad) + tensor([ 0., 2., 4., 6., 8., 10.], device='cuda:0') + """ + from torch._dynamo.external_utils import wrap_numpy as wrap + + return wrap(fn) + + +_is_compiling_flag: bool = False +_is_exporting_flag: bool = False + + +def is_compiling() -> bool: + """ + Indicates whether a graph is executed/traced as part of torch.compile() or torch.export(). + + Note that there are 2 other related flags that should deprecated eventually: + * torch._dynamo.external_utils.is_compiling() + * torch._utils.is_compiling() + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_compiling(): + >>> pass # ...logic that is not needed in a compiled/traced graph... + >>> + >>> # ...rest of the function... + """ + if torch.jit.is_scripting(): + return False + else: + return _is_compiling_flag + + +def is_dynamo_compiling() -> bool: + """ + Indicates whether a graph is traced via TorchDynamo. + + It's stricter than is_compiling() flag, as it would only be set to True when + TorchDynamo is used. + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_dynamo_compiling(): + >>> pass # ...logic that is not needed in a TorchDynamo-traced graph... + >>> + >>> # ...rest of the function... + """ + return False + + +def is_exporting() -> bool: + """ + Indicated whether we're under exporting. + + It's stricter than is_compiling() flag, as it would only be set to True when + torch.export is used. + + Example:: + + >>> def forward(self, x): + >>> if not torch.compiler.is_exporting(): + >>> pass # ...logic that is not needed in export... + >>> + >>> # ...rest of the function... + """ + return _is_exporting_flag + + +def save_cache_artifacts() -> Optional[tuple[bytes, "CacheInfo"]]: + """ + Serializes all the cache artifacts that were created during the compilation + + Example: + + - Execute torch.compile + - Call torch.compiler.save_cache_artifacts() + """ + from ._cache import CacheArtifactManager + + if torch._dynamo.config.caching_precompile: + from torch._dynamo.precompile_context import PrecompileContext + + PrecompileContext.save_to_dynamo_cache() + + return CacheArtifactManager.serialize() + + +def load_cache_artifacts(serialized_artifacts: bytes) -> Optional["CacheInfo"]: + """ + Hot loads cache artifacts that were previously serialized via + save_cache_artifacts + + Example: + + # From a previous invocation + artifacts = torch.compiler.save_cache_artifacts() + + torch.compiler.load_cache_artifacts(artifacts[0]) + """ + from ._cache import CacheArtifactManager, CacheInfo + + artifacts = CacheArtifactManager.deserialize(serialized_artifacts) + if artifacts is not None: + return CacheArtifactManager.populate_caches(artifacts) + return None + + +def skip_guard_on_inbuilt_nn_modules_unsafe(guard_entries): + """ + A common function to skip guards on the inbuilt nn modules like + torch.nn.Linear. This is unsafe to use by default. But for majority of + torch.compile users, the model code does not modify the inbuilt nn module + attributes. They can benefit from reduction in guard latency overhead using + this API. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe}, + >> ) + """ + return [ + not entry.orig_guard.source.is_unspecialized_builtin_nn_module() + for entry in guard_entries + ] + + +def skip_guard_on_all_nn_modules_unsafe(guard_entries): + """ + A common function to skip guards on all nn modules, both user defined as + well inbuilt nn modules (like torch.nn.Linear). This is unsafe to use by + default. But for majority of torch.compile users, the model code does not + modify the nn module attributes. They can benefit from reduction in guard + latency overhead using this API. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_all_nn_modules_unsafe}, + >> ) + """ + + return [ + not entry.orig_guard.source.is_unspecialized_nn_module() + for entry in guard_entries + ] + + +def keep_tensor_guards_unsafe(guard_entries, keep_parameters=False): + """ + A common function to keep tensor guards on all tensors. This is unsafe to + use by default. But if you don't expect any changes in the model code, you + can just keep the tensor guards. + + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.keep_tensor_guards}, + >> ) + """ + + keep_flags = [] + for entry in guard_entries: + if entry.guard_type == "TENSOR_MATCH": + if not isinstance(entry.value, torch.nn.Parameter): + keep_flags.append(True) + elif keep_parameters: + keep_flags.append(True) + else: + keep_flags.append(False) + else: + keep_flags.append(False) + return keep_flags + + +def skip_guard_on_globals_unsafe(guard_entries): + """ + A common function to skip guards on all globals. This is unsafe to use by + default. But if you don't expect any changes in the globals, you can just + keep the tensor guards. + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_guard_on_globals}, + >> ) + """ + + return [not entry.is_global for entry in guard_entries] + + +def skip_all_guards_unsafe(guard_entries): + """ + A function for skipping all guards on a compiled function. + + WARNING: This function will drop all the safety guarantees from Dynamo + compiled function. Use this with caution. + + To use this API, use guard_filter_fn argument while calling torch.compile + + >> opt_mod = torch.compile( + >> mod, + >> options={"guard_filter_fn": torch.compiler.skip_all_guards_unsafe}, + >> ) + """ + return [False for entry in guard_entries] + + +def nested_compile_region(fn=None): + """ + Tells **``torch.compile``** that the marked set of operations forms a nested + compile region (which is often repeated in the full model) whose code can be + compiled once and safely reused. ``nested_compile_region`` can also be used + as a decorator. + + During **``torch.compile``** tracing, the compiler applies *hierarchical + compilation* with ``nested_compile_region``: it emits optimized code for the + marked region the first time it is encountered and re-emits (or “stamps + out”) the previously compiled code on every subsequent invocation. This can + substantially reduce overall compile time for deeply-stacked, + structurally-identical components such as the transformer layers of a + large-language-model (LLM). + + Outside a ``torch.compile`` context—i.e., in standard eager execution—the + call is a no-op, so existing workflows remain unaffected. + + Note that ``nested_compile_region`` **does not** promise that a region will + be compiled exactly once. If the compiler detects that new input conditions + (shape, dtype, device, stride, globals etc.) make the cached version invalid + to reuse, it will transparently re-compile the region. Using it is + therefore *safe*: correctness is always preserved, and you pay the extra + compilation cost only when required. + """ + + from torch._higher_order_ops.invoke_subgraph import ( + mark_compile_region as _mark_compile_region, + ) + + return _mark_compile_region(fn) + + +def load_compiled_function( + file: io.IOBase, *, f_globals: Optional[dict[str, object]] = None +) -> Callable[..., Any]: + """ + Load an aot-compiled function from a file. + + .. warning:: + + This API is currently experimental and subject to change. + + Args: + file: A file-like object containing the serialized compiled function. + f_globals: Optional globals to be loaded into the compiled function. + + Returns: + A torch-compiled function with compilation preloaded from disk. + """ + from torch._dynamo.aot_compile import AOTCompiledFunction + + data = file.read() + return AOTCompiledFunction.deserialize(data, f_globals) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/_cache.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..6dc62eb914585fe2b2de453f2c8b8e64a77e29b1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/_cache.py @@ -0,0 +1,322 @@ +import copy +import dataclasses +import logging +from abc import ABC, abstractmethod +from collections import defaultdict +from collections.abc import Generator +from contextlib import contextmanager +from itertools import chain +from typing import Any, Optional + +from torch.utils._appending_byte_serializer import ( + AppendingByteSerializer, + BytesReader, + BytesWriter, +) +from torch.utils._ordered_set import OrderedSet + + +log = logging.getLogger(__name__) + + +@dataclasses.dataclass(frozen=True) +class CacheArtifact(ABC): + """ + Data for each cache artifact that will be serialized and deserialized + """ + + key: str + content: bytes = dataclasses.field(repr=False) # Do not display potential binary + + @staticmethod + def serialize(writer: BytesWriter, cls: "CacheArtifact") -> None: + writer.write_str(cls.key) + writer.write_bytes(cls.content) + + @staticmethod + def deserialize(artifact_type: str, reader: BytesReader) -> "CacheArtifact": + key = reader.read_str() + content = reader.read_bytes() + return CacheArtifactFactory.create(artifact_type, key, content) + + @staticmethod + def encode(content: Any) -> bytes: + assert isinstance(content, bytes), f"Expected bytes, got {type(content)}" + return content + + @abstractmethod + def populate_cache(self) -> None: + pass + + @staticmethod + def type() -> str: + """ + Returns the type of the artifact. Must be unique across all CacheArtifact classes. + + CacheArtifactFactory.register will add property method to CacheInfo based on this (def {type}_artifacts) + that returns all artifacts for specific cache. + """ + raise RuntimeError("CacheArtifact is an abstract class, please use a subclass") + + +class CacheArtifactFactory: + """ + Factory for creating CacheArtifact objects based on their type + """ + + _artifact_types: dict[str, type[CacheArtifact]] = {} + + @classmethod + def register(cls, artifact_cls: type[CacheArtifact]) -> type[CacheArtifact]: + artifact_type_key = artifact_cls.type() + assert artifact_cls.type() not in cls._artifact_types, ( + f"Artifact of type={artifact_type_key} already registered in mega-cache artifact factory" + ) + cls._artifact_types[artifact_type_key] = artifact_cls + setattr( + CacheInfo, + f"{artifact_type_key}_artifacts", + property(lambda self: self.artifacts[artifact_type_key]), + ) + return artifact_cls + + @classmethod + def _get_artifact_type(cls, artifact_type_key: str) -> type[CacheArtifact]: + assert artifact_type_key in cls._artifact_types, ( + f"Artifact of type={artifact_type_key} not registered in mega-cache artifact factory" + ) + return cls._artifact_types[artifact_type_key] + + @classmethod + def create(cls, artifact_type_key: str, key: str, content: bytes) -> CacheArtifact: + artifact_cls = cls._get_artifact_type(artifact_type_key) + # pyrefly: ignore [bad-instantiation] + return artifact_cls(key, content) + + @classmethod + def encode_create( + cls, artifact_type_key: str, key: str, content: Any + ) -> CacheArtifact: + artifact_cls = cls._get_artifact_type(artifact_type_key) + # pyrefly: ignore [bad-instantiation] + return artifact_cls(key, artifact_cls.encode(content)) + + +@dataclasses.dataclass +class CacheInfo: + """ + Return value of serialization and deserialization for the purpose of + instrumentation + """ + + artifacts: defaultdict[str, list[str]] = dataclasses.field( + default_factory=lambda: defaultdict(list) + ) + + # Methods set by CacheArtifactFactory.register based on CacheArtifact.type() + @property + def inductor_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def autotune_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def aot_autograd_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def pgo_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + @property + def precompile_artifacts(self) -> list[str]: # type: ignore[empty-body] + ... + + def add(self, artifact: CacheArtifact) -> None: + self.artifacts[artifact.type()].append(artifact.key) + + def clear(self) -> None: + self.artifacts.clear() + + def empty(self) -> bool: + return not self.artifacts + + +def _serialize_single_cache( + writer: BytesWriter, cls: "tuple[str, list[CacheArtifact]]" +) -> None: + writer.write_str(cls[0]) + writer.write_uint64(len(cls[1])) + for artifact in cls[1]: + CacheArtifact.serialize(writer, artifact) + + +def _deserialize_single_cache( + reader: BytesReader, +) -> "tuple[str, list[CacheArtifact]]": + artifacts = [] + artifact_type_key = reader.read_str() + num_artifacts = reader.read_uint64() + for _ in range(num_artifacts): + artifacts.append(CacheArtifact.deserialize(artifact_type_key, reader)) + + return artifact_type_key, artifacts + + +CacheArtifactsResult = dict[str, list[CacheArtifact]] + + +class CacheArtifactManager: + """ + Lightweight manager class for collecting and processing cache artifacts for + hot loading + + Intended Lifecycle: + - Execute code via torch.compile, this will call + CacheArtifactManager.record_artifact on each cache artifact + - Call CacheArtifactManager.serialize to convert all the cache artifacts + to portable format + - Call CacheArtifactManager.deserialize to hot load the cache artifacts on + a potentially different process + + NOTE: There's no FB/FC guarantees, results of cache artifacts will not be + used unless code version matches. + """ + + # Protected by the compile_lock + _new_cache_artifacts: CacheArtifactsResult = defaultdict(list) + # Keep a separate seen artifacts list to make avoid unnecessary duplicates + # This list will not be cleared between serialize() calls + _seen_artifacts: OrderedSet[CacheArtifact] = OrderedSet() + # When serialize() is called, artifacts are transferred from _cache_artifacts to + # internal data structure of the _serializer + # This allows us to only pay the cost of serialization if serialize() is called + _serializer: AppendingByteSerializer[tuple[str, list[CacheArtifact]]] = ( + AppendingByteSerializer(serialize_fn=_serialize_single_cache) + ) + _cache_info: CacheInfo = CacheInfo() + + @classmethod + def clear(cls) -> None: + cls._new_cache_artifacts.clear() + cls._seen_artifacts.clear() + cls._serializer.clear() + cls._cache_info.clear() + + @classmethod + @contextmanager + def with_fresh_cache(cls) -> Generator[None, None, None]: + original_new_cache_artifacts = cls._new_cache_artifacts + original_seen_artifacts = cls._seen_artifacts + original_serializer = cls._serializer + original_cache_info = cls._cache_info + + cls._new_cache_artifacts = defaultdict(list) + cls._seen_artifacts = OrderedSet() + cls._serializer = AppendingByteSerializer(serialize_fn=_serialize_single_cache) + cls._cache_info = cls._cache_info.__class__() + try: + yield + finally: + cls._new_cache_artifacts = original_new_cache_artifacts + cls._seen_artifacts = original_seen_artifacts + cls._serializer = original_serializer + cls._cache_info = original_cache_info + + @classmethod + def record_artifact( + cls, + artifact_type: str, + key: str, + content: Any, + ) -> None: + """ + Called from each caching operation to record the artifact in this + "mega" list + """ + artifact = CacheArtifactFactory.encode_create(artifact_type, key, content) + if artifact in cls._seen_artifacts: + return + log.debug("Recording %s", str(artifact)) + cls._new_cache_artifacts[artifact_type].append(artifact) + cls._seen_artifacts.add(artifact) + + @classmethod + def need_serialize(cls) -> bool: + """ + Have we seen new artifacts since last serialize call? + """ + return len(cls._new_cache_artifacts) != 0 + + @classmethod + def serialize(cls) -> Optional[tuple[bytes, CacheInfo]]: + """ + Converts the "mega" list into portable format + """ + for artifact in chain(*cls._new_cache_artifacts.values()): + log.debug("saving: %s", artifact) + cls._cache_info.add(artifact) + + if cls._cache_info.empty(): + # If there are not artifacts, dont just return bytes with + # version. + return None + + try: + # We deep copy cls._cache_info since later compilations + # can keep adding to cache_info + info = copy.deepcopy(cls._cache_info) + cls._serializer.extend(cls._new_cache_artifacts.items()) + artifact_bytes = cls._serializer.to_bytes() + cls._new_cache_artifacts.clear() + return artifact_bytes, info + except Exception: + log.warning("Failed to pickle cache artifacts", exc_info=True) + return None + + @staticmethod + def deserialize(serialized_artifacts: bytes) -> Optional[CacheArtifactsResult]: + """ + Converts the portable format back into CacheArtifacts + """ + try: + CacheArtifactManager._ensure_cache_artifacts_registered() + artifacts = dict( + AppendingByteSerializer.to_list( + serialized_artifacts, + deserialize_fn=_deserialize_single_cache, + ) + ) + except Exception: + log.warning("Failed to un-pickle cache artifacts", exc_info=True) + return None + + return artifacts + + @staticmethod + def populate_caches(artifacts: CacheArtifactsResult) -> CacheInfo: + info = CacheInfo() + for artifact in chain(*artifacts.values()): + log.debug("writing: %s", artifact) + info.add(artifact) + artifact.populate_cache() + + return info + + @classmethod + def _ensure_cache_artifacts_registered(cls) -> None: + """When deserializing caches in fresh process, we need to ensure that all + cache artifacts are registered in the cache registry. This is done by + simply importing all the cache artifacts already wrapped with register call. + """ + from torch._dynamo.package import PrecompileCacheArtifact # noqa: F401 + from torch._dynamo.pgo import PGOCacheArtifact # noqa: F401 + from torch._functorch._aot_autograd.autograd_cache import ( # noqa: F401 + AOTAutogradCacheArtifact, + ) + from torch._inductor.codecache import InductorCacheArtifact # noqa: F401 + from torch._inductor.runtime.autotune_cache import ( # noqa: F401 + AutotuneCacheArtifact, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/config.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4677f29d9074fcf68393f15bae448d7f28eb4ea2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/compiler/config.py @@ -0,0 +1,281 @@ +""" +This is the top-level configuration module for the compiler, containing +cross-cutting configuration options that affect all parts of the compiler +stack. + +You may also be interested in the per-component configuration modules, which +contain configuration options that affect only a specific part of the compiler: + +* :mod:`torch._dynamo.config` +* :mod:`torch._inductor.config` +* :mod:`torch._functorch.config` +* :mod:`torch.fx.experimental.config` +""" + +import sys +from typing import Optional + +from torch.utils._config_module import Config, install_config_module + + +__all__ = [ + "job_id", + "dynamic_shapes", + "assume_static_by_default", + "automatic_dynamic_shapes", + "recompile_limit", + "accumulated_recompile_limit", + "verbose", + "capture_scalar_outputs", + "capture_dynamic_output_shape_ops", + "log_file_name", + "fail_on_recompile_limit_hit", + "allow_unspec_int_on_nn_module", + "skip_tensor_guards_with_matching_dict_tags", + "enable_cpp_symbolic_shape_guards", + "wrap_top_frame", + "reorderable_logging_functions", + "force_disable_caches", +] + + +# NB: Docblocks go UNDER variable definitions! Use spacing to make the +# grouping clear. + +# FB-internal note: you do NOT have to specify this explicitly specify this if +# you run on MAST, we will automatically default this to +# mast:MAST_JOB_NAME:MAST_JOB_VERSION. +job_id: Optional[str] = Config( + env_name_default=["TORCH_COMPILE_JOB_ID", "TORCH_COMPILE_STICKY_PGO_KEY"], + default=None, +) +""" +Semantically, this should be an identifier that uniquely identifies, e.g., a +training job. You might have multiple attempts of the same job, e.g., if it was +preempted or needed to be restarted, but each attempt should be running +substantially the same workload with the same distributed topology. You can +set this by environment variable with :envvar:`TORCH_COMPILE_JOB_ID`. + +Operationally, this controls the effect of profile-guided optimization related +persistent state. PGO state can affect how we perform compilation across +multiple invocations of PyTorch, e.g., the first time you run your program we +may compile twice as we discover what inputs are dynamic, and then PGO will +save this state so subsequent invocations only need to compile once, because +they remember it is dynamic. This profile information, however, is sensitive +to what workload you are running, so we require you to tell us that two jobs +are *related* (i.e., are the same workload) before we are willing to reuse +this information. Notably, PGO does nothing (even if explicitly enabled) +unless a valid ``job_id`` is available. In some situations, PyTorch can +configured to automatically compute a ``job_id`` based on the environment it +is running in. + +Profiles are always collected on a per rank basis, so different ranks may have +different profiles. If you know your workload is truly SPMD, you can run with +:data:`torch._dynamo.config.enable_compiler_collectives` to ensure nodes get +consistent profiles across all ranks. +""" + +pgo_extra_read_key: Optional[str] = Config( + env_name_default="TORCH_COMPILE_STICKY_PGO_READ", default=None +) +pgo_extra_write_key: Optional[str] = Config( + env_name_default="TORCH_COMPILE_STICKY_PGO_WRITE", default=None +) +""" +Additional read/write keys for PGO. +Write key: Besides writing to the default local/remote PGO state, this also writes to the specified key. +Read key: Besides reading from the default state, this also reads from the specified key (if written to before) +and merges it with the default state. +""" + + +cache_key_tag: str = Config(env_name_default="TORCH_COMPILE_CACHE_KEY_TAG", default="") +""" +Tag to be included in the cache key generation for all torch compile caching. +A common use case for such a tag is to break caches. +""" + +force_disable_caches: bool = Config( + justknob="pytorch/remote_cache:force_disable_caches", + env_name_force=[ + "TORCHINDUCTOR_FORCE_DISABLE_CACHES", + "TORCH_COMPILE_FORCE_DISABLE_CACHES", + ], + default=False, +) +""" +Force disables all caching -- This will take precedence over and override any other caching flag +""" + +dynamic_sources: str = Config( + env_name_default="TORCH_COMPILE_DYNAMIC_SOURCES", default="" +) +""" +Comma delimited list of sources that should be marked as dynamic. Primarily useful for large +models with graph breaks where you need intermediate tensors and ints to be marked dynamic. + +This whitelist is dominant over all other flags dynamic=False, force_nn_module_property_static_shapes +and force_parameter_static_shapes. +""" + +unbacked_sources: str = Config( + env_name_default="TORCH_COMPILE_UNBACKED_SOURCES", default="" +) +""" +Comma delimited list of sources that should be marked as unbacked. Primarily useful for large +models with graph breaks where you need intermediate tensors marked unbacked. + +This whitelist is dominant over all other flags dynamic=False, force_nn_module_property_static_shapes +and force_parameter_static_shapes. +""" + +# force a python GC before recording cudagraphs +force_cudagraph_gc: bool = Config(env_name_default="TORCH_CUDAGRAPH_GC", default=False) +""" +If True (the backward-compatible behavior) then gc.collect() before recording +any cudagraph. +""" + + +# Cross-cutting configuration options that affect the entire compilation pipeline + +dynamic_shapes: bool = Config(alias="torch._dynamo.config.dynamic_shapes") +""" +Controls whether the compilation pipeline supports dynamic tensor shapes. +When enabled, the compiler can handle tensors with varying dimensions across +different invocations. This is a cross-cutting setting that affects shape +inference, guard generation, and code generation across the entire compilation +stack. +""" + +assume_static_by_default: bool = Config( + alias="torch._dynamo.config.assume_static_by_default" +) +""" +When enabled, all tensor dimensions are assumed to be static unless explicitly +marked as dynamic or detected as changing. This compilation-wide behavior affects +how the entire stack handles shape specialization and can improve performance +for static workloads. +""" + +automatic_dynamic_shapes: bool = Config( + alias="torch._dynamo.config.automatic_dynamic_shapes" +) +""" +Enables automatic detection and handling of dynamic shapes. When a tensor's +shape changes between compilations, the system automatically marks those +dimensions as dynamic rather than requiring manual specification. This +cross-cutting optimization improves the user experience by reducing recompilations. +""" + +recompile_limit: int = Config(alias="torch._dynamo.config.recompile_limit") +""" +Maximum number of recompilations allowed for a single function before falling +back to eager execution. This compilation performance control prevents excessive +recompilation overhead that can degrade overall performance. +""" + +accumulated_recompile_limit: int = Config( + alias="torch._dynamo.config.accumulated_recompile_limit" +) +""" +Global limit on total recompilations across all compiled functions to prevent +runaway recompilation scenarios. This safeguard protects against compilation +performance issues that could affect the entire program. +""" + +verbose: bool = Config(alias="torch._dynamo.config.verbose") +""" +Enables verbose debugging output for Dynamo. When enabled, provides detailed +information about Dynamo's compilation decisions, optimizations, and potential +issues. +""" + + +# TorchDynamo-specific configuration options + +capture_scalar_outputs: bool = Config( + alias="torch._dynamo.config.capture_scalar_outputs" +) +""" +Controls whether TorchDynamo captures operations that return scalar values (like .item()) +into the FX graph. When disabled, these operations cause graph breaks. This is a +TorchDynamo-specific tracing behavior that affects how the tracer handles +scalar-returning operations. +""" + +capture_dynamic_output_shape_ops: bool = Config( + alias="torch._dynamo.config.capture_dynamic_output_shape_ops" +) +""" +Controls whether TorchDynamo captures operations with dynamic output shapes (like +nonzero, unique) into the FX graph. When disabled, these operations cause graph breaks. +This is a TorchDynamo-specific setting for handling operations with unpredictable +output shapes during tracing. +""" + +log_file_name: Optional[str] = Config(alias="torch._dynamo.config.log_file_name") +""" +Specifies a file path for TorchDynamo-specific logging output. When set, internal +TorchDynamo debug information is written to this file rather than stdout. This is +useful for debugging TorchDynamo's internal tracing behavior. +""" + +fail_on_recompile_limit_hit: bool = Config( + alias="torch._dynamo.config.fail_on_recompile_limit_hit" +) +""" +Raises a hard error when recompile limits are exceeded instead of falling back +to eager execution. This is useful for detecting excessive recompilation in +performance-critical deployments where you want to ensure compilation overhead +is kept under control. +""" + +allow_unspec_int_on_nn_module: bool = Config( + alias="torch._dynamo.config.allow_unspec_int_on_nn_module" +) +""" +Allows integer attributes of nn.Module instances to be unspecialized through +the dynamic shape mechanism. By default, TorchDynamo specializes on all integer +module attributes, but this can cause excessive recompilation when integers +like step counters change frequently. +""" + +skip_tensor_guards_with_matching_dict_tags: bool = Config( + alias="torch._dynamo.config.skip_tensor_guards_with_matching_dict_tags" +) +""" +Optimizes guard generation by treating tensors as immutable when they are +dictionary values with consistent dictionary tags across invocations. This +reduces guard overhead for tensors stored in persistent data structures. +""" + +enable_cpp_symbolic_shape_guards: bool = Config( + alias="torch._dynamo.config.enable_cpp_symbolic_shape_guards" +) +""" +Uses C++ implementation for symbolic shape guard evaluation to improve performance. +The C++ guard manager can significantly speed up guard checking for symbolic shapes +in shape-polymorphic compilations. +""" + +wrap_top_frame: bool = Config(alias="torch._dynamo.config.wrap_top_frame") +""" +Wraps the top-level decorated function/module in a frame wrapper to ensure +nn.Module hooks are compiled within the same frame as the main function. This +improves compilation coverage for models that rely on hooks. +""" + +reorderable_logging_functions: set = Config( + alias="torch._dynamo.config.reorderable_logging_functions" +) +""" +A set of logging functions that can be reordered to execute after the compiled +portion of the graph, allowing larger graphs to be captured. Functions in this +set will have their execution deferred to avoid graph breaks, though this may +affect the timing of log output. In particular, mutated values will not be logged +at the right time, leading to incorrect logging. +""" + + +install_config_module(sys.modules[__name__]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/contrib/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/contrib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/contrib/_tensorboard_vis.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/contrib/_tensorboard_vis.py new file mode 100644 index 0000000000000000000000000000000000000000..77babfd8eac47f8c41572a5b6493b96f554bc1f9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/contrib/_tensorboard_vis.py @@ -0,0 +1,152 @@ +# mypy: allow-untyped-defs +import time +from collections import defaultdict +from functools import partial + +import torch + + +# Unfortunately it doesn't seem as if there was any way to get TensorBoard to do +# anything without having TF installed, and so this file has a hard dependency on it +# as well. It really is a debugging tool, so it doesn't matter. +try: + from tensorflow.core.framework import graph_pb2 + from tensorflow.core.util import event_pb2 + from tensorflow.python.summary.writer.writer import FileWriter +except ImportError: + raise ImportError( + "TensorBoard visualization of GraphExecutors requires having " + "TensorFlow installed" + ) from None + + +def dump_tensorboard_summary(graph_executor, logdir): + with FileWriter(logdir) as w: + pb_graph = visualize(graph_executor) + evt = event_pb2.Event( + wall_time=time.time(), graph_def=pb_graph.SerializeToString() + ) + w.add_event(evt) + + +def visualize(graph, name_prefix="", pb_graph=None, executors_it=None): + """Visualizes an independent graph, or a graph executor.""" + value_map = {} + pb_graph = pb_graph or graph_pb2.GraphDef() + + if isinstance(graph, torch._C.GraphExecutorState): + visualize_graph_executor( + graph, name_prefix, pb_graph, partial(visualize, pb_graph=pb_graph) + ) + return pb_graph + + # Set up an input node + pb_graph.node.add(op="input", name=name_prefix + "input") + for i, value in enumerate(graph.param_node().outputs()): + value_map[value.unique()] = name_prefix + "input:" + str(i) + + visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it) + + # Gather all outputs + return_node = pb_graph.node.add(op="output", name=name_prefix + "output") + for value in graph.return_node().inputs(): + return_node.input.append(value_map[value.unique()]) + + return pb_graph + + +def visualize_graph_executor(state, name_prefix, pb_graph, inline_graph): + """Append the state of a given GraphExecutor to the graph protobuf. + + Args: + state (GraphExecutor or GraphExecutorState): GraphExecutor to display. + name_prefix (str): Name prefix of the containing subgraph. + pb_graph (GraphDef): graph to append to. + inline_graph (Callable): a function that handles setting up a value_map, + so that some graphs in here can be inlined. This is necessary, because + this will simply be `visualize` for the top-level GraphExecutor, + or `inline_graph` for all nested ones. + + The signature should look like (Graph, name_prefix) -> (). + It will be called exactly once. + + The strategy is to embed all different configurations as independent subgraphs, + while inlining the original graph as the one that actually produces the values. + """ + if state.autograd_fallback_graph is not None: + visualize( + graph=state.autograd_fallback_graph, + name_prefix=name_prefix + "autograd_fallback/", + pb_graph=pb_graph, + executors_it=iter(state.autograd_fallback.executors()), + ) + + for i, (arg_spec, plan) in enumerate(state.execution_plans.items()): + subgraph_name = name_prefix + f"plan{i}/" + + # Create a disconnected node that will keep information regarding the input + # types of this trace. This is unfortunately a bit too verbose to be included + # in the subgraph name. + input_kinds = pb_graph.node.add(op="INPUT_KIND", name=subgraph_name) + input_kinds.attr["inputs"].s = repr(arg_spec).encode("ascii") + + visualize(plan.graph, subgraph_name, pb_graph, iter(plan.code.executors())) + + # Show gradient as an independent subgraph of this plan + if plan.grad_executor is not None: + grad_subgraph_name = subgraph_name + "grad/" + visualize(plan.grad_executor, grad_subgraph_name, pb_graph) + + return inline_graph(state.graph, name_prefix + "original/") + + +def visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it=None): + """Recursive part of visualize (basically skips setting up the input and output nodes).""" + + def inline_graph(subgraph, name, node): + rec_value_map = { + inp.unique(): value_map[val.unique()] + for inp, val in zip(subgraph.inputs(), node.inputs()) + } + visualize_rec( + graph=subgraph, value_map=rec_value_map, name_prefix=name, pb_graph=pb_graph + ) + for out, val in zip(subgraph.outputs(), node.outputs()): + value_map[val.unique()] = rec_value_map[out.unique()] + + op_id_counter: defaultdict[str, int] = defaultdict(int) + + def name_for(node): + kind = node.kind()[node.kind().index("::") + 2 :] + op_id_counter[kind] += 1 + return kind, name_prefix + kind + "_" + str(op_id_counter[kind]) + + def add_fusion_group(node): + op, name = name_for(node) + inline_graph(node.g("Subgraph"), name + "/", node) + + def add_graph_executor(node): + op, name = name_for(node) + if executors_it is None: + add_node(node) + else: + ge = next(executors_it) + visualize_graph_executor( + ge, name + "/", pb_graph, partial(inline_graph, node=node) + ) + + def add_node(node): + if node.kind() == "prim::FusionGroup": + return add_fusion_group(node) + elif node.kind() == "prim::GraphExecutor": + return add_graph_executor(node) + op, name = name_for(node) + pb_node = pb_graph.node.add(op=op, name=name) + for value in node.inputs(): + pb_node.input.append(value_map[value.unique()]) + # TODO: handle attrs + for i, value in enumerate(node.outputs()): + value_map[value.unique()] = name + ":" + str(i) + + for node in graph.nodes(): + add_node(node) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cpu/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cpu/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5015b8f400a58b78a5fdba2092c03c737236cf37 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cpu/__init__.py @@ -0,0 +1,202 @@ +# mypy: allow-untyped-defs +r""" +This package implements abstractions found in ``torch.cuda`` +to facilitate writing device-agnostic code. +""" + +from contextlib import AbstractContextManager +from typing import Any, Optional, Union + +import torch + +from .. import device as _device +from . import amp + + +__all__ = [ + "is_available", + "is_initialized", + "synchronize", + "current_device", + "current_stream", + "stream", + "set_device", + "device_count", + "Stream", + "StreamContext", + "Event", +] + + +def _is_avx2_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX2.""" + return torch._C._cpu._is_avx2_supported() + + +def _is_avx512_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX512.""" + return torch._C._cpu._is_avx512_supported() + + +def _is_avx512_bf16_supported() -> bool: + r"""Returns a bool indicating if CPU supports AVX512_BF16.""" + return torch._C._cpu._is_avx512_bf16_supported() + + +def _is_vnni_supported() -> bool: + r"""Returns a bool indicating if CPU supports VNNI.""" + # Note: Currently, it only checks avx512_vnni, will add the support of avx2_vnni later. + return torch._C._cpu._is_avx512_vnni_supported() + + +def _is_amx_tile_supported() -> bool: + r"""Returns a bool indicating if CPU supports AMX_TILE.""" + return torch._C._cpu._is_amx_tile_supported() + + +def _is_amx_fp16_supported() -> bool: + r"""Returns a bool indicating if CPU supports AMX FP16.""" + return torch._C._cpu._is_amx_fp16_supported() + + +def _init_amx() -> bool: + r"""Initializes AMX instructions.""" + return torch._C._cpu._init_amx() + + +def is_available() -> bool: + r"""Returns a bool indicating if CPU is currently available. + + N.B. This function only exists to facilitate device-agnostic code + + """ + return True + + +def synchronize(device: torch.types.Device = None) -> None: + r"""Waits for all kernels in all streams on the CPU device to complete. + + Args: + device (torch.device or int, optional): ignored, there's only one CPU device. + + N.B. This function only exists to facilitate device-agnostic code. + """ + + +class Stream: + """ + N.B. This class only exists to facilitate device-agnostic code + """ + + def __init__(self, priority: int = -1) -> None: + pass + + def wait_stream(self, stream) -> None: + pass + + def record_event(self) -> None: + pass + + def wait_event(self, event) -> None: + pass + + +class Event: + def query(self) -> bool: + return True + + def record(self, stream=None) -> None: + pass + + def synchronize(self) -> None: + pass + + def wait(self, stream=None) -> None: + pass + + +_default_cpu_stream = Stream() +_current_stream = _default_cpu_stream + + +def current_stream(device: torch.types.Device = None) -> Stream: + r"""Returns the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): Ignored. + + N.B. This function only exists to facilitate device-agnostic code + + """ + return _current_stream + + +class StreamContext(AbstractContextManager): + r"""Context-manager that selects a given stream. + + N.B. This class only exists to facilitate device-agnostic code + + """ + + cur_stream: Optional[Stream] + + def __init__(self, stream): + self.stream = stream + self.prev_stream = _default_cpu_stream + + def __enter__(self): + cur_stream = self.stream + if cur_stream is None: + return + + global _current_stream + self.prev_stream = _current_stream + _current_stream = cur_stream + + def __exit__(self, type: Any, value: Any, traceback: Any) -> None: + cur_stream = self.stream + if cur_stream is None: + return + + global _current_stream + _current_stream = self.prev_stream + + +def stream(stream: Stream) -> AbstractContextManager: + r"""Wrapper around the Context-manager StreamContext that + selects a given stream. + + N.B. This function only exists to facilitate device-agnostic code + """ + return StreamContext(stream) + + +def device_count() -> int: + r"""Returns number of CPU devices (not cores). Always 1. + + N.B. This function only exists to facilitate device-agnostic code + """ + return 1 + + +def set_device(device: torch.types.Device) -> None: + r"""Sets the current device, in CPU we do nothing. + + N.B. This function only exists to facilitate device-agnostic code + """ + + +def current_device() -> str: + r"""Returns current device for cpu. Always 'cpu'. + + N.B. This function only exists to facilitate device-agnostic code + """ + return "cpu" + + +def is_initialized() -> bool: + r"""Returns True if the CPU is initialized. Always True. + + N.B. This function only exists to facilitate device-agnostic code + """ + return True diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0aea9efa300aff923b56696d6ef878d2d408db82 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/__init__.py @@ -0,0 +1,1956 @@ +# mypy: allow-untyped-defs +r""" +This package adds support for CUDA tensor types. + +It implements the same function as CPU tensors, but they utilize +GPUs for computation. + +It is lazily initialized, so you can always import it, and use +:func:`is_available()` to determine if your system supports CUDA. + +:ref:`cuda-semantics` has more details about working with CUDA. +""" + +import importlib +import os +import sys +import threading +import traceback +import warnings +from collections.abc import Callable +from functools import lru_cache +from typing import Any, cast, NewType, Optional, TYPE_CHECKING, Union + +import torch +import torch._C +from torch import device as _device +from torch._utils import _dummy_type, _LazySeedTracker, classproperty + +from . import _device_limits, gds +from ._utils import _get_device_index +from .graphs import ( + CUDAGraph, + graph, + graph_pool_handle, + is_current_stream_capturing, + make_graphed_callables, +) +from .green_contexts import GreenContext +from .streams import Event, ExternalStream, Stream + + +if TYPE_CHECKING: + from torch.types import Device + +try: + from torch._C import _cudart # type: ignore[attr-defined] +except ImportError: + _cudart = None + +_initialized = False +_tls = threading.local() +_initialization_lock = threading.Lock() +_queued_calls: list[ + tuple[Callable[[], None], list[str]] +] = [] # don't invoke these until initialization occurs +_is_in_bad_fork = getattr(torch._C, "_cuda_isInBadFork", lambda: False) + +_HAS_PYNVML = False +_PYNVML_ERR = None +try: + from torch import version as _version + + try: + if not _version.hip: + import pynvml # type: ignore[import] + else: + import ctypes + from pathlib import Path + + # In ROCm (at least up through 6.3.2) there're 2 copies of libamd_smi.so: + # - One at lib/libamd_smi.so + # - One at share/amd_smi/amdsmi/libamd_smi.so + # + # The amdsmi python module hardcodes loading the second one in share- + # https://github.com/ROCm/amdsmi/blob/1d305dc9708e87080f64f668402887794cd46584/py-interface/amdsmi_wrapper.py#L174 + # + # See also https://github.com/ROCm/amdsmi/issues/72. + # + # This creates an ODR violation if the copy of libamd_smi.so from lib + # is also loaded (via `ld` linking, `LD_LIBRARY_PATH` or `rpath`). + # + # In order to avoid the violation we hook CDLL and try using the + # already loaded version of amdsmi, or any version in the processes + # rpath/LD_LIBRARY_PATH first, so that we only load a single copy + # of the .so. + class _amdsmi_cdll_hook: + def __init__(self) -> None: + self.original_CDLL = ctypes.CDLL # type: ignore[misc,assignment] + paths = ["libamd_smi.so"] + if rocm_home := os.getenv("ROCM_HOME", os.getenv("ROCM_PATH")): + paths = [os.path.join(rocm_home, "lib/libamd_smi.so")] + paths + self.paths: list[str] = paths + + def hooked_CDLL( + self, name: Union[str, Path, None], *args: Any, **kwargs: Any + ) -> ctypes.CDLL: + if name and Path(name).name == "libamd_smi.so": + for path in self.paths: + try: + return self.original_CDLL(path, *args, **kwargs) + except OSError: + pass + return self.original_CDLL(name, *args, **kwargs) # type: ignore[arg-type] + + def __enter__(self) -> None: + ctypes.CDLL = self.hooked_CDLL # type: ignore[misc,assignment] + + def __exit__(self, type: Any, value: Any, traceback: Any) -> None: + ctypes.CDLL = self.original_CDLL # type: ignore[misc] + + with _amdsmi_cdll_hook(): + import amdsmi # type: ignore[import] + + _HAS_PYNVML = True + except ModuleNotFoundError: + pass + finally: + del _version +except ImportError as err: + _PYNVML_ERR = err # sometimes a lib is installed but the import fails for some other reason, so we log the error for later + +_lazy_seed_tracker = _LazySeedTracker() + +# Define dummy _CudaDeviceProperties type if PyTorch was compiled without CUDA +if hasattr(torch._C, "_CudaDeviceProperties"): + _CudaDeviceProperties = torch._C._CudaDeviceProperties +else: + _CudaDeviceProperties = _dummy_type("_CudaDeviceProperties") # type: ignore[assignment, misc] + +if hasattr(torch._C, "_cuda_exchangeDevice"): + _exchange_device = torch._C._cuda_exchangeDevice +else: + + def _exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without CUDA support") + + +if hasattr(torch._C, "_cuda_maybeExchangeDevice"): + _maybe_exchange_device = torch._C._cuda_maybeExchangeDevice +else: + + def _maybe_exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without CUDA support") + + +has_half: bool = True +has_magma: bool = torch._C._has_magma + +default_generators: tuple[torch._C.Generator] = () # type: ignore[assignment] + + +def _is_compiled() -> bool: + r"""Return true if compile with CUDA support.""" + return hasattr(torch._C, "_cuda_getDeviceCount") + + +def _nvml_based_avail() -> bool: + return os.getenv("PYTORCH_NVML_BASED_CUDA_CHECK") == "1" + + +def is_available() -> bool: + r""" + Return a bool indicating if CUDA is currently available. + + .. note:: This function will NOT poison fork if the environment variable + ``PYTORCH_NVML_BASED_CUDA_CHECK=1`` is set. For more details, see + :ref:`multiprocessing-poison-fork-note`. + """ + if not _is_compiled(): + return False + if _nvml_based_avail(): + # The user has set an env variable to request this availability check that attempts to avoid fork poisoning by + # using NVML at the cost of a weaker CUDA availability assessment. Note that if NVML discovery/initialization + # fails, this assessment falls back to the default CUDA Runtime API assessment (`cudaGetDeviceCount`) + return device_count() > 0 + else: + # The default availability inspection never throws and returns 0 if the driver is missing or can't + # be initialized. This uses the CUDA Runtime API `cudaGetDeviceCount` which in turn initializes the CUDA Driver + # API via `cuInit` + return torch._C._cuda_getDeviceCount() > 0 + + +def is_bf16_supported(including_emulation: bool = True): + r"""Return a bool indicating if the current CUDA/ROCm device supports dtype bfloat16.""" + # Check for ROCm, if true return true, no ROCM_VERSION check required, + # since it is supported on AMD GPU archs. + if torch.version.hip: + return True + + # If CUDA is not available, than it does not support bf16 either + if not is_available(): + return False + + device = torch.cuda.current_device() + + # Check for CUDA version and device compute capability. + # This is a fast way to check for it. + cuda_version = torch.version.cuda + if cuda_version is not None and torch.cuda.get_device_properties(device).major >= 8: + return True + + if not including_emulation: + return False + + # Finally try to create a bfloat16 device. + return _check_bf16_tensor_supported(device) + + +@lru_cache(maxsize=16) +def _check_bf16_tensor_supported(device: "Device"): + try: + torch.tensor([1.0], dtype=torch.bfloat16, device=device) + return True + except Exception: + return False + + +def is_tf32_supported() -> bool: + r"""Return a bool indicating if the current CUDA/ROCm device supports dtype tf32.""" + if torch.version.hip: + prop_name = torch.cuda.get_device_properties().gcnArchName + archs = ("gfx94", "gfx95") + for arch in archs: + if arch in prop_name: + return True + return False + + # Otherwise, tf32 is supported on CUDA platforms that natively (i.e. no emulation) + # support bfloat16. + return is_bf16_supported(including_emulation=False) + + +def _sleep(cycles): + torch._C._cuda_sleep(cycles) + + +def _busy_wait_for_flag(): + torch._C._cuda_busy_wait_for_flag() + + +def _clear_flag(): + torch._C._cuda_clear_flag() + + +def _extract_arch_version(arch_string: str) -> int: + """Extracts the architecture string from a CUDA version""" + base = arch_string.split("_", maxsplit=2)[1] + base = base.removesuffix("a").removesuffix("f") + return int(base) + + +def _check_capability(): + incompatible_gpu_warn = """ + Found GPU%d %s which is of cuda capability %d.%d. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (%d.%d) - (%d.%d) + """ + matched_cuda_warn = """ + Please install PyTorch with a following CUDA + configurations: {} following instructions at + https://pytorch.org/get-started/locally/ + """ + + # Binary CUDA_ARCHES SUPPORTED by PyTorch + CUDA_ARCHES_SUPPORTED = { + "12.6": {"min": 50, "max": 90}, + "12.8": {"min": 70, "max": 120}, + "13.0": {"min": 75, "max": 120}, + } + + if ( + torch.version.cuda is not None and torch.cuda.get_arch_list() + ): # on ROCm we don't want this check + for d in range(device_count()): + capability = get_device_capability(d) + major = capability[0] + minor = capability[1] + name = get_device_name(d) + current_arch = major * 10 + minor + min_arch = min( + (_extract_arch_version(arch) for arch in torch.cuda.get_arch_list()), + default=50, + ) + max_arch = max( + (_extract_arch_version(arch) for arch in torch.cuda.get_arch_list()), + default=50, + ) + if current_arch < min_arch or current_arch > max_arch: + warnings.warn( + incompatible_gpu_warn + % ( + d, + name, + major, + minor, + min_arch // 10, + min_arch % 10, + max_arch // 10, + max_arch % 10, + ), + stacklevel=2, + ) + matched_arches = "" + for arch, arch_info in CUDA_ARCHES_SUPPORTED.items(): + if ( + current_arch >= arch_info["min"] + and current_arch <= arch_info["max"] + ): + matched_arches += f" {arch}" + if matched_arches != "": + warnings.warn( + matched_cuda_warn.format(matched_arches), stacklevel=2 + ) + + +def _check_cubins(): + incompatible_device_warn = """ +{} with CUDA capability sm_{} is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities {}. +If you want to use the {} GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ +""" + if torch.version.cuda is None: # on ROCm we don't want this check + return + arch_list = get_arch_list() + if len(arch_list) == 0: + return + supported_sm = [_extract_arch_version(arch) for arch in arch_list if "sm_" in arch] + for idx in range(device_count()): + cap_major, cap_minor = get_device_capability(idx) + # NVIDIA GPU compute architectures are backward compatible within major version + supported = any(sm // 10 == cap_major for sm in supported_sm) + if not supported: + device_name = get_device_name(idx) + capability = cap_major * 10 + cap_minor + warnings.warn( + incompatible_device_warn.format( + device_name, capability, " ".join(arch_list), device_name + ), + stacklevel=2, + ) + + +def is_initialized(): + r"""Return whether PyTorch's CUDA state has been initialized.""" + return _initialized and not _is_in_bad_fork() + + +def _lazy_call(callable, **kwargs): + with _initialization_lock: + if is_initialized(): + callable() + else: + # TODO(torch_deploy): this accesses linecache, which attempts to read the + # file system to get traceback info. Patch linecache or do something + # else here if this ends up being important. + global _lazy_seed_tracker + if kwargs.get("seed_all", False): + _lazy_seed_tracker.queue_seed_all(callable, traceback.format_stack()) + elif kwargs.get("seed", False): + _lazy_seed_tracker.queue_seed(callable, traceback.format_stack()) + else: + # Don't store the actual traceback to avoid memory cycle + _queued_calls.append((callable, traceback.format_stack())) + + +_lazy_call(_check_capability) +_lazy_call(_check_cubins) + + +class DeferredCudaCallError(Exception): + pass + + +AcceleratorError = torch._C.AcceleratorError +OutOfMemoryError = torch._C.OutOfMemoryError + + +def init(): + r"""Initialize PyTorch's CUDA state. + + You may need to call this explicitly if you are interacting with + PyTorch via its C API, as Python bindings for CUDA functionality + will not be available until this initialization takes place. + Ordinary users should not need this, as all of PyTorch's CUDA methods + automatically initialize CUDA state on-demand. + + Does nothing if the CUDA state is already initialized. + """ + _lazy_init() + + +def _lazy_init(): + global _initialized, _queued_calls + if is_initialized() or hasattr(_tls, "is_initializing"): + return + with _initialization_lock: + # We be double-checked locking, boys! This is OK because + # the above test was GIL protected anyway. The inner test + # is for when a thread blocked on some other thread which was + # doing the initialization; when they get the lock, they will + # find there is nothing left to do. + if is_initialized(): + return + # It is important to prevent other threads from entering _lazy_init + # immediately, while we are still guaranteed to have the GIL, because some + # of the C calls we make below will release the GIL + if _is_in_bad_fork(): + raise RuntimeError( + "Cannot re-initialize CUDA in forked subprocess. To use CUDA with " + "multiprocessing, you must use the 'spawn' start method" + ) + if not hasattr(torch._C, "_cuda_getDeviceCount"): + raise AssertionError("Torch not compiled with CUDA enabled") + if _cudart is None: + raise AssertionError( + "libcudart functions unavailable. It looks like you have a broken build?" + ) + # This function throws if there's a driver initialization error, no GPUs + # are found or any other error occurs + torch._C._cuda_init() + # Some of the queued calls may reentrantly call _lazy_init(); + # we need to just return without initializing in that case. + # However, we must not let any *other* threads in! + _tls.is_initializing = True + + _queued_calls.extend(calls for calls in _lazy_seed_tracker.get_calls() if calls) + + try: + for queued_call, orig_traceback in _queued_calls: + try: + queued_call() + except Exception as e: + msg = ( + f"CUDA call failed lazily at initialization with error: {str(e)}\n\n" + f"CUDA call was originally invoked at:\n\n{''.join(orig_traceback)}" + ) + raise DeferredCudaCallError(msg) from e + finally: + delattr(_tls, "is_initializing") + _initialized = True + + +def cudart(): + r"""Retrieves the CUDA runtime API module. + + + This function initializes the CUDA runtime environment if it is not already + initialized and returns the CUDA runtime API module (_cudart). The CUDA + runtime API module provides access to various CUDA runtime functions. + + Args: + ``None`` + + Returns: + module: The CUDA runtime API module (_cudart). + + Raises: + RuntimeError: If CUDA cannot be re-initialized in a forked subprocess. + AssertionError: If PyTorch is not compiled with CUDA support or if libcudart functions are unavailable. + + Example of CUDA operations with profiling: + >>> import torch + >>> from torch.cuda import cudart, check_error + >>> import os + >>> + >>> os.environ["CUDA_PROFILE"] = "1" + >>> + >>> def perform_cuda_operations_with_streams(): + >>> stream = torch.cuda.Stream() + >>> with torch.cuda.stream(stream): + >>> x = torch.randn(100, 100, device='cuda') + >>> y = torch.randn(100, 100, device='cuda') + >>> z = torch.mul(x, y) + >>> return z + >>> + >>> torch.cuda.synchronize() + >>> print("====== Start nsys profiling ======") + >>> check_error(cudart().cudaProfilerStart()) + >>> with torch.autograd.profiler.emit_nvtx(): + >>> result = perform_cuda_operations_with_streams() + >>> print("CUDA operations completed.") + >>> check_error(torch.cuda.cudart().cudaProfilerStop()) + >>> print("====== End nsys profiling ======") + + To run this example and save the profiling information, execute: + >>> $ nvprof --profile-from-start off --csv --print-summary -o trace_name.prof -f -- python cudart_test.py + + This command profiles the CUDA operations in the provided script and saves + the profiling information to a file named `trace_name.prof`. + The `--profile-from-start off` option ensures that profiling starts only + after the `cudaProfilerStart` call in the script. + The `--csv` and `--print-summary` options format the profiling output as a + CSV file and print a summary, respectively. + The `-o` option specifies the output file name, and the `-f` option forces the + overwrite of the output file if it already exists. + """ + _lazy_init() + return _cudart + + +class cudaStatus: + SUCCESS: int = 0 + ERROR_NOT_READY: int = 34 + + +class CudaError(RuntimeError): + def __init__(self, code: int) -> None: + # pyrefly: ignore [missing-attribute] + msg = _cudart.cudaGetErrorString(_cudart.cudaError(code)) + super().__init__(f"{msg} ({code})") + + +def check_error(res: int) -> None: + r"""Raise an error if the result of a CUDA runtime API call is not success.""" + # pyrefly: ignore [missing-attribute] + if res != _cudart.cudaError.success: + raise CudaError(res) + + +class _DeviceGuard: + def __init__(self, index: int): + self.idx = index + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch.cuda._exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) + return False + + +class device: + r"""Context-manager that changes the selected device. + + Args: + device (torch.device or int): device index to select. It's a no-op if + this argument is a negative integer or ``None``. + """ + + def __init__(self, device: Any): + self.idx = _get_device_index(device, optional=True) + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch.cuda._exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) + return False + + +class device_of(device): + r"""Context-manager that changes the current device to that of given object. + + You can use both tensors and storages as arguments. If a given object is + not allocated on a GPU, this is a no-op. + + Args: + obj (Tensor or Storage): object allocated on the selected device. + """ + + def __init__(self, obj): + idx = obj.get_device() if obj.is_cuda else -1 + super().__init__(idx) + + +def set_device(device: "Device") -> None: + r"""Set the current device. + + Usage of this function is discouraged in favor of :any:`device`. In most + cases it's better to use ``CUDA_VISIBLE_DEVICES`` environmental variable. + + Args: + device (torch.device or int): selected device. This function is a no-op + if this argument is negative. + """ + device = _get_device_index(device) + if device >= 0: + torch._C._cuda_setDevice(device) + + +def get_device_name(device: "Device" = None) -> str: + r"""Get the name of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + name. This function is a no-op if this argument is a negative + integer. It uses the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Returns: + str: the name of the device + """ + return get_device_properties(device).name + + +def get_device_capability(device: "Device" = None) -> tuple[int, int]: + r"""Get the cuda capability of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + device capability. This function is a no-op if this argument is + a negative integer. It uses the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + + Returns: + tuple(int, int): the major and minor cuda capability of the device + """ + prop = get_device_properties(device) + return prop.major, prop.minor + + +# pyrefly: ignore [not-a-type] +def get_device_properties(device: "Device" = None) -> _CudaDeviceProperties: + r"""Get the properties of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + properties of the device. It uses the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + + Returns: + _CudaDeviceProperties: the properties of the device + """ + _lazy_init() # will define _get_device_properties + device = _get_device_index(device, optional=True) + if device < 0 or device >= device_count(): + raise AssertionError("Invalid device id") + return _get_device_properties(device) # type: ignore[name-defined] + + +def can_device_access_peer(device: "Device", peer_device: "Device") -> bool: + r"""Check if peer access between two devices is possible.""" + _lazy_init() + device = _get_device_index(device, optional=True) + peer_device = _get_device_index(peer_device) + if device < 0 or device >= device_count(): + raise AssertionError("Invalid device id") + if peer_device < 0 or peer_device >= device_count(): + raise AssertionError("Invalid peer device id") + return torch._C._cuda_canDeviceAccessPeer(device, peer_device) + + +class StreamContext: + r"""Context-manager that selects a given stream. + + All CUDA kernels queued within its context will be enqueued on a selected + stream. + + Args: + Stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: Streams are per-device. + """ + + cur_stream: Optional["torch.cuda.Stream"] + + def __init__(self, stream: Optional["torch.cuda.Stream"]): + self.stream = stream + self.idx = _get_device_index(None, True) + if not torch.jit.is_scripting(): + if self.idx is None: + # pyrefly: ignore [bad-assignment] + self.idx = -1 + + self.src_prev_stream = ( + None if not torch.jit.is_scripting() else torch.cuda.default_stream(None) + ) + self.dst_prev_stream = ( + None if not torch.jit.is_scripting() else torch.cuda.default_stream(None) + ) + + def __enter__(self): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # Return if stream is None or CUDA device not available + if cur_stream is None or self.idx == -1: + return + self.src_prev_stream = torch.cuda.current_stream(None) + + # If the stream is not on the current device, then + # set the current stream on the device + if self.src_prev_stream.device != cur_stream.device: + with device(cur_stream.device): + self.dst_prev_stream = torch.cuda.current_stream(cur_stream.device) + torch.cuda.set_stream(cur_stream) + + def __exit__(self, type: Any, value: Any, traceback: Any): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # If stream is None or no CUDA device available, return + if cur_stream is None or self.idx == -1: + return + + # Reset the stream on the original device + # and destination device + if self.src_prev_stream.device != cur_stream.device: # type: ignore[union-attr] + torch.cuda.set_stream(self.dst_prev_stream) # type: ignore[arg-type] + torch.cuda.set_stream(self.src_prev_stream) # type: ignore[arg-type] + + +def stream(stream: Optional["torch.cuda.Stream"]) -> StreamContext: + r"""Wrap around the Context-manager StreamContext that selects a given stream. + + Arguments: + stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: + In eager mode stream is of type Stream class while in JIT it is + an object of the custom class ``torch.classes.cuda.Stream``. + """ + return StreamContext(stream) + + +def _set_stream_by_id(stream_id, device_index, device_type): + r"""set stream specified by the stream id, device index and + device type + + Args: stream_id (int): stream id in stream pool + device_index (int): device index in topo + device_type (int): enum device type + """ + torch._C._cuda_setStream( + stream_id=stream_id, + device_index=device_index, + device_type=device_type, + ) + + +def set_stream(stream: Stream): + r"""Set the current stream.This is a wrapper API to set the stream. + Usage of this function is discouraged in favor of the ``stream`` + context manager. + + Args: + stream (Stream): selected stream. This function is a no-op + if this argument is ``None``. + """ + if stream is None: + return + _set_stream_by_id( + stream_id=stream.stream_id, + device_index=stream.device_index, + device_type=stream.device_type, + ) + + +def _parse_visible_devices() -> Union[list[int], list[str]]: + r"""Parse CUDA_VISIBLE_DEVICES environment variable.""" + var = os.getenv("CUDA_VISIBLE_DEVICES") + + if torch.version.hip: + hip_devices = os.getenv("HIP_VISIBLE_DEVICES") + rocr_devices = os.getenv("ROCR_VISIBLE_DEVICES") + + # You must take care if both HIP and ROCR env vars are set as they have + # different meanings. Both env vars accept either a list of ints or a + # list of UUIDs. The ROCR env var is processed first which then reduces + # the number of GPUs that HIP can select from. + if rocr_devices is not None: + rocr_count = len(rocr_devices.split(",")) + if hip_devices is not None: + # sanity check if both env vars are set + if len(hip_devices.split(",")) > rocr_count: + raise RuntimeError( + "HIP_VISIBLE_DEVICES contains more devices than ROCR_VISIBLE_DEVICES" + ) + # HIP_VISIBLE_DEVICES is preferred over ROCR_VISIBLE_DEVICES + var = hip_devices + else: + return list(range(rocr_count)) + elif hip_devices is not None: + var = hip_devices + + if var is None: + return list(range(64)) + + def _strtoul(s: str) -> int: + """Return -1 or positive integer sequence string starts with.""" + if not s: + return -1 + for idx, c in enumerate(s): + if not (c.isdigit() or (idx == 0 and c in "+-")): + break + if idx + 1 == len(s): + idx += 1 + return int(s[:idx]) if idx > 0 else -1 + + def parse_list_with_prefix(lst: str, prefix: str) -> list[str]: + rcs: list[str] = [] + for elem in lst.split(","): + # Repeated id results in empty set + if elem in rcs: + return cast(list[str], []) + # Anything other but prefix is ignored + if not elem.startswith(prefix): + break + rcs.append(elem) + return rcs + + if var.startswith("GPU-"): + return parse_list_with_prefix(var, "GPU-") + if var.startswith("MIG-"): + return parse_list_with_prefix(var, "MIG-") + # CUDA_VISIBLE_DEVICES uses something like strtoul + # which makes `1gpu2,2ampere` is equivalent to `1,2` + rc: list[int] = [] + for elem in var.split(","): + x = _strtoul(elem.strip()) + # Repeated ordinal results in empty set + if x in rc: + return cast(list[int], []) + # Negative value aborts the sequence + if x < 0: + break + rc.append(x) + return rc + + +def _raw_device_count_amdsmi() -> int: + if not _HAS_PYNVML: # If amdsmi is not available + return -1 + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException as e: + warnings.warn( + f"Can't initialize amdsmi - Error code: {e.err_code}", stacklevel=2 + ) + return -1 + socket_handles = amdsmi.amdsmi_get_processor_handles() + return len(socket_handles) + + +def _raw_device_count_nvml() -> int: + r"""Return number of devices as reported by NVML or negative value if NVML discovery/initialization failed.""" + from ctypes import byref, c_int, CDLL + + nvml_h = CDLL("libnvidia-ml.so.1") + rc = nvml_h.nvmlInit() + if rc != 0: + warnings.warn("Can't initialize NVML", stacklevel=2) + return -1 + dev_count = c_int(-1) + rc = nvml_h.nvmlDeviceGetCount_v2(byref(dev_count)) + if rc != 0: + warnings.warn("Can't get nvml device count", stacklevel=2) + return -1 + del nvml_h + return dev_count.value + + +def _raw_device_uuid_amdsmi() -> Optional[list[str]]: + from ctypes import byref, c_int, c_void_p, CDLL, create_string_buffer + + if not _HAS_PYNVML: # If amdsmi is not available + return None + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException: + warnings.warn("Can't initialize amdsmi", stacklevel=2) + return None + try: + socket_handles = amdsmi.amdsmi_get_processor_handles() + dev_count = len(socket_handles) + except amdsmi.AmdSmiException: + warnings.warn("Can't get amdsmi device count", stacklevel=2) + return None + uuids: list[str] = [] + for idx in range(dev_count): + try: + handler = amdsmi.amdsmi_get_processor_handles()[idx] + except amdsmi.AmdSmiException: + warnings.warn("Cannot get amd device handler", stacklevel=2) + return None + try: + uuid = amdsmi.amdsmi_get_gpu_asic_info(handler)["asic_serial"][ + 2: + ] # Removes 0x prefix from serial + except amdsmi.AmdSmiException: + warnings.warn("Cannot get uuid for amd device", stacklevel=2) + return None + uuids.append( + str(uuid).lower() + ) # Lower-case to match expected HIP_VISIBLE_DEVICES uuid input + return uuids + + +def _raw_device_uuid_nvml() -> Optional[list[str]]: + r"""Return list of device UUID as reported by NVML or None if NVM discovery/initialization failed.""" + from ctypes import byref, c_int, c_void_p, CDLL, create_string_buffer + + nvml_h = CDLL("libnvidia-ml.so.1") + rc = nvml_h.nvmlInit() + if rc != 0: + warnings.warn("Can't initialize NVML", stacklevel=2) + return None + dev_count = c_int(-1) + rc = nvml_h.nvmlDeviceGetCount_v2(byref(dev_count)) + if rc != 0: + warnings.warn("Can't get nvml device count", stacklevel=2) + return None + uuids: list[str] = [] + for idx in range(dev_count.value): + dev_id = c_void_p() + rc = nvml_h.nvmlDeviceGetHandleByIndex_v2(idx, byref(dev_id)) + if rc != 0: + warnings.warn("Can't get device handle", stacklevel=2) + return None + buf_len = 96 + buf = create_string_buffer(buf_len) + rc = nvml_h.nvmlDeviceGetUUID(dev_id, buf, buf_len) + if rc != 0: + warnings.warn("Can't get device UUID", stacklevel=2) + return None + uuids.append(buf.raw.decode("ascii").strip("\0")) + del nvml_h + return uuids + + +def _transform_uuid_to_ordinals(candidates: list[str], uuids: list[str]) -> list[int]: + r"""Given the set of partial uuids and list of known uuids builds a set of ordinals excluding ambiguous partials IDs.""" + + def uuid_to_ordinal(candidate: str, uuids: list[str]) -> int: + best_match = -1 + for idx, uuid in enumerate(uuids): + if not uuid.startswith(candidate): + continue + # Ambiguous candidate + if best_match != -1: + return -1 + best_match = idx + return best_match + + rc: list[int] = [] + for candidate in candidates: + if torch.version.hip: + candidate = candidate.replace( + "GPU-", "", 1 + ) # Remove GPU-prefix to match amdsmi asic serial + idx = uuid_to_ordinal(candidate, uuids) + # First invalid ordinal stops parsing + if idx < 0: + break + # Duplicates result in empty set + if idx in rc: + return cast(list[int], []) + rc.append(idx) + return rc + + +def _device_count_amdsmi() -> int: + visible_devices = _parse_visible_devices() + if not visible_devices: + return 0 + try: + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_amdsmi() + if uuids is None: + return -1 + # Create string version of visible devices to avoid mypy warnings + visible_device_str = cast(list[str], visible_devices) + visible_devices = _transform_uuid_to_ordinals(visible_device_str, uuids) + else: + raw_cnt = _raw_device_count_amdsmi() + if raw_cnt <= 0: + return raw_cnt + # Trim the list up to a maximum available device + # pyrefly: ignore [bad-argument-type] + for idx, val in enumerate(visible_devices): + # pyrefly: ignore [redundant-cast] + if cast(int, val) >= raw_cnt: + return idx + except OSError: + return -1 + except AttributeError: + return -1 + return len(visible_devices) + + +def _device_count_nvml() -> int: + r"""Return number of devices as reported by NVML taking CUDA_VISIBLE_DEVICES into account. + + Negative value is returned if NVML discovery or initialization has failed. + """ + visible_devices = _parse_visible_devices() + if not visible_devices: + return 0 + try: + if type(visible_devices[0]) is str: + # Skip MIG parsing + if visible_devices[0].startswith("MIG-"): + return -1 + uuids = _raw_device_uuid_nvml() + if uuids is None: + return -1 + visible_devices = _transform_uuid_to_ordinals( + cast(list[str], visible_devices), uuids + ) + else: + raw_cnt = _raw_device_count_nvml() + if raw_cnt <= 0: + return raw_cnt + # Trim the list up to a maximum available device + # pyrefly: ignore [bad-argument-type] + for idx, val in enumerate(visible_devices): + # pyrefly: ignore [redundant-cast] + if cast(int, val) >= raw_cnt: + return idx + except OSError: + return -1 + except AttributeError: + return -1 + return len(visible_devices) + + +def _get_nvml_device_index(device: "Device") -> int: + r"""Return the NVML index of the device, taking CUDA_VISIBLE_DEVICES into account.""" + idx = _get_device_index(device, optional=True) + visible_devices = _parse_visible_devices() + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_nvml() + if uuids is None: + raise RuntimeError("Can't get device UUIDs") + visible_devices = _transform_uuid_to_ordinals( + cast(list[str], visible_devices), uuids + ) + visible_devices = cast(list[int], visible_devices) + if idx < 0 or idx >= len(visible_devices): + raise RuntimeError( + f"device {idx} is not visible (CUDA_VISIBLE_DEVICES={visible_devices})" + ) + return visible_devices[idx] + + +_cached_device_count: Optional[int] = None + + +def device_count() -> int: + r""" + Return the number of GPUs available. + + .. note:: This API will NOT poison fork if NVML discovery succeeds. + See :ref:`multiprocessing-poison-fork-note` for more details. + """ + global _cached_device_count + if not _is_compiled(): + return 0 + if _cached_device_count is not None: + return _cached_device_count + # bypass _device_count_nvml() if rocm (not supported) + nvml_count = _device_count_amdsmi() if torch.version.hip else _device_count_nvml() + r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count + # NB: Do not cache the device count prior to CUDA initialization, because + # the number of devices can change due to changes to CUDA_VISIBLE_DEVICES + # setting prior to CUDA initialization. + if _initialized: + _cached_device_count = r + return r + + +def get_arch_list() -> list[str]: + r"""Return list CUDA architectures this library was compiled for.""" + if not is_available(): + return [] + arch_flags = torch._C._cuda_getArchFlags() + if arch_flags is None: + return [] + return arch_flags.split() + + +def get_gencode_flags() -> str: + r"""Return NVCC gencode flags this library was compiled with.""" + arch_list = get_arch_list() + if len(arch_list) == 0: + return "" + arch_list_ = [arch.split("_") for arch in arch_list] + return " ".join( + [ + f"-gencode compute=compute_{arch},code={kind}_{arch}" + for (kind, arch) in arch_list_ + ] + ) + + +def current_device() -> int: + r"""Return the index of a currently selected device.""" + _lazy_init() + return torch._C._cuda_getDevice() + + +def synchronize(device: "Device" = None) -> None: + r"""Wait for all kernels in all streams on a CUDA device to complete. + + Args: + device (torch.device or int, optional): device for which to synchronize. + It uses the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + """ + _lazy_init() + with torch.cuda.device(device): + return torch._C._cuda_synchronize() + + +def ipc_collect(): + r"""Force collects GPU memory after it has been released by CUDA IPC. + + .. note:: + Checks if any sent CUDA tensors could be cleaned from the memory. Force + closes shared memory file used for reference counting if there is no + active counters. Useful when the producer process stopped actively sending + tensors and want to release unused memory. + """ + _lazy_init() + return torch._C._cuda_ipc_collect() + + +def current_stream(device: "Device" = None) -> Stream: + r"""Return the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the currently selected :class:`Stream` for the current device, given + by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + """ + _lazy_init() + streamdata = torch._C._cuda_getCurrentStream( + _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def default_stream(device: "Device" = None) -> Stream: + r"""Return the default :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the default :class:`Stream` for the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + """ + _lazy_init() + streamdata = torch._C._cuda_getDefaultStream( + _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def get_stream_from_external(data_ptr: int, device: "Device" = None) -> Stream: + r"""Return a :class:`Stream` from an externally allocated CUDA stream. + + This function is used to wrap streams allocated in other libraries in order + to facilitate data exchange and multi-library interactions. + + .. note:: This function doesn't manage the stream life-cycle, it is the user + responsibility to keep the referenced stream alive while this returned + stream is being used. + + Args: + data_ptr(int): Integer representation of the `cudaStream_t` value that + is allocated externally. + device(torch.device or int, optional): the device where the stream + was originally allocated. If device is specified incorrectly, + subsequent launches using this stream may fail. + """ + _lazy_init() + streamdata = torch._C._cuda_getStreamFromExternal( + data_ptr, _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def current_blas_handle(): + r"""Return cublasHandle_t pointer to current cuBLAS handle""" + _lazy_init() + return torch._C._cuda_getCurrentBlasHandle() + + +def set_sync_debug_mode(debug_mode: Union[int, str]) -> None: + r"""Set the debug mode for cuda synchronizing operations. + + Args: + debug_mode(str or int): if "default" or 0, don't error or warn on synchronizing operations, + if "warn" or 1, warn on synchronizing operations, if "error" or 2, error out synchronizing operations. + + Warning: + This is an experimental feature, and not all synchronizing operations will trigger warning or error. In + particular, operations in torch.distributed and torch.sparse namespaces are not covered yet. + """ + _lazy_init() + if isinstance(debug_mode, str): + if debug_mode == "default": + debug_mode = 0 + elif debug_mode == "warn": + debug_mode = 1 + elif debug_mode == "error": + debug_mode = 2 + else: + raise RuntimeError( + "invalid value of debug_mode, expected one of `default`, `warn`, `error`" + ) + + torch._C._cuda_set_sync_debug_mode(debug_mode) + + +def get_sync_debug_mode() -> int: + r"""Return current value of debug mode for cuda synchronizing operations.""" + _lazy_init() + return torch._C._cuda_get_sync_debug_mode() + + +def _get_pynvml_handler(device: "Device" = None): + if not _HAS_PYNVML: + raise ModuleNotFoundError( + "nvidia-ml-py does not seem to be installed or it can't be imported." + # pyrefly: ignore [invalid-inheritance] + ) from _PYNVML_ERR + # pyrefly: ignore [import-error,missing-module-attribute] + from pynvml import NVMLError_DriverNotLoaded + + try: + pynvml.nvmlInit() + except NVMLError_DriverNotLoaded as e: + raise RuntimeError("cuda driver can't be loaded, is cuda enabled?") from e + + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return handle + + +def _get_amdsmi_handler(device: "Device" = None): + if not _HAS_PYNVML: + raise ModuleNotFoundError( + "amdsmi does not seem to be installed or it can't be imported." + # pyrefly: ignore [invalid-inheritance] + ) from _PYNVML_ERR + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException as e: + raise RuntimeError( + "amdsmi driver can't be loaded, requires >=ROCm5.6 installation" + ) from e + device = _get_amdsmi_device_index(device) + handle = amdsmi.amdsmi_get_processor_handles()[device] + return handle + + +def _get_amdsmi_device_index(device: "Device") -> int: + r"""Return the amdsmi index of the device, taking visible_devices into account.""" + idx = _get_device_index(device, optional=True) + visible_devices = _parse_visible_devices() + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_amdsmi() + if uuids is None: + raise RuntimeError("Can't get device UUIDs") + visible_devices_str = cast( + list[str], visible_devices + ) # Create str variable for mypy + visible_devices = _transform_uuid_to_ordinals(visible_devices_str, uuids) + idx_map = dict(enumerate(cast(list[int], visible_devices))) + if idx not in idx_map: + raise RuntimeError( + f"device {idx} is not visible (HIP_VISIBLE_DEVICES={visible_devices})" + ) + return idx_map[idx] + + +def _get_amdsmi_device_memory_used(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + # amdsmi_get_gpu_vram_usage returns mem usage in megabytes + mem_mega_bytes = amdsmi.amdsmi_get_gpu_vram_usage(handle)["vram_used"] + mem_bytes = mem_mega_bytes * 1024 * 1024 + return mem_bytes + + +def _get_amdsmi_memory_usage(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_gpu_activity(handle)["umc_activity"] + + +def _get_amdsmi_utilization(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_gpu_activity(handle)["gfx_activity"] + + +def _get_amdsmi_temperature(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_temp_metric( + handle, + amdsmi.AmdSmiTemperatureType.JUNCTION, + amdsmi.AmdSmiTemperatureMetric.CURRENT, + ) + + +def _get_amdsmi_power_draw(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + socket_power = amdsmi.amdsmi_get_power_info(handle)["average_socket_power"] + if socket_power != "N/A": + return socket_power + else: + socket_power = amdsmi.amdsmi_get_power_info(handle)["current_socket_power"] + if socket_power != "N/A": + return socket_power + else: + return 0 + + +def _get_amdsmi_clock_rate(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + clock_info = amdsmi.amdsmi_get_clock_info(handle, amdsmi.AmdSmiClkType.GFX) + if "cur_clk" in clock_info: # ROCm 6.2 deprecation + clock_rate = clock_info["cur_clk"] + else: + clock_rate = clock_info["clk"] + if clock_rate != "N/A": + return clock_rate + else: + return 0 + + +def device_memory_used(device: "Device" = None) -> int: + r"""Return used global (device) memory in bytes as given by `nvidia-smi` or `amd-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + """ + if not torch.version.hip: + handle = _get_pynvml_handler() + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetMemoryInfo(handle).used + else: + return _get_amdsmi_device_memory_used(device) + + +def memory_usage(device: "Device" = None) -> int: + r"""Return the percent of time over the past sample period during which global (device) + memory was being read or written as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler() + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetUtilizationRates(handle).memory + else: + return _get_amdsmi_memory_usage(device) + + +def utilization(device: "Device" = None) -> int: + r"""Return the percent of time over the past sample period during which one or + more kernels was executing on the GPU as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetUtilizationRates(handle).gpu + else: + return _get_amdsmi_utilization(device) + + +def temperature(device: "Device" = None) -> int: + r"""Return the average temperature of the GPU sensor in Degrees C (Centigrades). + + The average temperature is computed based on past sample period as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + # 0 refers to the temperature sensor for the GPU die. + return pynvml.nvmlDeviceGetTemperature(handle, 0) + else: + return _get_amdsmi_temperature(device) + + +def power_draw(device: "Device" = None) -> int: + r"""Return the average power draw of the GPU sensor in mW (MilliWatts) + over the past sample period as given by `nvidia-smi` for Fermi or newer fully supported devices. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + return pynvml.nvmlDeviceGetPowerUsage(handle) + else: + return _get_amdsmi_power_draw(device) + + +def clock_rate(device: "Device" = None) -> int: + r"""Return the clock speed of the GPU SM in MHz (megahertz) over the past sample period as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + return pynvml.nvmlDeviceGetClockInfo(handle, 1) + else: + return _get_amdsmi_clock_rate(device) + + +def _get_device(device: Union[int, str, torch.device]) -> torch.device: + r"""Return the torch.device type object from the passed in device. + + Args: + device (torch.device or int): selected device. + """ + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + return device + + +def _get_generator(device: torch.device) -> torch._C.Generator: + r"""Return the CUDA Generator object for the given device. + + Args: + device (torch.device): selected device. + """ + idx = device.index + if idx is None: + idx = current_device() + return torch.cuda.default_generators[idx] + + +def _set_rng_state_offset( + offset: int, device: Union[int, str, torch.device] = "cuda" +) -> None: + r"""Set the random number generator state offset of the specified GPU. + + Args: + offset (int): The desired offset + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + """ + final_device = _get_device(device) + + def cb(): + default_generator = _get_generator(final_device) + default_generator.set_offset(offset) + + _lazy_call(cb) + + +def _get_rng_state_offset(device: Union[int, str, torch.device] = "cuda") -> int: + r"""Return the random number generator state offset of the specified GPU. + + Args: + device (torch.device or int, optional): The device to return the RNG state offset of. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + final_device = _get_device(device) + default_generator = _get_generator(final_device) + return default_generator.get_offset() + + +# pyrefly: ignore [deprecated] +from .memory import * # noqa: F403 +from .random import * # noqa: F403 + + +################################################################################ +# Define Storage and Tensor classes +################################################################################ + + +@staticmethod # type: ignore[misc] +def _lazy_new(cls, *args, **kwargs): + _lazy_init() + # We may need to call lazy init again if we are a forked child + # del _CudaBase.__new__ + return super(_CudaBase, cls).__new__(cls, *args, **kwargs) + + +class _CudaBase: + is_cuda = True + is_sparse = False + + def type(self, *args, **kwargs): + # We could use a Protocol here to tell mypy that self has `get_device` method + # but it is only available in the typing module on Python >= 3.8 + # or on typing_extensions module on Python >= 3.6 + with device(self.get_device()): # type: ignore[attr-defined] + return super().type(*args, **kwargs) # type: ignore[misc] + + __new__ = _lazy_new + + +from torch.storage import _LegacyStorage, _warn_typed_storage_removal + + +class _CudaLegacyStorage(_LegacyStorage): + @classmethod + def from_buffer(cls, *args, **kwargs): + _warn_typed_storage_removal() + raise RuntimeError("from_buffer: Not available for CUDA storage") + + @classmethod + def _new_with_weak_ptr(cls, *args, **kwargs): + raise RuntimeError("_new_with_weak_ptr: Not available for CUDA storage") + + @classmethod + def _new_shared_filename(cls, manager, obj, size, *, device=None, dtype=None): + raise RuntimeError("_new_shared_filename: Not available for CUDA storage") + + +class ByteStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.uint8 + + +class DoubleStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.double + + +class FloatStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.float + + +class HalfStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.half + + +class LongStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.long + + +class IntStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.int + + +class ShortStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.short + + +class CharStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.int8 + + +class BoolStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.bool + + +class BFloat16Storage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.bfloat16 + + +class ComplexDoubleStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.cdouble + + +class ComplexFloatStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.cfloat + + +del _LegacyStorage +del _CudaLegacyStorage + +torch._storage_classes.add(DoubleStorage) +torch._storage_classes.add(FloatStorage) +torch._storage_classes.add(LongStorage) +torch._storage_classes.add(IntStorage) +torch._storage_classes.add(ShortStorage) +torch._storage_classes.add(CharStorage) +torch._storage_classes.add(ByteStorage) +torch._storage_classes.add(HalfStorage) +torch._storage_classes.add(BoolStorage) +torch._storage_classes.add(BFloat16Storage) +torch._storage_classes.add(ComplexDoubleStorage) +torch._storage_classes.add(ComplexFloatStorage) + + +class _WrappedTritonKernel: + """Just a simple wrapper to store some metadata for testing purposes.""" + + def __init__(self, kernel): + self.kernel = kernel + self.kernel_invoked = False + + def __call__(self, *args, **kwargs): + res = self.kernel(*args, **kwargs) + self.kernel_invoked = True + return res + + +def _register_triton_kernels(): + @_WrappedTritonKernel + def kernel_impl(*args, **kwargs): + from torch.sparse._triton_ops import bsr_dense_mm + + # pyrefly: ignore [not-callable] + return bsr_dense_mm(*args, skip_checks=True, **kwargs) + + @_WrappedTritonKernel + def addmm_kernel_impl(*args, **kwargs): + from torch.sparse._triton_ops import bsr_dense_addmm + + return bsr_dense_addmm(*args, skip_checks=True, **kwargs) + + has_triton = importlib.util.find_spec("triton") is not None + if has_triton: + torch._TritonLibrary.registerOp( + "_triton_bsr_dense_mm_out", + "_triton_bsr_dense_mm_out(Tensor bsr, Tensor dense, *, Tensor(a!) out) -> Tensor(a!)", + kernel_impl, + "SparseCsrCUDA", + ) + + torch._TritonLibrary.registerOp( + "_triton_bsr_dense_addmm_out", + ( + "_triton_bsr_dense_addmm_out(Tensor input, Tensor bsr, Tensor dense," + " *, Scalar beta, Scalar alpha, Tensor(a!) out) -> Tensor(a!)" + ), + addmm_kernel_impl, + "SparseCsrCUDA", + ) + + +_lazy_call(_register_triton_kernels) + + +def _compile_kernel( + kernel_source: str, + kernel_name: str, + compute_capability: Optional[str] = None, + cuda_include_dirs: Optional[list] = None, + nvcc_options: Optional[list] = None, +): + """ + Compiles a CUDA kernel using NVRTC and returns a callable function. + + This function is a wrapper for NVRTC that enables runtime compilation of CUDA kernels. + Note that this returns a raw CUDA kernel that operates on raw memory pointers. + To use this kernel as a proper PyTorch operator, you should wrap it following the guide at: + pytorch.org/tutorials/advanced/python_custom_ops.html + + Args: + kernel_source (str): The CUDA kernel source code as a string + kernel_name (str): The name of the kernel function to compile + compute_capability (str, optional): The compute capability to target (e.g., "86"). + If None, will detect from current device. + cuda_include_dirs (list, optional): List of directories containing CUDA headers + nvcc_options (list, optional): Additional options to pass to NVRTC + + Returns: + callable: A Python function that can be called with PyTorch tensor arguments to execute the kernel + + Example: + >>> # xdoctest: +SKIP + >>> kernel_code = ''' + extern "C" + __global__ void add_tensors(const float* a, const float* b, float* c, int n) { + int i = threadIdx.x + blockIdx.x * blockDim.x; + if (i < n) + c[i] = a[i] + b[i]; + } + ''' + >>> add_kernel = torch.cuda.compile_kernel(kernel_code, "add_tensors") + >>> a = torch.randn(1024, device="cuda") + >>> b = torch.randn(1024, device="cuda") + >>> c = torch.empty_like(a) + >>> add_kernel(grid=(4, 1, 1), block=(256, 1, 1), args=[a, b, c, a.numel()]) + """ + from torch.cuda._utils import _cuda_load_module, _nvrtc_compile + + # Compile the kernel to PTX + ptx, mangled_name = _nvrtc_compile( + kernel_source, + kernel_name, + compute_capability, + cuda_include_dirs, + nvcc_options, + ) + + # Load the module and get the kernel + result = _cuda_load_module(ptx, [mangled_name]) + + if isinstance(result, dict): + return result[mangled_name] + else: + # This branch shouldn't be executed if kernel_names is provided, + # but MyPy needs this to understand type narrowing + return getattr(result, mangled_name) + + +from . import amp, jiterator, nvtx, profiler, sparse, tunable + + +_POOL_HANDLE = NewType("_POOL_HANDLE", tuple[int, int]) + + +__all__ = [ + # Typed storage and tensors + "BFloat16Storage", + "BFloat16Tensor", + "BoolStorage", + "BoolTensor", + "ByteStorage", + "ByteTensor", + "CharStorage", + "CharTensor", + "ComplexDoubleStorage", + "ComplexFloatStorage", + "DoubleStorage", + "DoubleTensor", + "FloatStorage", + "FloatTensor", + "HalfStorage", + "HalfTensor", + "IntStorage", + "IntTensor", + "LongStorage", + "LongTensor", + "ShortStorage", + "ShortTensor", + "CUDAGraph", + "CudaError", + "DeferredCudaCallError", + "Event", + "ExternalStream", + "Stream", + "StreamContext", + "GreenContext", + "amp", + "caching_allocator_alloc", + "caching_allocator_delete", + "caching_allocator_enable", + "can_device_access_peer", + "check_error", + "cudaStatus", + "cudart", + "current_blas_handle", + "current_device", + "current_stream", + "default_generators", + "default_stream", + "device", + "device_count", + "device_memory_used", + "device_of", + "empty_cache", + "get_allocator_backend", + "CUDAPluggableAllocator", + "change_current_allocator", + "get_arch_list", + "get_device_capability", + "get_device_name", + "get_device_properties", + "get_gencode_flags", + "get_per_process_memory_fraction", + "get_rng_state", + "get_rng_state_all", + "get_stream_from_external", + "get_sync_debug_mode", + "graph", + "graph_pool_handle", + "graphs", + "has_half", + "has_magma", + "host_memory_stats", + "host_memory_stats_as_nested_dict", + "init", + "initial_seed", + "ipc_collect", + "is_available", + "is_bf16_supported", + "is_current_stream_capturing", + "is_initialized", + "is_tf32_supported", + "jiterator", + "list_gpu_processes", + "make_graphed_callables", + "manual_seed", + "manual_seed_all", + "max_memory_allocated", + "max_memory_cached", + "max_memory_reserved", + "mem_get_info", + "memory", + "memory_allocated", + "memory_cached", + "memory_reserved", + "memory_snapshot", + "memory_stats", + "memory_stats_as_nested_dict", + "memory_summary", + "memory_usage", + "MemPool", + "use_mem_pool", + "temperature", + "power_draw", + "clock_rate", + "nccl", + "nvtx", + "profiler", + "random", + "reset_accumulated_host_memory_stats", + "reset_accumulated_memory_stats", + "reset_max_memory_allocated", + "reset_max_memory_cached", + "reset_peak_host_memory_stats", + "reset_peak_memory_stats", + "seed", + "seed_all", + "set_device", + "set_per_process_memory_fraction", + "set_rng_state", + "set_rng_state_all", + "set_stream", + "set_sync_debug_mode", + "sparse", + "stream", + "streams", + "synchronize", + "tunable", + "utilization", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_device_limits.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_device_limits.py new file mode 100644 index 0000000000000000000000000000000000000000..cd094e668a5bafacede9780e8e689006a72dcbf6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_device_limits.py @@ -0,0 +1,140 @@ +import torch +from torch._C import dtype + + +__all__ = ["GPULimits"] + + +class GPULimits: + r"""Utility class that provides the theoretical limits of Nvidia GPU devices. The + limits don't take into account thermal throttling (assume that the GPU run at its + peak rated frequency). This is because user hardware configuration may influence + power behavior. + """ + + def __init__(self, target_device: torch.device): + # The device properties object is obtained by calling 'cudaGetDeviceProperties' CUDA + # runtime function. We need the total memory bus width and the memory clock rate to + # calculate the memory bandwidth. + self.device_properties = torch.cuda.get_device_properties(target_device) + + # The compute capability is needed to determine the number of FLOPs per cycle per SM + self.compute_capability = int( + f"{self.device_properties.major}{self.device_properties.minor}" + ) + + # FLOPs per cycle information derived from Table 2 in: + # https://resources.nvidia.com/en-us-hopper-architecture/nvidia-h100-tensor-c + + # Returns the number of FMA instructions retired per cycle per SM for a given + # data type, when tensor cores are NOT used + def get_fma_per_cycle_per_sm_cuda_cores(self, data_type: dtype) -> int: + hardcoded_device_values = { + # Ampere Architecture + "fp16_80": 256, + "fp32_80": 64, + "fp64_80": 32, + # Hopper Architecture + "fp16_90": 64, + "fp32_90": 128, + "fp64_90": 64, + # Blackwell Architecture + "fp16_100": 256, + "fp32_100": 128, + "fp64_100": 64, + } + dict_key = "" + if data_type is torch.float16: + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.float32: + dict_key = f"fp32_{self.compute_capability}" + elif data_type is torch.float64: + dict_key = f"fp64_{self.compute_capability}" + else: + dict_key = "unknown" + + if dict_key not in hardcoded_device_values: + raise RuntimeError( + f"No data for sm_{self.compute_capability} and {data_type}." + ) + + return hardcoded_device_values[dict_key] + + # Returns the number of FMA instructions retired per cycle per SM for a given + # data type, when tensor cores ARE used + def get_fma_per_cycle_per_sm_tensor_cores(self, data_type: dtype) -> int: + hardcoded_device_values = { + # Ampere Architecture + "int8_80": 2048, + "fp16_80": 1024, + "fp32_80": 512, + "fp64_80": 64, + # Hopper Architecture + "int8_90": 4096, + "fp8_90": 4096, + "fp16_90": 2048, + "fp32_90": 1024, + "fp64_90": 128, + # Blackwell Architecture + "int8_100": 8192, + "fp8_100": 8192, + "fp16_100": 4096, + "fp32_100": 2048, + } + dict_key = "" + if data_type is torch.float16: + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.bfloat16: + # FP16 and BF16 are equivalent in terms of FLOPs per cycle per SM + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.float32: + dict_key = f"fp32_{self.compute_capability}" + elif data_type is torch.int8: + dict_key = f"int8_{self.compute_capability}" + elif data_type is torch.float64: + dict_key = f"fp64_{self.compute_capability}" + else: + dict_key = "unknown" + + if dict_key not in hardcoded_device_values: + raise RuntimeError( + f"No data for sm_{self.compute_capability} and {data_type}." + ) + + return hardcoded_device_values[dict_key] + + def get_tflops_per_second( + self, data_type: dtype, use_tensor_cores: bool = True + ) -> float: + num_sms = self.device_properties.multi_processor_count + clock_rate = self.device_properties.clock_rate # KHz + + fma_per_cycle = 0 + if use_tensor_cores: + fma_per_cycle = self.get_fma_per_cycle_per_sm_tensor_cores(data_type) + else: + fma_per_cycle = self.get_fma_per_cycle_per_sm_cuda_cores(data_type) + + # 1 FMA counts as 2 floating point operations + # Clock rate is in KHz + tflops_per_second = num_sms * fma_per_cycle * 2 * clock_rate / 1e9 + return tflops_per_second + + def get_memory_bandwidth_Bps(self) -> int: + # DRAM devices are Double-Data which means they provide an output at both fronts of + # a clock beat + bus_bytes_per_cycle = int(2 * self.device_properties.memory_bus_width / 8) + mem_clock_rate_Hz = self.device_properties.memory_clock_rate * 1000 + bytes_per_second = bus_bytes_per_cycle * mem_clock_rate_Hz * 2 + return bytes_per_second + + def get_shared_memory_bandwidth_Bps(self) -> int: + # Each warp can LD or ST 32 x 4 bytes per cycle. To calculate the + # device's throughput we need to multiply with frequency and number of SMs. + num_sms = self.device_properties.multi_processor_count + bytes_per_cycle_per_sm = 128 + bytes_per_cycle_per_device = num_sms * bytes_per_cycle_per_sm + bytes_per_second = ( + bytes_per_cycle_per_device * self.device_properties.clock_rate * 1000 + ) + return bytes_per_second diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_gpu_trace.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_gpu_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..2d8eaa85300aa948d07dd106314bfcae70d1d6bf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_gpu_trace.py @@ -0,0 +1,73 @@ +from collections.abc import Callable + +from torch._utils import CallbackRegistry + + +EventCreationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event creation" +) +EventDeletionCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event deletion" +) +EventRecordCallbacks: "CallbackRegistry[int, int]" = CallbackRegistry( + "CUDA event record" +) +EventWaitCallbacks: "CallbackRegistry[int, int]" = CallbackRegistry("CUDA event wait") +MemoryAllocationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA memory allocation" +) +MemoryDeallocationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA memory deallocation" +) +StreamCreationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA stream creation" +) +DeviceSynchronizationCallbacks: "CallbackRegistry[[]]" = CallbackRegistry( + "CUDA device synchronization" +) +StreamSynchronizationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA stream synchronization" +) +EventSynchronizationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event synchronization" +) + + +def register_callback_for_event_creation(cb: Callable[[int], None]) -> None: + EventCreationCallbacks.add_callback(cb) + + +def register_callback_for_event_deletion(cb: Callable[[int], None]) -> None: + EventDeletionCallbacks.add_callback(cb) + + +def register_callback_for_event_record(cb: Callable[[int, int], None]) -> None: + EventRecordCallbacks.add_callback(cb) + + +def register_callback_for_event_wait(cb: Callable[[int, int], None]) -> None: + EventWaitCallbacks.add_callback(cb) + + +def register_callback_for_memory_allocation(cb: Callable[[int], None]) -> None: + MemoryAllocationCallbacks.add_callback(cb) + + +def register_callback_for_memory_deallocation(cb: Callable[[int], None]) -> None: + MemoryDeallocationCallbacks.add_callback(cb) + + +def register_callback_for_stream_creation(cb: Callable[[int], None]) -> None: + StreamCreationCallbacks.add_callback(cb) + + +def register_callback_for_device_synchronization(cb: Callable[[], None]) -> None: + DeviceSynchronizationCallbacks.add_callback(cb) + + +def register_callback_for_stream_synchronization(cb: Callable[[int], None]) -> None: + StreamSynchronizationCallbacks.add_callback(cb) + + +def register_callback_for_event_synchronization(cb: Callable[[int], None]) -> None: + EventSynchronizationCallbacks.add_callback(cb) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_memory_viz.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_memory_viz.py new file mode 100644 index 0000000000000000000000000000000000000000..12b30de35cc97c83ab968514e5d484eb6dcc167a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_memory_viz.py @@ -0,0 +1,795 @@ +# mypy: allow-untyped-defs +import base64 +import io +import json +import operator +import os +import pickle +import subprocess +import sys +import warnings +from functools import lru_cache +from itertools import groupby +from typing import Any + + +cache = lru_cache(None) + +__all__ = ["format_flamegraph", "segments", "memory", "compare"] + + +def _frame_fmt(f, full_filename=False): + i = f["line"] + fname = f["filename"] + if not full_filename: + fname = fname.split("/")[-1] + func = f["name"] + return f"{fname}:{i}:{func}" + + +@cache +def _frame_filter(name, filename): + omit_functions = [ + "unwind::unwind", + "CapturedTraceback::gather", + "gather_with_cpp", + "_start", + "__libc_start_main", + "PyEval_", + "PyObject_", + "PyFunction_", + ] + omit_filenames = [ + "core/boxing", + "/Register", + "/Redispatch", + "pythonrun.c", + "Modules/main.c", + "Objects/call.c", + "Objects/methodobject.c", + "pycore_ceval.h", + "ceval.c", + "cpython/abstract.h", + ] + for of in omit_functions: + if of in name: + return False + for of in omit_filenames: + if of in filename: + return False + return True + + +def _frames_fmt(frames, full_filename=False, reverse=False): + if reverse: + frames = reversed(frames) + return [ + _frame_fmt(f, full_filename) + for f in frames + if _frame_filter(f["name"], f["filename"]) + ] + + +def _block_extra_legacy(b): + if "history" in b: + frames = b["history"][0].get("frames", []) + real_size = b["history"][0]["real_size"] + else: + real_size = b.get("requested_size", b["size"]) + frames = [] + return frames, real_size + + +def _block_extra(b): + if "frames" not in b: + # old snapshot format made it more complicated to get frames/allocated size + return _block_extra_legacy(b) + return b["frames"], b["requested_size"] + + +def format_flamegraph(flamegraph_lines, flamegraph_script=None): + if flamegraph_script is None: + cache_dir = os.path.expanduser("~/.cache/") + os.makedirs(cache_dir, exist_ok=True) + flamegraph_script = f"{cache_dir}/flamegraph.pl" + if not os.path.exists(flamegraph_script): + import tempfile + import urllib.request + + print(f"Downloading flamegraph.pl to: {flamegraph_script}") + with tempfile.NamedTemporaryFile(mode="wb", suffix=".pl") as f: + urllib.request.urlretrieve( + "https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl", + f.name, + ) + try: + os.chmod(f.name, 0o755) + os.rename(f.name, flamegraph_script) + except OSError: # noqa: B001,E722 + # Ok to skip, the file will be removed by tempfile + pass + args = [flamegraph_script, "--countname", "bytes"] + with subprocess.Popen( + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding="utf-8" + ) as p: + assert p.stdin is not None + assert p.stdout is not None + p.stdin.write(flamegraph_lines) + p.stdin.close() + result = p.stdout.read() + p.stdout.close() + p.wait() + assert p.wait() == 0 + return result + + +def _write_blocks(f, prefix, blocks): + def frames_fragment(frames): + if not frames: + return "" + return ";".join(_frames_fmt(frames, reverse=True)) + + for b in blocks: + if "history" not in b: + frames, accounted_for_size = _block_extra(b) + f.write( + f"{prefix};{b['state']};{frames_fragment(frames)} {accounted_for_size}\n" + ) + else: + accounted_for_size = 0 + for h in b["history"]: + sz = h["real_size"] + accounted_for_size += sz + if "frames" in h: + frames = h["frames"] + f.write(f"{prefix};{b['state']};{frames_fragment(frames)} {sz}\n") + else: + f.write(f"{prefix};{b['state']}; {sz}\n") + gaps = b["size"] - accounted_for_size + if gaps: + f.write(f"{prefix};{b['state']}; {gaps}\n") + + +def segments(snapshot, format_flamegraph=format_flamegraph): + f = io.StringIO() + for seg in snapshot["segments"]: + prefix = f"stream_{seg['stream']};seg_{seg['address']}" + _write_blocks(f, prefix, seg["blocks"]) + return format_flamegraph(f.getvalue()) + + +def memory(snapshot, format_flamegraph=format_flamegraph): + f = io.StringIO() + for seg in snapshot["segments"]: + prefix = f"stream_{seg['stream']}" + _write_blocks(f, prefix, seg["blocks"]) + return format_flamegraph(f.getvalue()) + + +def compare(before, after, format_flamegraph=format_flamegraph): + def _seg_key(seg): + return (seg["address"], seg["total_size"]) + + def _seg_info(seg): + return f"stream_{seg['stream']};seg_{seg['address']}" + + f = io.StringIO() + + before_segs = {_seg_key(seg) for seg in before} + after_segs = {_seg_key(seg) for seg in after} + + print(f"only_before = {[a for a, _ in (before_segs - after_segs)]}") + print(f"only_after = {[a for a, _ in (after_segs - before_segs)]}") + + for seg in before: + if _seg_key(seg) not in after_segs: + _write_blocks(f, f"only_before;{_seg_info(seg)}", seg["blocks"]) + + for seg in after: + if _seg_key(seg) not in before_segs: + _write_blocks(f, f"only_after;{_seg_info(seg)}", seg["blocks"]) + + return format_flamegraph(f.getvalue()) + + +def _format_size(num): + # https://stackoverflow.com/questions/1094841/get-human-readable-version-of-file-size + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}B" + num /= 1024.0 + return f"{num:.1f}YiB" + + +class Bytes: + def __init__(self, value): + self.value = value + + def __add__(self, rhs): + return Bytes(self.value + rhs) + + def __repr__(self): + return _format_size(self.value) + + +def calc_active(seg): + return sum(b["size"] for b in seg["blocks"] if b["state"] == "active_allocated") + + +def _report_free(free_external, free_internal): + total = free_external + free_internal + suffix = "" + if total != 0: + pct = (free_internal / total) * 100 + suffix = f" ({pct:.1f}% internal)" + return f"{Bytes(total)}{suffix}" + + +PAGE_SIZE = 1024 * 1024 * 20 +legend = f"""\ + +Legend: + [a ] - a segment in the allocator + ^-- a page {Bytes(PAGE_SIZE)} of memory in the segment + a-z: pages filled with a single block's content + ' ': page is completely free + *: page if completely full with multiple blocks + 0-9: page is partially full with tensors of multiple blocks (9 == 90% full) + (X% internal) - of the free memory, X% is free because we rounded the size of the allocation. +""" + + +def segsum(data): + r"""Visually reports how the allocator has filled its segments. + + This printout can help debug fragmentation issues since free fragments + will appear as gaps in this printout. The amount of free space is reported + for each segment. + We distinguish between internal free memory which occurs because the + allocator rounds the allocation size, and external free memory, which are + the gaps between allocations in a segment. + Args: + data: snapshot dictionary created from _snapshot() + """ + out = io.StringIO() + out.write(f"Summary of segments >= {Bytes(PAGE_SIZE)} in size\n") + total_reserved = 0 + total_allocated = 0 + free_external = 0 + free_internal = 0 + for seg in sorted( + data["segments"], key=lambda x: (x["total_size"], calc_active(x)) + ): + total_reserved += seg["total_size"] + + seg_free_external = 0 + seg_free_internal = 0 + seg_allocated = 0 + all_ranges = [] + boffset = 0 + for b in seg["blocks"]: + active = b["state"] == "active_allocated" + if active: + _, allocated_size = _block_extra(b) + all_ranges.append((boffset, allocated_size, True)) + seg_allocated += allocated_size + seg_free_internal += b["size"] - allocated_size + else: + seg_free_external += b["size"] + + boffset += b["size"] + + total_allocated += seg_allocated + free_external += seg_free_external + free_internal += seg_free_internal + + nseg = (seg["total_size"] - 1) // PAGE_SIZE + 1 + occupied = [" " for _ in range(nseg)] + frac = [0.0 for _ in range(nseg)] + active_size = 0 + for i, (start_, size, active) in enumerate(all_ranges): + active_size += size + finish_ = start_ + size + start = start_ // PAGE_SIZE + finish = (finish_ - 1) // PAGE_SIZE + 1 + m = chr(ord("a" if active else "A") + (i % 26)) + for j in range(start, finish): + s = max(start_, j * PAGE_SIZE) + e = min(finish_, (j + 1) * PAGE_SIZE) + frac[j] += (e - s) / PAGE_SIZE + if occupied[j] != " ": + occupied[j] = "0123456789*"[int(frac[j] * 10)] + else: + occupied[j] = m + stream = "" if seg["stream"] == 0 else f", stream_{seg['stream']}" + body = "".join(occupied) + assert ( + seg_free_external + seg_free_internal + seg_allocated == seg["total_size"] + ) + stream = f" stream_{seg['stream']}" if seg["stream"] != 0 else "" + if seg["total_size"] >= PAGE_SIZE: + out.write( + f"[{body}] {Bytes(seg['total_size'])} allocated, " + f"{_report_free(seg_free_external, seg_free_internal)} free{stream}\n" + ) + out.write(f"segments: {len(data['segments'])}\n") + out.write(f"total_reserved: {Bytes(total_reserved)}\n") + out.write(f"total_allocated: {Bytes(total_allocated)}\n") + out.write(f"total_free: {_report_free(free_external, free_internal)}\n") + out.write(legend) + assert free_internal + free_external + total_allocated == total_reserved + return out.getvalue() + + +def trace(data): + out = io.StringIO() + + def format(entries): + segment_intervals: list = [] + segment_addr_to_name = {} + allocation_addr_to_name = {} + + free_names: list = [] + next_name = 0 + + def _name(): + nonlocal next_name + if free_names: + return free_names.pop() + r, m = next_name // 26, next_name % 26 + next_name += 1 + return f"{chr(ord('a') + m)}{'' if r == 0 else r}" + + def find_segment(addr): + for name, saddr, size in segment_intervals: + if addr >= saddr and addr < saddr + size: + return name, saddr + for i, seg in enumerate(data["segments"]): + saddr = seg["address"] + size = seg["allocated_size"] + if addr >= saddr and addr < saddr + size: + return f"seg_{i}", saddr + return None, None + + count = 0 + out.write(f"{len(entries)} entries\n") + + total_reserved = 0 + for seg in data["segments"]: + total_reserved += seg["total_size"] + + for count, e in enumerate(entries): + if e["action"] == "alloc": + addr, size = e["addr"], e["size"] + n = _name() + seg_name, seg_addr = find_segment(addr) + if seg_name is None: + seg_name = "MEM" + offset = addr + else: + offset = addr - seg_addr + out.write(f"{n} = {seg_name}[{offset}:{Bytes(size)}]\n") + allocation_addr_to_name[addr] = (n, size, count) + count += size + elif e["action"] == "free_requested": + addr, size = e["addr"], e["size"] + name, _, _ = allocation_addr_to_name.get(addr, (addr, None, None)) + out.write(f"del {name} # {Bytes(size)}\n") + elif e["action"] == "free_completed": + addr, size = e["addr"], e["size"] + count -= size + name, _, _ = allocation_addr_to_name.get(addr, (addr, None, None)) + out.write(f"# free completed for {name} {Bytes(size)}\n") + if name in allocation_addr_to_name: + free_names.append(name) + del allocation_addr_to_name[name] + elif e["action"] == "segment_alloc": + addr, size = e["addr"], e["size"] + name = _name() + out.write(f"{name} = cudaMalloc({addr}, {Bytes(size)})\n") + segment_intervals.append((name, addr, size)) + segment_addr_to_name[addr] = name + elif e["action"] == "segment_free": + addr, size = e["addr"], e["size"] + name = segment_addr_to_name.get(addr, addr) + out.write(f"cudaFree({name}) # {Bytes(size)}\n") + if name in segment_addr_to_name: + free_names.append(name) + del segment_addr_to_name[name] + elif e["action"] == "oom": + size = e["size"] + free = e["device_free"] + out.write( + f"raise OutOfMemoryError # {Bytes(size)} requested, {Bytes(free)} free in CUDA\n" + ) + else: + out.write(f"{e}\n") + out.write(f"TOTAL MEM: {Bytes(count)}") + + for i, d in enumerate(data["device_traces"]): + if d: + out.write(f"Device {i} ----------------\n") + format(d) + return out.getvalue() + + +_memory_viz_template = r""" + + + + + + + +""" + + +def _format_viz(data, viz_kind, device): + if device is not None: + warnings.warn( + "device argument is deprecated, plots now contain all device", + FutureWarning, + stacklevel=3, + ) + buffer = pickle.dumps(data) + buffer += b"\x00" * (3 - len(buffer) % 3) + # Encode the buffer with base64 + encoded_buffer = base64.b64encode(buffer).decode("utf-8") + + json_format = json.dumps([{"name": "snapshot.pickle", "base64": encoded_buffer}]) + return _memory_viz_template.replace("$VIZ_KIND", repr(viz_kind)).replace( + "$SNAPSHOT", json_format + ) + + +def filter_alloc_free_pairs(data): + for dev_id in range(len(data["device_traces"])): + # set of indexes of trace events for alloc-free pairs + filterSet = set() + # map from addr to index of alloc event + allocMap = {} + # set of addrs from free_requested events + freeRequested = set() + for idx, event in enumerate(data["device_traces"][dev_id]): + if event["action"] == "alloc": + allocMap[event["addr"]] = idx + elif event["action"] == "free_requested": + freeRequested.add(event["addr"]) + if allocMap.get(event["addr"]) is not None: + filterSet.add(idx) + filterSet.add(allocMap[event["addr"]]) + allocMap.pop(event["addr"]) + elif event["action"] == "free_completed": + if event["addr"] in freeRequested: + freeRequested.remove(event["addr"]) + filterSet.add(idx) + else: + print(f"free_completed without free_requested: {event}") + + # Remove events whose index is in filterSet + if filterSet: + # Create a new list excluding events with indices in filterSet + data["device_traces"][dev_id] = [ + event + for idx, event in enumerate(data["device_traces"][dev_id]) + if idx not in filterSet + ] + + return data + + +def trace_plot(data, device=None, plot_segments=False, filter_freed=False): + """Generate a visualization over time of the memory usage recorded by the trace as an html file. + + Args: + data: Memory snapshot as generated from torch.cuda.memory._snapshot() + device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. + plot_segments (bool, optional): Plots memory returned from cudaMalloc, rather than individual allocations. + Defaults to False. + filter_freed (bool, optional): Filter out alloc-free paired events to only plot allocations that are not freed yet. + Defaults to False to plot all trace events. + + Returns: + str: HTML of visualization + """ + if filter_freed: + data = filter_alloc_free_pairs(data) + + return _format_viz( + data, + "Active Memory Timeline" + if not plot_segments + else "Active Cached Memory Timeline", + device, + ) + + +def _profile_to_snapshot(profile): + import torch + from torch._C._profiler import _EventType + from torch.profiler._memory_profiler import Action, TensorKey + + memory_profile = profile._memory_profile() + + allocation_stacks = {} + for event in memory_profile._op_tree.sorted_nodes: + if event.tag == _EventType.Allocation: + parent = event.parent + python_parents = [] + while parent: + if parent.tag in (_EventType.PyCall, _EventType.PyCCall): + python_parents.append(parent) + parent = parent.parent + key = TensorKey.from_allocation(event.extra_fields) + + # Corner case: If allocation doesn't have an ID (can't prove it was used as a Tensor) + # key will be None. I should add some way to identify these, I just haven't yet. + if key and event.extra_fields.alloc_size > 0: + allocation_stacks[key] = python_parents + + device_count = torch.cuda.device_count() + snapshot: dict[str, list[Any]] = { + "device_traces": [[] for _ in range(device_count + 1)], + "segments": [ + { + "device": device, + "address": None, + "total_size": 0, + "stream": 0, + "blocks": [], + } + for device in range(device_count + 1) + ], + } + + def to_device(device): + if device.type == "cuda": + return device.index + else: + return device_count + + def allocate(size, tensor_key, version, during_trace=True): + device = to_device(tensor_key.device) + addr = tensor_key.storage.ptr + + seg = snapshot["segments"][device] # type: ignore[index] + if seg["address"] is None or seg["address"] > addr: + seg["address"] = addr + seg["total_size"] = max( + seg["total_size"], addr + size + ) # record max addr for now, we will make it the size later + category = memory_profile._categories.get(tensor_key, version) + category = category.name.lower() if category is not None else "unknown" + stack = allocation_stacks.get(tensor_key, ()) + stack = [{"filename": "none", "line": 0, "name": p.name} for p in stack] + r = { + "action": "alloc", + "addr": addr, + "size": size, + "stream": 0, + "frames": stack, + "category": category, + } + if during_trace: + snapshot["device_traces"][device].append(r) + return r + + def free(alloc, device): + for e in ("free_requested", "free_completed"): + snapshot["device_traces"][device].append( + { + "action": e, + "addr": alloc["addr"], + "size": alloc["size"], + "stream": 0, + "frames": alloc["frames"], + } + ) + + kv_to_elem = {} + + # create the device trace + for _time, action, (tensor_key, version), size in memory_profile.timeline: + if not isinstance(tensor_key, TensorKey): + continue + if action == Action.CREATE: + kv_to_elem[(tensor_key, version)] = allocate(size, tensor_key, version) + elif action == Action.DESTROY: + free(kv_to_elem.pop((tensor_key, version)), to_device(tensor_key.device)) + elif action == Action.INCREMENT_VERSION: + free(kv_to_elem.pop((tensor_key, version)), to_device(tensor_key.device)) + kv_to_elem[(tensor_key, version + 1)] = allocate( + size, tensor_key, version + 1 + ) + elif action == Action.PREEXISTING: + kv_to_elem[(tensor_key, version)] = allocate( + size, tensor_key, version, during_trace=False + ) + + # create the final snapshot state + blocks_at_end = [ + (to_device(tensor_key.device), event["addr"], event["size"], event["frames"]) + for (tensor_key, version), event in kv_to_elem.items() + ] + for device, blocks in groupby(sorted(blocks_at_end), key=operator.itemgetter(0)): + seg = snapshot["segments"][device] # type: ignore[index] + last_addr = seg["address"] + for _, addr, size, frames in blocks: + if last_addr < addr: + seg["blocks"].append({"size": addr - last_addr, "state": "inactive"}) + seg["blocks"].append( + { + "size": size, + "state": "active_allocated", + "requested_size": size, + "frames": frames, + } + ) + last_addr = addr + size + if last_addr < seg["total_size"]: + seg["blocks"].append( + {"size": seg["total_size"] - last_addr, "state": "inactive"} + ) + + snapshot["segments"] = [seg for seg in snapshot["segments"] if seg["blocks"]] # type: ignore[attr-defined] + for seg in snapshot["segments"]: # type: ignore[attr-defined, name-defined, no-redef] + seg["total_size"] -= seg["address"] + if not seg["blocks"]: + seg["blocks"].append({"size": seg["total_size"], "state": "inactive"}) + + return snapshot + + +def profile_plot(profile, device=None): + """Generate a visualization over time of the memory usage recorded by kineto memory profiling as an html file. + + Args: + profile: profile as generated by `torch.profiler.profile(profile_memory=True)` + device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. + + Returns: + str: HTML of visualization + """ + snapshot = _profile_to_snapshot(profile) + return _format_viz(snapshot, "Active Memory Timeline", device) + + +def segment_plot(data: Any, device=None): + return _format_viz(data, "Allocator State History", device) + + +if __name__ == "__main__": + import os.path + + thedir = os.path.realpath(os.path.dirname(__file__)) + if thedir in sys.path: + # otherwise we find cuda/random.py as random... + sys.path.remove(thedir) + import argparse + + fn_name = "torch.cuda.memory._snapshot()" + pickled = f"pickled memory statistics from {fn_name}" + parser = argparse.ArgumentParser( + description=f"Visualize memory dumps produced by {fn_name}" + ) + + subparsers = parser.add_subparsers(dest="action") + + def _output(p): + p.add_argument( + "-o", + "--output", + default="output.svg", + help="flamegraph svg (default: output.svg)", + ) + + description = "Prints overall allocation statistics and a visualization of how the allocators segments are currently filled." + stats_a = subparsers.add_parser("stats", description=description) + stats_a.add_argument("input", help=pickled) + + description = "Prints buffer of the most recent allocation events embedded in the snapshot in a Pythonic style." + trace_a = subparsers.add_parser("trace", description=description) + trace_a.add_argument("input", help=pickled) + + description = "Generate a flamegraph that visualizes what memory is stored in each allocator segment (aka block)" + segments_a = subparsers.add_parser("segments", description=description) + segments_a.add_argument("input", help=pickled) + _output(segments_a) + + description = ( + "Generate a flamegraph the program locations contributing to CUDA memory usage." + ) + memory_a = subparsers.add_parser("memory", description=description) + memory_a.add_argument("input", help=pickled) + _output(memory_a) + + description = ( + "Generate a flamegraph that shows segments (aka blocks) that have been added " + "or removed between two different memorys snapshots." + ) + compare_a = subparsers.add_parser("compare", description=description) + compare_a.add_argument("before", help=pickled) + compare_a.add_argument("after", help=pickled) + _output(compare_a) + + plots = ( + ( + "trace_plot", + "Generate a visualization over time of the memory usage recorded by the trace as an html file.", + ), + ( + "segment_plot", + "Visualize how allocations are packed into allocator segments at each point in a trace as an html file.", + ), + ) + for cmd, description in plots: + trace_plot_a = subparsers.add_parser(cmd, description=description) + trace_plot_a.add_argument("input", help=pickled) + help = "visualize trace from this device (default: chooses the only device with trace info or errors)" + trace_plot_a.add_argument("-d", "--device", type=int, default=None, help=help) + help = "path to save the visualization(default: output.html)" + trace_plot_a.add_argument("-o", "--output", default="output.html", help=help) + if cmd == "trace_plot": + help = "visualize change to segments rather than individual allocations" + trace_plot_a.add_argument( + "-s", "--segments", action="store_true", help=help + ) + + help = ( + "filter out allocation-free pairs to only visualize the allocations that are not freed yet;" + "useful to reduce the number of events for large traces for debugging OOM" + ) + trace_plot_a.add_argument( + "-f", "--filter_freed", action="store_true", help=help + ) + + args = parser.parse_args() + + def _read(name): + if name == "-": + data = pickle.load(sys.stdin.buffer) + else: + with open(name, "rb") as f: + data = pickle.load(f) + if isinstance(data, list): # segments only... + data = {"segments": data, "traces": []} + return data + + def _write(name, data): + with open(name, "w") as f: + f.write(data) + + if args.action == "segments": + data = _read(args.input) + _write(args.output, segments(data)) + elif args.action == "memory": + data = _read(args.input) + _write(args.output, memory(data)) + elif args.action == "stats": + data = _read(args.input) + print(segsum(data)) + elif args.action == "trace": + data = _read(args.input) + print(trace(data)) + elif args.action == "compare": + before = _read(args.before) + after = _read(args.after) + _write(args.output, compare(before, after)) + elif args.action == "trace_plot": + data = _read(args.input) + _write( + args.output, + trace_plot( + data, + device=args.device, + plot_segments=args.segments, + filter_freed=args.filter_freed, + ), + ) + elif args.action == "segment_plot": + data = _read(args.input) + _write(args.output, segment_plot(data, device=args.device)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_pin_memory_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_pin_memory_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..faba7262af7de4c736660df78e9b0f5f9fc93b31 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_pin_memory_utils.py @@ -0,0 +1,24 @@ +import torch + + +def pin_memory(data_ptr: int, size: int) -> None: + cudart = torch.cuda.cudart() + succ = int( + cudart.cudaHostRegister( + data_ptr, + size, + 1, # lines up with 'cudaHostRegisterPortable' + ) + ) + + if succ != 0: + raise RuntimeError( + f"Registering memory failed with cudaError: {succ}." + " It's possible that this is an asynchronous error raised from a previous cuda operation." + " Consider launching with CUDA_LAUNCH_BLOCKING=1 to debug." + ) + + +def unpin_memory(data_ptr: int) -> None: + succ = int(torch.cuda.cudart().cudaHostUnregister(data_ptr)) + assert succ == 0, f"Unpinning shared memory failed with error-code: {succ}" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_sanitizer.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_sanitizer.py new file mode 100644 index 0000000000000000000000000000000000000000..b28437e554c4f08ad2eda92848449922bced9747 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_sanitizer.py @@ -0,0 +1,662 @@ +# mypy: allow-untyped-defs +r""" +This module introduces CUDA Sanitizer, a tool for detecting synchronization errors between kernels ran on different streams. + +It stores information on accesses to tensors to determine if they are synchronized +or not. When enabled in a python program and a possible data race is detected, a +detailed warning will be printed and the program will exit. + +It can be enabled either by importing this module and calling +:func:`enable_cuda_sanitizer()` or by exporting the ``TORCH_CUDA_SANITIZER`` +environment variable. +""" + +import enum +import functools +import inspect +import io +import logging +import re +import sys +import textwrap +import traceback +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any, Optional, TypeVar + +import torch +import torch.cuda._gpu_trace as gpu_trace +from torch.utils import _pytree as pytree +from torch.utils._python_dispatch import TorchDispatchMode + + +DEFAULT_STREAM_ID = 0 + +TK = TypeVar("TK") +TVa = TypeVar("TVa") +TVb = TypeVar("TVb") + +DataPtr = int +StreamId = int +EventId = int +SeqNum = int + +logger = logging.getLogger(__name__) + +# Note that this is only factories that take Tensor as input as they are +# the ones we care about. +FACTORY_FUNCTION_REGEX = re.compile("(new_.*|.*_like)") + + +class AccessType(enum.Enum): + READ = enum.auto() + WRITE = enum.auto() + + def __str__(self): + return "reading from" if self is AccessType.READ else "writing to" + + +@dataclass +class Access: + r"""Stores information about a single access to a tensor by a kernel. + + Args: + type: either AccessType.READ or AccessType.Write. + seq_num: the sequential number of the kernel performing the access. + stream: the stream id of the stream executing the kernel. + operator: the schema of the launched kernel, which lists the + arguments and return type. + aliases: the arguments in the schema this access corresponds to. + is_output: Whether the tensor was an output of the kernel. + stack_trace: the stack summary object captured during access. + """ + + type: AccessType + seq_num: SeqNum + stream: StreamId + operator: str + aliases: list[str] + is_output: bool + stack_trace: traceback.StackSummary + + +class SynchronizationError(Exception): + """Base class for errors detected by CUDA Sanitizer.""" + + +class UnsynchronizedAccessError(SynchronizationError): + """Stores information about two unsynchronized accesses to one data pointer.""" + + def __init__( + self, + data_ptr: DataPtr, + allocation_stack_trace: Optional[traceback.StackSummary], + current_access: Access, + previous_access: Access, + ): + self.data_ptr = data_ptr + self.allocation_stack_trace = allocation_stack_trace + self.current_access = current_access + self.previous_access = previous_access + + def __str__(self): + def format_access(access: Access): + message.write(f"{access.operator}\n{access.type}") + if access.aliases: + message.write(" argument(s) " + ", ".join(access.aliases)) + if access.is_output: + message.write(", and to") + if access.is_output: + message.write(" the output") + message.write( + f"\nWith stack trace:\n{''.join(access.stack_trace.format())}\n" + ) + + with io.StringIO() as message: + message.write( + textwrap.dedent( + f"""\ + ============================ + CSAN detected a possible data race on tensor with data pointer {self.data_ptr} + Access by stream {self.current_access.stream} during kernel: + """ + ) + ) + format_access(self.current_access) + + message.write( + f"Previous access by stream {self.previous_access.stream} during kernel:\n" + ) + format_access(self.previous_access) + + if self.allocation_stack_trace: + message.write( + "Tensor was allocated with stack trace:\n" + f"{''.join(self.allocation_stack_trace.format())}" + ) + else: + message.write("Trace for tensor allocation not found.") + return message.getvalue() + + +class CUDASanitizerErrors(Exception): + """Wrapper class for errors reported by CUDA Sanitizer.""" + + def __init__(self, errors: list[SynchronizationError]): + self.errors = errors + + def __str__(self): + return f"detected {len(self.errors)} errors" + + +@dataclass +class TensorInfo: + r"""Stores information about a single tensor and recent accesses to it. + + Args: + allocation_stack_trace: the stack summary object captured during tensor + allocation. Can be ``None`` if the allocation wasn't caught by CSAN. + reads: list of read accesses to the tensor that were performed since + the last write. + write: the last write access to the tensor. + """ + + allocation_stack_trace: Optional[traceback.StackSummary] + reads: list[Access] = field(default_factory=list) + write: Optional[Access] = None + + +class _TensorsAccessed: + def __init__(self) -> None: + self.accesses: dict[DataPtr, TensorInfo] = {} + + def ensure_tensor_exists(self, data_ptr: DataPtr) -> None: + if data_ptr not in self.accesses: + logger.info( + "Found tensor with pointer: %s, but no matching tensor " + "allocation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + data_ptr, + ) + self.create_tensor(data_ptr, None) + + def ensure_tensor_does_not_exist(self, data_ptr: DataPtr) -> None: + if data_ptr in self.accesses: + logger.info( + "Found duplicate tensor allocation in the trace for tensor with " + "pointer: %s. Assuming the trace for tensor deallocation " + "wasn't caught and backfilling it now. " + "Perhaps the sanitizer was enabled after some torch operations?", + data_ptr, + ) + self.delete_tensor(data_ptr) + + def create_tensor( + self, data_ptr: DataPtr, stack_trace: Optional[traceback.StackSummary] + ) -> None: + self.accesses[data_ptr] = TensorInfo(stack_trace) + + def delete_tensor(self, data_ptr: DataPtr) -> None: + del self.accesses[data_ptr] + + def were_there_reads_since_last_write(self, data_ptr: DataPtr) -> bool: + return bool(self.accesses[data_ptr].reads) + + def get_allocation_stack_trace( + self, data_ptr: DataPtr + ) -> Optional[traceback.StackSummary]: + return self.accesses[data_ptr].allocation_stack_trace + + def get_write(self, data_ptr: DataPtr) -> Optional[Access]: + return self.accesses[data_ptr].write + + def get_reads(self, data_ptr: DataPtr) -> list[Access]: + return self.accesses[data_ptr].reads + + def add_read(self, data_ptr: DataPtr, access: Access) -> None: + self.accesses[data_ptr].reads.append(access) + + def set_write(self, data_ptr: DataPtr, access: Access) -> None: + self.accesses[data_ptr].write = access + self.accesses[data_ptr].reads = [] + + +class StreamSynchronizations: + def __init__(self) -> None: + self.current_sync_states: dict[StreamId, dict[StreamId, SeqNum]] = {} + self.recorded_sync_states: dict[EventId, dict[StreamId, SeqNum]] = {} + self.host_sync_state: dict[StreamId, SeqNum] = {} + self.create_stream(DEFAULT_STREAM_ID) + + def _ensure_stream_exists(self, stream: StreamId) -> None: + if stream not in self.current_sync_states: + logger.info( + "Found Stream with id: %s, but no matching stream " + "creation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + stream, + ) + self.create_stream(stream) + + def _ensure_event_exists(self, event: EventId) -> None: + if event not in self.recorded_sync_states: + logger.info( + "Found Event with id: %s, but no matching event " + "creation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + event, + ) + self.create_event(event) + + def _ensure_event_does_not_exist(self, event: EventId) -> None: + if event in self.recorded_sync_states: + logger.info( + "Found duplicate event creation in the trace for event with " + "id: %s. Assuming the trace for event deletion wasn't caught " + "and backfilling it now. " + "Perhaps the sanitizer was enabled after some torch operations?", + event, + ) + self.delete_event(event) + + def create_stream(self, stream: StreamId) -> None: + if stream in self.current_sync_states: + logger.info( + "Found duplicate Stream creation in the trace for Stream with " + "id: %s. PyTorch Streams are only created once, so this " + "trace entry is ignored.", + stream, + ) + else: + self.host_sync_state[stream] = 0 + self.current_sync_states[stream] = self.host_sync_state.copy() + + def create_event(self, event: EventId) -> None: + self._ensure_event_does_not_exist(event) + self.recorded_sync_states[event] = {} + + def delete_event(self, event: EventId) -> None: + self._ensure_event_exists(event) + del self.recorded_sync_states[event] + + def update_seq_num(self, stream: StreamId, seq_num: SeqNum) -> None: + self._ensure_stream_exists(stream) + self.current_sync_states[stream][stream] = seq_num + + def record_state(self, event: EventId, stream: StreamId) -> None: + self._ensure_event_exists(event) + self._ensure_stream_exists(stream) + self.recorded_sync_states[event] = self.current_sync_states[stream].copy() + + def _state_wait_for_other( + self, state: dict[StreamId, SeqNum], other: dict[StreamId, SeqNum] + ) -> None: + for stream, seq_num in other.items(): + state[stream] = max(state.get(stream, -1), seq_num) + + def stream_wait_for_event(self, stream: StreamId, event: EventId) -> None: + self._ensure_stream_exists(stream) + self._ensure_event_exists(event) + self._state_wait_for_other( + self.current_sync_states[stream], self.recorded_sync_states[event] + ) + + def all_streams_wait_for_event(self, event: EventId) -> None: + self._ensure_event_exists(event) + for stream in self.current_sync_states: + self.stream_wait_for_event(stream, event) + + self._state_wait_for_other( + self.host_sync_state, self.recorded_sync_states[event] + ) + + def all_streams_wait_for_stream(self, stream: StreamId) -> None: + self._ensure_stream_exists(stream) + for state in self.current_sync_states.values(): + self._state_wait_for_other(state, self.current_sync_states[stream]) + + self._state_wait_for_other( + self.host_sync_state, self.current_sync_states[stream] + ) + + def sync_all_streams(self) -> None: + for stream, state in self.current_sync_states.items(): + self.host_sync_state[stream] = state[stream] + + for state in self.current_sync_states.values(): + self._state_wait_for_other(state, self.host_sync_state) + + def is_ordered_after( + self, current_stream: StreamId, seq_num: SeqNum, other_stream: StreamId + ) -> bool: + self._ensure_stream_exists(current_stream) + self._ensure_stream_exists(other_stream) + return seq_num <= self.current_sync_states[current_stream].get(other_stream, -1) + + +class EventHandler: + """Analyzes CSAN trace for synchronization errors. + + Stores information on each stream's synchronizations with other streams as well + as tensor accesses to determine whether a given kernel launch might cause a + data race. + """ + + def __init__(self) -> None: + self.tensors_accessed = _TensorsAccessed() + self.syncs = StreamSynchronizations() + self.seq_num: SeqNum = 0 + + def _handle_kernel_launch( + self, + stream: StreamId, + read_only: set[DataPtr], + read_write: set[DataPtr], + outputs: set[DataPtr], + operator: str, + tensor_aliases: dict[int, list[str]], + ) -> list[SynchronizationError]: + def check_conflict( + data_ptr: DataPtr, current_access: Access, previous_access: Optional[Access] + ) -> None: + if previous_access is None: + return + if not self.syncs.is_ordered_after( + current_access.stream, previous_access.seq_num, previous_access.stream + ): + error_list.append( + UnsynchronizedAccessError( + data_ptr, + self.tensors_accessed.get_allocation_stack_trace(data_ptr), + current_access, + previous_access, + ) + ) + + error_list: list[SynchronizationError] = [] + self.seq_num += 1 + self.syncs.update_seq_num(stream, self.seq_num) + stack_trace = traceback.StackSummary.extract( + traceback.walk_stack(inspect.currentframe()), lookup_lines=False + ) + # The stack trace generated in this way is in the inverse order, so it must be + # reversed. + stack_trace.reverse() + + for data_ptr in read_only: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + current_access = Access( + AccessType.READ, + self.seq_num, + stream, + operator, + tensor_aliases[data_ptr], + data_ptr in outputs, + stack_trace, + ) + check_conflict( + data_ptr, current_access, self.tensors_accessed.get_write(data_ptr) + ) + self.tensors_accessed.add_read(data_ptr, current_access) + + for data_ptr in read_write: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + current_access = Access( + AccessType.WRITE, + self.seq_num, + stream, + operator, + tensor_aliases[data_ptr], + data_ptr in outputs, + stack_trace, + ) + if self.tensors_accessed.were_there_reads_since_last_write(data_ptr): + for previous_access in self.tensors_accessed.get_reads(data_ptr): + check_conflict(data_ptr, current_access, previous_access) + else: + check_conflict( + data_ptr, current_access, self.tensors_accessed.get_write(data_ptr) + ) + self.tensors_accessed.set_write(data_ptr, current_access) + + return error_list + + def _handle_event_creation(self, event: EventId) -> None: + self.syncs.create_event(event) + + def _handle_event_deletion(self, event: EventId) -> None: + self.syncs.delete_event(event) + + def _handle_event_record(self, event: EventId, stream: StreamId) -> None: + self.syncs.record_state(event, stream) + + def _handle_event_wait(self, event: EventId, stream: StreamId) -> None: + self.syncs.stream_wait_for_event(stream, event) + + def _handle_memory_allocation(self, data_ptr: DataPtr) -> None: + self.tensors_accessed.ensure_tensor_does_not_exist(data_ptr) + stack_trace = traceback.StackSummary.extract( + traceback.walk_stack(inspect.currentframe()), lookup_lines=False + ) + # The stack trace generated in this way is in the inverse order, so it must be + # reversed. + stack_trace.reverse() + self.tensors_accessed.create_tensor( + data_ptr, + stack_trace, + ) + + def _handle_memory_deallocation(self, data_ptr: DataPtr) -> None: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + self.tensors_accessed.delete_tensor(data_ptr) + + def _handle_stream_creation(self, stream: StreamId) -> None: + self.syncs.create_stream(stream) + + def _handle_device_synchronization(self) -> None: + self.syncs.sync_all_streams() + + def _handle_stream_synchronization(self, stream: StreamId) -> None: + self.syncs.all_streams_wait_for_stream(stream) + + def _handle_event_synchronization(self, event: EventId) -> None: + self.syncs.all_streams_wait_for_event(event) + + +def zip_by_key(a: dict[TK, TVa], b: dict[TK, TVb]) -> Iterator[tuple[TK, TVa, TVb]]: + for arg, value in a.items(): + if arg in b: + yield arg, value, b[arg] + + +def zip_arguments( + schema: torch.FunctionSchema, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> Iterator[tuple[torch.Argument, Any]]: + schema_args = schema.arguments[: len(args)] + schema_kwargs = {arg.name: arg for arg in schema.arguments[len(args) :]} + + yield from zip(schema_args, args) + + for _, argument, value in zip_by_key(schema_kwargs, kwargs): + yield (argument, value) + + +class ArgumentHandler: + def __init__(self) -> None: + self.dataptrs_read: set[DataPtr] = set() + self.dataptrs_written: set[DataPtr] = set() + self.tensor_aliases: dict[DataPtr, list[str]] = {} + self.outputs: set[DataPtr] = set() + + def _handle_argument( + self, + value: Any, + is_write: bool, + metadata_only: bool, + name: Optional[str] = None, + is_output: bool = False, + ) -> None: + if isinstance(value, torch.Tensor) and value.is_cuda: + data_ptr = value.data_ptr() + if is_write: + self.dataptrs_written.add(data_ptr) + elif not metadata_only: + self.dataptrs_read.add(data_ptr) + + self.tensor_aliases.setdefault(data_ptr, []) + if name is not None: + self.tensor_aliases[data_ptr].append(name) + if is_output: + self.outputs.add(data_ptr) + + def parse_inputs( + self, + schema: torch.FunctionSchema, + args: tuple[Any, ...], + kwargs: dict[str, Any], + *, + is_factory: bool, + ) -> None: + for argument, value in zip_arguments(schema, args, kwargs): + is_write = argument.alias_info is not None and argument.alias_info.is_write + # A change is metadata only if it is a view or a factory function that + # reads only metadata + metadata_only = is_factory or ( + argument.alias_info is not None and not argument.alias_info.is_write + ) + pytree.tree_map_( + functools.partial( + self._handle_argument, + is_write=is_write, + name=argument.name, + metadata_only=metadata_only, + ), + value, + ) + + def parse_outputs( + self, schema: torch.FunctionSchema, outputs: Any, *, is_factory: bool + ) -> None: + for res, value in zip(schema.returns, (outputs,)): + metadata_only = is_factory or ( + res.alias_info is not None and not res.alias_info.is_write + ) + pytree.tree_map_( + functools.partial( + self._handle_argument, + is_write=not metadata_only, + is_output=True, + metadata_only=metadata_only, + ), + value, + ) + + +class CUDASanitizerDispatchMode(TorchDispatchMode): + def __init__(self) -> None: + self.event_handler = EventHandler() + torch._C._activate_gpu_trace() + gpu_trace.register_callback_for_event_creation( + self.event_handler._handle_event_creation + ) + gpu_trace.register_callback_for_event_deletion( + self.event_handler._handle_event_deletion + ) + gpu_trace.register_callback_for_event_record( + self.event_handler._handle_event_record + ) + gpu_trace.register_callback_for_event_wait( + self.event_handler._handle_event_wait + ) + gpu_trace.register_callback_for_memory_allocation( + self.event_handler._handle_memory_allocation + ) + gpu_trace.register_callback_for_memory_deallocation( + self.event_handler._handle_memory_deallocation + ) + gpu_trace.register_callback_for_stream_creation( + self.event_handler._handle_stream_creation + ) + gpu_trace.register_callback_for_device_synchronization( + self.event_handler._handle_device_synchronization + ) + gpu_trace.register_callback_for_stream_synchronization( + self.event_handler._handle_stream_synchronization + ) + gpu_trace.register_callback_for_event_synchronization( + self.event_handler._handle_event_synchronization + ) + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if kwargs is None: + kwargs = {} + + is_factory = bool(FACTORY_FUNCTION_REGEX.match(func._schema.name)) + + argument_handler = ArgumentHandler() + argument_handler.parse_inputs(func._schema, args, kwargs, is_factory=is_factory) + + outputs = func(*args, **kwargs) + + argument_handler.parse_outputs(func._schema, outputs, is_factory=is_factory) + errors = self.event_handler._handle_kernel_launch( + torch.cuda.current_stream().cuda_stream, + argument_handler.dataptrs_read - argument_handler.dataptrs_written, + argument_handler.dataptrs_written, + argument_handler.outputs, + func._schema, + argument_handler.tensor_aliases, + ) + if errors: + for error in errors: + print(error, file=sys.stderr) + raise CUDASanitizerErrors(errors) + + return outputs + + +class CUDASanitizer: + """Manages the lifetime of a CUDASanitizer dispatch mode object. + + The CUDASanitizer class wraps the entering/exiting functions of the dispatch mode + context manager in the enable function/destructor, respectively. This is to + explicitly set the lifetime of the dispatch mode object to that of the application. + This approach was deemed more elegant than using the atexit module. + """ + + def __init__(self) -> None: + self.dispatch = CUDASanitizerDispatchMode() + self.enabled = False + + def enable(self): + self.dispatch.__enter__() + self.enabled = True + + def disable(self): + self.dispatch.__exit__(None, None, None) + self.enabled = False + + def __del__(self): + # Since this object lifetime is linked to the `torch.cuda._sanitizer` python + # module, it often gets deleted as part of the overall `torch` module cleanup + # At that time, depending on CPython version, the torch.* module might be in + # different states of being already cleaned up. + # Similarly other imports might already have been cleaned up so `sys` might + # be already gone as well. + # Skip exiting the mode if it outlived the runtime. + if (sys is not None) and (not sys.is_finalizing()) and self.enabled: + self.disable() + + +def enable_cuda_sanitizer(): + """Enable CUDA Sanitizer. + + The sanitizer will begin to analyze low-level CUDA calls invoked by torch functions + for synchronization errors. All data races found will be printed to the standard + error output along with stack traces of suspected causes. For best results, the + sanitizer should be enabled at the very beginning of the program. + """ + cuda_sanitizer.enable() + + +cuda_sanitizer = CUDASanitizer() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c76c784f474fc69510dea240262770681a586def --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/_utils.py @@ -0,0 +1,529 @@ +import ctypes +import sys +from typing import Any, Optional, Union + +import torch + +# The _get_device_index has been moved to torch.utils._get_device_index +from torch._utils import _get_device_index as _torch_get_device_index + + +def _get_hip_runtime_library() -> ctypes.CDLL: + if sys.platform == "win32": + lib = ctypes.CDLL(f"amdhip64_{torch.version.hip[0]}.dll") + else: # Unix-based systems + lib = ctypes.CDLL("libamdhip64.so") + lib.cuGetErrorString = lib.hipGetErrorString # type: ignore[attr-defined] + lib.cuModuleLoadData = lib.hipModuleLoadData # type: ignore[attr-defined] + lib.cuModuleGetFunction = lib.hipModuleGetFunction # type: ignore[attr-defined] + lib.cuLaunchKernel = lib.hipModuleLaunchKernel # type: ignore[attr-defined] + lib.cuFuncSetAttribute = lib.hipFuncSetAttribute # type: ignore[attr-defined] + return lib + + +def _get_cuda_library() -> ctypes.CDLL: + if sys.platform == "win32": + return ctypes.CDLL("nvcuda.dll") + else: # Unix-based systems + return ctypes.CDLL("libcuda.so.1") + + +# Load GPU driver runtime +def _get_gpu_runtime_library() -> ctypes.CDLL: + if torch.version.hip: + return _get_hip_runtime_library() + else: + return _get_cuda_library() + + +# Helper: check CUDA errors +def _check_cuda(result: int) -> None: + if result == 0: + return + err_str = ctypes.c_char_p() + libcuda = _get_gpu_runtime_library() # Get reference to CUDA library + libcuda.cuGetErrorString(result, ctypes.byref(err_str)) + error_message = ( + err_str.value.decode() if err_str.value is not None else "Unknown CUDA error" + ) + raise RuntimeError(f"CUDA error: {error_message}") + + +def _get_hiprtc_library() -> ctypes.CDLL: + if sys.platform == "win32": + version_str = "".join(["0", torch.version.hip[0], "0", torch.version.hip[2]]) + lib = ctypes.CDLL(f"hiprtc{version_str}.dll") + else: + lib = ctypes.CDLL("libhiprtc.so") + + # Provide aliases for HIP RTC functions to match NVRTC API + lib.nvrtcGetErrorString = lib.hiprtcGetErrorString # type: ignore[attr-defined] + lib.nvrtcCreateProgram = lib.hiprtcCreateProgram # type: ignore[attr-defined] + lib.nvrtcDestroyProgram = lib.hiprtcDestroyProgram # type: ignore[attr-defined] + lib.nvrtcCompileProgram = lib.hiprtcCompileProgram # type: ignore[attr-defined] + lib.nvrtcGetPTXSize = lib.hiprtcGetCodeSize # type: ignore[attr-defined] + lib.nvrtcGetPTX = lib.hiprtcGetCode # type: ignore[attr-defined] + lib.nvrtcGetProgramLogSize = lib.hiprtcGetProgramLogSize # type: ignore[attr-defined] + lib.nvrtcGetProgramLog = lib.hiprtcGetProgramLog # type: ignore[attr-defined] + lib.nvrtcAddNameExpression = lib.hiprtcAddNameExpression # type: ignore[attr-defined] + lib.nvrtcGetLoweredName = lib.hiprtcGetLoweredName # type: ignore[attr-defined] + return lib + + +def _get_nvrtc_library() -> ctypes.CDLL: + major_version = int(torch.version.cuda.split(".")[0]) # type: ignore[union-attr] + if sys.platform == "win32": + nvrtc_libs = [ + f"nvrtc64_{major_version}0_0.dll", + ] + else: + nvrtc_libs = [ + f"libnvrtc.so.{major_version}", + "libnvrtc.so", # Fallback to unversioned + ] + for lib_name in nvrtc_libs: + try: + return ctypes.CDLL(lib_name) + except OSError: + continue + raise OSError("Could not find any NVRTC library") + + +def _get_gpu_rtc_library() -> ctypes.CDLL: + # Since PyTorch already loads the GPU RTC library, we can use the system library + # which should be compatible with PyTorch's version + if torch.version.hip: + return _get_hiprtc_library() + else: + return _get_nvrtc_library() + + +def _get_gpu_rtc_compatible_flags() -> list[str]: + """ + Get HIPCC/NVCC flags that are compatible with NVRTC compilation. + + Returns: + List of HIPCC/NVCC flags that can be safely used with NVRTC. + """ + from torch.utils.cpp_extension import COMMON_HIPCC_FLAGS, COMMON_NVCC_FLAGS + + nvrtc_unsupported_flags = { + "--expt-relaxed-constexpr", + } + + # Filter out unsupported flags + compatible_flags = [ + flag for flag in COMMON_NVCC_FLAGS if flag not in nvrtc_unsupported_flags + ] + + if torch.version.hip: + compatible_flags.extend(COMMON_HIPCC_FLAGS) + + return compatible_flags + + +def _nvrtc_compile( + kernel_source: str, + kernel_name: str, + compute_capability: Optional[str] = None, + cuda_include_dirs: Optional[list] = None, + nvcc_options: Optional[list] = None, + auto_pch: bool = False, +) -> tuple[bytes, str]: + """ + Compiles a CUDA kernel using NVRTC and returns the PTX code. + + Args: + kernel_source (str): The CUDA kernel source code as a string + kernel_name (str): The name of the kernel function to compile + compute_capability (str, None): The compute capability to target (e.g., "86"). + If None, will detect from current device. + cuda_include_dirs (list, None): List of directories containing CUDA headers + nvcc_options (list, None): Additional options to pass to NVRTC + auto_pch (bool): Enable automatic precompiled headers (CUDA 12.8+) + + Returns: + Tuple[bytes, str]: The compiled PTX code and mangled kernel name + """ + # Ensure CUDA is initialized + import torch.cuda + + # Load NVRTC library + libnvrtc = _get_gpu_rtc_library() + + # NVRTC constants + NVRTC_SUCCESS = 0 + + # Helper: check NVRTC errors + def check_nvrtc(result: int) -> None: + if result != NVRTC_SUCCESS: + err_str = ctypes.c_char_p() + libnvrtc.nvrtcGetErrorString(result, ctypes.byref(err_str)) + error_message = ( + err_str.value.decode() + if err_str.value is not None + else "Unknown CUDA error" + ) + raise RuntimeError(f"CUDA error: {error_message}") + + # Convert source to bytes + source_bytes = kernel_source.encode("utf-8") + + # Get compute capability if not provided + if compute_capability is None: + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + if torch.version.hip: + compute_capability = f"{props.gcnArchName}" + else: + compute_capability = f"{props.major}{props.minor}" + + # Prepare compilation options + options = [] + if torch.version.hip: + options.append(f"--offload-arch={compute_capability}".encode()) + else: + options.append(f"--gpu-architecture=sm_{compute_capability}".encode()) + + # Auto-detect and add CUDA include paths + from torch.utils.cpp_extension import include_paths + + cuda_include_paths = include_paths("cuda") + for cuda_path in cuda_include_paths: + options.append(f"-I{cuda_path}".encode()) + + # Add custom include directories + if cuda_include_dirs: + for directory in cuda_include_dirs: + options.append(f"-I{directory}".encode()) + + # Enable automatic precompiled headers (CUDA 12.8+) + if auto_pch: + assert str(torch.version.cuda) >= "12.8", "PCH requires CUDA 12.8+" + if nvcc_options is None: + nvcc_options = [] + nvcc_options.append("--pch") + + # Add custom NVCC options + if nvcc_options: + for option in nvcc_options: + options.append(option.encode("utf-8")) + + nvrtc_compatible_flags = _get_gpu_rtc_compatible_flags() + options.extend([flag.encode("utf-8") for flag in nvrtc_compatible_flags]) + + # Convert options to C array + num_options = len(options) + options_array = (ctypes.c_char_p * num_options)(*options) + + # Create program + prog = ctypes.c_void_p() + check_nvrtc( + libnvrtc.nvrtcCreateProgram( + ctypes.byref(prog), + source_bytes, + f"{kernel_name}.cu".encode(), + 0, + None, + None, + ) + ) + + # Add kernel name, which can be a template expression + c_kernel_name = kernel_name.encode("utf-8") + check_nvrtc(libnvrtc.nvrtcAddNameExpression(prog, c_kernel_name)) + + # Compile program + res = libnvrtc.nvrtcCompileProgram(prog, num_options, options_array) + + # Handle compilation errors + if res != NVRTC_SUCCESS: + # Get log + log_size = ctypes.c_size_t() + libnvrtc.nvrtcGetProgramLogSize(prog, ctypes.byref(log_size)) + log = ctypes.create_string_buffer(log_size.value) + libnvrtc.nvrtcGetProgramLog(prog, log) + raise RuntimeError(f"Kernel compilation failed:\n{log.value.decode()}") + + # Get PTX + ptx_size = ctypes.c_size_t() + check_nvrtc(libnvrtc.nvrtcGetPTXSize(prog, ctypes.byref(ptx_size))) + ptx = ctypes.create_string_buffer(ptx_size.value) + check_nvrtc(libnvrtc.nvrtcGetPTX(prog, ptx)) + + # Get mangled name + c_mangled_name = ctypes.c_char_p() + check_nvrtc( + libnvrtc.nvrtcGetLoweredName(prog, c_kernel_name, ctypes.byref(c_mangled_name)) + ) + if c_mangled_name.value is not None: + mangled_name = c_mangled_name.value.decode() # make a copy + else: + mangled_name = "" + + libnvrtc.nvrtcDestroyProgram(ctypes.byref(prog)) + + # For HIP, hipRTC generates raw CO binaries instead of PTX, + # and for some reason, ".value" causes the string to be truncated, + # likely due to the presence of '\0' in the string. So we use .raw instead. + ptx_bytes = ptx.raw if torch.version.hip else ptx.value + return ptx_bytes, mangled_name + + +class _CudaModule: + def __init__(self, module: ctypes.c_void_p) -> None: + self._module = module + self._kernels: dict[str, _CudaKernel] = {} + + def __getattr__(self, name: str) -> "_CudaKernel": + if name in self._kernels: + return self._kernels[name] + + # Import the CUDA library inside the method + # pyrefly: ignore [missing-module-attribute] + from torch.cuda._utils import _get_gpu_runtime_library + + libcuda = _get_gpu_runtime_library() + + func = ctypes.c_void_p() + try: + _check_cuda( + libcuda.cuModuleGetFunction( + ctypes.byref(func), self._module, name.encode("utf-8") + ) + ) + kernel = _CudaKernel(func, self._module) + self._kernels[name] = kernel + return kernel + + except RuntimeError as err: + raise AttributeError(f"No kernel named '{name}' in this module") from err + + +class _CudaKernel: + """ + Represents a compiled CUDA kernel that can be called with PyTorch tensors. + """ + + def __init__(self, func: ctypes.c_void_p, module: ctypes.c_void_p) -> None: + self.func = func + self.module = module + self._max_shared_mem_bytes = 0 + + def __call__( + self, + grid: tuple[int, int, int] = (1, 1, 1), + block: tuple[int, int, int] = (1, 1, 1), + args: Optional[list] = None, + shared_mem: int = 0, + stream: Optional[Any] = None, + ) -> None: + """ + Call the compiled CUDA kernel + + Args: + grid (tuple): Grid dimensions (grid_x, grid_y, grid_z) + block (tuple): Block dimensions (block_x, block_y, block_z) + args (list): List of arguments to pass to the kernel. + PyTorch tensor arguments will be automatically converted to pointers. + shared_mem (int): Shared memory size in bytes + stream (torch.cuda.Stream): CUDA stream to use. If None, uses current stream. + """ + import torch + + libcuda = torch.cuda._utils._get_gpu_runtime_library() + + if not args: + args = [] + + # Process arguments and convert tensors to pointers + processed_args: list[ctypes.c_void_p] = [] + c_args = [] + + for arg in args: + if isinstance(arg, torch.Tensor): + if not arg.is_cuda and not (arg.is_cpu and arg.is_pinned()): + raise ValueError( + "All tensor arguments must be CUDA tensors or pinned CPU tensors" + ) + # Get pointer to tensor data + ptr = ctypes.c_void_p(arg.data_ptr()) + processed_args.append(ptr) + c_args.append(ctypes.byref(ptr)) + elif isinstance(arg, int): + # Convert integers to C int + c_int = ctypes.c_int(arg) + # Store the C int for reference keeping, not in processed_args + c_args.append(ctypes.byref(c_int)) + elif isinstance(arg, float): + # Python floats are doubles - use double by default + c_double = ctypes.c_double(arg) + # Store the C double for reference keeping, not in processed_args + c_args.append(ctypes.byref(c_double)) + else: + raise TypeError(f"Unsupported argument type: {type(arg)}") + + # Convert to array of void pointers + c_args_array = (ctypes.c_void_p * len(c_args))() + for i, arg in enumerate(c_args): + c_args_array[i] = ctypes.cast(arg, ctypes.c_void_p) + + # Get the stream + if stream is None: + # Defer import to avoid circular imports + import torch.cuda + + stream = torch.cuda.current_stream() + + # Check if kernel requires large shared memory but hasn't been configured + if shared_mem >= 48 * 1024 and ( + self._max_shared_mem_bytes == 0 or shared_mem > self._max_shared_mem_bytes + ): + configured_msg = ( + "not configured" + if self._max_shared_mem_bytes == 0 + else f"only {self._max_shared_mem_bytes} bytes configured" + ) + raise RuntimeError( + f"Kernel requires {shared_mem} bytes of shared memory (>= 48KB), " + f"but {configured_msg}. " + "Call kernel.set_shared_memory_config(shared_mem) after compilation " + "and before launching the kernel." + ) + + _check_cuda( + libcuda.cuLaunchKernel( + self.func, + grid[0], + grid[1], + grid[2], + block[0], + block[1], + block[2], + shared_mem, + stream._as_parameter_, + c_args_array, + None, + ) + ) + + def set_shared_memory_config(self, shared_mem_bytes: int) -> None: + if shared_mem_bytes < 48 * 1024: + # No configuration needed for <= 48KB, just update the value + self._max_shared_mem_bytes = shared_mem_bytes + return + + libcuda = _get_gpu_runtime_library() + + # Get device properties to validate against limits + device_props = torch.cuda.get_device_properties() + # HIP doesn't have shared_memory_per_block_optin in device properties, so we hard-code it here + if torch.version.hip: + # navi, CDNA1-CDNA3 allows a max of 64KB shared memory + # CDNA4 allows a max of 160KB shared memory + max_shared_mem = ( + 65536 if device_props.gcnArchName != "gfx950" else 160 * 1024 + ) + else: + max_shared_mem = getattr( + device_props, "shared_memory_per_block_optin", 49152 + ) + + if shared_mem_bytes > max_shared_mem: + raise RuntimeError( + f"Requested shared memory ({shared_mem_bytes} bytes) exceeds " + f"device limit ({max_shared_mem} bytes). " + "Consider reducing block size or shared memory usage." + ) + + # Set the function attribute once + # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html + cudaFuncAttributeMaxDynamicSharedMemorySize = 8 + _check_cuda( + libcuda.cuFuncSetAttribute( + self.func, + cudaFuncAttributeMaxDynamicSharedMemorySize, + shared_mem_bytes, + ) + ) + + self._max_shared_mem_bytes = shared_mem_bytes + + +def _cuda_load_module( + ptx: Union[str, bytes], kernel_names: Optional[list[str]] = None +) -> Union[_CudaModule, dict[str, "_CudaKernel"]]: + """ + Loads a CUDA module from PTX code and returns a module object that can access kernels. + + Args: + ptx (bytes or str): The PTX code to load + kernel_names (list, optional): List of kernel names to extract from the module. + If None, will return a module object with __getattr__. + + Returns: + object: If kernel_names is None, returns a module object with __getattr__ to access kernels. + If kernel_names is provided, returns a dict mapping kernel names to _CudaKernel objects. + """ + # Ensure CUDA is initialized + import torch.cuda + + # Load CUDA driver library + libcuda = _get_gpu_runtime_library() + + # Convert PTX to bytes if it's a string + if isinstance(ptx, str): + ptx = ptx.encode("utf-8") + + # Load PTX module + module = ctypes.c_void_p() + # Get the current stream without directly importing torch.cuda at module level + stream = torch.cuda.current_stream() + with stream: + _check_cuda(libcuda.cuModuleLoadData(ctypes.byref(module), ptx)) + + if not kernel_names: + return _CudaModule(module) + + # Return specific kernels + kernels = {} + for name in kernel_names: + func = ctypes.c_void_p() + _check_cuda( + libcuda.cuModuleGetFunction( + ctypes.byref(func), module, name.encode("utf-8") + ) + ) + kernels[name] = _CudaKernel(func, module) + return kernels + + +def _get_device_index( + device: Any, optional: bool = False, allow_cpu: bool = False +) -> int: + r"""Get the device index from :attr:`device`, which can be a torch.device object, a Python integer, or ``None``. + + If :attr:`device` is a torch.device object, returns the device index if it + is a CUDA device. Note that for a CUDA device without a specified index, + i.e., ``torch.device('cuda')``, this will return the current default CUDA + device if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``, + CPU devices will be accepted and ``-1`` will be returned in this case. + + If :attr:`device` is a Python integer, it is returned as is. + + If :attr:`device` is ``None``, this will return the current default CUDA + device if :attr:`optional` is ``True``. + """ + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + if isinstance(device, torch.device): + if allow_cpu: + if device.type not in ["cuda", "cpu"]: + raise ValueError(f"Expected a cuda or cpu device, but got: {device}") + elif device.type != "cuda": + raise ValueError(f"Expected a cuda device, but got: {device}") + if not torch.jit.is_scripting(): + if isinstance(device, torch.cuda.device): + return device.idx + return _torch_get_device_index(device, optional, allow_cpu) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/comm.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..94b2e67b1d68ca7290c901151ee5decc4fd23aaa --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/comm.py @@ -0,0 +1,19 @@ +# The functions here have been moved to torch.nn.parallel.comm +from torch.nn.parallel.comm import ( + broadcast, + broadcast_coalesced, + gather, + reduce_add, + reduce_add_coalesced, + scatter, +) + + +__all__ = [ + "broadcast", + "broadcast_coalesced", + "reduce_add", + "reduce_add_coalesced", + "scatter", + "gather", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/gds.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/gds.py new file mode 100644 index 0000000000000000000000000000000000000000..16e566466bb7296be1675f018e340e3b04ae6fd7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/gds.py @@ -0,0 +1,167 @@ +import os +import sys +from collections.abc import Callable +from typing import Optional + +import torch +from torch.types import Storage + + +__all__: list[str] = [ + "gds_register_buffer", + "gds_deregister_buffer", + "GdsFile", +] + + +def _dummy_fn(name: str) -> Callable: + def fn(*args, **kwargs): # type: ignore[no-untyped-def] + raise RuntimeError(f"torch._C.{name} is not supported on this platform") + + return fn + + +if not hasattr(torch._C, "_gds_register_buffer"): + assert not hasattr(torch._C, "_gds_deregister_buffer") + assert not hasattr(torch._C, "_gds_register_handle") + assert not hasattr(torch._C, "_gds_deregister_handle") + assert not hasattr(torch._C, "_gds_load_storage") + assert not hasattr(torch._C, "_gds_save_storage") + # Define functions + torch._C.__dict__["_gds_register_buffer"] = _dummy_fn("_gds_register_buffer") + torch._C.__dict__["_gds_deregister_buffer"] = _dummy_fn("_gds_deregister_buffer") + torch._C.__dict__["_gds_register_handle"] = _dummy_fn("_gds_register_handle") + torch._C.__dict__["_gds_deregister_handle"] = _dummy_fn("_gds_deregister_handle") + torch._C.__dict__["_gds_load_storage"] = _dummy_fn("_gds_load_storage") + torch._C.__dict__["_gds_save_storage"] = _dummy_fn("_gds_save_storage") + + +def gds_register_buffer(s: Storage) -> None: + """Registers a storage on a CUDA device as a cufile buffer. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src = torch.randn(1024, device="cuda") + >>> s = src.untyped_storage() + >>> gds_register_buffer(s) + + Args: + s (Storage): Buffer to register. + """ + torch._C._gds_register_buffer(s) + + +def gds_deregister_buffer(s: Storage) -> None: + """Deregisters a previously registered storage on a CUDA device as a cufile buffer. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src = torch.randn(1024, device="cuda") + >>> s = src.untyped_storage() + >>> gds_register_buffer(s) + >>> gds_deregister_buffer(s) + + Args: + s (Storage): Buffer to register. + """ + torch._C._gds_deregister_buffer(s) + + +class GdsFile: + r"""Wrapper around cuFile. + + cuFile is a file-like interface to the GPUDirect Storage (GDS) API. + + See the `cufile docs `_ + for more details. + + Args: + filename (str): Name of the file to open. + flags (int): Flags to pass to ``os.open`` when opening the file. ``os.O_DIRECT`` will + be added automatically. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src1 = torch.randn(1024, device="cuda") + >>> src2 = torch.randn(2, 1024, device="cuda") + >>> file = torch.cuda.gds.GdsFile(f, os.O_CREAT | os.O_RDWR) + >>> file.save_storage(src1.untyped_storage(), offset=0) + >>> file.save_storage(src2.untyped_storage(), offset=src1.nbytes) + >>> dest1 = torch.empty(1024, device="cuda") + >>> dest2 = torch.empty(2, 1024, device="cuda") + >>> file.load_storage(dest1.untyped_storage(), offset=0) + >>> file.load_storage(dest2.untyped_storage(), offset=src1.nbytes) + >>> torch.equal(src1, dest1) + True + >>> torch.equal(src2, dest2) + True + + """ + + def __init__(self, filename: str, flags: int): + if sys.platform == "win32": + raise RuntimeError("GdsFile is not supported on this platform.") + self.filename = filename + self.flags = flags + self.fd = os.open(filename, flags | os.O_DIRECT) # type: ignore[attr-defined] + self.handle: Optional[int] = None + self.register_handle() + + def __del__(self) -> None: + if self.handle is not None: + self.deregister_handle() + os.close(self.fd) + + def register_handle(self) -> None: + """Registers file descriptor to cuFile Driver. + + This is a wrapper around ``cuFileHandleRegister``. + """ + assert self.handle is None, ( + "Cannot register a handle that is already registered." + ) + self.handle = torch._C._gds_register_handle(self.fd) + + def deregister_handle(self) -> None: + """Deregisters file descriptor from cuFile Driver. + + This is a wrapper around ``cuFileHandleDeregister``. + """ + assert self.handle is not None, ( + "Cannot deregister a handle that is not registered." + ) + torch._C._gds_deregister_handle(self.handle) + self.handle = None + + def load_storage(self, storage: Storage, offset: int = 0) -> None: + """Loads data from the file into the storage. + + This is a wrapper around ``cuFileRead``. ``storage.nbytes()`` of data + will be loaded from the file at ``offset`` into the storage. + + Args: + storage (Storage): Storage to load data into. + offset (int, optional): Offset into the file to start loading from. (Default: 0) + """ + assert self.handle is not None, ( + "Cannot load data from a file that is not registered." + ) + torch._C._gds_load_storage(self.handle, storage, offset) + + def save_storage(self, storage: Storage, offset: int = 0) -> None: + """Saves data from the storage into the file. + + This is a wrapper around ``cuFileWrite``. All bytes of the storage + will be written to the file at ``offset``. + + Args: + storage (Storage): Storage to save data from. + offset (int, optional): Offset into the file to start saving to. (Default: 0) + """ + assert self.handle is not None, ( + "Cannot save data to a file that is not registered." + ) + torch._C._gds_save_storage(self.handle, storage, offset) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/graphs.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/graphs.py new file mode 100644 index 0000000000000000000000000000000000000000..8a6e26281ca08d372d4fff35ea3784720478e508 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/graphs.py @@ -0,0 +1,613 @@ +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +import gc +import typing +from collections.abc import Callable +from typing import Optional, overload, TYPE_CHECKING, TypeAlias, Union +from typing_extensions import ParamSpec, Self, TypeVar + +import torch +from torch import Tensor + + +if TYPE_CHECKING: + # importing _POOL_HANDLE at runtime toplevel causes an import cycle + from torch.cuda import _POOL_HANDLE + +from .._utils import _dummy_type + + +__all__ = [ + "is_current_stream_capturing", + "graph_pool_handle", + "CUDAGraph", + "graph", + "make_graphed_callables", +] + + +_R = TypeVar("_R") +_P = ParamSpec("_P") + + +if not hasattr(torch._C, "_CudaStreamBase"): + # Define dummy base classes + torch._C.__dict__["_CUDAGraph"] = _dummy_type("_CUDAGraph") + torch._C.__dict__["_graph_pool_handle"] = _dummy_type("_graph_pool_handle") + torch._C.__dict__["_cuda_isCurrentStreamCapturing"] = _dummy_type( + "_cuda_isCurrentStreamCapturing" + ) + +from torch._C import ( # noqa: F401 + _cuda_isCurrentStreamCapturing, + _CUDAGraph, + _graph_pool_handle, +) + + +def is_current_stream_capturing() -> bool: + r"""Return True if CUDA graph capture is underway on the current CUDA stream, False otherwise. + + If a CUDA context does not exist on the current device, returns False without initializing the context. + """ + return _cuda_isCurrentStreamCapturing() + + +# Python shim helps Sphinx process docstrings more reliably. +def graph_pool_handle() -> _POOL_HANDLE: + r"""Return an opaque token representing the id of a graph memory pool. + + See :ref:`Graph memory management`. + + .. warning:: + This API is in beta and may change in future releases. + """ + return torch.cuda._POOL_HANDLE(_graph_pool_handle()) + + +# Python shim helps Sphinx process docstrings more reliably. +class CUDAGraph(torch._C._CUDAGraph): + r"""Wrapper around a CUDA graph. + + Arguments: + keep_graph (bool, optional): If ``keep_graph=False``, the + cudaGraphExec_t will be instantiated on GPU at the end of + ``capture_end`` and the underlying cudaGraph_t will be + destroyed. Users who want to query or otherwise modify the + underlying cudaGraph_t before instantiation can set + ``keep_graph=True`` and access it via ``raw_cuda_graph`` after + ``capture_end``. Note that the cudaGraphExec_t will not be + instantiated at the end of ``capture_end`` in this + case. Instead, it will be instantiated via an explicit called + to ``instantiate`` or automatically on the first call to + ``replay`` if ``instantiate`` was not already called. Calling + ``instantiate`` manually before ``replay`` is recommended to + prevent increased latency on the first call to ``replay``. It + is allowed to modify the raw cudaGraph_t after first calling + ``instantiate``, but the user must call ``instantiate`` again + manually to make sure the instantiated graph has these + changes. Pytorch has no means of tracking these changes. + + .. warning:: + This API is in beta and may change in future releases. + + """ + + def __new__(cls, keep_graph: bool = False) -> Self: + return super().__new__(cls, keep_graph) + + def capture_begin( + self, pool: Optional[_POOL_HANDLE] = None, capture_error_mode: str = "global" + ) -> None: + r"""Begin capturing CUDA work on the current stream. + + Typically, you shouldn't call ``capture_begin`` yourself. + Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`, + which call ``capture_begin`` internally. + + Arguments: + pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or + :meth:`other_Graph_instance.pool()`) that hints this graph may share memory + with the indicated pool. See :ref:`Graph memory management`. + capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream. + Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc, + may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for + actions in the current thread, and "relaxed" will not error on these actions. Do NOT change this setting + unless you're familiar with `cudaStreamCaptureMode `_ + """ # noqa: B950 + super().capture_begin(pool=pool, capture_error_mode=capture_error_mode) + + def capture_end(self) -> None: + r"""End CUDA graph capture on the current stream. + + After ``capture_end``, ``replay`` may be called on this instance. + + Typically, you shouldn't call ``capture_end`` yourself. + Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`, + which call ``capture_end`` internally. + """ + super().capture_end() + + def instantiate(self) -> None: + r"""Instantiate the CUDA graph. Will be called by + ``capture_end`` if ``keep_graph=False``, or by ``replay`` if + ``keep_graph=True`` and ``instantiate`` has not already been + explicitly called. Does not destroy the cudaGraph_t returned + by ``raw_cuda_graph``. + """ + super().instantiate() + + def replay(self) -> None: + r"""Replay the CUDA work captured by this graph.""" + super().replay() + + def reset(self) -> None: + r"""Delete the graph currently held by this instance.""" + super().reset() + + def pool(self) -> _POOL_HANDLE: + r"""Return an opaque token representing the id of this graph's memory pool. + + This id can optionally be passed to another graph's ``capture_begin``, + which hints the other graph may share the same memory pool. + """ + return super().pool() + + def enable_debug_mode(self) -> None: + r"""Enable debugging mode for CUDAGraph.debug_dump.""" + return super().enable_debug_mode() + + def debug_dump(self, debug_path: str) -> None: + r""" + Arguments: + debug_path (required): Path to dump the graph to. + + Calls a debugging function to dump the graph if the debugging is + enabled via CUDAGraph.enable_debug_mode() + """ + return super().debug_dump(debug_path) + + def raw_cuda_graph(self) -> int: + r"""Returns the underlying cudaGraph_t. ``keep_graph`` must be True. + + See the following for APIs for how to manipulate this object: `Graph Managmement `_ and `cuda-python Graph Management bindings `_ + """ # noqa: B950 + return super().raw_cuda_graph() + + def raw_cuda_graph_exec(self) -> int: + r"""Returns the underlying cudaGraphExec_t. ``instantiate`` must have been called if ``keep_graph`` is True, or ``capture_end`` must have been called if ``keep_graph`` is False. If you call ``instantiate()`` after ``raw_cuda_graph_exec()``, the previously returned cudaGraphExec_t will be destroyed. It is your responsibility not to use this object after destruction. + + See the following for APIs for how to manipulate this object: `Graph Execution `_ and `cuda-python Graph Execution bindings `_ + """ # noqa: B950 + return super().raw_cuda_graph_exec() + + +class graph: + r"""Context-manager that captures CUDA work into a :class:`torch.cuda.CUDAGraph` object for later replay. + + See :ref:`CUDA Graphs ` for a general introduction, + detailed use, and constraints. + + Arguments: + cuda_graph (torch.cuda.CUDAGraph): Graph object used for capture. + pool (optional): Opaque token (returned by a call to :func:`~torch.cuda.graph_pool_handle()` or + :meth:`other_Graph_instance.pool()`) hinting this graph's capture + may share memory from the specified pool. See :ref:`Graph memory management`. + stream (torch.cuda.Stream, optional): If supplied, will be set as the current stream in the context. + If not supplied, ``graph`` sets its own internal side stream as the current stream in the context. + capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream. + Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc, + may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for + actions in the current thread, and "relaxed" will not error on actions. Do NOT change this setting + unless you're familiar with `cudaStreamCaptureMode `_ + + .. note:: + For effective memory sharing, if you pass a ``pool`` used by a previous capture and the previous capture + used an explicit ``stream`` argument, you should pass the same ``stream`` argument to this capture. + + .. warning:: + This API is in beta and may change in future releases. + + .. _cudaStreamCaptureMode: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g9d0535d93a214cbf126835257b16ba85 + """ # noqa: B950 + + default_capture_stream: Optional[torch.cuda.Stream] = None + + def __init__( + self, + cuda_graph: CUDAGraph, + pool: Optional[_POOL_HANDLE] = None, + stream: Optional[torch.cuda.Stream] = None, + capture_error_mode: str = "global", + ): + # Lazy-init of default_capture_stream helps avoid circular-import errors. + # Not thread safe, but graphs already have the general (explicitly documented) + # restriction that only one capture may be underway at a time in the process. + if self.__class__.default_capture_stream is None: + self.__class__.default_capture_stream = torch.cuda.Stream() + + self.pool: Union[tuple[()], tuple[_POOL_HANDLE]] = ( + () if pool is None else (pool,) + ) + self.capture_stream = ( + stream if stream is not None else self.__class__.default_capture_stream + ) + assert self.capture_stream is not None + self.stream_ctx = torch.cuda.stream(self.capture_stream) + self.cuda_graph = cuda_graph + self.capture_error_mode = capture_error_mode + + def __enter__(self) -> None: + # Free as much memory as we can for the graph + torch.cuda.synchronize() + + if torch.compiler.config.force_cudagraph_gc: + # Originally we unconditionally garbage collected here. On one hand + # that's nice because we have a chance to collect more memory, but + # on the other hand it is REALLY expensive, especially for doing + # multiple cudagraph captures in a row. In theory it will only help + # when a dead python cycle is holding onto CUDA memory. + gc.collect() + + torch.cuda.empty_cache() + + # Stackoverflow seems comfortable with this pattern + # https://stackoverflow.com/questions/26635684/calling-enter-and-exit-manually#39172487 + self.stream_ctx.__enter__() + + self.cuda_graph.capture_begin( + # type: ignore[misc] + *self.pool, + # pyrefly: ignore [bad-keyword-argument] + capture_error_mode=self.capture_error_mode, + ) + + def __exit__(self, *args: object) -> None: + self.cuda_graph.capture_end() + self.stream_ctx.__exit__(*args) + # returning None should propagate exceptions from either capture_end or stream_ctx.__exit__() + + +_ModuleOrCallable: TypeAlias = Union["torch.nn.Module", Callable[..., object]] + + +@overload +def make_graphed_callables( + callables: _ModuleOrCallable, + sample_args: tuple[Tensor, ...], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> _ModuleOrCallable: ... + + +@overload +def make_graphed_callables( + callables: tuple[_ModuleOrCallable, ...], + sample_args: tuple[tuple[Tensor, ...], ...], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> tuple[_ModuleOrCallable, ...]: ... + + +def make_graphed_callables( + callables: Union[_ModuleOrCallable, tuple[_ModuleOrCallable, ...]], + sample_args: Union[tuple[Tensor, ...], tuple[tuple[Tensor, ...], ...]], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> Union[_ModuleOrCallable, tuple[_ModuleOrCallable, ...]]: + r"""Accept callables (functions or :class:`nn.Module`\ s) and returns graphed versions. + + Each graphed callable's forward pass runs its source callable's + forward CUDA work as a CUDA graph inside a single autograd node. + + The graphed callable's forward pass also appends + a backward node to the autograd graph. During backward, this node runs the + callable's backward work as a CUDA graph. + + Therefore, each graphed callable should be a drop-in replacement for its source callable + in an autograd-enabled training loop. + + See :ref:`Partial-network capture` for detailed use and constraints. + + If you pass a tuple of several callables, their captures will use the same memory pool. + See :ref:`Graph memory management` for when this is appropriate. + + Arguments: + callables (torch.nn.Module or Python function, or tuple of these): Callable or callables to graph. + See :ref:`Graph memory management` for when passing a tuple of callables + is appropriate. If you pass a tuple of callables, their order in the tuple must be the same order + they'll run in the live workload. + sample_args (tuple of Tensors, or tuple of tuples of Tensors): Samples args for each callable. + If a single callable was passed, ``sample_args`` must be a single tuple of argument Tensors. + If a tuple of callables was passed, ``sample_args`` must be tuple of tuples of argument Tensors. + num_warmup_iters (int): The number of warmup iterations. Currently, ``DataDistributedParallel`` needs + 11 iterations for warm up. Default: ``3``. + allow_unused_input (bool): If False, specifying inputs that were not used when computing outputs + (and therefore their grad is always zero) is an error. Defaults to False. + pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or + :meth:`other_Graph_instance.pool()`) that hints this graph may share memory + with the indicated pool. See :ref:`Graph memory management`. + .. note:: + The ``requires_grad`` state of each Tensor in ``sample_args`` must match the state + that's expected for the corresponding real input in the training loop. + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + ``sample_args`` for each callable must contain only Tensors. Other types are not allowed. + + .. warning:: + Returned callables do not support higher order differentiation (e.g., double backward). + + .. warning:: + In any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters + may be trainable. Buffers must have ``requires_grad=False``. + + .. warning:: + After you pass a :class:`torch.nn.Module` through :func:`~make_graphed_callables`, + you may not add or remove any of that Module's parameters or buffers. + + .. warning:: + :class:`torch.nn.Module`\s passed to :func:`~torch.cuda.make_graphed_callables` must not have module hooks + registered on them at the time they are passed. However, registering hooks on modules *after* passing them + through :func:`~torch.cuda.make_graphed_callables` is allowed. + + .. warning:: + When running a graphed callable, you must pass its arguments in the same order and format + they appeared in that callable's ``sample_args``. + + .. warning:: + The automatic mixed precision is supported in :func:`~torch.cuda.make_graphed_callables` only with disabled + caching. The context manager `torch.cuda.amp.autocast()` must have `cache_enabled=False`. + """ + if torch.is_autocast_enabled() and torch.is_autocast_cache_enabled(): + raise RuntimeError( + "make_graphed_callables does not support the autocast caching. Please set `cache_enabled=False`." + ) + + just_one_callable = False + + _sample_args: tuple[tuple[Tensor, ...], ...] + if not isinstance(callables, tuple): + just_one_callable = True + callables = (callables,) + _sample_args = (typing.cast(tuple[Tensor, ...], sample_args),) + else: + _sample_args = typing.cast(tuple[tuple[Tensor, ...], ...], sample_args) + + flatten_sample_args = [] + + for c, args in zip(callables, _sample_args): + if isinstance(c, torch.nn.Module): + assert ( + len(c._backward_hooks) == 0 + and len(c._forward_hooks) == 0 + and len(c._forward_pre_hooks) == 0 + ), ( + "Modules must not have hooks registered at the time they are passed. However, registering hooks " + + "on modules after passing them through make_graphed_callables is allowed." + ) + assert all(b.requires_grad is False for b in c.buffers()), ( + "In any :class:`~torch.nn.Module` passed to " + + ":func:`~make_graphed_callables`, only parameters may be trainable. All buffers must have " + + "``requires_grad=False``." + ) + flatten_arg = torch.utils._pytree.arg_tree_leaves(*args) + flatten_sample_args.append(tuple(flatten_arg)) + assert all(isinstance(arg, torch.Tensor) for arg in flatten_arg), ( + "In the beta API, sample_args " + + "for each callable must contain only Tensors. Other types are not allowed." + ) + + # If a callable is an nn.Module, its graph's full input surface is the args the user explicitly + # passes to forward (ie, its sample_args) AND the module's parameter attributes. + per_callable_len_user_args = [len(args) for args in flatten_sample_args] + per_callable_module_params = [ + tuple(c.parameters()) if isinstance(c, torch.nn.Module) else () + for c in callables + ] + per_callable_static_input_surfaces = [ + flatten_sample_args[i] + per_callable_module_params[i] + for i in range(len(callables)) + ] + + fwd_graphs = [torch.cuda.CUDAGraph() for _ in range(len(callables))] + bwd_graphs = [torch.cuda.CUDAGraph() for _ in range(len(callables))] + + mempool = graph_pool_handle() if pool is None else pool + + # Warmup + # Hopefully prevents cudnn benchmarking and other lazy-initialization cuda work + # from ending up in any captures. + torch.cuda.synchronize() + with torch.cuda.stream(torch.cuda.Stream()): + for func, args, static_input_surface in zip( + callables, _sample_args, per_callable_static_input_surfaces + ): + grad_inputs, outputs, outputs_grad = None, None, None + for _ in range(num_warmup_iters): + outputs = torch.utils._pytree.tree_leaves(func(*args)) + outputs_grad = tuple(o for o in outputs if o.requires_grad) + if len(outputs_grad) > 0: + grad_inputs = torch.autograd.grad( + outputs=outputs_grad, + inputs=tuple( + i for i in static_input_surface if i.requires_grad + ), + grad_outputs=tuple( + torch.empty_like(o) for o in outputs if o.requires_grad + ), + only_inputs=True, + allow_unused=allow_unused_input, + ) + for v in [outputs, outputs_grad, grad_inputs]: + del v + + torch.cuda.synchronize() + + # All captures here share a mempool. To avoid replays corrupting each other's memory, + # the safest approach is to capture all passes in the same order they'll run: + # fwd 1, fwd 2, ... fwd N, then bwd N, bwd N-1, ... bwd 1. + + # Capture forward graphs + per_callable_static_outputs = [] + per_callable_output_unflatten_spec = [] + for func, args, fwd_graph in zip(callables, _sample_args, fwd_graphs): + with torch.cuda.graph(fwd_graph, pool=mempool): + func_outputs = func(*args) + + flatten_outputs, spec = torch.utils._pytree.tree_flatten(func_outputs) + per_callable_static_outputs.append(tuple(flatten_outputs)) + per_callable_output_unflatten_spec.append(spec) + + # Capture backward graphs in reverse order + per_callable_static_grad_outputs = [] + per_callable_static_grad_inputs = [] + for static_input_surface, static_outputs, bwd_graph in zip( + reversed(per_callable_static_input_surfaces), + reversed(per_callable_static_outputs), + reversed(bwd_graphs), + ): + # For now, assumes all static_outputs require grad + # assert all(o.requires_grad for o in static_outputs), "Outputs of graphed callables must require grad." + static_grad_outputs = tuple( + torch.empty_like(o) if o.requires_grad else None for o in static_outputs + ) + + outputs_grad = tuple(o for o in static_outputs if o.requires_grad) + grad_inputs = None + if len(outputs_grad) > 0: + with torch.cuda.graph(bwd_graph, pool=mempool): + grad_inputs = torch.autograd.grad( + outputs=outputs_grad, + inputs=tuple(i for i in static_input_surface if i.requires_grad), + grad_outputs=tuple(o for o in static_grad_outputs if o is not None), + only_inputs=True, + allow_unused=allow_unused_input, + ) + + # Constructs a tuple suitable for returning from Graphed.backward: + # Pads out the actually-needed grads with Nones in gradient slots for inputs that don't require grad. + # I couldn't think of a slick one-liner for this pattern. + static_grad_inputs = [] + grad_idx = 0 + for arg in static_input_surface: + if arg.requires_grad and grad_inputs is not None: + static_grad_inputs.append(grad_inputs[grad_idx]) + grad_idx += 1 + else: + static_grad_inputs.append(None) # type: ignore[arg-type] + static_grad_inputs = tuple(static_grad_inputs) # type: ignore[assignment] + + per_callable_static_grad_outputs.append(static_grad_outputs) + per_callable_static_grad_inputs.append(static_grad_inputs) + + # Reverses the most recent two lists + per_callable_static_grad_outputs.reverse() + per_callable_static_grad_inputs.reverse() + # Now for every per_callable list, per_callable_*[i] holds the stuff for the ith callable. + + def make_graphed_autograd_function( + fwd_graph: CUDAGraph, + bwd_graph: CUDAGraph, + module_params: tuple[torch.nn.Parameter, ...], + len_user_args: int, + output_unflatten_spec: torch.utils._pytree.TreeSpec, + static_input_surface: tuple[Tensor, ...], + static_outputs: tuple[Tensor, ...], + static_grad_outputs: tuple[Optional[Tensor], ...], + static_grad_inputs: tuple[Tensor, ...], + ) -> Callable[..., object]: + class Graphed(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx: object, *inputs: Tensor) -> tuple[Tensor, ...]: + # At this stage, only the user args may (potentially) be new tensors. + for i in range(len_user_args): + if static_input_surface[i].data_ptr() != inputs[i].data_ptr(): + static_input_surface[i].copy_(inputs[i]) + fwd_graph.replay() + assert isinstance(static_outputs, tuple) + return tuple(o.detach() for o in static_outputs) + + @staticmethod + @torch.autograd.function.once_differentiable + # pyrefly: ignore [bad-override] + def backward(ctx: object, *grads: Tensor) -> tuple[Tensor, ...]: + assert len(grads) == len(static_grad_outputs) + for g, grad in zip(static_grad_outputs, grads): + if g is not None: + # don't copy if autograd gods have been kind and the + # incoming grad is already in the right place + if g.data_ptr() != grad.data_ptr(): + g.copy_(grad) + bwd_graph.replay() + + # Input args that didn't require grad expect a None gradient. + assert isinstance(static_grad_inputs, tuple) + return tuple( + # pyrefly: ignore [bad-argument-type] + b.detach() if b is not None else b + for b in static_grad_inputs + ) + + def functionalized(*user_args: object) -> object: + # Runs the autograd function with inputs == all inputs to the graph that might require grad + # (explicit user args + module parameters) + # Assumes module params didn't change since capture. + flatten_user_args = torch.utils._pytree.arg_tree_leaves(*user_args) + out = Graphed.apply(*(tuple(flatten_user_args) + module_params)) + return torch.utils._pytree.tree_unflatten(out, output_unflatten_spec) + + return functionalized + + # Put together the final graphed callables + ret: list[_ModuleOrCallable] = [] + for i, func in enumerate(callables): + graphed = make_graphed_autograd_function( + fwd_graphs[i], + bwd_graphs[i], + per_callable_module_params[i], + per_callable_len_user_args[i], + per_callable_output_unflatten_spec[i], + per_callable_static_input_surfaces[i], + per_callable_static_outputs[i], + per_callable_static_grad_outputs[i], + per_callable_static_grad_inputs[i], + ) + + if isinstance(func, torch.nn.Module): + + def make_graphed_forward( + func: torch.nn.Module, + graph_training_state: bool, + graphed: Callable[_P, _R], + orig_fwd: Callable[_P, _R], + ) -> Callable[_P, _R]: + def new_fwd(*user_args: _P.args, **user_kwargs: _P.kwargs) -> _R: + # If the module's training-or-eval state matches what we graphed, + # run the graph, otherwise run the original forward method + if func.training == graph_training_state: + return graphed(*user_args, **user_kwargs) + else: + return orig_fwd(*user_args, **user_kwargs) + + return new_fwd + + func.forward = make_graphed_forward( + func, func.training, graphed, func.forward + ) + ret.append(func) + else: + ret.append(graphed) + + if just_one_callable: + return ret[0] + + return tuple(ret) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/green_contexts.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/green_contexts.py new file mode 100644 index 0000000000000000000000000000000000000000..0c9a9c857cd3ac982f9a86e1577fbaea06a99513 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/green_contexts.py @@ -0,0 +1,43 @@ +import torch + + +_GreenContext = object +SUPPORTED = False + +if hasattr(torch._C, "_CUDAGreenContext"): + _GreenContext = torch._C._CUDAGreenContext # type: ignore[misc] + SUPPORTED = True + + +# Python shim helps Sphinx process docstrings more reliably. +# pyrefly: ignore [invalid-inheritance] +class GreenContext(_GreenContext): + r"""Wrapper around a CUDA green context. + + .. warning:: + This API is in beta and may change in future releases. + """ + + @staticmethod + def create(num_sms: int, device_id: int = 0) -> _GreenContext: + r"""Create a CUDA green context. + + Arguments: + num_sms (int): The number of SMs to use in the green context. + device_id (int, optional): The device index of green context. + """ + if not SUPPORTED: + raise RuntimeError("PyTorch was not built with Green Context support!") + return _GreenContext.create(num_sms, device_id) # type: ignore[attr-defined] + + # Note that these functions are bypassed by we define them here + # for Sphinx documentation purposes + def set_context(self) -> None: # pylint: disable=useless-parent-delegation + r"""Make the green context the current context.""" + return super().set_context() # type: ignore[misc] + + def pop_context(self) -> None: # pylint: disable=useless-parent-delegation + r"""Assuming the green context is the current context, pop it from the + context stack and restore the previous context. + """ + return super().pop_context() # type: ignore[misc] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/jiterator.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/jiterator.py new file mode 100644 index 0000000000000000000000000000000000000000..09192622cce10e7bcdd8c89ac784b1f35eb85402 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/jiterator.py @@ -0,0 +1,189 @@ +# mypy: allow-untyped-defs +import re +from collections.abc import Callable + +import torch +from torch import Tensor + + +__all__: list[str] = [] + + +class _CodeParser: + def __init__(self, code_string: str): + optional_ws = r"\s*" + required_ws = r"\s+" + template_params = r"(?P\<.+\>)" + return_type = r"(?P\w+)" + function_name = r"(?P\w+)" + function_params = r"(?P\(.+\))" + function_body = r"(?P\{.+\})" + + pattern = ( + optional_ws + + "template" + + optional_ws + + template_params + + optional_ws + + return_type + + required_ws + + function_name + + optional_ws + + function_params + + optional_ws + + function_body + + optional_ws + ) + + result = re.match( + pattern, code_string, re.DOTALL + ) # DOTALL for matching multiline + + if result is None: + raise Exception( # noqa: TRY002 + f"Couldn't parse code, please check correctness:\n {code_string}" + ) + + self.template_params = result["template_params"] + self.return_type = result["return_type"] + self.function_name = result["function_name"] + self.function_params = result["function_params"] + self.function_body = result["function_body"] + + +class _JittedFunction: + def __init__( + self, code_string: str, return_by_ref: bool, num_outputs: int, **kwargs + ): + self.code_string = code_string + + assert return_by_ref or num_outputs == 1, ( + "Return by value only works for single output. " + ) + self.return_by_ref = return_by_ref + self.num_outputs = num_outputs + + parsed_code = _CodeParser(code_string) + self.kernel_name = parsed_code.function_name + + self.kwargs_dict = kwargs + self.is_cuda_available = torch.cuda.is_available() + + def __call__(self, *tensors: Tensor, **kwargs): + # Jiterator follow torch.cuda's lazy initialization behavior + # Defer checking cuda's availability at the function invocation time + assert self.is_cuda_available, ( + "Jiterator is only supported on CUDA and ROCm GPUs, none are available." + ) + + assert len(tensors) <= 8, "jiterator only supports up to 8 tensor inputs." + + expanded_kwargs = self.kwargs_dict.copy() + for key, value in kwargs.items(): + if key in self.kwargs_dict: + expanded_kwargs[key] = value + else: + raise KeyError(f"{key} is not declared in function definition") + + return torch._C._cuda_jiterator_compile_and_launch_kernel( + self.code_string, + self.kernel_name, + self.return_by_ref, + self.num_outputs, + tensors, + expanded_kwargs, + ) + + +def _create_jit_fn(code_string: str, **kwargs) -> Callable: + """ + Create a jiterator-generated cuda kernel for an elementwise op. + + The code string has to be a valid CUDA function that describes the computation for a single element. The code + string has to follow the c++ template pattern, as shown in the example below. This function will be inlined + into elementwise kernel template, and compiled on the fly. Compiled kernel will be cached in memory, as well as + local temp dir. + + Jiterator-generated kernels accepts noncontiguous tensors, and supports broadcasting and type promotion. + + Args: + code_string (str): CUDA code string to be compiled by jiterator. The entry functor must return by value. + kwargs (Dict, optional): Keyword arguments for generated function + + Example:: + + code_string = "template T my_kernel(T x, T y, T alpha) { return -x + alpha * y; }" + jitted_fn = create_jit_fn(code_string, alpha=1.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b, alpha=3.14) + + code_string also allows multiple function definitions, and the last function will be treated as the entry function. + + Example:: + + code_string = ( + "template T util_fn(T x, T y) { return ::sin(x) + ::cos(y); }" + ) + code_string += "template T my_kernel(T x, T y, T val) { return ::min(val, util_fn(x, y)); }" + jitted_fn = create_jit_fn(code_string, val=0.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b) # using default val=0.0 + + Jiterator can be used together with python registration to override an operator's cuda kernel. + Following example is overriding gelu's cuda kernel with relu. + + Example:: + + code_string = "template T my_gelu(T a) { return a > 0 ? a : 0; }" + my_gelu = create_jit_fn(code_string) + my_lib = torch.library.Library("aten", "IMPL") + my_lib.impl("aten::gelu", my_gelu, "CUDA") + # torch.nn.GELU and torch.nn.function.gelu are now overridden + a = torch.rand(3, device="cuda") + torch.allclose(torch.nn.functional.gelu(a), torch.nn.functional.relu(a)) + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + This API only supports up to 8 inputs and 1 output + + .. warning:: + All input tensors must live in CUDA device + """ + return _JittedFunction(code_string, return_by_ref=False, num_outputs=1, **kwargs) + + +def _create_multi_output_jit_fn( + code_string: str, num_outputs: int, **kwargs +) -> Callable: + """ + Create a jiterator-generated cuda kernel for an elementwise op that supports returning one or more outputs. + + Args: + code_string (str): CUDA code string to be compiled by jiterator. The entry functor must return value by reference. + num_outputs(int): number of outputs return by the kernel + kwargs (Dict, optional): Keyword arguments for generated function + + Example:: + + code_string = "template void my_kernel(T x, T y, T alpha, T& out) { out = -x + alpha * y; }" + jitted_fn = create_jit_fn(code_string, alpha=1.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b, alpha=3.14) + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + This API only supports up to 8 inputs and 8 outputs + """ + return _JittedFunction( + code_string, return_by_ref=True, num_outputs=num_outputs, **kwargs + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/memory.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..0b7ace97ba6b302b935eff3c01ae28cde6199db6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/memory.py @@ -0,0 +1,1463 @@ +# mypy: allow-untyped-defs +r"""This package adds support for device memory management implemented in CUDA.""" + +import collections +import contextlib +import ctypes +import os +import pickle +import re +import sys +import warnings +from inspect import signature +from typing import Any, cast, Literal, Optional, TYPE_CHECKING, TypedDict +from typing_extensions import deprecated, NotRequired + +import torch +from torch import _C +from torch._utils import _dummy_type + +from . import ( + _get_amdsmi_device_index, + _get_device_index, + _get_nvml_device_index, + _lazy_init, + is_initialized, +) +from ._memory_viz import memory as _memory, segments as _segments + + +if TYPE_CHECKING: + from torch.types import Device + + +# Type definitions for memory profiler +class _Frame(TypedDict): + """Frame information from memory profiler snapshots.""" + + filename: str + line: int + name: str + # Fields added by FX augmentation (optional) + fx_node_op: NotRequired[str] + fx_node_name: NotRequired[str] + fx_node_target: NotRequired[str] + fx_original_trace: NotRequired[str] + + +class _Block(TypedDict): + """Memory block information.""" + + size: int + requested_size: int + address: int + state: str + frames: list[_Frame] + + +class _Segment(TypedDict): + """Memory segment information.""" + + address: int + total_size: int + stream: int + segment_type: str + allocated_size: int + active_size: int + blocks: list[_Block] + + +class _TraceEntry(TypedDict): + """Memory trace entry information.""" + + action: str + addr: NotRequired[int] + frames: list[_Frame] + size: int + stream: int + device_free: NotRequired[int] + + +class _Snapshot(TypedDict): + """Memory snapshot structure.""" + + segments: list[_Segment] + device_traces: NotRequired[list[list[_TraceEntry]]] + + +__all__ = [ + "caching_allocator_alloc", + "caching_allocator_delete", + "caching_allocator_enable", + "get_per_process_memory_fraction", + "set_per_process_memory_fraction", + "empty_cache", + "memory_stats", + "memory_stats_as_nested_dict", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", + "reset_max_memory_allocated", + "reset_max_memory_cached", + "host_memory_stats", + "host_memory_stats_as_nested_dict", + "reset_accumulated_host_memory_stats", + "reset_peak_host_memory_stats", + "memory_allocated", + "max_memory_allocated", + "memory_reserved", + "max_memory_reserved", + "memory_cached", + "max_memory_cached", + "memory_snapshot", + "memory_summary", + "list_gpu_processes", + "mem_get_info", + "get_allocator_backend", + "CUDAPluggableAllocator", + "change_current_allocator", + "MemPool", + "use_mem_pool", +] + + +if not hasattr(torch._C, "_cuda_CUDAAllocator"): + # Define dummy base classes + torch._C.__dict__["_cuda_CUDAAllocator"] = _dummy_type("_cuda_CUDAAllocator") + + +if not hasattr(torch._C, "_MemPool"): + # Define dummy base classes + torch._C.__dict__["_MemPool"] = _dummy_type("_MemPool") + torch._C.__dict__["_cuda_beginAllocateToPool"] = _dummy_type( + "_cuda_beginAllocateToPool" + ) + torch._C.__dict__["_cuda_beginAllocateCurrentThreadToPool"] = _dummy_type( + "_cuda_beginAllocateCurrentThreadToPool" + ) + torch._C.__dict__["_cuda_endAllocateToPool"] = _dummy_type( + "_cuda_endAllocateToPool" + ) + torch._C.__dict__["_cuda_releasePool"] = _dummy_type("_cuda_releasePool") + +from torch._C import ( # noqa: F401 + _cuda_beginAllocateCurrentThreadToPool, + _cuda_beginAllocateToPool, + _cuda_CUDAAllocator, + _cuda_endAllocateToPool, + _cuda_releasePool, + _MemPool, +) + + +def _host_allocator(): + _lazy_init() + return torch._C._cuda_cudaHostAllocator() + + +@contextlib.contextmanager +def _free_mutex(): + torch._C._cuda_lock_mutex() + try: + yield + finally: + torch._C._cuda_unlock_mutex() + + +def caching_allocator_alloc(size, device: "Device" = None, stream=None): + r"""Perform a memory allocation using the CUDA memory allocator. + + Memory is allocated for a given device and a stream, this + function is intended to be used for interoperability with other + frameworks. Allocated memory is released through + :func:`~torch.cuda.caching_allocator_delete`. + + Args: + size (int): number of bytes to be allocated. + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + stream (torch.cuda.Stream or int, optional): selected stream. If is ``None`` then + the default stream for the selected device is used. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + if stream is None: + stream = torch.cuda.current_stream(device) + if isinstance(stream, torch.cuda.streams.Stream): + stream = stream.cuda_stream + if not isinstance(stream, int): + raise TypeError( + "Invalid type for stream argument, must be " + "`torch.cuda.Stream` or `int` representing a pointer " + "to a existing stream" + ) + with torch.cuda.device(device): + return torch._C._cuda_cudaCachingAllocator_raw_alloc(size, stream) + + +def caching_allocator_delete(mem_ptr): + r"""Delete memory allocated using the CUDA memory allocator. + + Memory allocated with :func:`~torch.cuda.caching_allocator_alloc`. + is freed here. The associated device and stream are tracked inside + the allocator. + + Args: + mem_ptr (int): memory address to be freed by the allocator. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + torch._C._cuda_cudaCachingAllocator_raw_delete(mem_ptr) + + +def caching_allocator_enable(value: bool = True) -> None: + r"""Enable or disable the CUDA memory allocator. On by default.""" + if is_initialized(): + torch._C._cuda_cudaCachingAllocator_enable(value) + + +def set_per_process_memory_fraction(fraction, device: "Device" = None) -> None: + r"""Set memory fraction for a process. + + The fraction is used to limit an caching allocator to allocated memory on a CUDA device. + The allowed value equals the total visible memory multiplied fraction. + If trying to allocate more than the allowed value in a process, will raise an out of + memory error in allocator. + + Args: + fraction(float): Range: 0~1. Allowed memory equals total_memory * fraction. + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + .. note:: + In general, the total available free memory is less than the total capacity. + """ + _lazy_init() + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + if not isinstance(fraction, float): + raise TypeError("Invalid type for fraction argument, must be `float`") + if fraction < 0 or fraction > 1: + raise ValueError(f"Invalid fraction value: {fraction}. Allowed range: 0~1") + + torch._C._cuda_setMemoryFraction(fraction, device) + + +def get_per_process_memory_fraction(device: "Device" = None) -> float: + r"""Get memory fraction for a process. + + Args: + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + Returns: + memory fraction, in range 0~1. Allowed memory equals total_memory * fraction. + """ + _lazy_init() + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + return torch._C._cuda_getMemoryFraction(device) + + +def empty_cache() -> None: + r"""Release all unoccupied cached memory currently held by the caching + allocator so that those can be used in other GPU application and visible in + `nvidia-smi`. + + .. note:: + :func:`~torch.cuda.empty_cache` doesn't increase the amount of GPU + memory available for PyTorch. However, it may help reduce fragmentation + of GPU memory in certain cases. See :ref:`cuda-memory-management` for + more details about GPU memory management. + """ + if is_initialized(): + torch._C._cuda_emptyCache() + + +def memory_stats(device: "Device" = None) -> dict[str, Any]: + r"""Return a dictionary of CUDA memory allocator statistics for a given device. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of reserved segments from ``cudaMalloc()``. + - ``"reserved_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of reserved memory. + - ``"active.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of active memory blocks. + - ``"active_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of active memory. + - ``"inactive_split.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of inactive, non-releasable memory blocks. + - ``"inactive_split_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of inactive, non-releasable memory. + + For these core statistics, values are broken down as follows. + + Pool type: + + - ``all``: combined statistics across all memory pools. + - ``large_pool``: statistics for the large allocation pool + (as of June 2025, for size >= 1MB allocations). + - ``small_pool``: statistics for the small allocation pool + (as of June 2025, for size < 1MB allocations). + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_alloc_retries"``: number of failed ``cudaMalloc`` calls that + result in a cache flush and retry. + - ``"num_ooms"``: number of out-of-memory errors thrown. + - ``"num_sync_all_streams"``: number of ``synchronize_and_free_events`` calls. + - ``"num_device_alloc"``: number of CUDA allocation calls. This includes both + cuMemMap and cudaMalloc. + - ``"num_device_free"``: number of CUDA free calls. This includes both cuMemUnmap + and cudaFree. + + The caching allocator can be configured via ENV to not split blocks larger than a + defined size (see Memory Management section of the Cuda Semantics documentation). + This helps avoid memory fragmentation but may have a performance + penalty. Additional outputs to assist with tuning and evaluating impact: + + - ``"max_split_size"``: blocks above this size will not be split. + - ``"oversize_allocations.{current,peak,allocated,freed}"``: + number of over-size allocation requests received by the memory allocator. + - ``"oversize_segments.{current,peak,allocated,freed}"``: + number of over-size reserved segments from ``cudaMalloc()``. + + The caching allocator can be configured via ENV to round memory allocations in order + to reduce fragmentation. Sometimes the overhead from rounding can be higher than + the fragmentation it helps reduce. The following stat can be used to check if + rounding adds too much overhead: + + - ``"requested_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + memory requested by client code, compare this with allocated_bytes to check if + allocation rounding adds too much overhead. + + Args: + device (torch.device or int, optional): selected device. Returns + statistics for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + + .. note:: + With :ref:`backend:cudaMallocAsync`, some stats are not + meaningful, and are always reported as zero. + """ + result = [] + + def _recurse_add_to_result(prefix, obj): + if isinstance(obj, dict): + if len(prefix) > 0: + prefix += "." + for k, v in obj.items(): + _recurse_add_to_result(prefix + k, v) + else: + result.append((prefix, obj)) + + stats = memory_stats_as_nested_dict(device=device) + _recurse_add_to_result("", stats) + result.sort() + + return collections.OrderedDict(result) + + +def memory_stats_as_nested_dict(device: "Device" = None) -> dict[str, Any]: + r"""Return the result of :func:`~torch.cuda.memory_stats` as a nested dictionary.""" + if not is_initialized(): + return {} + device = _get_device_index(device, optional=True) + return torch._C._cuda_memoryStats(device) + + +def reset_accumulated_memory_stats(device: "Device" = None) -> None: + r"""Reset the "accumulated" (historical) stats tracked by the CUDA memory allocator. + + See :func:`~torch.cuda.memory_stats` for details. Accumulated stats correspond to + the `"allocated"` and `"freed"` keys in each individual stat dict, as well as + `"num_alloc_retries"` and `"num_ooms"`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + return torch._C._cuda_resetAccumulatedMemoryStats(device) + + +def reset_peak_memory_stats(device: "Device" = None) -> None: + r"""Reset the "peak" stats tracked by the CUDA memory allocator. + + See :func:`~torch.cuda.memory_stats` for details. Peak stats correspond to the + `"peak"` key in each individual stat dict. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + return torch._C._cuda_resetPeakMemoryStats(device) + + +def host_memory_stats() -> dict[str, Any]: + r"""Return a dictionary of CUDA memory allocator statistics for a given device. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{current,peak,allocated,freed}"``: + number of reserved segments from ``cudaMalloc()``. + - ``"reserved_bytes.{current,peak,allocated,freed}"``: + amount of reserved memory. + + For these core statistics, values are broken down as follows. + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_host_alloc"``: number of CUDA allocation calls. This includes both + cudaHostAlloc and cudaHostRegister. + - ``"num_host_free"``: number of CUDA free calls. This includes both cudaHostFree + and cudaHostUnregister. + + Finally, we also provide some simple timing counters: + + - ``"host_alloc_time.{total,max,min,count,avg}"``: + timing of allocation requests going through CUDA calls. + - ``"host_free_time.{total,max,min,count,avg}"``: + timing of free requests going through CUDA calls. + + For these timing statistics, values are broken down as follows. + + Metric type: + + - ``total``: total time spent. + - ``max``: maximum value per call. + - ``min``: minimum value per call. + - ``count``: number of times it was called. + - ``avg``: average time per call. + """ + result = [] + + def _recurse_add_to_result(prefix, obj): + if isinstance(obj, dict): + if len(prefix) > 0: + prefix += "." + for k, v in obj.items(): + _recurse_add_to_result(prefix + k, v) + else: + result.append((prefix, obj)) + + stats = host_memory_stats_as_nested_dict() + _recurse_add_to_result("", stats) + result.sort() + + return collections.OrderedDict(result) + + +def host_memory_stats_as_nested_dict() -> dict[str, Any]: + r"""Return the result of :func:`~torch.cuda.host_memory_stats` as a nested dictionary.""" + if not is_initialized(): + return {} + return torch._C._cuda_hostMemoryStats() + + +def reset_accumulated_host_memory_stats() -> None: + r"""Reset the "accumulated" (historical) stats tracked by the host memory allocator. + + See :func:`~torch.cuda.host_memory_stats` for details. Accumulated stats correspond to + the `"allocated"` and `"freed"` keys in each individual stat dict. + """ + return torch._C._cuda_resetAccumulatedHostMemoryStats() + + +def reset_peak_host_memory_stats() -> None: + r"""Reset the "peak" stats tracked by the host memory allocator. + + See :func:`~torch.cuda.host_memory_stats` for details. Peak stats correspond to the + `"peak"` key in each individual stat dict. + """ + return torch._C._cuda_resetPeakHostMemoryStats() + + +def reset_max_memory_allocated(device: "Device" = None) -> None: + r"""Reset the starting point in tracking maximum GPU memory occupied by tensors for a given device. + + See :func:`~torch.cuda.max_memory_allocated` for details. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. warning:: + This function now calls :func:`~torch.cuda.reset_peak_memory_stats`, which resets + /all/ peak memory stats. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + warnings.warn( + "torch.cuda.reset_max_memory_allocated now calls torch.cuda.reset_peak_memory_stats, " + "which resets /all/ peak memory stats.", + FutureWarning, + stacklevel=2, + ) + return reset_peak_memory_stats(device=device) + + +def reset_max_memory_cached(device: "Device" = None) -> None: + r"""Reset the starting point in tracking maximum GPU memory managed by the caching allocator for a given device. + + See :func:`~torch.cuda.max_memory_cached` for details. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. warning:: + This function now calls :func:`~torch.cuda.reset_peak_memory_stats`, which resets + /all/ peak memory stats. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + warnings.warn( + "torch.cuda.reset_max_memory_cached now calls torch.cuda.reset_peak_memory_stats, " + "which resets /all/ peak memory stats.", + FutureWarning, + stacklevel=2, + ) + return reset_peak_memory_stats(device=device) + + +def memory_allocated(device: "Device" = None) -> int: + r"""Return the current GPU memory occupied by tensors in bytes for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + This is likely less than the amount shown in `nvidia-smi` since some + unused memory can be held by the caching allocator and some context + needs to be created on GPU. See :ref:`cuda-memory-management` for more + details about GPU memory management. + """ + return memory_stats(device=device).get("allocated_bytes.all.current", 0) + + +def max_memory_allocated(device: "Device" = None) -> int: + r"""Return the maximum GPU memory occupied by tensors in bytes for a given device. + + By default, this returns the peak allocated memory since the beginning of + this program. :func:`~torch.cuda.reset_peak_memory_stats` can be used to + reset the starting point in tracking this metric. For example, these two + functions can measure the peak allocated memory usage of each iteration in a + training loop. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("allocated_bytes.all.peak", 0) + + +def memory_reserved(device: "Device" = None) -> int: + r"""Return the current GPU memory managed by the caching allocator in bytes for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("reserved_bytes.all.current", 0) + + +def max_memory_reserved(device: "Device" = None) -> int: + r"""Return the maximum GPU memory managed by the caching allocator in bytes for a given device. + + By default, this returns the peak cached memory since the beginning of this + program. :func:`~torch.cuda.reset_peak_memory_stats` can be used to reset + the starting point in tracking this metric. For example, these two functions + can measure the peak cached memory amount of each iteration in a training + loop. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("reserved_bytes.all.peak", 0) + + +@deprecated( + "`torch.cuda.memory_cached` has been renamed to `torch.cuda.memory_reserved`", + category=FutureWarning, +) +def memory_cached(device: "Device" = None) -> int: + r"""Deprecated; see :func:`~torch.cuda.memory_reserved`.""" + return memory_reserved(device=device) + + +@deprecated( + "`torch.cuda.max_memory_cached` has been renamed to `torch.cuda.max_memory_reserved`", + category=FutureWarning, +) +def max_memory_cached(device: "Device" = None) -> int: + r"""Deprecated; see :func:`~torch.cuda.max_memory_reserved`.""" + return max_memory_reserved(device=device) + + +def memory_snapshot(mempool_id=None): + r"""Return a snapshot of the CUDA memory allocator state across all devices. + + Interpreting the output of this function requires familiarity with the + memory allocator internals. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return torch._C._cuda_memorySnapshot(mempool_id)["segments"] + + +def memory_summary(device: "Device" = None, abbreviated: bool = False) -> str: + r"""Return a human-readable printout of the current memory allocator statistics for a given device. + + This can be useful to display periodically during training, or when + handling out-of-memory exceptions. + + Args: + device (torch.device or int, optional): selected device. Returns + printout for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + abbreviated (bool, optional): whether to return an abbreviated summary + (default: False). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + stats = memory_stats(device=device) + + def _format_size(sz, pref_sz): + prefixes = ["B ", "KiB", "MiB", "GiB", "TiB", "PiB"] + prefix = prefixes[0] + for new_prefix in prefixes[1:]: + if pref_sz < 768 * 1024: + break + prefix = new_prefix + sz //= 1024 + pref_sz /= 1024 + return f"{sz:6d} {prefix}" + + def _format_count(cnt, pref_cnt): + prefixes = [" ", "K", "M"] + prefix = prefixes[0] + for new_prefix in prefixes[1:]: + if pref_cnt < 750 * 1000: + break + prefix = new_prefix + cnt //= 1000 + pref_cnt /= 1000 + return f"{cnt:7d} {prefix} " + + metrics_to_display = [ + ("allocated_bytes", "Allocated memory", _format_size), + ("active_bytes", "Active memory", _format_size), + ("requested_bytes", "Requested memory", _format_size), + ("reserved_bytes", "GPU reserved memory", _format_size), + ("inactive_split_bytes", "Non-releasable memory", _format_size), + ("allocation", "Allocations", _format_count), + ("active", "Active allocs", _format_count), + ("segment", "GPU reserved segments", _format_count), + ("inactive_split", "Non-releasable allocs", _format_count), + ] + + lines = [] + lines.append("=" * 75) + lines.append(" {_:16} PyTorch CUDA memory summary, device ID {device:<17d} ") + lines.append("-" * 75) + lines.append( + " {_:9} CUDA OOMs: {num_ooms:<12d} | {_:6} cudaMalloc retries: {num_alloc_retries:<8d} " + ) + lines.append("=" * 75) + lines.append( + " Metric | Cur Usage | Peak Usage | Tot Alloc | Tot Freed " + ) + + for metric_key, metric_name, formatter in metrics_to_display: + lines.append("-" * 75) + submetrics = [("all", metric_name)] + if not abbreviated: + submetrics.append(("large_pool", " from large pool")) + submetrics.append(("small_pool", " from small pool")) + + current_prefval, peak_prefval, allocated_prefval, freed_prefval = ( + None, + None, + None, + None, + ) + + for submetric_key, submetric_name in submetrics: + prefix = metric_key + "." + submetric_key + "." + + current = stats[prefix + "current"] + peak = stats[prefix + "peak"] + allocated = stats[prefix + "allocated"] + freed = stats[prefix + "freed"] + + if current_prefval is None: + current_prefval = current + peak_prefval = peak + allocated_prefval = allocated + freed_prefval = freed + + lines.append( + f" {submetric_name:<21} | {formatter(current, current_prefval)} | {formatter(peak, peak_prefval)} | " + f"{formatter(allocated, allocated_prefval)} | {formatter(freed, freed_prefval)} ", + ) + + metrics_to_display = [ + ("oversize_allocations", "Oversize allocations", _format_count), + ("oversize_segments", "Oversize GPU segments", _format_count), + ] + + for metric_key, metric_name, formatter in metrics_to_display: + lines.append("-" * 75) + + prefix = metric_key + "." + + current = stats[prefix + "current"] + peak = stats[prefix + "peak"] + allocated = stats[prefix + "allocated"] + freed = stats[prefix + "freed"] + + lines.append( + f" {metric_name:<21} | {formatter(current, current)} | {formatter(peak, peak)} | " + f"{formatter(allocated, allocated)} | {formatter(freed, freed)} ", + ) + + lines.append("=" * 75) + + fmt_dict = {"_": "", "device": device} + for k, v in stats.items(): + fmt_dict[k.replace(".", "-")] = v + return "|" + "|\n|".join(lines).format(**fmt_dict) + "|\n" + + +def list_gpu_processes(device: "Device" = None) -> str: + r"""Return a human-readable printout of the running processes and their GPU memory use for a given device. + + This can be useful to display periodically during training, or when + handling out-of-memory exceptions. + + Args: + device (torch.device or int, optional): selected device. Returns + printout for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + """ + if not torch.version.hip: + try: + import pynvml # type: ignore[import] + except ModuleNotFoundError: + return "pynvml module not found, please install nvidia-ml-py" + # pyrefly: ignore [import-error,missing-module-attribute] + from pynvml import NVMLError_DriverNotLoaded + + try: + pynvml.nvmlInit() + except NVMLError_DriverNotLoaded: + return "cuda driver can't be loaded, is cuda enabled?" + + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + else: + try: + import amdsmi # type: ignore[import] + except ModuleNotFoundError: + return "amdsmi module not found, please install amdsmi" + try: + amdsmi.amdsmi_init() # type: ignore[attr-defined] + except amdsmi.AmdSmiException: # type: ignore[attr-defined] + return "amdsmi driver can't be loaded, is ROCm installed?" + + device = _get_amdsmi_device_index(device) + + try: + handle = amdsmi.amdsmi_get_processor_handles()[device] # type: ignore[attr-defined] + procs = amdsmi.amdsmi_get_gpu_process_list(handle) # type: ignore[attr-defined] + except amdsmi.AmdSmiException: # type: ignore[attr-defined] + return "amdsmi cannot list processes from other users" + + lines = [] + lines.append(f"GPU:{device}") + if len(procs) == 0: + lines.append("no processes are running") + for p in procs: + if not torch.version.hip: + mem = p.usedGpuMemory / (1024 * 1024) + pid = p.pid + else: + try: + proc_info = amdsmi.amdsmi_get_gpu_process_info(handle, p) # type: ignore[possibly-undefined] + except AttributeError: + # https://github.com/ROCm/amdsmi/commit/c551c3caedbd903ba828e7fdffa5b56d475a15e7 + # is a BC-breaking change that removes amdsmi_get_gpu_process_info API from amdsmi + proc_info = p + mem = proc_info["memory_usage"]["vram_mem"] / (1024 * 1024) + pid = proc_info["pid"] + lines.append(f"process {pid:>10d} uses {mem:>12.3f} MB GPU memory") + return "\n".join(lines) + + +def mem_get_info(device: "Device" = None) -> tuple[int, int]: + r"""Return the global free and total GPU memory for a given device using cudaMemGetInfo. + + Args: + device (torch.device or int or str, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default) or if the device index is not specified. + + .. note:: + See :ref:`cuda-memory-management` for more + details about GPU memory management. + """ + if device is None: + device = torch.cuda.current_device() + # optional=True allows `device = torch.device('cuda')` for which device.index is None + device = _get_device_index(device, optional=True) + return torch.cuda.cudart().cudaMemGetInfo(device) + + +def _record_memory_history_legacy( + enabled: bool, + record_context=True, + trace_alloc_max_entries=1, + trace_alloc_record_context=False, + device: "Device" = None, + record_context_cpp=False, + clear_history=False, + compile_context=False, + global_record_annotations=False, +): + _C._cuda_record_memory_history_legacy( # type: ignore[call-arg] + enabled, + record_context, + # pyrefly: ignore [bad-argument-type] + trace_alloc_max_entries, + trace_alloc_record_context, + record_context_cpp, + clear_history, + compile_context, + global_record_annotations, + ) + + +def _record_memory_history( + enabled: Optional[Literal["state", "all"]] = "all", *args, **kwargs +) -> None: + """Enable recording of stack traces associated with memory + allocations, so you can tell what allocated any piece of memory in + :func:`torch.cuda.memory._snapshot()`. + + In addition to keeping stack traces with each current allocation and free, + this will also enable recording of a history of all alloc/free events. + + Use :func:`torch.cuda.memory._snapshot()` to retrieve this information, + and the tools in `_memory_viz.py` to visualize snapshots. + + Buffer behavior + --------------- + + This will store up to `max_entries` instances of `TraceEntry` when enabled. + Python trace collection defaults to `sys.maxsize`, meaning long-running + or indefinitely running jobs should set a reasonable limit to avoid excessive + memory use. Expect each entry to be several KB. + + Longer running workflows or those with smaller `max_entries` values will only + store the last accumulated `max_entries` entries, meaning new entries overwrite + older entries. + + C++ implementation for reference to ring buffer implementation: + + .. code-block:: cpp + + if (record_history) { + if (alloc_trace->size() < alloc_trace_max_entries_) { + alloc_trace->emplace_back(te); + } else { + (*alloc_trace)[alloc_trace_next++] = te; + if (alloc_trace_next == alloc_trace_max_entries_) { + alloc_trace_next = 0; + } + } + } + + Latency impact + -------------- + + The Python trace collection is fast (2us per trace), so you may consider + enabling this on production jobs if you anticipate ever having to debug + memory issues. + + C++ trace collection is also fast (~50ns/frame), which for many typical programs + works out to ~2us per trace, but can vary depending on stack depth. + + Args: + enabled (Literal[None, "state", "all"], optional): + `None`, disable recording memory history. + `"state"`, keep information for currently allocated memory. + `"all"`, additionally keep a history of all alloc/free calls. + Defaults to "all". + context (Literal[None, "state", "alloc", "all"], optional): + `None`, Do not record any tracebacks. + `"state"`, Record tracebacks for currently allocated memory. + `"alloc"`, additionally keep tracebacks for alloc calls. + `"all"`, additionally keep tracebacks for free calls. + Defaults to "all". + stacks (Literal["python", "all"], optional): + `"python"`, include Python, TorchScript, and inductor frames in tracebacks + `"all"`, additionally include C++ frames + Defaults to "all". + max_entries (int, optional): Keep a maximum of `max_entries` + alloc/free events in the recorded history recorded. + """ + if isinstance(enabled, bool): + return _record_memory_history_legacy(enabled, *args, **kwargs) + else: + return _record_memory_history_impl(enabled, *args, **kwargs) + + +def _record_memory_history_impl( + enabled: Optional[str] = "all", + context: Optional[str] = "all", + stacks: str = "all", + max_entries: int = sys.maxsize, + device: "Device" = None, + clear_history: bool = False, + compile_context: bool = False, + global_record_annotations: bool = False, +): + _C._cuda_record_memory_history( # type: ignore[call-arg] + enabled, + context, + stacks, + max_entries, + clear_history, + compile_context, + global_record_annotations, + ) + + +_record_memory_history.__signature__ = signature(_record_memory_history_impl) # type: ignore[attr-defined] + + +def _augment_frames(frames: list[_Frame]) -> int: + """ + Augment a list of frames with FX debug information. + + Args: + frames: List of frame dictionaries to augment + + Returns: + The count of frames that were augmented. + """ + from torch.fx.graph_module import FX_GRAPH_MODULE_FILE_PREFIX + + # Regex pattern to match FX generated files + _FX_GENERATED_PATTERN = re.compile( + rf"{re.escape(FX_GRAPH_MODULE_FILE_PREFIX)}.*\.py$" + ) + + count = 0 + if not frames: + return count + + for frame in frames: + if "filename" in frame and "line" in frame: + filename = frame["filename"] + lineno = frame["line"] + + # Check if this looks like an FX generated file + if not _FX_GENERATED_PATTERN.search(os.path.basename(filename)): + continue + + # Look up metadata from the global registry + from torch.fx.traceback import _FX_METADATA_REGISTRY + + metadata = _FX_METADATA_REGISTRY.get(filename) + if metadata is None: + continue + + lineno_map = metadata.get("lineno_map", {}) + node_metadata = metadata.get("node_metadata", {}) + prologue_start = metadata.get("prologue_start", 0) + + # Get the node index for this line + node_idx = lineno_map.get(lineno - prologue_start) + + if node_idx is not None and node_idx in node_metadata: + node_info = node_metadata[node_idx] + original_trace = node_info.get("stack_trace") + node_op = node_info.get("op") + node_name = node_info.get("name") + node_target = node_info.get("target") + + # Always add node metadata + frame["fx_node_op"] = node_op + frame["fx_node_name"] = node_name + frame["fx_node_target"] = str(node_target) + + # Add original trace if available + if original_trace: + frame["fx_original_trace"] = original_trace + + count += 1 + + return count + + +def _augment_memory_snapshot_stack_traces( + snapshot: str | _Snapshot, +) -> _Snapshot: + """ + Augment a memory snapshot with original source stack traces from FX metadata. + + IMPORTANT: This function reads from a global in-memory registry (_FX_METADATA_REGISTRY) + that is populated during graph module compilation. It must be called in the same + Python process where the FX graphs were compiled. It cannot be used to augment + snapshots loaded from disk in a different process. + + Args: + snapshot: Either a memory snapshot dict or path to a snapshot pickle file + + Returns: + The augmented snapshot dictionary with fx_node_op, fx_node_name, + fx_original_trace, and fx_node_info fields added to frames + """ + + snapshot_dict: _Snapshot + if isinstance(snapshot, str): + # Load the memory snapshot + with open(snapshot, "rb") as f: + snapshot_dict = cast(_Snapshot, pickle.load(f)) + else: + snapshot_dict = snapshot + + # Process stack traces in the snapshot + augmented_count = 0 + + # Process blocks in segments (for regular allocations) + if "segments" in snapshot_dict: + for segment in snapshot_dict["segments"]: + if "blocks" in segment: + for block in segment["blocks"]: + if "frames" in block: + augmented_count += _augment_frames(block["frames"]) + + # Process device traces (for memory history) + if "device_traces" in snapshot_dict: + for trace_list in snapshot_dict["device_traces"]: + for trace_entry in trace_list: + if isinstance(trace_entry, dict) and "frames" in trace_entry: + augmented_count += _augment_frames(trace_entry["frames"]) + + return snapshot_dict + + +def _snapshot(device: "Device" = None, augment_with_fx_traces=False): + """Save a snapshot of CUDA memory state at the time it was called. + + The state is represented as a dictionary with the following structure. + + .. code-block:: python + + class Snapshot(TypedDict): + segments: List[Segment] + device_traces: List[List[TraceEntry]] + + + class Segment(TypedDict): + # Segments are memory returned from a cudaMalloc call. + # The size of reserved memory is the sum of all Segments. + # Segments are cached and reused for future allocations. + # If the reuse is smaller than the segment, the segment + # is split into more then one Block. + # empty_cache() frees Segments that are entirely inactive. + address: int + total_size: int # cudaMalloc'd size of segment + stream: int + segment_type: Literal["small", "large"] # 'large' (>1MB) + allocated_size: int # size of memory in use + active_size: int # size of memory in use or in active_awaiting_free state + blocks: List[Block] + + + class Block(TypedDict): + # A piece of memory returned from the allocator, or + # current cached but inactive. + size: int + requested_size: int # size requested during malloc, may be smaller than + # size due to rounding + address: int + state: Literal[ + "active_allocated", # used by a tensor + "active_awaiting_free", # waiting for another stream to finish using + # this, then it will become free + "inactive", + ] # free for reuse + frames: List[Frame] # stack trace from where the allocation occurred + + + class Frame(TypedDict): + filename: str + line: int + name: str + # Optional FX debug fields (present when augment_with_fx_traces=True + # and the frame corresponds to FX-generated code) + fx_node_op: str # FX node operation type (e.g., 'call_function', 'output') + fx_node_name: str # FX node name (e.g., 'linear', 'relu_1') + fx_original_trace: str # Original model source code stack trace + + + class TraceEntry(TypedDict): + # When `torch.cuda.memory._record_memory_history()` is enabled, + # the snapshot will contain TraceEntry objects that record each + # action the allocator took. + action: Literal[ + "alloc" # memory allocated + "free_requested", # the allocated received a call to free memory + "free_completed", # the memory that was requested to be freed is now + # able to be used in future allocation calls + "segment_alloc", # the caching allocator ask cudaMalloc for more memory + # and added it as a segment in its cache + "segment_free", # the caching allocator called cudaFree to return memory + # to cuda possibly trying free up memory to + # allocate more segments or because empty_caches was called + "oom", # the allocator threw an OOM exception. 'size' is + # the requested number of bytes that did not succeed + "snapshot", # the allocator generated a memory snapshot + # useful to coorelate a previously taken + # snapshot with this trace + ] + addr: int # not present for OOM + frames: List[Frame] + size: int + stream: int + device_free: int # only present for OOM, the amount of + # memory cuda still reports to be free + + Args: + device: Device to capture snapshot for. If None, captures for current device. + augment_with_fx_traces: If True, augment stack trace frames with FX debug information + that maps generated FX code back to original model source code. + This adds fx_node_op, fx_node_name, fx_original_trace, and + fx_node_info fields to Frame objects. Default: False. + + Returns: + The Snapshot dictionary object + """ + s = _C._cuda_memorySnapshot(None) + if augment_with_fx_traces: + s = _augment_memory_snapshot_stack_traces(s) # type: ignore[assignment, arg-type] + return s + + +def _dump_snapshot(filename="dump_snapshot.pickle", augment_with_fx_traces=False): + """ + Save a pickled version of the `torch.memory._snapshot()` dictionary to a file. + + This file can be opened by the interactive snapshot viewer at pytorch.org/memory_viz + + Snapshot file sizes scale with `max_entries` and stack trace depth per entry, + with several KB per entry. These can easily be in the GB range for longer running + workflows with large `max_entries`. + + Args: + filename (str, optional): Name of the file to create. Defaults to "dump_snapshot.pickle". + augment_with_fx_traces (bool, optional): If True, augment the snapshot with FX debug information + before dumping. This maps generated FX code stack traces + back to original model source code. Defaults to False. + verbose (bool, optional): If True and augment_with_fx_traces is True, print verbose debug output + during augmentation. Defaults to False. + """ + s = _snapshot(augment_with_fx_traces=augment_with_fx_traces) + + with open(filename, "wb") as f: + pickle.dump(s, f) + + +def _set_memory_metadata(metadata: str): + """ + Set custom metadata that will be attached to all subsequent CUDA memory allocations. + + This metadata will be recorded in the memory snapshot for all allocations made + after this call until the metadata is cleared or changed. + + Args: + metadata (str): Custom metadata string to attach to allocations. + Pass an empty string to clear the metadata. + """ + # pyrefly: ignore [missing-attribute] + torch._C._cuda_setMemoryMetadata(metadata) + + +def _get_memory_metadata() -> str: + """ + Get the current custom metadata that is being attached to CUDA memory allocations. + + Returns: + str: The current metadata string, or empty string if no metadata is set. + """ + # pyrefly: ignore [missing-attribute] + return torch._C._cuda_getMemoryMetadata() + + +def _save_segment_usage(filename="output.svg", snapshot=None): + if snapshot is None: + snapshot = _snapshot() + with open(filename, "w") as f: + f.write(_segments(snapshot)) + + +def _save_memory_usage(filename="output.svg", snapshot=None): + if snapshot is None: + snapshot = _snapshot() + with open(filename, "w") as f: + f.write(_memory(snapshot)) + + +@deprecated( + "torch.cuda._set_allocator_settings is deprecated. Use torch._C._accelerator_setAllocatorSettings instead.", + category=FutureWarning, +) +def _set_allocator_settings(env: str): + # pyrefly: ignore [missing-attribute] + return torch._C._accelerator_setAllocatorSettings(env) + + +def get_allocator_backend() -> str: + r"""Return a string describing the active allocator backend as set by + ``PYTORCH_ALLOC_CONF``. Currently available backends are + ``native`` (PyTorch's native caching allocator) and `cudaMallocAsync`` + (CUDA's built-in asynchronous allocator). + + .. note:: + See :ref:`cuda-memory-management` for details on choosing the allocator backend. + """ + return torch._C._cuda_getAllocatorBackend() + + +class _CUDAAllocator: + r"""Wrapper over internal CUDA memory allocators.""" + + def __init__(self, allocator: torch._C._cuda_CUDAAllocator): + self._allocator = allocator + + def allocator(self): + return self._allocator + + +class CUDAPluggableAllocator(_CUDAAllocator): + r"""CUDA memory allocator loaded from a so file.""" + + def __init__(self, path_to_so_file: str, alloc_fn_name: str, free_fn_name: str): + r"""Memory allocators are compiled in .so files and loaded dynamically using ctypes. + + To change the active allocator use the :func:`torch.memory.cuda.change_current_allocator` function. + + Args: + path_to_so_file(str): Path in the filesystem to the `.so` file containing + the allocator functions + alloc_fn_name(str): Name of the function to perform the memory allocation + in the so file. The signature must be: + void* alloc_fn_name(ssize_t size, int device, cudaStream_t stream); + free_fn_name(str): Name of the function to perform the memory release + in the so file. The signature must be: + void free_fn_name(void* ptr, size_t size, cudaStream_t stream); + + .. warning:: + This is currently supported only in unix OSs + + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + allocator = ctypes.CDLL(path_to_so_file) + alloc_fn = ctypes.cast(getattr(allocator, alloc_fn_name), ctypes.c_void_p).value + free_fn = ctypes.cast(getattr(allocator, free_fn_name), ctypes.c_void_p).value + assert alloc_fn is not None + assert free_fn is not None + self._allocator = torch._C._cuda_customAllocator(alloc_fn, free_fn) + + +def change_current_allocator(allocator: _CUDAAllocator) -> None: + r"""Change the currently used memory allocator to be the one provided. + + If the current allocator has already been used/initialized, this function will error. + + + Args: + allocator (torch.cuda.memory._CUDAAllocator): allocator to be set as the active one. + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + torch._C._cuda_changeCurrentAllocator(allocator.allocator()) + + +def _get_current_allocator() -> _CUDAAllocator: + r"""Return the allocator being currently used. + + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + return _CUDAAllocator(torch._C._cuda_getAllocator()) + + +class MemPool(_MemPool): + r"""MemPool represents a pool of memory in a caching allocator. Currently, + it's just the ID of the pool object maintained in the CUDACachingAllocator. + + Args: + allocator(torch._C._cuda_CUDAAllocator, optional): a + torch._C._cuda_CUDAAllocator object that can be used to + define how memory gets allocated in the pool. If :attr:`allocator` + is ``None`` (default), memory allocation follows the default/ + current configuration of the CUDACachingAllocator. + use_on_oom(bool): a bool that indicates if this pool can be used + as a last resort if a memory allocation outside of the pool fails due + to Out Of Memory. This is False by default. + no_split(bool): a bool that indicates if this pool should not split a segment. + This is False by default. + """ + + def __init__( + self, + allocator: Optional[_cuda_CUDAAllocator] = None, + use_on_oom: bool = False, + no_split: bool = False, + ): + super().__init__(allocator, True, use_on_oom, no_split) + + @property + def id(self) -> tuple[int, int]: + r"""Returns the ID of this pool as a tuple of two ints.""" + return super().id + + @property + def allocator(self) -> Optional[_cuda_CUDAAllocator]: + r"""Returns the allocator this MemPool routes allocations to.""" + return super().allocator + + def use_count(self) -> int: # pylint: disable=useless-parent-delegation + r"""Returns the reference count of this pool.""" + return super().use_count() + + def snapshot(self): + r"""Return a snapshot of the CUDA memory allocator pool state across all + devices. + + Interpreting the output of this function requires familiarity with the + memory allocator internals. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + snapshot = torch.cuda.memory_snapshot(self.id) + return snapshot + + +@contextlib.contextmanager +def use_mem_pool(pool: MemPool, device: "Device" = None): + r"""A context manager that routes allocations to a given pool. + + Args: + pool(torch.cuda.MemPool): a MemPool object to be made active so that + allocations route to this pool. + device (torch.device or int, optional): selected device. Uses MemPool on + the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + This context manager makes only current thread's allocations route to + the given pool. If a new thread is spawned inside the context manager + (e.g. by calling backward) the allocations in that thread will not + route to the given pool. + """ + device_index = ( + torch.cuda.current_device() if device is None else _get_device_index(device) + ) + _cuda_beginAllocateCurrentThreadToPool(device_index, pool.id) + try: + yield + finally: + _cuda_endAllocateToPool(device_index, pool.id) + _cuda_releasePool(device_index, pool.id) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nccl.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nccl.py new file mode 100644 index 0000000000000000000000000000000000000000..f71e843a55df5dff02f7c046effdbed13667d6ca --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nccl.py @@ -0,0 +1,152 @@ +# mypy: allow-untyped-defs +import collections +import warnings +from collections.abc import Sequence +from typing import Optional, Union + +import torch.cuda + + +__all__ = ["all_reduce", "reduce", "broadcast", "all_gather", "reduce_scatter"] + +SUM = 0 # ncclRedOp_t + + +def is_available(tensors): + if not hasattr(torch._C, "_nccl_all_reduce"): + warnings.warn("PyTorch is not compiled with NCCL support", stacklevel=2) + return False + + devices = set() + for tensor in tensors: + if tensor.is_sparse: + return False + if not tensor.is_contiguous(): + return False + if not tensor.is_cuda: + return False + device = tensor.get_device() + if device in devices: + return False + devices.add(device) + + return True + + +def version(): + """ + Returns the version of the NCCL. + + + This function returns a tuple containing the major, minor, and patch version numbers of the NCCL. + The suffix is also included in the tuple if a version suffix exists. + Returns: + tuple: The version information of the NCCL. + """ + ver = torch._C._nccl_version() + major = ver >> 32 + minor = (ver >> 16) & 65535 + patch = ver & 65535 + suffix = torch._C._nccl_version_suffix().decode("utf-8") + if suffix == "": + return (major, minor, patch) + else: + return (major, minor, patch, suffix) + + +def unique_id(): + return torch._C._nccl_unique_id() + + +def init_rank(num_ranks, uid, rank): + return torch._C._nccl_init_rank(num_ranks, uid, rank) + + +def _check_sequence_type(inputs: Union[torch.Tensor, Sequence[torch.Tensor]]) -> None: + if not isinstance(inputs, collections.abc.Container) or isinstance( + inputs, torch.Tensor + ): + raise TypeError("Inputs should be a collection of tensors") + + +def all_reduce(inputs, outputs=None, op=SUM, streams=None, comms=None): + _check_sequence_type(inputs) + if outputs is None: + outputs = inputs + _check_sequence_type(outputs) + torch._C._nccl_all_reduce(inputs, outputs, op, streams, comms) + + +# `output` used to be `outputs`, taking in a list of tensors. So we have two +# arguments for BC reasons. +def reduce( + inputs: Sequence[torch.Tensor], + output: Optional[Union[torch.Tensor, Sequence[torch.Tensor]]] = None, + root: int = 0, + op: int = SUM, + streams: Optional[Sequence[torch.cuda.Stream]] = None, + comms=None, + *, + outputs: Optional[Sequence[torch.Tensor]] = None, +) -> None: + _check_sequence_type(inputs) + _output: torch.Tensor + if outputs is not None: + if output is not None: + raise ValueError( + "'output' and 'outputs' can not be both specified. 'outputs' is deprecated in " + "favor of 'output', taking in a single output tensor. The signature of reduce is: " + "reduce(inputs, output=None, root=0, op=SUM, streams=None, comms=None)." + ) + else: + warnings.warn( + "`nccl.reduce` with an output tensor list is deprecated. " + "Please specify a single output tensor with argument 'output' instead instead.", + FutureWarning, + stacklevel=2, + ) + _output = outputs[root] + elif not isinstance(output, torch.Tensor) and isinstance( + output, collections.abc.Sequence + ): + # User called old API with positional arguments of list of output tensors. + warnings.warn( + "nccl.reduce with an output tensor list is deprecated. " + "Please specify a single output tensor.", + FutureWarning, + stacklevel=2, + ) + _output = output[root] + else: + _output = inputs[root] if output is None else output + torch._C._nccl_reduce(inputs, _output, root, op, streams, comms) + + +def broadcast( + inputs: Sequence[torch.Tensor], root: int = 0, streams=None, comms=None +) -> None: + _check_sequence_type(inputs) + torch._C._nccl_broadcast(inputs, root, streams, comms) + + +def all_gather( + inputs: Sequence[torch.Tensor], + outputs: Sequence[torch.Tensor], + streams=None, + comms=None, +) -> None: + _check_sequence_type(inputs) + _check_sequence_type(outputs) + torch._C._nccl_all_gather(inputs, outputs, streams, comms) + + +def reduce_scatter( + inputs: Sequence[torch.Tensor], + outputs: Sequence[torch.Tensor], + op: int = SUM, + streams=None, + comms=None, +) -> None: + _check_sequence_type(inputs) + _check_sequence_type(outputs) + torch._C._nccl_reduce_scatter(inputs, outputs, op, streams, comms) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nvtx.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nvtx.py new file mode 100644 index 0000000000000000000000000000000000000000..4efa946215f0c576bb1c4639766355411e77b48c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/nvtx.py @@ -0,0 +1,129 @@ +# mypy: allow-untyped-defs +r"""This package adds support for NVIDIA Tools Extension (NVTX) used in profiling.""" + +from contextlib import contextmanager + + +try: + from torch._C import _nvtx +except ImportError: + + class _NVTXStub: + @staticmethod + def _fail(*args, **kwargs): + raise RuntimeError( + "NVTX functions not installed. Are you sure you have a CUDA build?" + ) + + rangePushA = _fail + rangePop = _fail + markA = _fail + + _nvtx = _NVTXStub() # type: ignore[assignment] + +__all__ = ["range_push", "range_pop", "range_start", "range_end", "mark", "range"] + + +def range_push(msg): + """ + Push a range onto a stack of nested range span. Returns zero-based depth of the range that is started. + + Args: + msg (str): ASCII message to associate with range + """ + return _nvtx.rangePushA(msg) + + +def range_pop(): + """Pop a range off of a stack of nested range spans. Returns the zero-based depth of the range that is ended.""" + return _nvtx.rangePop() + + +def range_start(msg) -> int: + """ + Mark the start of a range with string message. It returns an unique handle + for this range to pass to the corresponding call to rangeEnd(). + + A key difference between this and range_push/range_pop is that the + range_start/range_end version supports range across threads (start on one + thread and end on another thread). + + Returns: A range handle (uint64_t) that can be passed to range_end(). + + Args: + msg (str): ASCII message to associate with the range. + """ + # pyrefly: ignore [missing-attribute] + return _nvtx.rangeStartA(msg) + + +def range_end(range_id) -> None: + """ + Mark the end of a range for a given range_id. + + Args: + range_id (int): an unique handle for the start range. + """ + # pyrefly: ignore [missing-attribute] + _nvtx.rangeEnd(range_id) + + +def _device_range_start(msg: str, stream: int = 0) -> object: + """ + Marks the start of a range with string message. + It returns an opaque heap-allocated handle for this range + to pass to the corresponding call to device_range_end(). + + A key difference between this and range_start is that the + range_start marks the range right away, while _device_range_start + marks the start of the range as soon as all the tasks on the + CUDA stream are completed. + + Returns: An opaque heap-allocated handle that should be passed to _device_range_end(). + + Args: + msg (str): ASCII message to associate with the range. + stream (int): CUDA stream id. + """ + # pyrefly: ignore [missing-attribute] + return _nvtx.deviceRangeStart(msg, stream) + + +def _device_range_end(range_handle: object, stream: int = 0) -> None: + """ + Mark the end of a range for a given range_handle as soon as all the tasks + on the CUDA stream are completed. + + Args: + range_handle: an unique handle for the start range. + stream (int): CUDA stream id. + """ + # pyrefly: ignore [missing-attribute] + _nvtx.deviceRangeEnd(range_handle, stream) + + +def mark(msg): + """ + Describe an instantaneous event that occurred at some point. + + Args: + msg (str): ASCII message to associate with the event. + """ + return _nvtx.markA(msg) + + +@contextmanager +def range(msg, *args, **kwargs): + """ + Context manager / decorator that pushes an NVTX range at the beginning + of its scope, and pops it at the end. If extra arguments are given, + they are passed as arguments to msg.format(). + + Args: + msg (str): message to associate with the range + """ + range_push(msg.format(*args, **kwargs)) + try: + yield + finally: + range_pop() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/profiler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..d99ef0ff3cdc4541f124aac7461f1bdf02631a2d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/profiler.py @@ -0,0 +1,86 @@ +# mypy: allow-untyped-defs +import contextlib +import tempfile + +import torch + +from . import check_error, cudart + + +__all__ = ["init", "start", "stop", "profile"] + +DEFAULT_FLAGS = [ + "gpustarttimestamp", + "gpuendtimestamp", + "gridsize3d", + "threadblocksize", + "streamid", + "enableonstart 0", + "conckerneltrace", +] + + +def init(output_file, flags=None, output_mode="key_value"): + rt = cudart() + if not hasattr(rt, "cudaOutputMode"): + raise AssertionError("HIP does not support profiler initialization!") + if ( + hasattr(torch.version, "cuda") + and torch.version.cuda is not None + and int(torch.version.cuda.split(".")[0]) >= 12 + ): + # Check https://github.com/pytorch/pytorch/pull/91118 + # cudaProfilerInitialize is no longer needed after CUDA 12 + raise AssertionError("CUDA12+ does not need profiler initialization!") + flags = DEFAULT_FLAGS if flags is None else flags + if output_mode == "key_value": + output_mode_enum = rt.cudaOutputMode.KeyValuePair + elif output_mode == "csv": + output_mode_enum = rt.cudaOutputMode.CSV + else: + raise RuntimeError( + "supported CUDA profiler output modes are: key_value and csv" + ) + with tempfile.NamedTemporaryFile(delete=True) as f: + f.write(b"\n".join(f.encode("ascii") for f in flags)) + f.flush() + check_error(rt.cudaProfilerInitialize(f.name, output_file, output_mode_enum)) + + +def start(): + r"""Starts cuda profiler data collection. + + .. warning:: + Raises CudaError in case of it is unable to start the profiler. + """ + check_error(cudart().cudaProfilerStart()) + + +def stop(): + r"""Stops cuda profiler data collection. + + .. warning:: + Raises CudaError in case of it is unable to stop the profiler. + """ + check_error(cudart().cudaProfilerStop()) + + +@contextlib.contextmanager +def profile(): + """ + Enable profiling. + + Context Manager to enabling profile collection by the active profiling tool from CUDA backend. + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> import torch + >>> model = torch.nn.Linear(20, 30).cuda() + >>> inputs = torch.randn(128, 20).cuda() + >>> with torch.cuda.profiler.profile() as prof: + ... model(inputs) + """ + try: + start() + yield + finally: + stop() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/random.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/random.py new file mode 100644 index 0000000000000000000000000000000000000000..d5dee5052d284c0309d2bbd2224f5688726402d2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/random.py @@ -0,0 +1,184 @@ +# mypy: allow-untyped-defs +from collections.abc import Iterable +from typing import Union + +import torch +from torch import Tensor + +from . import _lazy_call, _lazy_init, current_device, device_count, is_initialized + + +__all__ = [ + "get_rng_state", + "get_rng_state_all", + "set_rng_state", + "set_rng_state_all", + "manual_seed", + "manual_seed_all", + "seed", + "seed_all", + "initial_seed", +] + + +def get_rng_state(device: Union[int, str, torch.device] = "cuda") -> Tensor: + r"""Return the random number generator state of the specified GPU as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + idx = device.index + if idx is None: + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + return default_generator.get_state() + + +def get_rng_state_all() -> list[Tensor]: + r"""Return a list of ByteTensor representing the random number states of all devices.""" + results = [get_rng_state(i) for i in range(device_count())] + return results + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "cuda" +) -> None: + r"""Set the random number generator state of the specified GPU. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + """ + if not is_initialized(): + with torch._C._DisableFuncTorch(): + # Clone the state because the callback will be triggered + # later when CUDA is lazy initialized. + new_state = new_state.clone(memory_format=torch.contiguous_format) + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + + def cb(): + idx = device.index + if idx is None: + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.set_state(new_state) + + _lazy_call(cb) + + +def set_rng_state_all(new_states: Iterable[Tensor]) -> None: + r"""Set the random number generator state of all devices. + + Args: + new_states (Iterable of torch.ByteTensor): The desired state for each device. + """ + for i, state in enumerate(new_states): + set_rng_state(state, i) + + +def manual_seed(seed: int) -> None: + r"""Set the seed for generating random numbers for the current GPU. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + Args: + seed (int): The desired seed. + + .. warning:: + If you are working with a multi-GPU model, this function is insufficient + to get determinism. To seed all GPUs, use :func:`manual_seed_all`. + """ + seed = int(seed) + + def cb(): + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.manual_seed(seed) + + _lazy_call(cb, seed=True) + + +def manual_seed_all(seed: int) -> None: + r"""Set the seed for generating random numbers on all GPUs. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + Args: + seed (int): The desired seed. + """ + seed = int(seed) + + def cb(): + for i in range(device_count()): + default_generator = torch.cuda.default_generators[i] + default_generator.manual_seed(seed) + + _lazy_call(cb, seed_all=True) + + +def seed() -> None: + r"""Set the seed for generating random numbers to a random number for the current GPU. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + .. warning:: + If you are working with a multi-GPU model, this function will only initialize + the seed on one GPU. To initialize all GPUs, use :func:`seed_all`. + """ + + def cb(): + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.seed() + + _lazy_call(cb) + + +def seed_all() -> None: + r"""Set the seed for generating random numbers to a random number on all GPUs. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + """ + + def cb(): + random_seed = 0 + seeded = False + for i in range(device_count()): + default_generator = torch.cuda.default_generators[i] + if not seeded: + default_generator.seed() + random_seed = default_generator.initial_seed() + seeded = True + else: + default_generator.manual_seed(random_seed) + + _lazy_call(cb) + + +def initial_seed() -> int: + r"""Return the current random seed of the current GPU. + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + return default_generator.initial_seed() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/sparse.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..702def052945ad1bd54ab221ae517a9c01361e34 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/sparse.py @@ -0,0 +1 @@ +# The Tensor classes are added to this module by python_tensor.cpp diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/streams.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/streams.py new file mode 100644 index 0000000000000000000000000000000000000000..a84deb4596f3706898480103609b0948c20f8f8d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/streams.py @@ -0,0 +1,262 @@ +# mypy: allow-untyped-defs +# pylint: disable=useless-parent-delegation +import ctypes + +import torch +from torch._utils import _dummy_type + + +if not hasattr(torch._C, "_CudaStreamBase"): + # Define dummy base classes + torch._C.__dict__["_CudaStreamBase"] = _dummy_type("_CudaStreamBase") + torch._C.__dict__["_CudaEventBase"] = _dummy_type("_CudaEventBase") + + +class Stream(torch._C._CudaStreamBase): + r"""Wrapper around a CUDA stream. + + A CUDA stream is a linear sequence of execution that belongs to a specific + device, independent from other streams. It supports with statement as a + context manager to ensure the operators within the with block are running + on the corresponding stream. See :ref:`cuda-semantics` for details. + + Args: + device(torch.device or int, optional): a device on which to allocate + the stream. If :attr:`device` is ``None`` (default) or a negative + integer, this will use the current device. + priority(int, optional): priority of the stream, which can be positive, 0, or negative. + A lower number indicates a higher priority. By default, the priority is set to 0. + If the value falls outside of the allowed priority range, it will automatically be + mapped to the nearest valid priority (lowest for large positive numbers or + highest for large negative numbers). + + """ + + def __new__(cls, device=None, priority=0, **kwargs): + # Check CUDA availability + if not torch.backends.cuda.is_built(): + raise RuntimeError("torch.cuda.Stream requires CUDA support") + # setting device manager is expensive, so we avoid it unless necessary + if device is None or ("stream_id" in kwargs and "device_index" in kwargs): + return super().__new__(cls, priority=priority, **kwargs) + else: + with torch.cuda.device(device): + return super().__new__(cls, priority=priority, **kwargs) + + def wait_event(self, event) -> None: + r"""Make all future work submitted to the stream wait for an event. + + Args: + event (torch.cuda.Event): an event to wait for. + + .. note:: This is a wrapper around ``cudaStreamWaitEvent()``: see + `CUDA Stream documentation`_ for more info. + + This function returns without waiting for :attr:`event`: only future + operations are affected. + + .. _CUDA Stream documentation: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html + """ + event.wait(self) + + def wait_stream(self, stream) -> None: + r"""Synchronize with another stream. + + All future work submitted to this stream will wait until all kernels + submitted to a given stream at the time of call complete. + + Args: + stream (Stream): a stream to synchronize. + + .. note:: This function returns without waiting for currently enqueued + kernels in :attr:`stream`: only future operations are affected. + """ + self.wait_event(stream.record_event()) + + def record_event(self, event=None): + r"""Record an event. + + Args: + event (torch.cuda.Event, optional): event to record. If not given, a new one + will be allocated. + + Returns: + Recorded event. + """ + if event is None: + event = Event() + event.record(self) + return event + + def query(self) -> bool: + r"""Check if all the work submitted has been completed. + + Returns: + A boolean indicating if all kernels in this stream are completed. + """ + return super().query() + + def synchronize(self) -> None: + r"""Wait for all the kernels in this stream to complete. + + .. note:: This is a wrapper around ``cudaStreamSynchronize()``: see + `CUDA Stream documentation`_ for more info. + """ + super().synchronize() + + @property + def _as_parameter_(self): + return ctypes.c_void_p(self.cuda_stream) + + def __eq__(self, o) -> bool: + if isinstance(o, Stream): + return super().__eq__(o) + return False + + def __hash__(self): + return hash((self.cuda_stream, self.device)) + + def __repr__(self): + return f"" + + def __cuda_stream__(self): + """Implements the CUDA Stream Protocol: + https://nvidia.github.io/cuda-python/cuda-core/latest/interoperability.html#cuda-stream-protocol + + Returns: + tuple: A 2-tuple of (version, handle) where version is the protocol version + and handle is the address of cudaStream_t (CUDA) or hipStream_t (ROCm) as a Python int. + """ + return (0, self.cuda_stream) + + +class ExternalStream(Stream): + r"""Wrapper around an externally allocated CUDA stream. + + This class is used to wrap streams allocated in other libraries in order + to facilitate data exchange and multi-library interactions. + + .. note:: This class doesn't manage the stream life-cycle, it is the user + responsibility to keep the referenced stream alive while this class is + being used. + + Args: + stream_ptr(int): Integer representation of the `cudaStream_t` value. + allocated externally. + device(torch.device or int, optional): the device where the stream + was originally allocated. If device is specified incorrectly, + subsequent launches using this stream may fail. + """ + + def __new__(cls, stream_ptr, device=None, **kwargs): + with torch.cuda.device(device): + return super().__new__(cls, stream_ptr=stream_ptr, **kwargs) + + +class Event(torch._C._CudaEventBase): + r"""Wrapper around a CUDA event. + + CUDA events are synchronization markers that can be used to monitor the + device's progress, to accurately measure timing, and to synchronize CUDA + streams. + + The underlying CUDA events are lazily initialized when the event is first + recorded or exported to another process. After creation, only streams on the + same device may record the event. However, streams on any device can wait on + the event. + + Args: + enable_timing (bool, optional): indicates if the event should measure time + (default: ``False``) + blocking (bool, optional): if ``True``, :meth:`wait` will be blocking (default: ``False``) + interprocess (bool): if ``True``, the event can be shared between processes + (default: ``False``) + external (bool, optional): indicates whether this event should create event record and event wait nodes, or create an internal cross-stream dependency, when captured in a cuda graph. See `cross-stream dependencies `_, `cudaEventRecordExternal `_, and `cudaEventWaitExternal `_ for more information about internal vs. external events. (default: ``False``) + + .. _CUDA Event Documentation: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EVENT.html + """ # noqa: B950 + + def __new__( + cls, enable_timing=False, blocking=False, interprocess=False, external=False + ): + return super().__new__( + cls, + enable_timing=enable_timing, + blocking=blocking, + interprocess=interprocess, + external=external, + ) + + @classmethod + def from_ipc_handle(cls, device, handle): + r"""Reconstruct an event from an IPC handle on the given device.""" + return super().from_ipc_handle(device, handle) + + def record(self, stream=None): + r"""Record the event in a given stream. + + Uses ``torch.cuda.current_stream()`` if no stream is specified. The + stream's device must match the event's device. + """ + if stream is None: + stream = torch.cuda.current_stream() + super().record(stream) + + def wait(self, stream=None) -> None: + r"""Make all future work submitted to the given stream wait for this event. + + Use ``torch.cuda.current_stream()`` if no stream is specified. + + .. note:: This is a wrapper around ``cudaStreamWaitEvent()``: see + `CUDA Event documentation`_ for more info. + """ + if stream is None: + stream = torch.cuda.current_stream() + super().wait(stream) + + def query(self): + r"""Check if all work currently captured by event has completed. + + Returns: + A boolean indicating if all work currently captured by event has + completed. + """ + return super().query() + + def elapsed_time(self, end_event): + r"""Return the time elapsed. + + Time reported in milliseconds after the event was recorded and + before the end_event was recorded. + """ + return super().elapsed_time(end_event) + + def synchronize(self) -> None: + r"""Wait for the event to complete. + + Waits until the completion of all work currently captured in this event. + This prevents the CPU thread from proceeding until the event completes. + + .. note:: This is a wrapper around ``cudaEventSynchronize()``: see + `CUDA Event documentation`_ for more info. + """ + super().synchronize() + + def ipc_handle(self): + r"""Return an IPC handle of this event. + + If not recorded yet, the event will use the current device. + """ + return super().ipc_handle() + + @property + def _as_parameter_(self): + return ctypes.c_void_p(self.cuda_event) + + def __repr__(self) -> str: + if self.cuda_event: + return f"" + else: + return "" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/tunable.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/tunable.py new file mode 100644 index 0000000000000000000000000000000000000000..e6d00f51e7dc21970edbc55ad1eb1fd00b354ad4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/cuda/tunable.py @@ -0,0 +1,802 @@ +r""" +This module exposes a TunableOp interface. + +Some operations, such as GEMMs, could be implemented using more than one library +or more than one technique. For example, a GEMM could be implemented for CUDA or +ROCm using either the blas or blasLt libraries. Further, ROCm's rocblas and +hipblaslt libraries allow the user to query for all possible algorithms and then +choose one. How does one know which implementation is the fastest and should be +chosen? That's what TunableOp provides. + +Enabling TunableOp and Tuning Separately +======================================== + +The TunableOp feature is enabled separately from enabling the tuning phase +itself. Enabling TunableOp means that PyTorch will replace any standard +operators with their Tunable implementations. Any call to a TunableOp first +checks whether it has already been tuned for the given operator inputs. If so, +it will immediately call the tuned operation; no further tuning will take place +even when the tuning setting is enabled. Instead if no tuning result is found, +and tuning is enabled, the TunableOp will benchmark every registered +implementation of that operator for the given set of inputs and select the +fastest. + +File Input and Output +===================== + +The first time any TunableOp is invoked, the internal database of tuned +operations will be prepared by attempting to read the results from the given +file. The default filename is 'tunableop_results.csv'. To support tuning when +multiple GPUs are used across multiple processes, the GPU device ordinal is +automatically inserted into the filename to avoid multiple processes overwriting +the same file. + +If tuning is enabled and new tunings are discovered during the course of your +workload, it will also write out to this same filename with all tunings, both +the ones it read in at startup as well as the new ones found at runtime. This +can be used, for example, to build up a tunings file across many workloads by +reusing the same file. The output file is automatically created when the +application terminates. This behavior can be controlled by the C++ and Python +APIs but not the environment variables. + +Assuming you specified a filename, you'll end up with a CSV file with contents +like so:: + + Validator,PT_VERSION,2.2.0 + Validator,ROCM_VERSION,6.0.0.0-12969-1544e39 + Validator,HIPBLASLT_VERSION,0.6.0-a9c5cc7 + Validator,ROCBLAS_VERSION,4.0.0-72e57364-dirty + GemmTunableOp_float_NT,nt_25088_4096_64,Gemm_Hipblaslt_1219,1.262 + GemmTunableOp_float_NT,nt_4096_4096_64,Gemm_Rocblas_1216,0.033 + +Note the "Validator" lines. If you change a library version, or ROCm version, or +PyTorch version, TunableOp will detect this and reject the tunings file because +the prior tunings are likely affected by other software changes. + +The remaining lines are the tuned solutions for each TunableOp encountered +during your execution. Each line consists of 4 comma-separated fields: operator +name, operator parameters, solution name, and average execution time. The +execution time is an optional field. The CSV file can be edited, but with +caution. For example, the solution name (field 3) can be changed to "Default" +and it will fall back to the original PyTorch untuned implementation. Or, in the +case of ROCm's hipBLAS or hipBLASLt libraries, if you know the specific solution +index you can override the solution that TunableOp selected by replacing the +value. The operator name and parameters (fields 1 and 2) are internally named +and should not be modified. In the case of GemmTunableOp, field 1 indicates the +datatype and whether the inputs are transposed (T) or not (N) and field 2 +indicates the M, N, K input shapes. + +There is an option to enable verbose output but it is only recommended for +debugging purposes. This will produce a lot of diagnostic messages but may be +useful to see if TunableOp is being used at all. Otherwise, TunableOp is +completely silent, besides file output, unless there is a warning or error +during its use. The verbose option is only available by setting the environment +variable PYTORCH_TUNABLEOP_VEROBSE=1. + +A Note on Tuning Behavior, Warmup, and Cache Effects +==================================================== + +Tuning an operator consists of iterating through the list or registered +implementations and profiling each one. The profile is established by running a +single implementation in a loop multiple times and taking the average execution +time. There is also an optional warmup phase prior to tuning that can help with +reaching stable power states by the hardware. During tuning of a workload the +various hardware caches will more likely produce hits than when not tuning. +There are options for flushing the instruction cache and rotate the input tensors +which might help produce a more faithful profile of the tuned operator as if the +operator were run within a larger workload instead of in a tight, repetitive loop. + +By default, each possible solution for a given operator will be run for either +100 iterations or as many iterations that can be run within 30ms, whichever is +smaller, and its average execution will be calculated. The fastest solution +among all that were successfully profiled will be chosen. A profile might fail +if the given solution doesn't achieve the same accuracy as the default +implementation or if the solution returns an error code. + +Current Tunable Operators +========================= + +TunableGemm for ROCm +-------------------- + +Currently only a TunableGemm for ROCm is implemented. Note that CUDA builds of +PyTorch will function correctly when using TunableOp but the only solution +available to CUDA builds is the 'Default' implementation i.e. the original +cuBLAS default, now called through TunableOp. Any call to at::cuda::blas::gemm() +or ::bgemm() will be routed through TunableOp when enabled. Calling gemm() for a +given set of input arguments (transa, transb, m, n, k) will attempt to use the +fastest available implementation across both rocblas and hipblaslt. + +Offline Tuning +============== + +Motivation +---------- +There are several use cases for offline tuning. + +One use case involves a workload with a high-memory utilization, where regular tuning might lead to running out of memory. + +Another use case is for compute-intensive workloads. In such cases, it is more resource-efficient to collect +the GEMMs for the workload once and then tune repeatedly with different tuning parameters or libraries. + +Workflow +-------- +There are basically two steps: +1) Set the environment variables to collect the untuned GEMM and this will generate ``tunableop_untuned0.csv``: + +.. code-block:: bash + + export PYTORCH_TUNABLEOP_ENABLED=1 + export PYTORCH_TUNABLEOP_TUNING=0 + export PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 + ... + +2) Run a Python script that reads the ``tunableop_untuned0.csv`` and generates the ``tunableop_results0.csv``, like this: + +.. code-block:: python + + import torch.cuda.tunable as tunable + import os + + os.putenv("PYTORCH_TUNABLEOP_ENABLED", "1") + os.putenv("PYTORCH_TUNABLEOP_TUNING", "1") + os.putenv("PYTORCH_TUNABLEOP_RECORD_UNTUNED", "0") + tunable.tune_gemm_in_file("tunableop_untuned0.csv") + + +It is also possible to take multiple untuned files and distribute the GEMMs for tuning to multiple GPUs +within a single node. In the first step, the GEMMs are first gathered and duplicate GEMMs are eliminated. +Next, the GEMMs are distributed to different GPUs for tuning. After all GEMMs are tuned, the results from +all the GPUs are then gathered into a single file whose base filename has ``_full0`` appended to it +(for example ``tunableop_results_full0.csv``). Finally, this new file, containing the gathered results, will be +duplicated N times, once for each GPU as convenience to the user will run the workload with the tuned +configuration on N GPUs. + +.. code-block:: python + + if __name__ == "__main__": + num_gpus = 8 # number of GPUs that will be used during the tuning process + tunable.mgpu_tune_gemm_in_file("tunableop_untuned?.csv", num_gpus) + +Note that the usage of the ``mgpu_tune_gemm_in_file`` API is different from its single GPU counterpart +(``tune_gemm_in_file``). The body of the Python script that calls the API must be wrapped in ``main()`` as shown +due to the use of concurrent futures module. The argument to ``mgpu_tune_gemm_in_file`` must contain a wild card +expression (``?`` or ``*``) to generate the list of untuned files containing the GEMMs to be processed. The ``num_gpus`` +must between 1 and the total number of GPUs available. + +Tuning Context +============== + +The behavior of TunableOp is currently manipulated through environment +variables, the C++ interface of at::cuda::tunable::getTuningContext(), or the +torch.cuda.tunable python interfaces. The environment variables take precedence +over any setting you manipulate using the C++ or Python APIs. + +Environment Variable Interface +------------------------------ +Environment variables are cached the first time they are read. You cannot use the +environment variable interface programmatically since the settings become fixed. +Use the C++ or Python APIs instead. + +""" + +import concurrent.futures +import glob +import multiprocessing as mp +import os +import shutil +import warnings +from typing import Optional + +import torch + + +__all__ = [ + "enable", + "is_enabled", + "tuning_enable", + "tuning_is_enabled", + "record_untuned_enable", + "record_untuned_is_enabled", + "set_max_tuning_duration", + "get_max_tuning_duration", + "set_max_tuning_iterations", + "get_max_tuning_iterations", + "set_filename", + "get_filename", + "get_results", + "get_validators", + "read_file", + "tune_gemm_in_file", + "mgpu_tune_gemm_in_file", + "set_rotating_buffer_size", + "get_rotating_buffer_size", + "set_numerical_check_tolerances", +] + + +def enable(val: bool = True) -> None: + r"""This is the big on/off switch for all TunableOp implementations.""" + torch._C._cuda_tunableop_enable(val) # type: ignore[attr-defined] + + +def is_enabled() -> bool: + r"""Returns whether the TunableOp feature is enabled.""" + return torch._C._cuda_tunableop_is_enabled() # type: ignore[attr-defined] + + +def tuning_enable(val: bool = True) -> None: + r"""Enable tuning of TunableOp implementations. + + When enabled, if a tuned entry isn't found, run the tuning step and record + the entry. + """ + torch._C._cuda_tunableop_tuning_enable(val) # type: ignore[attr-defined] + + +def tuning_is_enabled() -> bool: + r"""Returns whether TunableOp implementations can be tuned.""" + return torch._C._cuda_tunableop_tuning_is_enabled() # type: ignore[attr-defined] + + +def record_untuned_enable(val: bool = True) -> None: + r"""Enable recording untuned of TunableOp perations for offline tuning. + + When enabled, if a tuned entry isn't found, write it to the untuned file. + """ + torch._C._cuda_record_untuned_enable(val) # type: ignore[attr-defined] + + +def record_untuned_is_enabled() -> bool: + r"""Returns whether TunableOp operations are recorded for offline tuning.""" + return torch._C._cuda_record_untuned_is_enabled() # type: ignore[attr-defined] + + +def set_max_tuning_duration(duration: int) -> None: + r"""Set max time in milliseconds to spend tuning a given solution. + + If both max tuning duration and iterations are set, the smaller of the two + will be honored. At minimum 1 tuning iteration will always be run. + """ + torch._C._cuda_tunableop_set_max_tuning_duration(duration) # type: ignore[attr-defined] + + +def get_max_tuning_duration() -> int: + r"""Get max time to spend tuning a given solution.""" + return torch._C._cuda_tunableop_get_max_tuning_duration() # type: ignore[attr-defined] + + +def set_max_tuning_iterations(iterations: int) -> None: + r"""Set max number of iterations to spend tuning a given solution. + + If both max tuning duration and iterations are set, the smaller of the two + will be honored. At minimum 1 tuning iteration will always be run. + """ + torch._C._cuda_tunableop_set_max_tuning_iterations(iterations) # type: ignore[attr-defined] + + +def get_max_tuning_iterations() -> int: + r"""Get max iterations to spend tuning a given solution.""" + return torch._C._cuda_tunableop_get_max_tuning_iterations() # type: ignore[attr-defined] + + +def set_filename(filename: str, insert_device_ordinal: bool = False) -> None: + r"""Set the filename to use for input/output of tuning results. + + If :attr:`insert_device_ordinal` is ``True`` then the current device ordinal + will be added to the given filename automatically. This can be used in a + 1-process-per-gpu scenario to ensure all processes write to a separate file. + """ + torch._C._cuda_tunableop_set_filename(filename, insert_device_ordinal) # type: ignore[attr-defined] + + +def get_filename() -> str: + r"""Get the results filename.""" + return torch._C._cuda_tunableop_get_filename() # type: ignore[attr-defined] + + +def get_results() -> tuple[str, str, str, float]: + r"""Return all TunableOp results.""" + return torch._C._cuda_tunableop_get_results() # type: ignore[attr-defined] + + +def get_validators() -> tuple[str, str]: + r"""Return the TunableOp validators.""" + return torch._C._cuda_tunableop_get_validators() # type: ignore[attr-defined] + + +def read_file(filename: Optional[str] = None) -> bool: + r"""Read results from a TunableOp CSV file. + + If :attr:`filename` is not given, ``get_filename()`` is called. + """ + if filename is None: + filename = get_filename() + return torch._C._cuda_tunableop_read_file(filename) # type: ignore[attr-defined] + + +def set_rotating_buffer_size(buffer_size: int) -> None: + r"""Set rotating buffer size to this value in MB, if the buffer size is greater than zero. + + If less than zero, query L2 cache size. If equal to zero, means deactivate rotating buffer. + """ + return torch._C._cuda_tunableop_set_rotating_buffer_size(buffer_size) # type: ignore[attr-defined] + + +def get_rotating_buffer_size() -> int: + r"""Get the rotating buffer size in kilobytes.""" + return torch._C._cuda_tunableop_get_rotating_buffer_size() # type: ignore[attr-defined] + + +def set_numerical_check_tolerances( + enable: bool, atol: float = 1e-5, rtol: float = 1e-5 +) -> None: + r"""Set the atol and rtol values in numeric check""" + return torch._C._cuda_tunableop_set_numerical_check_tolerances(enable, atol, rtol) # type: ignore[attr-defined] + + +def tune_gemm_in_file(filename: str) -> None: + r"""tune GEMM in file.""" + + assert is_enabled() + assert tuning_is_enabled() + + deviceid = torch.cuda.current_device() + + with open(filename) as file: + for line in file: + if line.startswith(("Gemm", "ScaledGemm")): + _process_single_offline_gemm(line, deviceid) + + +def _gather_unique_untuned_gemm_from_files(filename_pattern: str) -> set[str]: + r"""Process multiple untuned results file and return a set with duplicates removed.""" + unique_gemm_entries = set() # set will avoid duplicates + + for file_path in glob.glob(filename_pattern): + with open(file_path) as file: + for line in file: + if line.startswith(("Gemm", "ScaledGemm")): + unique_gemm_entries.add(line) + + return unique_gemm_entries + + +def _gather_tunableop_results() -> None: + r"""Gather results from multiple tunableop results file and create a single file.""" + gemm_lines = set() + validator_lines = [] + + # Need to allow for the possibility that results filename was + # set with the Python API instead of with environment variable. + # Also possible that results filename was not set at all. + # There are several test cases to check, but ultimately we + # need a glob-able expression + results_filename = get_filename() # Note empty string could be returned here + + if ( + results_filename is not None and results_filename != "" + ): # Case were the Python API was used to set the filename + dot_pos = results_filename.find(".") + if dot_pos != -1 and dot_pos > 0: + # Replace the character just to the left of the dot + filename_pattern = ( + results_filename[: dot_pos - 1] + "?" + results_filename[dot_pos:] + ) + else: + filename_pattern = "" # Needed to make linter happy + else: # Case where the environment variable was used to set the filename. + results_filename_env = os.getenv("PYTORCH_TUNABLEOP_FILENAME") + if results_filename_env is None or results_filename_env == "": + filename_pattern = "tunableop_results?.csv" + elif "%d" in results_filename_env: + filename_pattern = results_filename_env.replace("%d", "?") + else: + filename_pattern = results_filename_env.replace(".", "?.") + + assert "?" in filename_pattern + + FirstFile = False + matching_files = glob.glob(filename_pattern) + num_matching_files = len(matching_files) + for file_path in matching_files: + with open(file_path) as file: + for line in file: + if line.startswith("Validator"): + if not (FirstFile): + # Only read Validator from first file + validator_lines.append(line) + else: + gemm_lines.add(line) + + FirstFile = True + + output_file = filename_pattern.replace("?", "_full0") + + with open(output_file, "w") as out_file: + for line in validator_lines: + out_file.write(line) + for line in gemm_lines: + out_file.write(line) + + # Create num_matching_copies of the results file + for i in range(1, num_matching_files): + duplicate_file = output_file.replace("0", str(i)) + shutil.copy(output_file, duplicate_file) + + +def _create_matrices( + m: int, + n: int, + k: int, + lda: int, + ldb: int, + ldc: int, + transA: bool, + transB: bool, + dtypeA: torch.dtype, + deviceid: str, + dtypeB: Optional[torch.dtype] = None, + randn: bool = True, + subMatrix: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Helper function for _process_single_offline_gemm. + Creates matrices that are then consumed by one of the Torch GEMM APIs. + """ + # Fill parameters set for use with ScaledGEMM + fillA = 0.25 + fillB = 0.75 + + if dtypeB is None: + dtypeB = dtypeA + + if subMatrix: + # User reference for understanding leading dimension: + # https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/dgemm.f + # TO DO: According to lines 108 - 133, there is no lower bound on rowsA, + # but there is a restriction on rowsB. Using this formula for now as it + # seems to work for all UTs. + rowsA = rowsB = max(ldc, k) + + if randn: + matA = torch.randn(rowsA, lda, dtype=dtypeA, device=deviceid) + matB = torch.randn(rowsB, ldb, dtype=dtypeA, device=deviceid) + else: + matA = torch.full((rowsA, lda), fillA, dtype=dtypeB, device=deviceid) + matB = torch.full((rowsB, ldb), fillB, dtype=dtypeB, device=deviceid) + + subA = matA[:k, :m].t() if transA else matA[:m, :k] + subB = matB[:n, :k].t() if transB else matB[:k, :n] + return subA, subB + else: + if randn: + matA = ( + torch.rand(k, m, dtype=dtypeA, device=deviceid).t() + if transA + else torch.rand(m, k, dtype=dtypeA, device=deviceid) + ) + matB = ( + torch.rand(n, k, dtype=dtypeB, device=deviceid).t() + if transB + else torch.rand(k, n, dtype=dtypeB, device=deviceid) + ) + else: + matA = ( + torch.full((k, m), fillA, dtype=dtypeA, device=deviceid).t() + if transA + else torch.full((m, k), fillA, dtype=dtypeA, device=deviceid) + ) + matB = ( + torch.full((n, k), fillB, dtype=dtypeB, device=deviceid).t() + if transB + else torch.full((k, n), fillB, dtype=dtypeB, device=deviceid) + ) + return matA, matB + + +def _create_batch_matrices( + m: int, + n: int, + k: int, + b: int, + lda: int, + ldb: int, + ldc: int, + transA: bool, + transB: bool, + dtype: torch.dtype, + deviceid: str, + subMatrix: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Helper function for _process_single_offline_gemm. + Creates batch matrices that are then consumed by one of the Torch GEMM APIs. + Similar to _create_matrices but for 3D batch matrices. + """ + if subMatrix: + # User reference for understanding leading dimension: + # https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/dgemm.f + # TO DO: According to lines 108 - 133, there is no lower bound on rowsA, + # but there is a restriction on rowsB. Using this formula for now as it + # seems to work for all UTs. + rowsA = rowsB = max(ldc, k) + + matA = torch.randn(b, rowsA, lda, dtype=dtype, device=deviceid) + matB = torch.randn(b, rowsB, ldb, dtype=dtype, device=deviceid) + + subA = matA[:b, :k, :m].transpose(1, 2) if transA else matA[:b, :m, :k] + subB = matB[:b, :n, :k].transpose(1, 2) if transB else matB[:b, :k, :n] + return subA, subB + else: + matA = ( + torch.rand(b, k, m, dtype=dtype, device=deviceid) + if transA + else torch.rand(b, m, k, dtype=dtype, device=deviceid) + ) + matB = ( + torch.rand(b, n, k, dtype=dtype, device=deviceid) + if transB + else torch.rand(b, k, n, dtype=dtype, device=deviceid) + ) + matA = matA.transpose(1, 2) if transA else matA + matB = matB.transpose(1, 2) if transB else matB + return matA, matB + + +def _process_single_offline_gemm(untuned_gemm_line: str, gpu_id: int) -> None: + r"""Process a single untuned GEMM.""" + + deviceid = "cuda:" + str(gpu_id) + + dtype_dict = { + "float": torch.float32, + "tf32": torch.float32, + "double": torch.float64, + "BFloat16": torch.bfloat16, + "Half": torch.half, + "c10::complex": torch.complex128, + "c10::complex": torch.complex64, + "Float8_e4m3fn": torch.float8_e4m3fn, + "Float8_e5m2": torch.float8_e5m2, + "Float8_e4m3fnuz": torch.float8_e4m3fnuz, + "Float8_e5m2fnuz": torch.float8_e5m2fnuz, + } + + untuned_gemm = untuned_gemm_line.strip().split(",")[:] + + underscore_count = untuned_gemm[0].count("_") + + # Initialize dtype to make linter happy + dtype = None + dtypeA = None + dtypeB = None + dtypeC = None + + # Extract BLAS parameters + if underscore_count == 2: + [op_sig, data_type, layout] = untuned_gemm[0].split("_") + transB = layout[0] == "T" + transA = layout[1] == "T" + dtype = dtype_dict.get(data_type) + if data_type == "tf32": + torch.backends.cuda.matmul.allow_tf32 = True + else: + torch.backends.cuda.matmul.allow_tf32 = False + + else: # ScaledGEMM + count = untuned_gemm[0].count("_") + assert count in [6, 7] + untuned_gemm_temp = untuned_gemm[0].split("_") + # dtypeC = might not be FP8 type, keep track + # of the number of underscores + op_sig = untuned_gemm_temp[0] + data_typeA = untuned_gemm_temp[1] + "_" + untuned_gemm_temp[2] + data_typeB = untuned_gemm_temp[3] + "_" + untuned_gemm_temp[4] + if count == 7: + data_typeC = untuned_gemm_temp[5] + "_" + untuned_gemm_temp[6] + else: + data_typeC = untuned_gemm_temp[5] + transB = untuned_gemm_temp[count][0] == "T" + transA = untuned_gemm_temp[count][1] == "T" + dtypeA = dtype_dict.get(data_typeA) + dtypeB = dtype_dict.get(data_typeB) + dtypeC = dtype_dict.get(data_typeC) + + untuned_gemm_temp = untuned_gemm[1].split("_") + [n, m, k] = [int(g) for g in untuned_gemm_temp[1:4]] + if op_sig == "GemmStridedBatchedTunableOp": + assert untuned_gemm_temp[6] == "ld" + [ldb, lda, ldc] = [int(g) for g in untuned_gemm_temp[7:10]] + else: + assert untuned_gemm_temp[4] == "ld" + [ldb, lda, ldc] = [int(g) for g in untuned_gemm_temp[5:8]] + + # Detect subMatrix case + if all(item in [n, m, k] for item in [lda, ldb, ldc]): + subMatrix = False + else: + subMatrix = True + + if op_sig == "GemmTunableOp": + # Warnings for unsupported cases: + if m == 1 or n == 1 or k == 1: + if (not transA) and (not transB): + pass # case is supported + elif transA and n == 1: + pass # case is supported + else: + warnings.warn( + "Offline tuning is not supported for this GEMM. Use online tuning instead. " + + f"Skipped tuning for: {untuned_gemm[1]}", + stacklevel=2, + ) + return + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + matA, matB = _create_matrices( + m, n, k, lda, ldb, ldc, transA, transB, dtype, deviceid, subMatrix=subMatrix + ) + torch.mm(matA, matB) + + elif op_sig == "GemmStridedBatchedTunableOp": + # Warnings for unsupported cases: + if m == 1 or n == 1 or k == 1: + warnings.warn( + "Offline tuning is not support for this GEMM. Use online tuning instead. " + + f"Skipped tuning for: {untuned_gemm[1]}", + stacklevel=2, + ) + return + + [b] = [int(g) for g in untuned_gemm_temp[5:6]] + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + matA, matB = _create_batch_matrices( + m, + n, + k, + b, + lda, + ldb, + ldc, + transA, + transB, + dtype, + deviceid, + subMatrix=subMatrix, + ) + torch.bmm(matA, matB) + elif op_sig == "ScaledGemmTunableOp": + # Only combination supported by PyTorch + assert transB is True + assert transA is False + + # Resolve linter issue + if dtypeA is None or not isinstance(dtypeA, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtypeA}") + + matA, matB = _create_matrices( + m, + n, + k, + lda, + ldb, + ldc, + transA, + transB, + dtypeA, + deviceid, + dtypeB=dtypeB, + randn=False, + subMatrix=subMatrix, + ) + + assert untuned_gemm_temp[8] == "rw" + if untuned_gemm_temp[9] == "1": + rowwise = True + else: + rowwise = False + if rowwise: + scaleA = ( + torch.ones((1, m), device=deviceid) + if transA + else torch.ones((m, 1), device=deviceid) + ) + scaleB = ( + torch.ones((1, n), device=deviceid) + if transB + else torch.ones((n, 1), device=deviceid) + ) + else: + scaleA = torch.tensor(0.8, device=deviceid) + scaleB = torch.tensor(0.9, device=deviceid) + + assert untuned_gemm_temp[10] == "bias" + if untuned_gemm_temp[11] == "None": # no bias vector + torch._scaled_mm( + matA, matB, scale_a=scaleA, scale_b=scaleB, out_dtype=dtypeC + ) + else: # bias vector present + fillbias = 0.10 + bias_dtype = dtype_dict.get(untuned_gemm_temp[11]) + bias = ( + torch.full((n,), fillbias, dtype=bias_dtype, device=deviceid) + if transB + else torch.full((m,), fillbias, dtype=bias_dtype, device=deviceid) + ) + torch._scaled_mm( + matA, matB, scale_a=scaleA, scale_b=scaleB, out_dtype=dtypeC, bias=bias + ) + + elif op_sig == "GemmAndBiasTunableOp": + # y = x*A^T + b + assert transA != transB + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + bias = torch.rand(n, dtype=dtype, device=deviceid) + + X, matA = _create_matrices( + m, n, k, lda, ldb, ldc, transA, transB, dtype, deviceid, subMatrix=subMatrix + ) + matA = matA.t() + torch.nn.functional.linear(X, matA, bias) + else: + warnings.warn(f"error: unknown op {op_sig}", stacklevel=2) + + +def _check_tuning_assertions() -> None: + r"""Helper function for multi-GPU tuning case. Need to check that TunableOp feature + is enabled and that tuning is enabled. + """ + + if is_enabled() is False: + warnings.warn("TunableOp was disabled. Trying to enable now.", stacklevel=2) + enable(True) + assert is_enabled() is True + assert tuning_is_enabled() is True + assert record_untuned_is_enabled() is False + + +def mgpu_tune_gemm_in_file(filename_pattern: str, num_gpus: int) -> None: + r"""Process one or more files and distribute work over one or more GPUs.""" + unique_gemm_entries = _gather_unique_untuned_gemm_from_files(filename_pattern) + + total_gpus = torch.cuda.device_count() + + assert 1 <= num_gpus <= total_gpus + + mp_context = mp.get_context("spawn") + + futures = [] # empty list to hold futures + + # GEMM are assigned to GPUs in a round robin manner + h = 0 + with concurrent.futures.ProcessPoolExecutor( + max_workers=num_gpus, + mp_context=mp_context, + initializer=_check_tuning_assertions, + ) as executor: + # The workers are a separate process. TunableOp will be + # enabled in the child processes if PYTORCH_TUNABLEOP_ENABLED=1 + # In the initializer, we also try to enable TunableOP if th + # environment variable was NOT set. + + for line in unique_gemm_entries: + future = executor.submit(_process_single_offline_gemm, line, h) + futures.append(future) + h = (h + 1) % num_gpus + + for future in concurrent.futures.as_completed(futures): + future.result() + + torch.cuda.synchronize() + + _gather_tunableop_results() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6148873cd5bf82fa2677421ed9b1c994757c2e7b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/__init__.py @@ -0,0 +1,168 @@ +# mypy: allow-untyped-defs +import logging +import pdb +import sys +import traceback +import typing +from datetime import timedelta + +import torch + + +log = logging.getLogger(__name__) + + +def is_available() -> bool: + """ + Return ``True`` if the distributed package is available. + + Otherwise, + ``torch.distributed`` does not expose any other APIs. Currently, + ``torch.distributed`` is available on Linux, MacOS and Windows. Set + ``USE_DISTRIBUTED=1`` to enable it when building PyTorch from source. + Currently, the default value is ``USE_DISTRIBUTED=1`` for Linux and Windows, + ``USE_DISTRIBUTED=0`` for MacOS. + """ + return hasattr(torch._C, "_c10d_init") + + +if is_available() and not torch._C._c10d_init(): + raise RuntimeError("Failed to initialize torch.distributed") + +# Custom Runtime Errors thrown from the distributed package +DistError = torch._C._DistError +DistBackendError = torch._C._DistBackendError +DistNetworkError = torch._C._DistNetworkError +DistStoreError = torch._C._DistStoreError +QueueEmptyError = torch._C._DistQueueEmptyError + +if is_available(): + from torch._C._distributed_c10d import ( + _broadcast_coalesced, + _compute_bucket_assignment_by_size, + _ControlCollectives, + _DEFAULT_FIRST_BUCKET_BYTES, + _make_nccl_premul_sum, + _register_builtin_comm_hook, + _register_comm_hook, + _StoreCollectives, + _test_python_store, + _verify_params_across_processes, + Backend as _Backend, + BuiltinCommHookType, + DebugLevel, + FileStore, + get_debug_level, + GradBucket, + Logger, + PrefixStore, + ProcessGroup as ProcessGroup, + Reducer, + set_debug_level, + set_debug_level_from_env, + Store, + TCPStore, + Work as _Work, + ) + + class _DistributedPdb(pdb.Pdb): + """ + Supports using PDB from inside a multiprocessing child process. + + Usage: + _DistributedPdb().set_trace() + """ + + def interaction(self, *args, **kwargs): + _stdin = sys.stdin + try: + with open("/dev/stdin") as sys.stdin: + pdb.Pdb.interaction(self, *args, **kwargs) + finally: + sys.stdin = _stdin + + _breakpoint_cache: dict[int, typing.Any] = {} + + def breakpoint(rank: int = 0, skip: int = 0, timeout_s=3600): + """ + Set a breakpoint, but only on a single rank. All other ranks will wait for you to be + done with the breakpoint before continuing. + + Args: + rank (int): Which rank to break on. Default: ``0`` + skip (int): Skip the first ``skip`` calls to this breakpoint. Default: ``0``. + """ + if skip > 0: + key = hash(str(traceback.format_exc())) + counter = _breakpoint_cache.get(key, 0) + 1 + _breakpoint_cache[key] = counter + if counter <= skip: + log.warning("Skip the breakpoint, counter=%d", counter) + return + + # avoid having the default timeout (if short) interrupt your debug session + if timeout_s is not None: + for group in torch.distributed.distributed_c10d._pg_map: + torch.distributed.distributed_c10d._set_pg_timeout( + timedelta(seconds=timeout_s), group + ) + + if get_rank() == rank: + pdb = _DistributedPdb() + pdb.message( + "\n!!! ATTENTION !!!\n\n" + f"Type 'up' to get to the frame that called dist.breakpoint(rank={rank})\n" + ) + pdb.set_trace() + # If Meta/Python keys are in the TLS, we want to make sure that we ignore them + # and hit the (default) CPU/CUDA implementation of barrier. + meta_in_tls = torch._C._meta_in_tls_dispatch_include() + guard = torch._C._DisableTorchDispatch() # type: ignore[attr-defined] + torch._C._set_meta_in_tls_dispatch_include(False) + try: + barrier() + finally: + torch._C._set_meta_in_tls_dispatch_include(meta_in_tls) + del guard + + if sys.platform != "win32": + from torch._C._distributed_c10d import HashStore + + from .device_mesh import DeviceMesh, init_device_mesh + + # Variables prefixed with underscore are not auto imported + # See the comment in `distributed_c10d.py` above `_backend` on why we expose + # this. + # pyrefly: ignore [deprecated] + from .distributed_c10d import * # noqa: F403 + from .distributed_c10d import ( # pyrefly: ignore # deprecated; pyrefly: ignore [deprecated] + _all_gather_base, + _coalescing_manager, + _CoalescingManager, + _create_process_group_wrapper, + _get_process_group_name, + _rank_not_in_group, + _reduce_scatter_base, + _time_estimator, + get_node_local_rank, + ) + from .remote_device import _remote_device + from .rendezvous import ( + _create_store_from_options, + register_rendezvous_handler, + rendezvous, + ) + + set_debug_level_from_env() + +else: + # This stub is sufficient to get + # python test/test_public_bindings.py -k test_correct_module_names + # working even when USE_DISTRIBUTED=0. Feel free to add more + # stubs as necessary. + # We cannot define stubs directly because they confuse pyre + + class _ProcessGroupStub: + pass + + sys.modules["torch.distributed"].ProcessGroup = _ProcessGroupStub # type: ignore[attr-defined] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_checkpointable.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_checkpointable.py new file mode 100644 index 0000000000000000000000000000000000000000..01afad0b0746b09694d987873457910140acff75 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_checkpointable.py @@ -0,0 +1,37 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates +from typing_extensions import Protocol, runtime_checkable + +import torch + + +@runtime_checkable +class _Checkpointable(Protocol): # noqa: PYI046 + """ + Interface for checkpointable objects. + Implemented as a protocol, implicit subtyping is supported so subclasses do not need to inherit this explicitly. + This is to allow arbitrary objects/tensor subclasses to hook into DCP seamlessly through implementing the interface. + """ + + def __create_write_items__(self, fqn: str, object: object) -> list[object]: + """ + Return a list of WriteItems based on object's contents. + """ + raise NotImplementedError( + "_Checkpointable._create_write_items is not implemented" + ) + + def __create_chunk_list__(self) -> list[object]: + """ + Return a list of `ChunkStorageMetadata` based on object's contents. + """ + raise NotImplementedError( + "_Checkpointable._create_chunk_list is not implemented" + ) + + def __get_tensor_shard__(self, index: int) -> torch.Tensor: + """ + Return a 'torch.Tensor' shard based on 'MetadataIndex'. + """ + raise NotImplementedError( + "_Checkpointable._get_tensor_shard is not implemented" + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_composable_state.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_composable_state.py new file mode 100644 index 0000000000000000000000000000000000000000..4117e179b0614db5873289fd549f4d45fbbc68a0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_composable_state.py @@ -0,0 +1,46 @@ +import weakref +from typing import cast + +import torch.nn as nn + + +class _State: + pass + + +_module_state_mapping: weakref.WeakKeyDictionary[ + nn.Module, weakref.ReferenceType[_State] +] = weakref.WeakKeyDictionary() + + +def _insert_module_state(module: nn.Module, state: _State) -> None: + global _module_state_mapping + if module in _module_state_mapping: + raise AssertionError(f"Inserting {module} more than once.") + _module_state_mapping[module] = weakref.ref(state) + + +def _get_module_state(module: nn.Module) -> _State | None: + """ + Return the ``_State`` in ``model``. + + Given a ``module``, this API finds out if the module is also a ``_State`` + instance or if the module is managed by a composable API. If the module + is also a ``_State``, ``module`` will be casted to ``_State` and returned. + If it is managed by a composable API, the corresponding ``_State`` will + be returned. + """ + global _module_state_mapping + if isinstance(module, _State): + # pyrefly: ignore [redundant-cast] + return cast(_State, module) + else: + # https://github.com/pytorch/pytorch/issues/107054 + if module in _module_state_mapping: + state_ref = _module_state_mapping[module] + state = state_ref() + if state is None: + raise AssertionError("State has already been garbage collected") + return state + else: + return None diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_dist2.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_dist2.py new file mode 100644 index 0000000000000000000000000000000000000000..dd8a2b3e274c654afe3cd86fde40a6aba31e5571 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_dist2.py @@ -0,0 +1,183 @@ +""" +This is an experimental new API for PyTorch Distributed. This is actively in development and subject to change or deletion entirely. + +This is intended as a proving ground for more flexible and object oriented distributed APIs. +""" + +from collections.abc import Generator +from contextlib import contextmanager +from datetime import timedelta +from typing import Protocol + +import torch +from torch._C._distributed_c10d import ( + _current_process_group, + _set_process_group, + ProcessGroup, + ReduceOp, + Store, +) +from torch.distributed.rendezvous import rendezvous + + +_BACKENDS: dict[str, "ProcessGroupFactory"] = {} + +__all__ = [ + "ProcessGroup", + "ReduceOp", + "ProcessGroupFactory", + "register_backend", + "new_group", + "current_process_group", + "process_group", +] + + +class ProcessGroupFactory(Protocol): + """Protocol for process group factories.""" + + def __call__( + self, + store: Store, + rank: int, + world_size: int, + timeout: timedelta, + device: torch.device, + **kwargs: object, + ) -> ProcessGroup: ... + + +def register_backend(name: str, func: ProcessGroupFactory) -> None: + """ + Register a new process group backend. + + Args: + name: The name of the backend. + func: The function to create the process group. + """ + if name in _BACKENDS: + raise ValueError(f"Backend {name} already registered") + + _BACKENDS[name] = func + + +def _gloo_factory( + store: Store, + rank: int, + world_size: int, + timeout: timedelta, + device: torch.device, + **kwargs: object, +) -> ProcessGroup: + from torch.distributed import ProcessGroupGloo + + if len(kwargs) != 0: + raise AssertionError("Gloo backend received unexpected kwargs") + + backend_class = ProcessGroupGloo(store, rank, world_size, timeout) + backend_class._set_sequence_number_for_group() + + pg = ProcessGroup(store, rank, world_size) + pg._set_default_backend(ProcessGroup.BackendType.GLOO) + + # register devices + pg._register_backend(device, ProcessGroup.BackendType.GLOO, backend_class) + pg._register_backend( + torch.device("cpu"), ProcessGroup.BackendType.GLOO, backend_class + ) + if torch.cuda.is_available(): + pg._register_backend( + torch.device("cuda"), ProcessGroup.BackendType.GLOO, backend_class + ) + return pg + + +def _nccl_factory( + store: Store, + rank: int, + world_size: int, + timeout: timedelta, + device: torch.device, + **kwargs: object, +) -> ProcessGroup: + from torch.distributed import ProcessGroupNCCL + + opts = ProcessGroupNCCL.Options() + opts._timeout = timeout + for k, v in kwargs.items(): + if not hasattr(opts, k): + raise KeyError(f"Unknown option {k}") + setattr(opts, k, v) + + backend_class = ProcessGroupNCCL(store, rank, world_size, opts) + backend_class._set_sequence_number_for_group() + backend_class.eager_connect_single_device(device) + + pg = ProcessGroup(store, rank, world_size) + pg._set_default_backend(ProcessGroup.BackendType.NCCL) + pg._register_backend(device, ProcessGroup.BackendType.NCCL, backend_class) + + return pg + + +register_backend("gloo", _gloo_factory) +register_backend("nccl", _nccl_factory) + + +def new_group( + backend: str, + timeout: timedelta, + device: str | torch.device, + **kwargs: object, +) -> ProcessGroup: + """ + Create a new process group with the given backend and options. This group is + independent and will not be globally registered and thus not usable via the + standard torch.distributed.* APIs. + + Args: + backend: The backend to use for the process group. + timeout: The timeout for collective operations. + device: The device to use for the process group. + **kwargs: All remaining arguments are passed to the backend constructor. + See the backend specific documentation for details. + + Returns: + A new process group. + """ + if backend not in _BACKENDS: + raise ValueError(f"Backend {backend} not registered") + + device = torch.device(device) + + store, rank, world_size = next(iter(rendezvous("env://"))) + store.set_timeout(timeout) + + return _BACKENDS[backend](store, rank, world_size, timeout, device, **kwargs) + + +def current_process_group() -> ProcessGroup: + """ + Get the current process group. Thread local method. + + Returns: + The current process group. + """ + return _current_process_group() + + +@contextmanager +def process_group(pg: ProcessGroup) -> Generator[None, None, None]: + """ + Context manager for process groups. Thread local method. + + Args: + pg: The process group to use. + """ + prev_pg = current_process_group() + + _set_process_group(pg) + try: + yield + finally: + _set_process_group(prev_pg) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives.py new file mode 100644 index 0000000000000000000000000000000000000000..994b0044a15ab25d3d7ccb81fc3bdb4a5fdfade7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives.py @@ -0,0 +1,1251 @@ +# mypy: allow-untyped-defs +import contextlib +import sys +import warnings +from typing import Any, cast, TYPE_CHECKING, Union + +import torch +import torch.distributed as dist +import torch.distributed.distributed_c10d as c10d +from torch.distributed.device_mesh import DeviceMesh +from torch.fx.experimental.proxy_tensor import get_proxy_mode + +from . import _functional_collectives_impl as fun_col_impl + + +try: + from torch.utils._cxx_pytree import tree_map_only +except ImportError: + from torch.utils._pytree import tree_map_only # type: ignore[no-redef] + + +try: + from torch.compiler import is_dynamo_compiling as is_torchdynamo_compiling +except Exception: + warnings.warn( + "Unable to import torchdynamo util `is_torchdynamo_compiling`, so won't support torchdynamo correctly", + stacklevel=2, + ) + + def is_torchdynamo_compiling(): # type: ignore[misc] + return False + return False + + +""" +New traceable, functional collectives. +RFC: https://github.com/pytorch/pytorch/issues/93173 + + compiler: trace these ops with plain-old-data schemas, then choose how to lower them. + eager: execute these 'functional' ops which in eager return AsyncCollectiveTensor subclasses, + automatically calling .wait() on underlying/hidden async 'work' obj only when fed to + a downstream op. + +Issues: +* Where should these ops live? Couldn't `import torch` if putting these ops in existing torch.distributed files +* Proper support for eager requires inplace ops. We should explore having it as an option for the API. +""" + +""" +Functional collectives are asynchronous only and we perform implicit stream synchronization +on behalf of the user. + +We use AsyncCollectiveTensor to wrap the result tensor of a collective and it lets us witness +first usage of the tensor and insert cross stream sync at the right place. + +The above are the easy bits, the hard one is how we match the Work object returned by +c10d and the tensor AsyncCollectiveTensor wraps. We alloc the tensor inside the collective +op implementation (see ``clone()`` call in ``_all_reduce``) and then it's handled by the +dispatcher which might call other implementations that are allowed to change the returned +tensor - even return a tensor with a different shape (see ``torch.vmap``). + +This means the caller of our ops receives a Tensor that is not guaranteed to be the same +allocated by our implementations and that makes pairing The AsyncTensor to the original +tensor a lot harder. This pairing is needed so we can lookup the Work object to use. + +Originally, we tried WeakKeyDictionary to map from Tensor to Work, but because Tensor's +identity is not stable across dispatch, the op caller would end up with a different Tensor +instance that would not match any in the dictionary. + +With Tensor identity out of the question, we decided use the tensor data pointer, which +should be stable across all the Tensor changes done during dispatch. + +We have a dictionary of tensor::data_ptr -> Work that we insert right after we call into c10d. + +We use this dictionary when AsyncCollectiveTensor is used to invoke Work::wait() + +Finally, we setup a finalizer against the tensor wrapper to observe it getting collected so we +can clean up stale entries in the dictionary. + +To eliminate the possibility of races we have a global version counter that is used by the finalizer. + +As a wise man said once: Don't cross the streams (https://www.youtube.com/watch?v=wyKQe_i9yyo) + +""" + +""" +Functional collectives can accept any of these types to describe the ranks participating in collectives. + +The different types will be desugared to a canonical format +""" +RANK_TYPES = Union[ + list[int], + list[list[int]], + dist.ProcessGroup, + DeviceMesh, + tuple["dist.tensor.DeviceMesh", int], + c10d.GroupName, +] + + +""" +User facing APIs for functional collectives +------------------------------------------- + +These apis are called by user code and expected to work both in eager execution and compilation, +but there are significant differences to how the two modes are implemented underneath. + +Eager execution is 'optimized' using a tensor subclass that schedules the synchronization (via wait_tensor() op) +just before the tensor is first used. Compiled tracing currently relies on the compiler to perform this optimization, +and cannot yet correctly trace the AsyncTensor wrapper class. In the future, these paths may be unified +if sufficient subclass support is added in dynamo. + +Example: all_reduce is an entrypoint API, and other collectives follow a similar pattern. + +Here's how it works under torch.compile/dynamo: +all_reduce(...) + |--> _expand_group(...) - desugars processgroup into canonical/traceable format + |--> c10d_functional.all_reduce(...) - dynamo captures this op call, doesn't trace deeper + |--> _maybe_wrap_tensor(...) - wait_tensor() op is immediately called, no AsyncTensor subclass needed + +And under eager execution: +all_reduce(...) + |--> _expand_group(...) - same as above, but less critical for eager + |--> c10d_functional.all_reduce(...) - dispatches to real kernel OR records op in trace + |--> _maybe_wrap_tensor(...) - AsyncTensor wrapper applied to returned tensor, + which issues wait_tensor() at the time of first use +""" + + +def wait_tensor(tensor): + """ + Wait on a tensor returned by the collectives ops. + + Waiting follows device semantics, which means blocking on CPU and synchronizing streams on CUDA. + """ + return torch.ops._c10d_functional.wait_tensor(tensor) # type: ignore[attr-defined] + + +def broadcast(self: torch.Tensor, src: int, group: RANK_TYPES, tag: str = ""): + """ + Broadcasts the tensor to all processes in the given process group. + + Args: + src (int): Source rank + group (ProcessGroup or List[int]): The process group to work on. + tag (str, optional): A unique identifier for the collective. Default: empty string + """ + group_name = _resolve_group_name(group, tag) + tensor = torch.ops._c10d_functional.broadcast(self, src, group_name) + return _maybe_wrap_tensor(tensor) + + +def all_reduce(self: torch.Tensor, reduceOp: str, group: RANK_TYPES, tag: str = ""): + """ + Reduces the tensor data across all machines in such a way that all get + the final result. + + The input tensor is left unmodified. + + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + group_name = _resolve_group_name(group, tag) + tensor = torch.ops._c10d_functional.all_reduce(self, reduceOp.lower(), group_name) + return _maybe_wrap_tensor(tensor) + + +def all_gather_tensor( + self: torch.Tensor, + gather_dim: int, + group: RANK_TYPES, + tag: str = "", +) -> torch.Tensor: + """ + Gather tensor data across from all machines and concatenate over ``gather_dim``. + + Note that it currently only supports gather_dim = 0. + + The input tensor is left unmodified. + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + if not self.is_contiguous(): + raise AssertionError("Tensor must be contiguous for all_gather_tensor") + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + tensor = torch.ops._c10d_functional.all_gather_into_tensor( + self, group_size, group_name + ) + res = _maybe_wrap_tensor(tensor) + # TODO this should be done inside AsyncCollectiveTensor to delay the wait() call + if gather_dim != 0: + # torch.cat access the data so we already need to wait here, first do wait + # and then chunk + cat avoid us going through ACT dispatching logic again + if isinstance(res, AsyncCollectiveTensor): + res = res.wait() # type: ignore[attr-defined] + res = torch.cat(torch.chunk(res, group_size, dim=0), dim=gather_dim) + return res + + +def all_gather_tensor_autograd( + self: torch.Tensor, + gather_dim: int, + group: RANK_TYPES, + tag: str = "", +): + """ + Gather tensor data across from all machines and concatenate over ``gather_dim``. + + Note that it currently only supports gather_dim = 0. + + This function is the same as all_gather_tensor but will propagate the + backwards gradient across workers. + + See all_gather_tensor for more details on usage. + """ + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + + tensor = torch.ops._c10d_functional_autograd.all_gather_into_tensor( + self, group_size, group_name + ) + res = _FromTorchTensor.apply(tensor) + # TODO this should be done inside AsyncCollectiveTensor to delay the wait() call + if gather_dim != 0: + # torch.cat access the data so we already need to wait here, first do wait + # and then chunk + cat avoid us going through ACT dispatching logic again + if isinstance(res, AsyncCollectiveTensor): + res = res.wait() # type: ignore[attr-defined] + res = torch.cat(torch.chunk(res, group_size, dim=0), dim=gather_dim) + return res + + +def reduce_scatter_tensor( + self: torch.Tensor, + reduceOp: str, + scatter_dim: int, + group: RANK_TYPES, + tag: str = "", +): + """ + Reduces the tensor data across all machines in such a way that all get + the final result, then scatter the results to corresponding ranks. + + + The input tensor is left unmodified. + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + + if self.size(scatter_dim) % group_size != 0: + raise AssertionError( + f"input dimension 0 ({self.size(0)} must be a multiple of group_size {group_size})" + ) + if scatter_dim != 0: + tensor_list = torch.chunk(self, group_size, dim=scatter_dim) + self = torch.cat(tensor_list) + + tensor = torch.ops._c10d_functional.reduce_scatter_tensor( + self, + reduceOp.lower(), + group_size, + group_name, # type: ignore[possibly-undefined] + ) + res = _maybe_wrap_tensor(tensor) + return res + + +def reduce_scatter_tensor_autograd( + self: torch.Tensor, + reduceOp: str, + scatter_dim: int, + group: RANK_TYPES, + tag: str = "", +): + """ + Reduces the tensor data across all machines in such a way that all get + the final result, then scatter the results to corresponding ranks. + + This function is the same as reduce_scatter_tensor but will propagate the + backwards gradient across workers. + + Currently only the "sum" reduceOp is supported. + + See reduce_scatter_tensor for more details on usage. + """ + + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + + if self.size(scatter_dim) % group_size != 0: + raise AssertionError( + f"input dimension 0 ({self.size(0)} must be a multiple of group_size {group_size}" + ) + if scatter_dim != 0: + tensor_list = torch.chunk(self, group_size, dim=scatter_dim) + self = torch.cat(tensor_list) + + tensor = torch.ops._c10d_functional_autograd.reduce_scatter_tensor( + self, + reduceOp.lower(), + group_size, + group_name, # type: ignore[possibly-undefined] + ) + res = _FromTorchTensor.apply(tensor) + return res + + +def all_reduce_coalesced( + self: list[torch.Tensor], reduceOp: str, group: RANK_TYPES, tag: str = "" +) -> list[torch.Tensor]: + """ + Reduces a list of tensors across all machines in such a way that all get + the final result. + + The all tensors in the input list are left unmodified. + + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + group_name = _resolve_group_name(group, tag) + tensor_list = torch.ops._c10d_functional.all_reduce_coalesced( # type: ignore[attr-defined] + self, + reduceOp.lower(), + group_name, + ) + return list(map(_maybe_wrap_tensor, tensor_list)) + + +def all_gather_into_tensor_coalesced( + self: list[torch.Tensor], group: RANK_TYPES, tag: str = "" +) -> list[torch.Tensor]: + """ + Gather a list of tensors across from all machines. + + Note that it currently only supports gather_dim = 0. + + The input tensor is left unmodified. + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + tensor_list = torch.ops._c10d_functional.all_gather_into_tensor_coalesced( # type: ignore[attr-defined] + self, + group_size, + group_name, + ) + return list(map(_maybe_wrap_tensor, tensor_list)) + + +def reduce_scatter_tensor_coalesced( + inputs: list[torch.Tensor], + reduceOp: str, + scatter_dim: list[int], + group: RANK_TYPES, + tag: str = "", +) -> list[torch.Tensor]: + """ + Reduces a list of tensors across all machines in such a way that all get + the final result, then scatter the results to corresponding ranks. + + The input tensors are left unmodified. + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + + if len(scatter_dim) != len(inputs): + raise AssertionError( + f"Length of scatter_dim ({len(scatter_dim)}) must equal length of inputs ({len(inputs)})" + ) + for idx, (dim, tensor) in enumerate(zip(scatter_dim, inputs)): + if tensor.size(dim) % group_size != 0: + raise AssertionError( + f"input dimension {dim} ({tensor.size(dim)} must be a multiple of group_size {group_size} for tensor at index {idx}" + ) + if dim != 0: + tensor_list = torch.chunk(tensor, group_size, dim=dim) + inputs[idx] = torch.cat(tensor_list) + + tensor_list = torch.ops._c10d_functional.reduce_scatter_tensor_coalesced( # type: ignore[attr-defined] + inputs, + reduceOp.lower(), + group_size, + group_name, # type: ignore[possibly-undefined] + ) + + return list(map(_maybe_wrap_tensor, tensor_list)) + + +# This is a bit unsafe: it checks if the first argument in the schema reports as a non-mutable alias. +# Today, this maps 1:1 with "aten ops that are views". +def _is_view_op(tgt): + if not isinstance(tgt, torch._ops.OpOverload): + raise AssertionError(f"Expected torch._ops.OpOverload, got {type(tgt)}") + # Don't apply the view optimization to any `CompositeImplicitAutograd` ops. + # See issue: https://github.com/pytorch/pytorch/issues/133421 + if torch._C._dispatch_has_kernel_for_dispatch_key( + tgt.name(), torch.DispatchKey.CompositeImplicitAutograd + ): + return False + schema = tgt._schema + if len(schema.arguments) > 0: + first_arg = schema.arguments[0] + # check if op is a view + return first_arg.alias_info is not None and not first_arg.alias_info.is_write + + +def all_to_all_single( + self: torch.Tensor, + output_split_sizes: list[int] | None, + input_split_sizes: list[int] | None, + group: RANK_TYPES, + tag: str = "", +) -> torch.Tensor: + """ + Each process splits input tensor and then scatters the split list + to all processes in a group. Then concatenate the received tensors from all + the processes in the group and return single output tensor. + + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one dimension of the DeviceMesh + + :: N.B. If you pass a PG or a 1D list to perform a MPMD collective, the compiler won't be able to recover + that information and perform collective algebraic optimization. Use other forms of input for that. + """ + if output_split_sizes is not None: + if not all( + isinstance(size, (int, torch.SymInt)) for size in output_split_sizes + ): + raise AssertionError( + f"All output_split_sizes must be int or SymInt, got {output_split_sizes}" + ) + if input_split_sizes is not None: + if not all(isinstance(size, (int, torch.SymInt)) for size in input_split_sizes): + raise AssertionError( + f"All input_split_sizes must be int or SymInt, got {input_split_sizes}" + ) + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + if output_split_sizes is None or input_split_sizes is None: + if not (output_split_sizes is None and input_split_sizes is None): + raise AssertionError( + "output_split_sizes and input_split_sizes must either be " + "specified together or both set to None" + ) + output_split_sizes = [self.shape[0] // group_size] * group_size + input_split_sizes = output_split_sizes + tensor = torch.ops._c10d_functional.all_to_all_single( # type: ignore[attr-defined] + self, + output_split_sizes, + input_split_sizes, + group_name, + ) + return _maybe_wrap_tensor(tensor) + + +def all_to_all_single_autograd( + self: torch.Tensor, + output_split_sizes: list[int] | None, + input_split_sizes: list[int] | None, + group: RANK_TYPES, + tag: str = "", +) -> torch.Tensor: + """ + Same as all_to_all_single but supports autograd. + """ + if output_split_sizes is not None: + if not all( + isinstance(size, (int, torch.SymInt)) for size in output_split_sizes + ): + raise AssertionError( + f"All output_split_sizes must be int or SymInt, got {output_split_sizes}" + ) + if input_split_sizes is not None: + if not all(isinstance(size, (int, torch.SymInt)) for size in input_split_sizes): + raise AssertionError( + f"All input_split_sizes must be int or SymInt, got {input_split_sizes}" + ) + + group_name = _resolve_group_name(group, tag) + group_size = c10d._get_group_size_by_name(group_name) + if output_split_sizes is None or input_split_sizes is None: + if not (output_split_sizes is None and input_split_sizes is None): + raise AssertionError( + "output_split_sizes and input_split_sizes must either be " + "specified together or both set to None" + ) + output_split_sizes = [self.shape[0] // group_size] * group_size + input_split_sizes = output_split_sizes + tensor = torch.ops._c10d_functional_autograd.all_to_all_single( # type: ignore[attr-defined] + self, + output_split_sizes, + input_split_sizes, + group_name, + ) + return _FromTorchTensor.apply(tensor) + + +def permute_tensor( + self: torch.Tensor, + src_dst: list[int], + group: RANK_TYPES, + tag: str = "", +) -> torch.Tensor: + """ + Permutes the elements of the tensor according to the given source/destination pairs. `src_dst` should + be defined such that src_dst[m] == n means m sends to n. + + Group can be one of: + List[int]: ranks participating in the collective. + List[List[int]]: 2D mesh of ranks taking part of this collective in MPMD. + ProcessGroup: Will perform a collective using the ranks and tag of the PG. + DeviceMesh: Do a SPMD collective over all ranks of the mesh + (DeviceMesh, int): Do a MPMD collective over one + """ + t, rankset, group_size = _expand_group(group, tag) + local_pg = c10d._find_or_create_pg_by_ranks_and_tag(t, rankset, group_size) + + output_split_sizes = [0] * group_size + input_split_sizes = [0] * group_size + for src, dst in enumerate(src_dst): + if src == dist.get_rank(local_pg): + input_split_sizes[dst] = self.numel() + if dst == dist.get_rank(local_pg): + output_split_sizes[src] = self.numel() + + return all_to_all_single(self, output_split_sizes, input_split_sizes, group, tag) + + +class AsyncCollectiveTensor(torch.Tensor): + r""" + A Tensor wrapper subclass that is used to trigger a call to wait + prior to first use of the underlying tensor. + Use it inside functional collective pytorch wrappers like the following: + def functional_collective(self, group, tag): + tag, rankset, group_size = _expand_group(group, tag) + tensor = torch.ops.c10d_functional.{collective}(self, tag, rankset, group_size) + return _maybe_wrap_tensor(tensor) + """ + + elem: torch.Tensor + completed: bool + + __slots__ = ["elem", "completed"] + + @staticmethod + def __new__(cls, elem: torch.Tensor): + r = torch.Tensor._make_wrapper_subclass( + cls, + elem.size(), + strides=elem.stride(), + storage_offset=elem.storage_offset(), + dtype=elem.dtype, + layout=elem.layout, + device=elem.device, + requires_grad=elem.requires_grad, + ) + r.elem = elem + r.completed = False + return r + + def __tensor_flatten__(self): + return ["elem"], None + + def tolist(self): + return self.trigger_wait().tolist() + + @staticmethod + def __tensor_unflatten__(inner_tensors, meta, outer_size, outer_stride): + if meta is not None: + raise AssertionError( + "meta must be None for AsyncCollectiveTensor unflatten" + ) + elem = inner_tensors["elem"] + return AsyncCollectiveTensor(elem) + + def __coerce_same_metadata_as_tangent__( + self, expected_metadata: Any, expected_type: type | None = None + ): + if expected_type is not torch.Tensor: + return None + + return self.trigger_wait() + + def __repr__(self) -> str: # type: ignore[override] + return f"AsyncCollectiveTensor({self.trigger_wait()})" + + def trigger_wait(self): + if not self.completed: + out = wait_tensor(self.elem) + self.completed = True + return out + else: + return self.elem + + def wait(self) -> torch.Tensor: + return wait_tensor(self.elem) + + def _get_acs_underlying_tensor(self): + """This method enables _functional_collectives_impl to test if a tensor is an ACS""" + return self.elem + + @classmethod + def __torch_dispatch__(cls, func, types, args=(), kwargs=None): # type: ignore[override] + if func is torch.ops.aten.view.default: + # Fast handle aten.view as a lot of view related op goes to aten.view + # eventually, this avoids pytree slowdown + # pyrefly: ignore [index-error] + res = func(args[0].elem, args[1]) + wrapper_res = AsyncCollectiveTensor(res) + return wrapper_res + + is_view_op = _is_view_op(func) + + def unwrap(e: AsyncCollectiveTensor): + # wait_tensor is idepotent and will do stream sync only once + if not is_view_op: + return e.trigger_wait() + return e.elem + + def wrap(e: torch.Tensor): + # wait_tensor is idepotent and will do stream sync only once + if isinstance(e, AsyncCollectiveTensor): + raise AssertionError( + "Cannot wrap an AsyncCollectiveTensor inside another AsyncCollectiveTensor" + ) + res = AsyncCollectiveTensor(e) + return res + + unwrapped_args = tree_map_only(AsyncCollectiveTensor, unwrap, args) + unwrapped_kwargs = tree_map_only(AsyncCollectiveTensor, unwrap, kwargs) + + # we don't wrap the result as it doesn't need to be waited on. + out = func(*unwrapped_args, **unwrapped_kwargs) + + # View ops dont require a sync, so we should re-wrap the outputs. + if is_view_op: + out = tree_map_only(torch.Tensor, wrap, out) + + return out + + def numpy(self): # type: ignore[override] + return self.wait().numpy() + + +""" +Utils and infrastructure for tracing support +""" + + +def _expand_group(group: RANK_TYPES, tag: str = "") -> tuple[str, list[int], int]: + """ + _expand_group desugars the different RANK_TYPES types into a canonical format that is traceable. + + By having this be part of the explicit eager codepath, we avoid having to specialize behavior inside + torchdynamo and can still interoperate with processgroup objects or other untraceable forms. + """ + # had to define this hack _inside_ expand_group to avoid + # graph_break [('torch.* op returned non-Tensor int + # caused by 'cast_*` functions being treated as 'torch.*' ops (iiuc) + if TYPE_CHECKING: + + def cast_listlistint(x): + return cast(list[list[int]], x) + + def cast_listint(x): + return cast(list[int], x) + + else: + # fake cast op for use at runtime since dynamo doesn't support real cast + # also, dynamo didn't like encountering 'typing' objects () + # NotImplementedError: argument of type: + def cast_listlistint(x): + return x + + def cast_listint(x): + return x + + rankset: list[int] + if isinstance(group, list): + if isinstance(group[0], list): + nested_list = cast_listlistint(group) + rankset = [] + group_size = -1 + for rs in nested_list: + rankset.extend(rs) + if group_size != -1 and group_size != len(rs): + raise ValueError( + f"group sizes must be identical found {group_size} and {len(rs)}" + ) + group_size = len(rs) + else: + rankset = cast_listint(group) + group_size = len(rankset) + elif isinstance(group, dist.ProcessGroup): + rankset = dist.get_process_group_ranks(group) + group_size = len(rankset) + tag = tag or c10d._get_group_tag(group) + elif isinstance(group, DeviceMesh): + if group.ndim != 1: + raise AssertionError( + "Only 1D mesh is supported, pass in (DeviceMesh, int) together if mesh > 1D" + ) + # TODO: it should run collective in the whole mesh instead of dim 0 + pg = group.get_group() + rankset = dist.get_process_group_ranks(pg) + group_size = len(rankset) + tag = tag or c10d._get_group_tag(pg) + elif isinstance(group, tuple): + if ( + len(group) == 2 + and isinstance(group[0], DeviceMesh) + and isinstance(group[1], int) + ): + dmesh = group[0] + dim = group[1] + pg = dmesh.get_group(dim) + rankset = dist.get_process_group_ranks(pg) + group_size = len(rankset) + tag = tag or c10d._get_group_tag(pg) + else: + raise ValueError("Invalid tuple for group must be (DeviceMesh, int)") + else: + raise ValueError( + "Invalid type for group, must be one of List, Processgroup, DeviceMesh or (DeviceMesh, int)." + ) + + return (tag, rankset, group_size) + + +def _resolve_group_name(group: RANK_TYPES, tag: str = "") -> c10d.GroupName: + """ + Given group in RANK_TYPES, return the group name. + """ + # `tag` will be deprecated. See details in: + # https://github.com/pytorch/pytorch/issues/93173#issuecomment-1907095208 + if isinstance(group, dist.ProcessGroup): + return group.group_name + elif isinstance(group, str): + # In some cases Dynamo doesn't like tracing through NewType constructors + # - so use a cast instead (the actual newtype representation is + # literally the underlying type so this is fine). I haven't been able to + # reproduce it in isolation (see T247631668). + return cast(c10d.GroupName, group) # c10d.GroupName(group) + elif isinstance(group, DeviceMesh): + if group.ndim != 1: + raise AssertionError( + "Only 1D mesh is supported, pass in (DeviceMesh, int) together if mesh > 1D" + ) + return group._dim_group_names[0] + elif isinstance(group, tuple): + if ( + len(group) == 2 + and isinstance(group[0], DeviceMesh) + and isinstance(group[1], int) + ): + dmesh = group[0] + dim = group[1] + return dmesh._dim_group_names[dim] + else: + raise ValueError("Invalid tuple for group must be (DeviceMesh, int)") + elif isinstance(group, list): + if not is_torchdynamo_compiling(): + warnings.warn( + "The combination of ranks + tag as process group " + "identifier has been deprecated. Please switch to " + "using ProcessGroup, DeviceMesh, or group name instead.", + FutureWarning, + stacklevel=3, + ) + # pyrefly: ignore [redundant-cast] + return c10d._resolve_group_name_by_ranks_and_tag(cast(list[int], group), tag) + else: + raise ValueError(f"Unsupported group type: {type(group)}, {group}") + + +class _FromTorchTensor(torch.autograd.Function): + """ + _FromTorchTensor allows autograd to propagate from a normal Tensor to an + AsyncCollectiveTensor. + """ + + @staticmethod + def forward( # type: ignore[override] + ctx, # pyre-ignore[2]: Parameter must be annotated. + input: torch.Tensor, + ) -> torch.Tensor: + return _maybe_wrap_tensor(input) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor: # type: ignore[override] + return grad_output + + +def _are_we_tracing() -> bool: + if is_torchdynamo_compiling(): + return True + # If fake mode is turned on, we are almost definitely compiling/tracing. + if torch._C._get_dispatch_mode(torch._C._TorchDispatchModeKey.FAKE) is not None: + return True + # See Note [enable_python_dispatcher in dynamo] + if torch._C._dispatch_tls_is_dispatch_key_included( + torch._C.DispatchKey.PythonDispatcher + ): + return True + return get_proxy_mode() is not None + + +def _maybe_wrap_tensor(self) -> torch.Tensor: + if _are_we_tracing(): + return wait_tensor(self) + res = AsyncCollectiveTensor(self) + return cast(torch.Tensor, res) + + +@contextlib.contextmanager +def allow_inflight_collective_as_graph_input_ctx(value: bool = True): + """ + Context manager to temporarily set whether inflight collectives are allowed as torch.compile graph inputs. + Common use case is when the collective is issued in eager (with `async_op=True`) but waited in compiled region: + ``` + def all_reduce_eager(x): + y = x * x + req = dist.all_reduce(y, op=dist.ReduceOp.SUM, async_op=True) + return y + + + @torch.compile(fullgraph=True) + def all_reduce_wait_compiled(y): + torch.ops.c10d_functional.wait_tensor(y) + return y * y + + + x = torch.ones(1280, 1280, device="cuda") + self.rank + # the context manager ensures that `wait_tensor(y)` will wait on the correct work object + with allow_inflight_collective_as_graph_input_ctx(): + y = all_reduce_eager(x) + z = all_reduce_wait_compiled(y) + ``` + With this context manager, when a collective is called, under the hood the work object of the collective + will be registered in the work registry, and the wait_tensor() in compiled region called on + the output tensor of the collective will wait on the correct work object. + """ + previous = torch._C._distributed_c10d._allow_inflight_collective_as_graph_input() + + try: + torch._C._distributed_c10d._set_allow_inflight_collective_as_graph_input(value) + yield + finally: + torch._C._distributed_c10d._set_allow_inflight_collective_as_graph_input( + previous + ) + + +def _make_all_gather_out_tensor(input, group_size): + out_size = list(input.size()) + if len(out_size) == 0: + out_size.append(group_size) + else: + out_size[0] *= group_size + out_tensor = input.new_empty(out_size) + return out_tensor + + +def _all_gather_into_tensor_coalesced_meta(self, tag, rankset, group_size): + return [_make_all_gather_out_tensor(t, group_size) for t in self] + + +# We now register meta kernels to deal with tracing +def _broadcast_meta(self, *args): + return torch.empty_like(self) + + +def _all_reduce_meta(self, *args): + return torch.empty_like(self) + + +def _wait_tensor_meta(self, *args): + return torch.empty_like(self) + + +def _all_gather_into_tensor_meta(shard, tag, rankset, group_size): + return _make_all_gather_out_tensor(shard, group_size) + + +def _reduce_scatter_tensor_meta(input, reduce_op, tag, rankset, group_size): + out_size = list(input.size()) + out_size[0] //= group_size + return input.new_empty(out_size) + + +def _all_reduce_coalesced_meta(self, *args): + return [torch.empty_like(t) for t in self] + + +def _all_reduce__meta(inp, *args): + return inp + + +def _broadcast__meta(inp, *args): + return inp + + +def _all_reduce_coalesced__meta(inputs, *args): + return inputs + + +def _reduce_scatter_tensor_coalesced_meta(inputs, reduceOp, tag, rankset, group_size): + def mk_out_tensor(input): + out_size = list(input.size()) + out_size[0] //= group_size + out_tensor = input.new_empty(out_size) + return out_tensor + + return [mk_out_tensor(t) for t in inputs] + + +# NB: We often say all_to_all has dynamic output size, but this is not +# technically true: instead, what typically happens is you manually +# communicate the output_split_sizes ahead of time (which is dynamic), +# but then you pass those sizes explicitly, and the all to all itself +# isn't dynamic, it just follows the specified output splits +def _all_to_all_single_meta( + input, output_split_sizes, input_split_sizes, *args, **kwargs +): + if output_split_sizes is None: + return input.new_empty(input.size()) + else: + for s in output_split_sizes: + torch._check(s >= 0) + out_size = list(input.size()) + out_size[0] = sum(output_split_sizes) + return input.new_empty(out_size) + + +def _all_gather_into_tensor_out_native_meta(input, group_size, group_name, *, out): + return _make_all_gather_out_tensor(input, group_size) + + +def _all_gather_into_tensor_native_meta(input, group_size, group_name): + return _make_all_gather_out_tensor(input, group_size) + + +def _all_gather_into_tensor_coalesced_native_meta(inputs, group_size, group_name): + return [ + _all_gather_into_tensor_native_meta(input, group_size, group_name) + for input in inputs + ] + + +def _reduce_scatter_tensor_native_meta(inp, reduce_op, group_size, group_name): + shape = list(inp.size()) + shape[0] //= group_size + return inp.new_empty(shape) + + +def _reduce_scatter_tensor_out_native_meta( + inp, reduce_op, group_size, group_name, *, out +): + shape = list(inp.size()) + shape[0] //= group_size + return inp.new_empty(shape) + + +def _reduce_scatter_tensor_coalesced_native_meta( + inputs, reduce_op, group_size, group_name +): + return [ + _reduce_scatter_tensor_native_meta(inp, reduce_op, group_size, group_name) + for inp in inputs + ] + + +# Library MUST be defined at module scope or it doesn't work +lib_impl = torch.library.Library("_c10d_functional", "IMPL") +lib_impl.impl("all_reduce", _all_reduce_meta, "Meta") +lib_impl.impl("all_reduce_", _all_reduce__meta, "Meta") +lib_impl.impl("all_reduce_coalesced", _all_reduce_coalesced_meta, "Meta") +lib_impl.impl("all_reduce_coalesced_", _all_reduce_coalesced__meta, "Meta") +lib_impl.impl("wait_tensor", _wait_tensor_meta, "Meta") +lib_impl.impl( + "all_gather_into_tensor_out", _all_gather_into_tensor_out_native_meta, "Meta" +) +lib_impl.impl("all_gather_into_tensor", _all_gather_into_tensor_native_meta, "Meta") +lib_impl.impl( + "all_gather_into_tensor_coalesced", + _all_gather_into_tensor_coalesced_native_meta, + "Meta", +) +lib_impl.impl("reduce_scatter_tensor", _reduce_scatter_tensor_native_meta, "Meta") +lib_impl.impl( + "reduce_scatter_tensor_out", _reduce_scatter_tensor_out_native_meta, "Meta" +) +lib_impl.impl( + "reduce_scatter_tensor_coalesced", + _reduce_scatter_tensor_coalesced_native_meta, + "Meta", +) +lib_impl.impl("all_to_all_single", _all_to_all_single_meta, "Meta") +lib_impl.impl("broadcast", _broadcast_meta, "Meta") +lib_impl.impl("broadcast_", _broadcast__meta, "Meta") + +# mark these ops has side effect so that they won't be removed by DCE +torch.fx.node.has_side_effect(torch.ops._c10d_functional.wait_tensor.default) # type: ignore[has-type] +torch.fx.node.has_side_effect(torch.ops._c10d_functional.wait_tensor) # type: ignore[has-type] + +# Register legacy ops for backward compatibility +# TODO(yifu): remove these in functional collective beta release +legacy_lib = torch.library.Library("c10d_functional", "DEF") +legacy_lib_impl = torch.library.Library("c10d_functional", "IMPL") +ops_defs = [ + "broadcast(Tensor self, int src, str tag, int[] ranks, int group_size) -> Tensor", + "all_reduce(Tensor self, str reduceOp, str tag, int[] ranks, int group_size) -> Tensor", + "all_reduce_coalesced(Tensor[] self, str reduceOp, str tag, int[] ranks, int group_size) -> Tensor[]", + "wait_tensor(Tensor self) -> Tensor", + "all_gather_into_tensor(Tensor shard, str tag, int[] ranks, int group_size) -> Tensor", + "all_gather_into_tensor_coalesced(Tensor[] input, str tag, int[] ranks, int group_size) -> Tensor[]", + "reduce_scatter_tensor(Tensor input, str reduceOp, str tag, int[] ranks, int group_size) -> Tensor", + "reduce_scatter_tensor_coalesced(Tensor[] inputs, str reduceOp, str tag, int[] ranks, int group_size) -> Tensor[]", + "all_to_all_single(Tensor input, SymInt[]? output_split_sizes, SymInt[]? input_split_sizes, str tag, int[] ranks, int group_size) -> Tensor", # noqa: B950 +] + +my_module = sys.modules[__name__] +for op_def in ops_defs: + op_name = op_def[0 : op_def.index("(")] + backend_impl = getattr(fun_col_impl, f"_{op_name}") + legacy_lib.define(op_def, tags=torch.Tag.pt2_compliant_tag) + legacy_lib_impl.impl(op_name, backend_impl, "CompositeImplicitAutograd") + + +""" +Dynamo Remappings allow seamless translation from non-functional collectives of supportable form into +functional collective calls followed by inplace copy ops, allowing them to be traced into a functional graph. + +We implement this by writing a decomposition and teaching dynamo how to associate it to a corresponding op via +the mapping dict below. + +These schemas intentionally match torch.distributed.distributed_c10d.* ops that we are trying to remap from +""" + + +def all_gather_tensor_inplace( + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + group=None, # TODO add a type, + async_op: bool = False, + tag: str = "", + gather_dim: int = 0, +): + if async_op: + raise AssertionError( + "Can't remap async version of inplace op to functional collective" + ) + + group = group or dist.group.WORLD + if group is None: + raise AssertionError("group cannot be None") + + return output_tensor.copy_(all_gather_tensor(input_tensor, gather_dim, group, tag)) + + +def reduce_scatter_tensor_inplace( + output: torch.Tensor, + input: torch.Tensor, + op: str = "sum", # TODO type is actually c10d ReduceOp. is this ok? + group=None, # TODO add a type + async_op: bool = False, + scatter_dim: int = 0, + tag: str = "", +): + if async_op: + raise AssertionError( + "Can't remap async version of inplace op to functional collective" + ) + + group = group or dist.group.WORLD + if group is None: + raise AssertionError("group cannot be None") + + return output.copy_(reduce_scatter_tensor(input, op, scatter_dim, group, tag)) + + +REDUCE_OP_TO_STR = { + dist.ReduceOp.SUM: "sum", + dist.ReduceOp.AVG: "avg", + dist.ReduceOp.PRODUCT: "product", + dist.ReduceOp.MIN: "min", + dist.ReduceOp.MAX: "max", + dist.ReduceOp.BAND: "band", + dist.ReduceOp.BOR: "bor", + dist.ReduceOp.BXOR: "bxor", +} + + +def all_reduce_inplace( + tensor: torch.Tensor, + op: str = "sum", + group=None, + async_op: bool = False, + tag: str = "", +): + if async_op: + raise AssertionError( + "Can't remap async version of inplace op to functional collective" + ) + + group = group or dist.group.WORLD + if group is None: + raise AssertionError("group cannot be None") + + return tensor.copy_(all_reduce(tensor, op, group, tag)) + + +def all_to_all_inplace( + output: torch.Tensor, + input: torch.Tensor, + output_split_sizes=None, + input_split_sizes=None, + group=None, + async_op=False, + tag: str = "", +): + if async_op: + raise AssertionError( + "Can't remap async version of inplace op to functional collective" + ) + + group = group or dist.group.WORLD + if group is None: + raise AssertionError("group cannot be None") + + return output.copy_( + all_to_all_single( + input, + output_split_sizes, + input_split_sizes, + group, + tag, + ) + ) + + +def all_gather_inplace( + tensor_list: list[torch.Tensor], + tensor: torch.Tensor, + group=None, + async_op=False, + tag: str = "", +): + if async_op: + raise AssertionError( + "Can't remap async version of inplace op to functional collective" + ) + if tensor.dim() != 0 and not all(t.size(0) == tensor.size(0) for t in tensor_list): + raise AssertionError("Remapping variable size all_gather is not yet supported") + + group = group or dist.group.WORLD + if group is None: + raise AssertionError("group cannot be None") + + output = all_gather_tensor(tensor, 0, group, tag) + + # Use aten.slice instead of aten.split because the latter causes + # tensor.shape(0) to be unnecessarily baked in when it's a SymInt. + output_splits = [] + offset = 0 + for t in tensor_list: + is_scalar = t.dim() == 0 + t_offset = 1 if is_scalar else t.size(0) + # pyrefly: ignore [unsupported-operation] + out = output[offset] if is_scalar else output[offset : offset + t_offset] + output_splits.append(out) + # pyrefly: ignore [unsupported-operation] + offset += t_offset + for dst, src in zip(tensor_list, output_splits): + dst.copy_(src) + return tensor_list + + +from torch.distributed.distributed_c10d import ( # pyrefly: ignore # deprecated; pyrefly: ignore [deprecated] + _all_gather_base as legacy_all_gather_base, + _reduce_scatter_base as legacy_reduce_scatter_base, + all_gather as legacy_all_gather, + all_gather_into_tensor as legacy_allgather, + all_reduce as legacy_allreduce, + all_to_all_single as legacy_all_to_all_single, + reduce_scatter_tensor as legacy_reducescatter, +) + + +# This dict should contain sets of functions that dynamo is allowed to remap. +# Functions in this set should accept the same args/kwargs 1:1 as their mapping. +traceable_collective_remaps = { + legacy_allgather: all_gather_tensor_inplace, # type: ignore[has-type] + legacy_reducescatter: reduce_scatter_tensor_inplace, # type: ignore[has-type] + legacy_allreduce: all_reduce_inplace, # type: ignore[has-type] + legacy_all_to_all_single: all_to_all_inplace, # type: ignore[has-type] + legacy_all_gather: all_gather_inplace, # type: ignore[has-type] + legacy_reduce_scatter_base: reduce_scatter_tensor_inplace, # type: ignore[has-type] + legacy_all_gather_base: all_gather_tensor_inplace, # type: ignore[has-type] +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives_impl.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..7b66ec1d4d913ed260b9f7ab1ec62a459e764e1f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_functional_collectives_impl.py @@ -0,0 +1,117 @@ +# mypy: allow-untyped-defs + +import torch +import torch.distributed.distributed_c10d as c10d + + +""" +This file contains the op impls for the legacy (c10d_functional) functional collectives. +These impls simply call into the native (_c10d_functional) functional collectives. +""" + + +def _broadcast(input, src, tag, ranks, group_size): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.broadcast( + input, + src, + group_name, + ) + + +def _all_reduce(input, reduce_op, tag, ranks, group_size): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.all_reduce( + input, + reduce_op, + group_name, + ) + + +def _all_reduce_coalesced(inputs, reduce_op, tag, ranks, group_size): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.all_reduce_coalesced( + inputs, + reduce_op, + group_name, + ) + + +def _all_gather_into_tensor(input, tag, ranks, group_size): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.all_gather_into_tensor( + input, + group_size, + group_name, + ) + + +def _all_gather_into_tensor_coalesced(input, tag, ranks, group_size): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.all_gather_into_tensor_coalesced( + input, + group_size, + group_name, + ) + + +def _reduce_scatter_tensor( + input: torch.Tensor, + reduce_op: str, + tag: str, + ranks: list[int], + group_size: int, +): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.reduce_scatter_tensor( + input, + reduce_op, + group_size, + group_name, + ) + + +def _reduce_scatter_tensor_coalesced( + inputs: list[torch.Tensor], + reduce_op: str, + tag: str, + ranks: list[int], + group_size: int, +): + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.reduce_scatter_tensor_coalesced( + inputs, + reduce_op, + group_size, + group_name, + ) + + +def _all_to_all_single( + input: torch.Tensor, + output_split_sizes: list[int] | None, + input_split_sizes: list[int] | None, + tag: str, + ranks: list[int], + group_size: int, +): + if output_split_sizes is None or input_split_sizes is None: + if not (output_split_sizes is None and input_split_sizes is None): + raise AssertionError( + "output_split_sizes and input_split_sizes must either be " + "specified together or both set to None" + ) + output_split_sizes = [input.shape[0] // group_size] * group_size + input_split_sizes = output_split_sizes + + group_name = c10d._resolve_group_name_by_ranks_and_tag(ranks, tag) + return torch.ops._c10d_functional.all_to_all_single( + input, + output_split_sizes, + input_split_sizes, + group_name, + ) + + +def _wait_tensor(tensor: torch.Tensor) -> torch.Tensor: + return torch.ops._c10d_functional.wait_tensor(tensor) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_mesh_layout.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_mesh_layout.py new file mode 100644 index 0000000000000000000000000000000000000000..4cd9c00a043dd7c49273bad3c4a2a1e8e00d42eb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_mesh_layout.py @@ -0,0 +1,309 @@ +""" +Definition of CuTe inspired Layouts for DeviceMesh internal bookkeeping and functions to manipulate them +""" + +import math +from collections.abc import Iterator +from dataclasses import dataclass +from itertools import product + +import torch +from torch.distributed._pycute import ( + as_tuple, + coalesce, + complement, + composition, + flatten, + IntTuple, + is_int, + is_tuple, + Layout, + match_structure, +) + + +@dataclass(frozen=True, init=True) +class _MeshLayout(Layout): + """ + Utility class for representing an integer layout by borrowing ideas from CuTe Layout Algebra. + See https://docs.nvidia.com/cutlass/media/docs/cpp/cute/02_layout_algebra.html for more details. + + Each layout is represented as a list of sizes and strides. We use it as a way for mechanical bookkeeping + of the integers such as ranks in a SPMD mesh, and the transformation on top of it. + + Lots of methods of layout like coalesce, composition, complement, etc. are borrowed from pycute. + https://github.com/NVIDIA/cutlass/blob/6dd13d42784ee5bfa232d2441e6b9a021c5c6290/python/pycute/layout.py#L137,L257 + + Note this is a CuTe-inspired layout, because CuTe uses co-lexicographic way in linearization while PyTorch + is using lexicographic. So even though the CuTe documentation can still be referenced, the implementation will be + different from that of PyCute's. + """ + + # pyrefly: ignore [bad-override] + shape: IntTuple + # pyrefly: ignore [bad-override] + stride: IntTuple + + def __post_init__(self) -> None: + if not is_tuple(self.shape) and not is_int(self.shape): + raise TypeError(f"shape must be a tuple or int, got {type(self.shape)}") + if not is_tuple(self.stride) and not is_int(self.stride): + raise TypeError(f"stride must be a tuple or int, got {type(self.stride)}") + if not match_structure(self.shape, self.stride): + raise ValueError( + f"sizes {self.shape} and strides {self.stride} don't match" + ) + + @property + def sizes(self) -> IntTuple: + return self.shape + + @property + def strides(self) -> IntTuple: + return self.stride + + @property + def sizes_and_strides(self) -> Iterator[tuple[int, int]]: + return zip(flatten(self.shape), flatten(self.stride)) + + @property + def top_level_sizes(self) -> tuple[int, ...]: + return tuple(self[i].numel() for i in range(len(self))) + + def numel(self) -> int: + return math.prod(flatten(self.shape)) + + # # operator [] (get-i like tuples) + def __getitem__(self, i: int) -> "_MeshLayout": + if i < -len(self) or i >= len(self): + raise IndexError( + f"Dim {i} is out of range for layout with {len(self)} dimensions. " + f"Expected dim to be in range [{-len(self)}, {len(self) - 1}]." + ) + layout = super().__getitem__(i) + return _MeshLayout(layout.shape, layout.stride) + + def nest(self) -> "_MeshLayout": + return _MeshLayout((self.shape,), (self.stride,)) + + def coalesce(self) -> "_MeshLayout": + """ + A layout is represented by (sizes):(strides), e.g. (3,2):(4,2). + Two consecutive dimensions can be "merged" into one if their + strides are contiguous/multiplicative (i.e., the inner stride * inner size + equals the next stride), we perform this kind of merge inside coalesce. + + Example 1 (simple): (3,2):(2,1) + - inner dimension: has stride=1, size=2 + - outer dimension: stride = inner_stride * inner_size = 2 + → coalesced = (6:1) # acts like a flat 1D array of length 6 + + Example 2 (non-coalescible): (3,2):(4,1) + - inner dimension: stride=1, size=2 → 2*1 = 2 + - outer dimension: stride=4, mismatch (≠ 2) + → cannot merge; result stays (3,2):(4,1) + """ + layout = coalesce(self) + return _MeshLayout(layout.shape, layout.stride) + + def composition(self, layout: "_MeshLayout") -> "_MeshLayout": + """ + By-dimension composition allows one layout to "select from" or "filter through" another layout. + Think of it as function composition: (self ∘ layout)(input) = self(layout(input)) + between two layouts. This function is a wrapper of pycute's composition. + + Mental model about how to understand the composition logic: + - The LEFT layout (self) defines the "output space" - what indices are possible + - The RIGHT layout (layout parameter) acts as a "selector" - which specific indices to pick + - The composition only generates indices that the left layout could originally produce, + but the right layout determines which indices to be picked. + - The stride of the composition layout will not be smaller than the stride of the right layout, + because when picking the indices the composition will at least follow the the right layout's stride + to move forward. + + Example: + self = (6,2):(2,1) # sizes=(6,2), strides=(2,1) + layout = (3:2) # sizes=(3,), stride=(2,) + self o layout = (3:2) + + Returns: + Layout being composed. + """ + result = composition(self, layout) + return _MeshLayout(result.shape, result.stride) + + def complement(self, world_size: int) -> "_MeshLayout": + """ + Compute the "complement layout" relative to a given world_size. + A complement layout fills in the "missing" factor so that: self repeat a layout of complement(self, world_size) + will get a complete world_size. We use ⊗ to denote the repeat operation. + + Example: + self = (4:1) # size=4, stride=1 + world_size = 8 + Then: + complete needed factor = 8 / 4 = 2 + complement(self, 8) = (2:1) + + Together they form: + (4:1) ⊗ (2:1) = (4,2):(2,1) + which has world_size = 4 * 2 = 8, as required. + + In distributed terms, complement() is often used to derive the "other" + rank grouping when splitting processes into 2D meshes. + + For a visualized explanation, see https://x.com/ezyang/status/1962364978393981433/ + """ + layout = complement(self, world_size) + return _MeshLayout(layout.shape, layout.stride) + + def splice(self, start: int, end: int, layout: "_MeshLayout") -> "_MeshLayout": + sizes = list(as_tuple(self.sizes)) + strides = list(as_tuple(self.strides)) + sizes[start:end] = list(as_tuple(layout.sizes)) + strides[start:end] = list(as_tuple(layout.strides)) + return _MeshLayout(tuple(sizes), tuple(strides)) + + def all_ranks_from_zero(self) -> list[int]: + """ + This function computes the all ranks specified by the layout staring from zero. + + How it works: + 1. we enumerates every possible coordinate (like a nested for-loop). + If sizes = (2, 3), we get the following coordinates: + (0,0), (0,1), (0,2), (1,0), (1,1), (1,2) + + 2. For each coordinate, we compute a linear rank index as: + all_ranks_from_zero = sum(coord[i] * strides[i] for i in range(ndim)) + + Example A: + sizes = (2, 3) # 2 rows, 3 cols + strides = (3, 1) # row-major layout + coords = (0,0) -> 0*3 + 0*1 = 0 + (0,1) -> 0*3 + 1*1 = 1 + (0,2) -> 0*3 + 2*1 = 2 + (1,0) -> 1*3 + 0*1 = 3 + (1,1) -> 1*3 + 1*1 = 4 + (1,2) -> 1*3 + 2*1 = 5 + result = [0, 1, 2, 3, 4, 5] + + Example B: + sizes = (2, 3) + strides = (1, 2) # non-standard / strided layout + coords = (0,0) -> 0*1 + 0*2 = 0 + (0,1) -> 0*1 + 1*2 = 2 + (0,2) -> 0*1 + 2*2 = 4 + (1,0) -> 1*1 + 0*2 = 1 + (1,1) -> 1*1 + 1*2 = 3 + (1,2) -> 1*1 + 2*2 = 5 + result = [0, 2, 4, 1, 3, 5] + """ + return [ + sum(c * s for c, s in zip(coord, flatten(self.strides))) + for coord in product(*(range(s) for s in flatten(self.sizes))) + ] + + def global_ranks(self, world_size: int) -> list[list[int]]: + """ + Build global ranks specified by the layout via two-level ranks composition. + + The nested list forms the Cartesian product of all ranks for one layout and offset + regarding filling up the world_size with the layout. + The final global ranks are the addition of these two. The result is a + list of lists: one sublist per layout. This rank list will be used to build + the communicator underlying the layout and the given `world_size`. + + Example: + world_size = 16 + self.size = 4 + self.stride = 1 + ranks = [0, 1, 2, 3] + offsets = [0, 4, 8, 12] + result = [ + [0+0, 0+1, 0+2, 0+3], # → [0, 1, 2, 3] + [4+0, 4+1, 4+2, 4+3], # → [4, 5, 6, 7] + [8+0, 8+1, 8+2, 8+3], # → [8, 9, 10,11] + [12+0, 12+1, 12+2, 12+3], # → [12,13,14,15] + ] + """ + return [ + [offset + rank for rank in self.all_ranks_from_zero()] + for offset in self.complement(world_size).all_ranks_from_zero() + ] + + def check_non_overlap(self) -> bool: + """ + Check if the layout has any overlap between the ranks it generates. If there is overlap, + we return False, otherwise True. + + The layout is supposed to be injective i.e, aside from indice 0, indices from each + dim of the layout must be non-overlapping. + + Example 1 - Valid (no overlap): + Layout: sizes=(2,3), strides=(6,1) + - Dim 1: stride=1, span=3*1=3, covers indices [0,1,2] + - Dim 0: stride=6, span=2*6=12, covers indices [0,6] + → No overlap since 6 > 3 + + Example 2 - Invalid (overlap): + Layout: sizes=(2,3), strides=(2,1) + - Dim 1: stride=1, span=3*1=3, covers indices [0,1,2] + - Dim 0: stride=2, span=2*2=4, covers indices [0,2] + → Overlap! stride=2 < span=3, so indices [0,2] are duplicated + + Example 3 - Invalid (overlap): + Layout: sizes=(4,2), strides=(1,1) + - Dim 1: stride=1, span=4, covers indices [0,1,2,3] + - Dim 0: stride=1, span=2, covers indices [0,1] + → Overlap! stride is same for two dims, so indices [0,2] are duplicated + + Returns: + bool: True if no overlap, False if overlap detected + """ + ranks = self.all_ranks_from_zero() + return len(ranks) == len(set(ranks)) + + def remap_to_tensor(self, rank_map: torch.Tensor) -> torch.Tensor: + """ + Leverage layout as an index for mesh tensor that re-maps the indexes after layout + transformation to actual device ranks. + + With this method, the cute layout serves as the backend of indices bookkeeping for the + mesh tensor when it comes to flatten, unflatten and slicing operations. The actual mesh + tensor still represents the actual device assignment and ranks. We need this function + to specify device allocation and create backend for a mesh. Although any transform of mesh tensors + can be treated as a view or subset of mesh tensor, we do need to use the actual view or + sub-tensor for DeviceMesh and its backend creation. + + The shape of the `rank_map` must be 1D and contiguous. + + Examples: + + Case 1 - Consecutive ranks, full world: + original_mesh_tensor = [[0,1],[2,3]] # 2x2 mesh, ranks 0-3 + world_size = 4 + layout = Layout(2:2) + Return: [[0,2],[1,3]] + + Case 2 - Non-consecutive ranks: + original_mesh_tensor = [[10,20],[30,40]] # custom rank assignment + world_size = 4 + layout = Layout(2:2) + Return: [[[10,30],[20,40]]] + + Args: + rank_map: The concrete mesh tensor with actual device ranks + + Returns: + torch.Tensor: A tensor representing the actual device allocation from rank_map + """ + assert rank_map.ndim == 1 + assert rank_map.is_contiguous() + assert rank_map.numel() >= self.cosize() + + complement_layout = self.complement(rank_map.numel()) + + return rank_map.as_strided( + flatten(complement_layout.sizes) + flatten(self.sizes), + flatten(complement_layout.strides) + flatten(self.strides), + ).reshape(-1, *self.top_level_sizes) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_serialization.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_serialization.py new file mode 100644 index 0000000000000000000000000000000000000000..8ebe7a29eb62d1f0cc705150bcbc5d24698f89cb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_serialization.py @@ -0,0 +1,158 @@ +import pickle +from dataclasses import dataclass +from io import BufferedIOBase +from typing import Any + +import torch +import torch._weights_only_unpickler as _weights_only_unpickler +from torch.serialization import _load, _save, DEFAULT_PROTOCOL, MAP_LOCATION + + +__all__: list[str] = [] + + +@dataclass +class _Entry: + key: str + is_storage: bool + length: int + + +_weights_only_unpickler._add_safe_globals([_Entry]) + + +class _PseudoZipFile: + def __init__(self) -> None: + self.records: dict[str, tuple[object, int]] = {} + + def write_record(self, key: str, data: object, length: int) -> None: + self.records[key] = (data, length) + + def write_to(self, f: BufferedIOBase) -> None: + entries = [] + for key, (data, length) in self.records.items(): + entries.append( + _Entry( + key=key, + is_storage=isinstance(data, torch.UntypedStorage), + length=length, + ) + ) + + pickle.dump(entries, f, protocol=DEFAULT_PROTOCOL) + + for data, _ in self.records.values(): + if isinstance(data, bytes): + f.write(data) + elif isinstance(data, str): + f.write(data.encode("utf-8")) + elif isinstance(data, torch.UntypedStorage): + data._write_file(f, False, False, 1) + else: + raise TypeError(f"unknown type: {type(data)}") + + def read_from(self, f: BufferedIOBase) -> None: + entries = _weights_only_unpickler.load(f) + + for entry in entries: + data = f.read(entry.length) + if entry.is_storage: + if entry.length == 0: + storage = torch.UntypedStorage(0) + else: + storage = torch.frombuffer( + data, + dtype=torch.uint8, + ).untyped_storage() + + self.records[entry.key] = ( + storage, + entry.length, + ) + else: + self.records[entry.key] = (data, entry.length) + + def has_record(self, key: str) -> bool: + return key in self.records + + def get_record(self, key: str) -> object: + return self.records[key][0] + + def get_storage_from_record( + self, key: str, _length: int, _type: int + ) -> torch.Tensor: + return torch.tensor(self.records[key][0], dtype=torch.uint8) + + def serialization_id(self) -> str: + return "torchft" + + +def _streaming_save( + obj: object, + f: BufferedIOBase, + pickle_module: Any = pickle, + pickle_protocol: int = DEFAULT_PROTOCOL, +) -> None: + """ + Save the object to a file-like object in a streaming fashion compatible with + network sockets. + + This behaves similarly to :func:`torch.save` with a few notable differences: + + * A non-seekable file like object can be used when loading. + * No forwards/backwards compatibility is provided for the serialization + format. This is only intended to be used with a single version of PyTorch + with transient storage (i.e. sockets or temp files). + * mmap is not supported + + See :func:`torch.save` for more details on specific arguments. + """ + + zip_file = _PseudoZipFile() + _save( + obj, + zip_file=zip_file, + pickle_module=pickle_module, + pickle_protocol=pickle_protocol, + _disable_byteorder_record=False, + ) + zip_file.write_to(f) + + +def _streaming_load( + f: BufferedIOBase, + map_location: MAP_LOCATION = None, + pickle_module: Any = None, + *, + weights_only: bool = True, + **pickle_load_args: Any, +) -> object: + """ + Load the object from a file-like object in a streaming fashion compatible with + network sockets. + + See :func:`_streaming_save` for more details about the streaming behavior. + + See :func:`torch.load` for more details on specific arguments. + """ + if weights_only: + if pickle_module is not None: + raise RuntimeError( + "Can not safely load weights when explicit pickle_module is specified" + ) + pickle_module = _weights_only_unpickler + else: + if pickle_module is None: + pickle_module = pickle + + if "encoding" not in pickle_load_args: + pickle_load_args["encoding"] = "utf-8" + + zip_file = _PseudoZipFile() + zip_file.read_from(f) + return _load( + zip_file=zip_file, + map_location=map_location, + pickle_module=pickle_module, + **pickle_load_args, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_state_dict_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_state_dict_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..771ea485a7d3c6aaaed19b0b4998e1309541d9b1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/_state_dict_utils.py @@ -0,0 +1,830 @@ +# mypy: allow-untyped-defs +import copy +import io +import math +import weakref +from collections.abc import Callable, Mapping, MutableMapping +from typing import Any, cast, NamedTuple, TYPE_CHECKING, Union + +import torch +import torch.cuda._pin_memory_utils as pin_memory_utils +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed._functional_collectives import AsyncCollectiveTensor + + +if dist.is_available() or TYPE_CHECKING: + from torch.distributed import distributed_c10d + from torch.distributed._shard.sharded_tensor import ShardedTensor + from torch.distributed.tensor import distribute_tensor, DTensor, Replicate + from torch.distributed.tensor._utils import compute_local_shape_and_global_offset + + +def _identity_func( + obj: torch.Tensor, + pg: dist.ProcessGroup | None, + device: torch.device | None, + companion_obj: Any, +) -> torch.Tensor: + return obj + + +def _all_gather_sharded_tensor( + sharded_tensor: "ShardedTensor", + pg: dist.ProcessGroup | None = None, + device: torch.device | None = None, +) -> torch.Tensor: + if pg is None: + pg = distributed_c10d._get_default_group() + world_size = dist.get_world_size(pg) + shards = sharded_tensor.local_shards() + dim_0_size = sharded_tensor.size()[0] # type: ignore[index] + tensor_numel = sharded_tensor.size().numel() # type: ignore[union-attr] + chunk_size = math.ceil(dim_0_size / world_size) * tensor_numel // dim_0_size + pg_device = ( + distributed_c10d._get_pg_default_device(pg) if device is None else device + ) + if shards: + local_tensor = shards[0].tensor.flatten() + if local_tensor.device.type != pg_device.type: + local_tensor = local_tensor.to(pg_device) + num_padding = chunk_size - local_tensor.numel() + if num_padding > 0: + local_tensor = F.pad(local_tensor, [0, num_padding]) + else: + local_tensor = torch.zeros( + chunk_size, dtype=sharded_tensor.dtype, device=pg_device + ) + + tensor = torch.empty( + chunk_size * world_size, + dtype=local_tensor.dtype, + device=pg_device, + ) + dist.all_gather_into_tensor(tensor, local_tensor, group=pg) + + tensor = tensor.narrow(0, 0, tensor_numel).reshape(sharded_tensor.size()) + return tensor + + +class CompanionMismatch(Exception): + pass + + +def _iterate_state_dict( + iter_object: Any, + sharded_tensor_func: Callable, + dtensor_func: Callable, + tensor_func: Callable, + *, + pg: dist.ProcessGroup | None = None, + device: torch.device | None = None, + cpu_offload: bool = False, + companion_obj: Any = None, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, + non_blocking: bool = True, +) -> dict[str, Any]: + """Iterate through the state dict, applying the given functions to each tensor type. + + Args: + iter_object (Any): the target state_dict. + sharded_tensor_func (Callable): the function to apply to ShardedTensor + dtensor_func (Callable): the function to apply to DTensor + tensor_func (Callable): the function to apply to Tensor + pg (Optional[dist.ProcessGroup]): process group passed to tensor functions + device (Optional[torch.device]): device passed to tensor functions + cpu_offload (bool): whether to offload the tensors to CPU memory. This option is ignored + if a companion_obj is supplied. + companion_obj (Any): A companion object to the state dict. If this object + is supplied, we attempt to copy the tensor to the companion object. + ranks_only (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + non_blocking (bool): whether to use non-blocking copy when copying to the companion object. + """ + # TODO: should we use pytree? + cpu_device = torch.device("cpu") + if isinstance(iter_object, ShardedTensor): + ret = sharded_tensor_func(iter_object, pg, device, companion_obj) + elif isinstance(iter_object, DTensor): + ret = dtensor_func(iter_object, pg, device, companion_obj) + elif isinstance(iter_object, torch.Tensor): + ret = tensor_func(iter_object, pg, device, companion_obj) + elif ( + isinstance(iter_object, (int, float, str, bytes, io.BytesIO)) + or iter_object is None + ): + ret = iter_object + elif isinstance(iter_object, dict): + if companion_obj is not None and ( + not isinstance(companion_obj, dict) + or set(companion_obj.keys()) != set(iter_object.keys()) + ): + msg = ( + "" + if isinstance(companion_obj, dict) + else f"{set(companion_obj.keys())=} {set(iter_object.keys())=}" + ) + raise CompanionMismatch(msg) + + ret = { + key: _iterate_state_dict( + value, + sharded_tensor_func, + dtensor_func, + tensor_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + companion_obj=companion_obj[key] if companion_obj is not None else None, + ranks_only=ranks_only, + type_check=type_check, + non_blocking=non_blocking, + ) + for key, value in iter_object.items() + } + elif isinstance(iter_object, (list, tuple)): + if companion_obj is not None and ( + not isinstance(companion_obj, (list, tuple)) + or len(companion_obj) != len(iter_object) + ): + raise CompanionMismatch + + ret = [ + _iterate_state_dict( + v, + sharded_tensor_func, + dtensor_func, + tensor_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + companion_obj=companion_obj[idx] if companion_obj is not None else None, + ranks_only=ranks_only, + type_check=type_check, + non_blocking=non_blocking, + ) + for idx, v in enumerate(iter_object) + ] + if isinstance(iter_object, tuple): + ret = tuple(ret) + elif not type_check: + ret = copy.deepcopy(iter_object) + else: + raise ValueError(f"Unexpected value type {type(iter_object)}") + + if not ranks_only or dist.get_rank(pg) in ranks_only: + if isinstance(ret, torch.Tensor): + if cpu_offload and companion_obj is None: + ret = ret.to(cpu_device) + + if companion_obj is not None: + if isinstance(companion_obj, DTensor): + if not isinstance(ret, DTensor): + raise AssertionError( + "ret must be a DTensor when companion_obj is a DTensor" + ) + companion_obj._local_tensor.copy_( + ret._local_tensor, non_blocking=non_blocking + ) + elif isinstance(companion_obj, ShardedTensor): + if not isinstance(ret, ShardedTensor): + raise AssertionError( + "ret must be a ShardedTensor when companion_obj is a ShardedTensor" + ) + for idx, shard in enumerate(companion_obj.local_shards()): + shard.tensor.copy_( + ret.local_shards()[idx].tensor, non_blocking=non_blocking + ) + else: + # pyrefly: ignore [missing-attribute] + companion_obj.copy_(ret, non_blocking=non_blocking) + ret = companion_obj + else: + ret = {} if isinstance(ret, dict) else None + + # pyrefly: ignore [bad-return] + return ret + + +def _gather_state_dict( + state_dict: dict[str, Any], + *, + pg: dist.ProcessGroup | None = None, + device: torch.device | None = None, + cpu_offload: bool = False, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, +) -> dict[str, Any]: + """ + Given a state_dict, this API gathers all the ShardedTensors or DTensors in + the state_dict. + + + Args: + state_dict (Dict[str, Any]): the target sharded state_dict. + pg (Optional[dist.ProcessGroup]): the process group that is used to + gather ShardedTensor. Note that gathering a DTensor will use + the DeviceMesh. So this argument will be ignored when gathering a + DTensor. + device: (Optional[torch.device]): the device that is used to + perform allgather for ShardedTensor. Note that gathering a DTensor + will use the DeviceMesh. So this argument will be ignored when + gathering a DTensor. + cpu_offload (bool): whether to offload the tensors to CPU memory. The + default value is False. + ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check: (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + The gathered state dictionary. + """ + + def sharded_tensor_func(value, pg, device, companion_obj): + # ShardedTensor does not seem to record the original device type. + # So if the tensor is moved to CPU, we won't know the original type. + # As a result, we have to rely on the user to tell us the correct one. + cpu_device = torch.device("cpu") + output_tensor = _all_gather_sharded_tensor(value, pg, device) + local_shard_device = ( + value.local_shards()[0].tensor.device + if value.local_shards() + else cpu_device + ) + if output_tensor.device != local_shard_device: + value = output_tensor.to(local_shard_device) + else: + value = output_tensor + return value + + def dtensor_func(value, pg, device, companion_obj): + if value.device != value.device_mesh.device_type: + value = value.to(value.device_mesh.device_type) + # FSDP all_gather: [Shard(0)] -> [Replicate()] + # HSDP all_gather: [Replicate(), Shard(0)] -> [Replicate(), Replicate()] + # 2D FSDP + TP all_gather: + # - [Shard(0), Shard(n)] -> [Replicate(), Replicate()] + # - [Shard(0), Replicate()] -> [Replicate(), Replicate()] + placements = [Replicate() for _ in value.placements] + value = value.redistribute( + device_mesh=value.device_mesh, + placements=placements, + ) + # Call `wait()` to force the tensor to be synchronous with respect + # to the main stream. + # See the discussion in https://github.com/pytorch/pytorch/pull/117799. + value = value.to_local() + if isinstance(value, AsyncCollectiveTensor): + value = value.wait() + return value + + return _iterate_state_dict( + state_dict, + sharded_tensor_func, + dtensor_func, + _identity_func, + pg=pg, + device=device, + cpu_offload=cpu_offload, + ranks_only=ranks_only, + type_check=type_check, + ) + + +def _offload_state_dict_to_cpu( + state_dict: dict[str, Any], + *, + ranks_only: tuple[int, ...] = (), + type_check: bool = True, +) -> dict[str, Any]: + """ + Given a state_dict, this API offload all the tensors to CPU memory. + + Args: + state_dict (Dict[str, Any]): the target state_dict. + pg (Optional[dist.ProcessGroup]): the process group that is used to + gather ShardedTensor. Note that gathering a DTensor will use + the DeviceMesh. So this argument will be ignored when gathering a + DTensor. + ranks_only: (Tuple[int, ...]): if this tuple is empty, all ranks will + have the same state_dicts. Otherwise only ranks that in ``ranks_only`` + have the same state_dicts. Other ranks will get empty state_dicts. + type_check: (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + The gathered state dictionary. + """ + + ret = _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + _identity_func, + pg=None, + device=None, + cpu_offload=True, + ranks_only=ranks_only, + type_check=type_check, + ) + return ret + + +@torch.no_grad() +def _copy_state_dict( + state_dict: dict[str, Any], + copy_state_dict: dict[str, Any], + non_blocking: bool = False, + type_check: bool = True, +) -> dict[str, Any]: + """ + Copies all tensors in a given state dict into a different state_dict with the + same structure. Additionally, a copied state dict with the same value references + is returned. Editing the keys on this state dict will not affect the + passed in copy_state_dict (but the value references are the same). + + .. warning:: + It is expected by this function that state_dict and copy_state_dict share + the same structure and data types. + + .. warning:: + The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Args: + state_dict (Dict[str, Any]): the target state_dict. + copy_state_dict (Dict[str, Any]): + The state dict we are copying into. This state_dict must have exactly + the same structure as the source `state_dict`. + non_blocking: (bool): Whether copy ops should be performed asynchronously + type_check (bool): check if the instance data type is a supported type + that can be saved by DCP. The current supported data types are + torch.Tensor, DTensor, int, float, str, list, dict, None. + + Returns: + State Dict copy + """ + + return _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + _identity_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + companion_obj=copy_state_dict, + type_check=type_check, + non_blocking=non_blocking, + ) + + +@torch.no_grad() +def _create_cpu_state_dict( + state_dict: dict[str, Any], pin_memory: bool = False, share_memory: bool = False +) -> dict[str, Any]: + """ + Given a state_dict, create another state_dict with the same structure and elements. + However, all tensors in the returned state_dict are new tensors on CPU. These + tensors can be placed on pin_memory or share_memory based on the provided arguments. + + .. warning:: + Setting both `pin_memory` and `share_memory` to True significantly increases the + latency of this method because of the nuances which require us to register memory + as pinned directly as opposed to relying on the pin_memory cache allocator. This + option should only be used for long lived tensors which are required to be shared. + This is not the case as long as at least one of `pin_memory` or `share_memory` is + set to False. + + """ + + def tensor_func( + obj: torch.Tensor, + pg: dist.ProcessGroup | None, + device: torch.device | None, + _: Any, + ) -> torch.Tensor: + if len(obj.size()) == 0: + return torch.tensor(0, dtype=obj.dtype) + + # sometimes, a tensor might have non-zero size and 0 numel. In this case, pinning memory will fail + # so we take a best guess at how to replicate the tensor below to maintain symmetry in the returned + # state dict. + if obj.numel() == 0 or obj.data_ptr() == 0: + t = torch.zeros_like(obj, device="cpu") + if share_memory: + t = t.share_memory_() + return t + + if share_memory: + t = torch.empty(*tuple(obj.size()), dtype=obj.dtype) + t = t.share_memory_() + if pin_memory: + pin_memory_utils.pin_memory(t.data_ptr(), t.numel() * t.element_size()) + weakref.finalize(t, pin_memory_utils.unpin_memory, t.data_ptr()) + + return t + elif pin_memory: + return torch.empty(*tuple(obj.size()), dtype=obj.dtype).pin_memory() + else: + return torch.empty(*tuple(obj.size()), dtype=obj.dtype) + + def dtensor_func( + obj: DTensor, + pg: dist.ProcessGroup | None, + device: torch.device | None, + _: Any, + ) -> DTensor: + if len(obj.size()) == 0: + return obj + + if obj.device != torch.device("cpu"): + ret = cast(DTensor, obj.to(device="cpu")) + else: + ret = copy.deepcopy(obj) + ret._local_tensor = tensor_func(ret._local_tensor, pg, device, None) + return ret + + def sharded_tensor_func( + obj: ShardedTensor, + pg: dist.ProcessGroup | None, + device: torch.device | None, + _: Any, + ) -> ShardedTensor: + if not obj.local_shards(): + return obj + + if obj.device != torch.device("cpu"): + ret = obj.to(device="cpu") + else: + ret = copy.deepcopy(obj) + + for shards in ret.local_shards(): + shards.tensor = tensor_func(shards.tensor, pg, device, None) + + return ret + + ret = _iterate_state_dict( + state_dict, + sharded_tensor_func, + dtensor_func, + tensor_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + type_check=False, + ) + return ret + + +def _check_state_dict_similarity( + state_dict: dict[str, Any], + compared_state_dict: dict[str, Any], +) -> bool: + """ + Given two state_dicts, check if the structures are the same. And + if a [key, tensor] pair exist in one state_dict there must be + the a corresponding pait, [key, other_tensor], in the other state_dict, + where tensor and other_tensor have the same size and dtype. + + Return the check result. + """ + + def tensor_func( + obj: torch.Tensor, + pg: dist.ProcessGroup | None, + device: torch.device | None, + companion_obj: Any, + ) -> torch.Tensor: + if companion_obj.dtype != obj.dtype or companion_obj.size() != obj.size(): + raise CompanionMismatch + return obj + + try: + _iterate_state_dict( + state_dict, + _identity_func, + _identity_func, + tensor_func, + pg=None, + device=None, + cpu_offload=False, + ranks_only=(), + companion_obj=compared_state_dict, + type_check=False, + ) + except CompanionMismatch: + return False + + return True + + +class _TensorInfo(NamedTuple): + size: torch.Size + dtype: torch.dtype + + +def _broadcast_tensors( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + keys: list[str], + device: torch.device, + pg: dist.ProcessGroup | None = None, +) -> None: + if pg is None: + pg = dist.distributed_c10d._get_default_group() + pg_device = ( + device + if device.type in {pg_device.type for pg_device in pg._device_types} + else pg._device_types[0] + ) + + tensors: list[torch.Tensor] = [] + for key in keys: + if dist.get_rank() == 0: + full_state = full_state_dict[key] + if not isinstance(full_state, torch.Tensor): + raise AssertionError("full_state must be a torch.Tensor") + full_tensor = full_state.detach().to(pg_device) + else: + tensor_info = full_state_dict[key] + full_tensor = torch.empty( + size=tensor_info.size, + device=pg_device, + dtype=tensor_info.dtype, + ) + + tensors.append(full_tensor) + + if (local_state := local_state_dict.get(key)) is None: + continue + + local_state_dict[key] = ( + (local_state, full_tensor) + if isinstance(local_state, DTensor) + else full_tensor + ) + + if len(tensors) > 1: + dist._broadcast_coalesced(pg, tensors, 500, 0) + else: + dist.broadcast(tensors[0], src=0, group=pg) + + if pg_device != device: + for key, full_tensor in zip(keys, tensors): + if (local_state := local_state_dict.get(key)) is not None: + local_state_dict[key] = ( + (local_state[0], full_tensor.to(device)) + if ( + isinstance(local_state, tuple) + and isinstance(local_state[0], DTensor) + ) + else full_tensor.to(device) + ) + + _distribute_tensors(local_state_dict, keys, device, pg) + + +def _distribute_tensors( + local_state_dict: dict[str, Any], + keys: list[str], + device: torch.device, + pg: dist.ProcessGroup | None = None, +) -> None: + if pg is None: + pg = dist.distributed_c10d._get_default_group() + for key in keys: + _local_state = local_state_dict.get(key) + if _local_state is None or torch.is_tensor(_local_state): + continue + + local_state = _local_state[0] + full_tensor = _local_state[1] + + shape, offset = compute_local_shape_and_global_offset( + full_tensor.shape, local_state.device_mesh, local_state.placements + ) + slices = [ + slice(cur_offset, cur_offset + cur_shape) + for cur_shape, cur_offset in zip(shape, offset) + ] + if local_state.is_meta: + # Use .clone() here rather than view to clone and return only the sliced portion, minimizing memory access and cost. + local_tensor = full_tensor[tuple(slices)].detach().clone() + # TODO: currently, we cannot handle strided sharding if the dp dimension is not even. For example, + # one of the case that is not yet supported is when placements = (Shard(0), _StridedShard(0, sf=2)). + ret = DTensor.from_local( + local_tensor, + local_state.device_mesh, + local_state.placements, + shape=local_state.shape, + stride=local_state.stride(), + ) + else: + ret = local_state + # Copy full_tensor[slices] into local_state.to_local() to reduce memory footprint. + ret.to_local().copy_(full_tensor[tuple(slices)]) + local_state_dict[key] = ret + + +def _broadcast_state_dict( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + device: torch.device, + pg: dist.ProcessGroup | None = None, + strict: bool = False, + cpu_offload: bool = False, +) -> None: + # Broadcast from rank0's `full_state_dict` to all ranks' `local_state_dict`. + # If strict is True, any keys in `local_state_dict` but not in `full_state_dict` + # will be removed from `local_state_dict`. + ret = {} + if dist.get_rank() == 0: + for key, value in full_state_dict.items(): + if not torch.is_tensor(value): + ret[key] = value + elif value.dim() == 0: + ret[key] = value.cpu() + else: + ret[key] = _TensorInfo(value.size(), value.dtype) + + broadcast_list = [ret] + dist.broadcast_object_list(broadcast_list, src=0, group=pg) + ret = broadcast_list[0] + # Gather values + keys = [] + local_state_dict_keys = set(local_state_dict.keys()) + global_keys = set() + for key, value in ret.items(): + global_keys.add(key) + if not isinstance(value, _TensorInfo): + if key in local_state_dict: + local_state_dict[key] = value + continue + + if dist.get_rank() == 0: + ret[key] = full_state_dict[key] + + keys.append(key) + # Broadcast every tensor to avoid OOM for now. + if len(keys) >= 1: + _broadcast_tensors(ret, local_state_dict, keys, device, pg) + if cpu_offload: + for key in keys: + local_state_dict[key] = local_state_dict[key].cpu() + keys.clear() + + if strict: + if missing_keys := (local_state_dict_keys - global_keys): + for key in missing_keys: + local_state_dict.pop(key) + + if keys: + _broadcast_tensors(ret, local_state_dict, keys, device, pg) + if cpu_offload: + for key in keys: + local_state_dict[key] = local_state_dict[key].cpu() + + +def _distribute_state_dict( + full_state_dict: dict[str, Any], + local_state_dict: dict[str, Any], + device: torch.device, + pg: dist.ProcessGroup | None = None, +) -> None: + # Full_state_dict = True, broadcast_from_rank0 = False here. Each rank has + # full_state_dict. Skip the broadcast in ``_broadcast_state_dict`` and + # distribute tensors in each rank + for key, value in full_state_dict.items(): + if key not in full_state_dict: + continue + if not torch.is_tensor(value): + local_state_dict[key] = value + elif value.dim() == 0: + local_state_dict[key] = value.cpu() + else: + if not isinstance(value, torch.Tensor): + raise AssertionError("value must be a torch.Tensor") + local_state = local_state_dict.get(key) + if local_state is None: + continue + elif isinstance(local_state, DTensor): + local_state_dict[key] = distribute_tensor( + value.detach().to(device), + local_state.device_mesh, + local_state.placements, + ) + else: + local_state_dict[key] = value.detach().to(device) + + +# These APIs are from torch.distributed.checkpoint. +# TODO: We should consolidate the code here as some not all modules can depend on +# DCP. +PATH_ITEM = Union[str, int] +OBJ_PATH = tuple[PATH_ITEM, ...] +FLATTEN_MAPPING = dict[str, OBJ_PATH] +STATE_DICT_TYPE = dict[str, Any] +CONTAINER_TYPE = MutableMapping[PATH_ITEM, Any] + + +def _traverse_state_dict( + state_dict: STATE_DICT_TYPE, + visitor: Callable[[OBJ_PATH, Any], None], +) -> None: + """ + Invoke ``visitor`` for each value recursively in ``state_dict``. + Mapping, list, and tuple will be flattened and other value types are treated + as the terminal values and will invoke ``visitor``. + """ + + def _traverse_obj(path: OBJ_PATH, value: Any) -> None: + if isinstance(value, Mapping): + for k, v in value.items(): + _traverse_obj(path + (str(k),), v) + elif isinstance(value, (list, tuple)): + for i, v in enumerate(value): + _traverse_obj(path + (i,), v) + else: + visitor(path, value) + + for key, value in state_dict.items(): + _traverse_obj((str(key),), value) + + +def _flatten_state_dict( + state_dict: STATE_DICT_TYPE, +) -> tuple[STATE_DICT_TYPE, FLATTEN_MAPPING]: + """ + Flatten ``state_dict`` made of nested dicts and lists into a top level dictionary. + + Use ``unflatten_state_dict`` to revert this process. + Returns: + A tuple with the flatten state_dict and a mapping from original to new state_dict. + N.B. The new keys are derived from the object paths, joined by dot. + For example: ``{ 'a': {'b':...}}`` results in the key `a.b`. + """ + flattened: STATE_DICT_TYPE = {} + mappings: FLATTEN_MAPPING = {} + + def flat_copy(path: OBJ_PATH, value: Any) -> None: + new_fqn = ".".join(map(str, path)) + if new_fqn in flattened: + raise ValueError(f"duplicated flatten key {new_fqn}") + flattened[new_fqn] = value + mappings[new_fqn] = path + + _traverse_state_dict(state_dict, flat_copy) + return flattened, mappings + + +def _set_element(root_dict: STATE_DICT_TYPE, path: OBJ_PATH, value: Any) -> None: + """Set ``value`` in ``root_dict`` along the ``path`` object path.""" + cur_container = cast(CONTAINER_TYPE, root_dict) + + def extend_list(lst: list[Any], idx: int) -> None: + while len(lst) <= idx: + lst.append(None) + + for i in range(1, len(path)): + prev_key = path[i - 1] + key = path[i] + def_val: CONTAINER_TYPE | list[Any] = {} if type(key) is str else [] + + if isinstance(cur_container, Mapping): + cur_container = cast( + CONTAINER_TYPE, cur_container.setdefault(prev_key, def_val) + ) + else: + # pyrefly: ignore [bad-argument-type] + extend_list(cur_container, prev_key) + if cur_container[prev_key] is None: + cur_container[prev_key] = def_val + cur_container = cur_container[prev_key] + + key = path[-1] + if type(key) is int: + extend_list(cast(list[Any], cur_container), key) + + cur_container[key] = value + + +def _unflatten_state_dict( + state_dict: STATE_DICT_TYPE, mapping: FLATTEN_MAPPING +) -> STATE_DICT_TYPE: + """Restore the original nested state_dict according to ``mapping`` and the flattened ``state_dict``.""" + nested: STATE_DICT_TYPE = {} + for key, value in state_dict.items(): + _set_element(nested, mapping[key], value) + return nested diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/argparse_util.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/argparse_util.py new file mode 100644 index 0000000000000000000000000000000000000000..5a4030479d301537998f30a7ca67a8dc69ccd2e3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/argparse_util.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import os +from argparse import Action + + +class env(Action): + """ + Get argument values from ``PET_{dest}`` before defaulting to the given ``default`` value. + + For flags (e.g. ``--standalone``) + use ``check_env`` instead. + + .. note:: when multiple option strings are specified, ``dest`` is + the longest option string (e.g. for ``"-f", "--foo"`` + the env var to set is ``PET_FOO`` not ``PET_F``) + + Example: + :: + + parser.add_argument("-f", "--foo", action=env, default="bar") + + ./program -> args.foo="bar" + ./program -f baz -> args.foo="baz" + ./program --foo baz -> args.foo="baz" + PET_FOO="env_bar" ./program -f baz -> args.foo="baz" + PET_FOO="env_bar" ./program --foo baz -> args.foo="baz" + PET_FOO="env_bar" ./program -> args.foo="env_bar" + + parser.add_argument("-f", "--foo", action=env, required=True) + + ./program -> fails + ./program -f baz -> args.foo="baz" + PET_FOO="env_bar" ./program -> args.foo="env_bar" + PET_FOO="env_bar" ./program -f baz -> args.foo="baz" + """ + + def __init__(self, dest, default=None, required=False, **kwargs) -> None: + env_name = f"PET_{dest.upper()}" + default = os.environ.get(env_name, default) + + # ``required`` means that it NEEDS to be present in the command-line args + # rather than "this option requires a value (either set explicitly or default" + # so if we found default then we don't "require" it to be in the command-line + # so set it to False + if default: + required = False + + super().__init__(dest=dest, default=default, required=required, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class check_env(Action): + """ + Check whether the env var ``PET_{dest}`` exists before defaulting to the given ``default`` value. + + Equivalent to + ``store_true`` argparse built-in action except that the argument can + be omitted from the commandline if the env var is present and has a + non-zero value. + + .. note:: it is redundant to pass ``default=True`` for arguments + that use this action because a flag should be ``True`` + when present and ``False`` otherwise. + + Example: + :: + + parser.add_argument("--verbose", action=check_env) + + ./program -> args.verbose=False + ./program --verbose -> args.verbose=True + PET_VERBOSE=1 ./program -> args.verbose=True + PET_VERBOSE=0 ./program -> args.verbose=False + PET_VERBOSE=0 ./program --verbose -> args.verbose=True + + Anti-pattern (don't do this): + + :: + + parser.add_argument("--verbose", action=check_env, default=True) + + ./program -> args.verbose=True + ./program --verbose -> args.verbose=True + PET_VERBOSE=1 ./program -> args.verbose=True + PET_VERBOSE=0 ./program -> args.verbose=False + + """ + + def __init__(self, dest, default=False, **kwargs) -> None: + env_name = f"PET_{dest.upper()}" + default = bool(int(os.environ.get(env_name, "1" if default else "0"))) + super().__init__(dest=dest, const=True, default=default, nargs=0, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/c10d_logger.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/c10d_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..fb5f887bdefa01518436362a9014b34056412218 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/c10d_logger.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import functools +import logging +from collections.abc import Callable +from typing import Any, TypeVar +from typing_extensions import ParamSpec + +import torch +import torch.distributed as dist +from torch.distributed.logging_handlers import _log_handlers +from torch.monitor import _WaitCounter + + +__all__: list[str] = [] + +_DEFAULT_DESTINATION = "default" + + +def _get_or_create_logger(destination: str = _DEFAULT_DESTINATION) -> logging.Logger: + logging_handler, log_handler_name = _get_logging_handler(destination) + logger = logging.getLogger(f"c10d-{log_handler_name}") + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter( + "%(asctime)s %(filename)s:%(lineno)s %(levelname)s p:%(processName)s t:%(threadName)s: %(message)s" + ) + logging_handler.setFormatter(formatter) + logger.propagate = False + logger.addHandler(logging_handler) + return logger + + +def _get_logging_handler( + destination: str = _DEFAULT_DESTINATION, +) -> tuple[logging.Handler, str]: + log_handler = _log_handlers[destination] + log_handler_name = f"{type(log_handler).__name__}-{destination}" + return (log_handler, log_handler_name) + + +# pyrefly: ignore [unknown-name] +global _c10d_logger +_c10d_logger = _get_or_create_logger() + + +def _get_msg_dict(func_name, *args, **kwargs) -> dict[str, Any]: + if dist.is_initialized(): + group = kwargs.get("group") or kwargs.get("process_group") + msg_dict = { + "func_name": f"{func_name}", + "pg_name": f"{dist._get_process_group_name(kwargs.get('pg'))}", # type: ignore[arg-type] + "backend": f"{dist.get_backend(group)}", + "world_size": f"{dist.get_world_size()}", + "group_size": f"{dist.get_world_size(group)}", + "global_rank": f"{dist.get_rank()}", + "local_rank": f"{dist.get_rank(group)}", + } + if msg_dict["backend"] == "nccl": + nccl_version = torch.cuda.nccl.version() + msg_dict["nccl_version"] = ".".join(str(v) for v in nccl_version) + else: + msg_dict = { + "func_name": f"{func_name}", + } + return msg_dict + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def _exception_logger(func: Callable[_P, _T]) -> Callable[_P, _T]: + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: + try: + return func(*args, **kwargs) + except Exception as error: + msg_dict = _get_msg_dict(func.__name__, *args, **kwargs) + msg_dict["error"] = f"{error}" + _c10d_logger.debug(msg_dict) + raise + + return wrapper + + +def _time_logger(func: Callable[_P, _T]) -> Callable[_P, _T]: + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: + with _WaitCounter(f"pytorch.wait_counter.c10d.{func.__name__}").guard(): + func_return = func(*args, **kwargs) + return func_return + + return wrapper diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/collective_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/collective_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a28b2ad15fe0d5114adc04ef9593176c457819ff --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/collective_utils.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 + + +""" +A set of primitive functions for performing collective ops. + +Each should also handle single rank scenario. +""" + +from __future__ import annotations + +import importlib +import logging +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, cast, Generic, TYPE_CHECKING, TypeVar + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + +import torch +import torch.distributed as dist + + +__all__: list[str] = [ + "SyncPayload", + "broadcast", + "all_gather", + "all_gather_object_enforce_type", +] + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +@dataclass +class SyncPayload(Generic[T]): + stage_name: str | None + success: bool + payload: T + exception: Exception | None = None + + +def broadcast( + data_or_fn: T | Callable[[], T], + *, + success: bool = True, + stage_name: str | None = None, + rank: int = 0, + pg: dist.ProcessGroup | None = None, +) -> T: + """ + Broadcasts the data payload from rank 0 to all other ranks. + Or if a function is passed, execute it in rank 0 and broadcast result to all other ranks. + + Can be used to broadcast a failure signal to stop all ranks. + + If the function raises an exception, all ranks will raise. + + Args: + data_or_fn: the data to broadcast or function to execute and broadcast result. + success: False to stop all ranks. + stage_name: the name of the logical stage for synchronization and debugging + rank: rank to broadcast data or execute function and broadcast results. + pg: the process group for sync + Throws: + RuntimeError from original exception trace + Returns: + the value after synchronization + + Example usage: + >> id = broadcast(data_or_fn=allocate_id, rank=0, pg=ext_pg.my_pg) + """ + + if not success and data_or_fn is not None: + raise AssertionError( + "Data or Function is expected to be None if not successful" + ) + + payload: T | None = None + exception: Exception | None = None + # if no pg is passed then execute if rank is 0 + if (pg is None and rank == 0) or (pg is not None and pg.rank() == rank): + # determine if it is an executable function or data payload only + if callable(data_or_fn): + try: + payload = data_or_fn() + except Exception as e: + success = False + exception = e + else: + payload = data_or_fn + + # broadcast the exception type if any to all ranks for failure categorization + sync_obj = SyncPayload( + stage_name=stage_name, + success=success, + payload=payload, + exception=exception, + ) + + if pg is not None: + broadcast_list = [sync_obj] + dist.broadcast_object_list(broadcast_list, src=rank, group=pg) + if len(broadcast_list) != 1: + raise AssertionError( + f"Expected broadcast_list to have exactly 1 element, got {len(broadcast_list)}" + ) + sync_obj = broadcast_list[0] + + # failure in any rank will trigger a throw in every rank. + if not sync_obj.success: + error_msg = f"Rank {rank} failed" + if stage_name is not None: + error_msg += f": stage {sync_obj.stage_name}" + if sync_obj.exception is not None: + error_msg += f": exception {sync_obj.exception}" + # pyrefly: ignore [invalid-inheritance] + raise RuntimeError(error_msg) from sync_obj.exception + + return cast(T, sync_obj.payload) + + +def all_gather( + data_or_fn: T | Callable[[], T], + stage_name: str | None = None, + pg: dist.ProcessGroup | None = None, +) -> list[T]: + """ + A simple all_gather primitive with basic synchronization guard logic, + by checking payload from all ranks has the same stage name. + + Args: + data_or_fn: the data to be all gathered across ranks or function to be executed + stage_name: the sync stage name for out-of-sync protection + pg: the process group for sync + Throws: + RuntimeError from original exception trace + Returns: + a list of synced data from all ranks + + Example usage: + >> all_ids = all_gather(data_or_fn=allocate_id, pg=ext_pg.my_pg) + """ + payload: T | None = None + exception: Exception | None = None + success = True + # determine if it is an executable function or data payload only + if callable(data_or_fn): + try: + payload = data_or_fn() + except Exception as e: + success = False + exception = e + else: + payload = data_or_fn + + sync_obj = SyncPayload( + stage_name=stage_name, + success=success, + payload=payload, + exception=exception, + ) + + if pg is not None: + # List of success/failure across all ranks. + total_list = [None] * dist.get_world_size(pg) + all_gather_object_enforce_type(pg, total_list, sync_obj) + # Each rank will throw RuntimeError in case of failure on any rank. + stage_name = cast(SyncPayload[T], total_list[0]).stage_name + exception_list: list[tuple[int, Exception]] = [] + ret_list: list[T] = [] + error_msg: str = "" + + for i, sp in enumerate(cast(list[SyncPayload[T]], total_list)): + if sp.stage_name != stage_name: + error_msg += ( + f"Unexpected stage name received from rank {i}: {sp.stage_name} " + ) + continue + if not sp.success and sp.exception is not None: + exception_list.append((i, sp.exception)) + continue + ret_list.append(sp.payload) + + if len(exception_list) > 0: + raise RuntimeError( # type: ignore[misc] + error_msg, + exception_list, + # pyrefly: ignore [invalid-inheritance] + ) from exception_list[0] + return ret_list + else: + if not sync_obj.success: + raise RuntimeError( + f"all_gather failed with exception {sync_obj.exception}", + # pyrefly: ignore [invalid-inheritance] + ) from sync_obj.exception + return [sync_obj.payload] # type: ignore[list-item] + + +# Note: use Any for typing for now so users can pass in +# either a list of None or target type placeholders +# otherwise pyre would complain +def all_gather_object_enforce_type( + pg: dist.ProcessGroup, + # pyre-fixme[2]: Parameter must have a type that does not contain `Any` + object_list: list[Any], + # pyre-fixme[2]: Parameter must have a type other than `Any` + obj: Any, + # pyre-fixme[2]: Parameter must have a type that does not contain `Any` + type_checker: Callable[[Any, Any], bool] = lambda x, y: type(x) is type(y), +) -> None: + """ + Similar to plain all_gather_object but with additional type checking + AFTER gather is done to ensure basic consistency. + If check does not pass, all ranks will fail with exception. + + This is generally to prevent conditional logic leading to + unexpected messages being received. This is considered fatal code error, + but due to logic stacks this might happen implicitly in practice. + + The default check does not check sub type (considered different) + or covariance (considered same) but users can pass in custom checker + if more complicated check is needed. + """ + dist.all_gather_object(object_list, obj, group=pg) + + # conservative check + list_len = len(object_list) + if list_len == 0: + return + first_obj = object_list[0] + for i in range(1, list_len): + if not type_checker(first_obj, object_list[i]): + raise TypeError( + f"Object type at index {i} is {type(object_list[i])}, " + f"while first object type is {type(first_obj)}" + ) + + +def _summarize_ranks(ranks: Iterable[int]) -> str: + ranks = sorted(ranks) + if min(ranks) < 0: + raise AssertionError("ranks should all be positive") + if len(set(ranks)) != len(ranks): + raise AssertionError("ranks should not contain duplicates") + curr: int | range | None = None + ranges = [] + while ranks: + x = ranks.pop(0) + if curr is None: + curr = x + elif isinstance(curr, int): + if x == curr + 1: + curr = range(curr, x + 1, 1) + else: + step = x - curr + curr = range(curr, x + step, step) + else: + if not isinstance(curr, range): + raise AssertionError("curr must be an instance of range") + if x == curr.stop: + curr = range(curr.start, curr.stop + curr.step, curr.step) + else: + ranges.append(curr) + curr = x + + if isinstance(curr, int): + ranges.append(range(curr, curr + 1, 1)) + elif isinstance(curr, range): + ranges.append(curr) + + result = [] + for r in ranges: + if len(r) == 1: + # pyrefly: ignore [bad-argument-type] + result.append(f"{r.start}") + elif r.step == 1: + # pyrefly: ignore [bad-argument-type] + result.append(f"{r.start}:{r.stop}") + else: + # pyrefly: ignore [bad-argument-type] + result.append(f"{r.start}:{r.stop}:{r.step}") + return ",".join(result) + + +def _check_philox_rng_sync( + generator: torch.Generator, group: dist.ProcessGroup +) -> tuple[dict[Any, set], str]: + local_state = generator.get_state() + all_states = [torch.empty_like(local_state) for _ in range(group.size())] + torch.distributed.all_gather(all_states, local_state) + seeds_offsets = [ + (state[:8].view(torch.uint64).item(), state[8:].view(torch.uint64).item()) + for state in all_states + ] + seed_offset_ranks = defaultdict(set) + for rank, (seed, offset) in enumerate(seeds_offsets): + seed_offset_ranks[(seed, offset)].add(rank) + return seed_offset_ranks, "(Seed, Offset)" + + +def _check_cpu_rng_sync( + generator: torch.Generator, group: dist.ProcessGroup +) -> tuple[dict[Any, set], str]: + # seed is returned as uint64_t from C impl, so may not fit in torch int64 tensor directly. + state_tensor = generator.get_state() + all_state_tensors = [torch.empty_like(state_tensor) for _ in range(group.size())] + torch.distributed.all_gather(all_state_tensors, state_tensor) + state_ranks = defaultdict(set) + for rank, state_tensor in enumerate(all_state_tensors): + # Summarize the state vector of the CPU rng. + # The properties that matter most are (1) its different if there is a state difference, (2) its printable + # (see desync table- not viable to print whole state vector of size 5k) + state_ranks[torch.hash_tensor(state_tensor).item()].add(rank) + return state_ranks, "Generator state hash" + + +def _check_rng_sync_internal( + generator: torch.Generator, group: dist.ProcessGroup +) -> tuple[dict[Any, set], str]: + if generator.device.type == "cuda": + return _check_philox_rng_sync(generator, group) + elif generator.device.type == "cpu": + return _check_cpu_rng_sync(generator, group) + else: + raise NotImplementedError( + f"Unsupported generator device: {generator.device.type}" + ) + + +def _desync_table_str(tag: str, value_ranks: dict[Any, set[int]]) -> str: + headers = ["Ranks", f"{tag} values"] + rank_values = [ + [_summarize_ranks(ranks), str(value)] for value, ranks in value_ranks.items() + ] + if importlib.util.find_spec("tabulate"): + from tabulate import tabulate + + return tabulate(rank_values, headers=headers) + row_str = "\n".join([str(row) for row in rank_values]) + return str(f"{headers}\n{row_str}") + + +def _check_rng_sync(generator: torch.Generator, group: dist.ProcessGroup) -> str | None: + value_ranks, value_header = _check_rng_sync_internal(generator, group) + log_str = None + if len(value_ranks) > 1: + log_str = f"Generator desync detected:\n{_desync_table_str(value_header, value_ranks)}" + logger.error(log_str) + return log_str diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/constants.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..7cd9b17cf6bcc554c0a0f4e8ebb735175b5f64a5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/constants.py @@ -0,0 +1,25 @@ +from datetime import timedelta + +from torch._C._distributed_c10d import _DEFAULT_PG_TIMEOUT + + +__all__ = ["default_pg_timeout", "default_pg_nccl_timeout"] + +# Default process group wide timeout, if applicable. +# This only applies to the non-nccl backends +# To make an attempt at backwards compatibility with THD, we use an +# extraordinarily high default timeout, given that THD did not have timeouts. +default_pg_timeout: timedelta = _DEFAULT_PG_TIMEOUT +# Separate timeout for PGNCCL mainly because it's always been that way in the C++ layer, but until recently +# there was one default that applied across all backends in the python layer. +# Later, we could consider merging them back together at the c++ layer if we can align on a same value. +# (only if TORCH_NCCL_BLOCKING_WAIT or TORCH_NCCL_ASYNC_ERROR_HANDLING is set to 1). + +try: + from torch._C._distributed_c10d import _DEFAULT_PG_NCCL_TIMEOUT + + default_pg_nccl_timeout: timedelta | None = _DEFAULT_PG_NCCL_TIMEOUT +except ImportError: + # if C++ NCCL support is not compiled, we don't have access to the default nccl value. + # if anyone is actually trying to use nccl in this state, it should error. + default_pg_nccl_timeout = None diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/device_mesh.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/device_mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..09574e65dbb017e5e88573a22d026a5614f91950 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/device_mesh.py @@ -0,0 +1,1370 @@ +# mypy: allow-untyped-defs +# Copyright (c) Meta Platforms, Inc. and affiliates +import logging +import os +import threading +import warnings +from collections.abc import Iterator +from itertools import zip_longest +from typing import Optional, TYPE_CHECKING, Union + +import torch +from torch.distributed import is_available +from torch.distributed._mesh_layout import _MeshLayout +from torch.distributed._pycute import IntTuple, is_int, suffix_product +from torch.utils._typing_utils import not_none + + +__all__ = ["init_device_mesh", "DeviceMesh"] + + +if not is_available(): + import sys + + # We need to create the stubs when distributed is not available. + # Otherwise, we would fail the doc tests (```./.ci/pytorch/docs-test.sh```), + # since it would try to import ``torch.distributed.device_mesh`` or + # ``torch.distributed.init_device_mesh`` but cannot find them. + + class _DeviceMeshStub: + pass + + def _init_device_mesh_stub(): + pass + + sys.modules["torch.distributed.device_mesh"].DeviceMesh = _DeviceMeshStub # type: ignore[attr-defined] + sys.modules[ + "torch.distributed.device_mesh" + ].init_device_mesh = _init_device_mesh_stub # type: ignore[attr-defined] + + +else: + from torch._C._distributed_c10d import Backend as C10dBackend + from torch.distributed.distributed_c10d import ( + _get_default_group, + _resolve_process_group, + get_backend, + get_process_group_ranks, + get_rank, + get_world_size, + GroupName, + init_process_group, + is_initialized, + new_group, + ProcessGroup, + split_group, + ) + + logger = logging.getLogger(__name__) + + # only import numpy typing when type checking + if TYPE_CHECKING: + try: + from numpy.typing import ArrayLike + except ImportError: + logger.warning( + "DeviceMesh requires numpy >= 1.21 to be installed for type checking" + ) + + BackendConfig = tuple[str | None, C10dBackend.Options | None] + torch.serialization.add_safe_globals([_MeshLayout]) + + class _MeshEnv(threading.local): + def __init__(self) -> None: + self.mesh_stack: list[DeviceMesh] = [] + + def get_current_mesh(self) -> "DeviceMesh": + if len(self.mesh_stack) == 0: + raise RuntimeError("No device mesh is currently active!") + return self.mesh_stack[-1] + + # TODO: to remove it once we move all use cases into new API. + def get_root_mesh(self, device_mesh: "DeviceMesh") -> "DeviceMesh": + # If a mesh could not be found in the child_to_root_mapping, it is a root mesh itself. + # A root mesh is not created through slicing. + # We considers the root mesh of a root mesh is itself. + # We keep this function for backward compatibility. + warnings.warn( + "This get_root_mesh API will be deprecated soon." + "Please use `get_root_mesh` inside DeviceMesh instead.", + stacklevel=2, + ) + if not device_mesh: + return device_mesh + return device_mesh._get_root_mesh() + + @staticmethod + def num_devices_per_host(device_type: str) -> int: + return _get_device_handle(device_type).device_count() + + @staticmethod + def num_hosts(device_type: str) -> int: + # ProcessGroup can't tell us this info so we have to infer it, assume + # homogeneous hardware for now + return get_world_size() // _MeshEnv.num_devices_per_host(device_type) + + # TODO: to remove it once we move all use cases into new API. + # We keep this API for backward compatibility. + def _get_all_submeshes( + self, device_mesh: "DeviceMesh", mesh_dim_name: str + ) -> list["DeviceMesh"]: + warnings.warn( + "This _get_all_submeshes API will be deprecated soon." + "Please use `_get_all_submeshes` inside DeviceMesh instead.", + stacklevel=2, + ) + return device_mesh._get_all_submeshes(mesh_dim_name) + + _mesh_resources: _MeshEnv = _MeshEnv() + + def _get_device_handle(device_type: str = "cuda"): + """ + Get the module corresponding to the device_type which is cuda or cuda-like device. + For example, when the device_type is cuda, the module `torch.cuda` is returned. + Return None when there is no corresponding module for device_type, otherwise + return the corresponding module. + """ + return getattr(torch, device_type, None) + + class DeviceMesh: + """ + DeviceMesh represents a mesh of devices, where layout of devices could be + represented as a n-d dimension array, and each value of the n-d dimensional + array is the global id of the default process group ranks. + + DeviceMesh could be used to setup the N dimensional device connections across the cluster, + and manage the ProcessGroups for N dimensional parallelisms. Communications could happen on + each dimension of the DeviceMesh separately. DeviceMesh respects the device that user selects + already (i.e. if user call `torch.cuda.set_device` before the DeviceMesh initialization), + and will select/set the device for the current process if user does not set the device + beforehand. Note that manual device selection should happen BEFORE the DeviceMesh initialization. + + DeviceMesh can also be used as a context manager when using together with DTensor APIs. + + .. note:: + DeviceMesh follows SPMD programming model, which means the same PyTorch Python program + is running on all processes/ranks in the cluster. Therefore, users need to make sure the + `mesh` array (which describes the layout of devices) should be identical across all ranks. + Inconsistent `mesh` will lead to silent hang. + + Args: + device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like". + mesh (ndarray): A multi-dimensional array or an integer tensor describing the layout + of devices, where the IDs are global IDs of the default process group. + _rank (int): (experimental/internal) + The global rank of the current process. If not provided, it will + be inferred from the default process group. + + Returns: + DeviceMesh: A :class:`DeviceMesh` object representing the device layout. + + The following program runs on each process/rank in an SPMD manner. In this example, we have 2 + hosts with 4 GPUs each. + A reduction over the first dimension of mesh will reduce across + columns (0, 4), .. and (3, 7), a reduction over the second dimension + of mesh reduces across rows (0, 1, 2, 3) and (4, 5, 6, 7). + + Example:: + + >>> # xdoctest: +SKIP("no rank") + >>> from torch.distributed.device_mesh import DeviceMesh + >>> + >>> # Initialize device mesh as (2, 4) to represent the topology + >>> # of cross-host(dim 0), and within-host (dim 1). + >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) + """ + + _device_type: str + _rank_map: torch.Tensor + _mesh_dim_names: tuple[str, ...] | None + _layout: _MeshLayout + _root_mesh: Optional["DeviceMesh"] = None + # Record flatten mesh name to its flattened mesh in root mesh. + _flatten_mapping: dict[str, "DeviceMesh"] + + def __init__( + self, + device_type: str, + mesh: Union[torch.Tensor, "ArrayLike"] | None = None, + *, + mesh_dim_names: tuple[str, ...] | None = None, + backend_override: tuple[BackendConfig, ...] | None = None, + _init_backend: bool = True, + _rank: int | None = None, + _layout: _MeshLayout | None = None, + _rank_map: torch.Tensor | None = None, + _root_mesh: Optional["DeviceMesh"] = None, + ) -> None: + # no-op in OSS, logs API usage metrics in meta-internal runs + torch._C._log_api_usage_once( + "torch.distributed.device_mesh.DeviceMesh.__init__" + ) + if mesh is not None: + if _layout is not None or _rank_map is not None: + raise TypeError( + "Cannot provide _layout and/or _rank_map if passing explicit mesh" + ) + if isinstance(mesh, torch.Tensor) and mesh.device.type != "cpu": + raise ValueError(f"`mesh` must be a CPU tensor, got {mesh}") + mesh_tensor = ( + mesh.detach().to(dtype=torch.int).contiguous() + if isinstance(mesh, torch.Tensor) + else torch.tensor(mesh, device="cpu", dtype=torch.int) + ) + _layout = _MeshLayout(mesh_tensor.size(), mesh_tensor.stride()) + _rank_map = mesh_tensor.flatten() + else: + if _layout is None or _rank_map is None: + raise TypeError( + "The mesh argument is required except for PRIVATE USAGE ONLY!" + ) + + assert _layout.check_non_overlap(), ( + "Please use a non-overlapping layout when creating a DeviceMesh." + ) + assert _rank_map.ndim == 1, "The rank map must be 1-dimensional" + assert _rank_map.is_contiguous(), "The rank map must be contiguous" + assert _rank_map.numel() >= _layout.cosize(), ( + f"The rank map contains {_rank_map.numel()} element, " + f"which isn't large enough for layout {_layout}" + ) + + self._device_type = device_type + self._layout = _layout + self._rank_map = _rank_map + self._mesh_dim_names = tuple(mesh_dim_names) if mesh_dim_names else None + self._root_mesh = _root_mesh + + if backend_override is None: + backend_override = ((None, None),) * len(self._layout) + elif len(backend_override) != len(self._layout): + raise ValueError( + f"backend_override should have the same length as the number of mesh dimensions, " + f"but got {len(backend_override)} and {len(self._layout)}." + ) + # Internal bookkeeping for the device mesh. + self._layout = ( + _layout + if _layout + else _MeshLayout(self.mesh.size(), self.mesh.stride()) + ) + if not self._layout.check_non_overlap(): + raise AssertionError( + "Please use a non-overlapping layout when creating a DeviceMesh." + ) + # Because we still need to support slicing of flattened dim from root mesh, so we don't check stride here. + if self._layout.numel() != self.mesh.numel(): + raise AssertionError( + "Please use a valid layout when creating a DeviceMesh." + f"The layout {self._layout} is not consistent with the mesh size {self.mesh.size()}." + ) + + # private field to pre-generate DeviceMesh's hash + self._flatten_rank_map = tuple(self._rank_map.tolist()) + self._thread_id = None + # Initialize instance-specific flatten mapping + self._flatten_mapping = {} + + # Skip process group initialization if xla device or init backend is False + # TODO(yeounoh) implement DeviceMesh backend and register XLA backend. + if device_type != "xla": + # always try to create default (world) pg, even if it is not initialized + # already. The world pg is used for device mesh identity (rank) on each + # process (we need to know if the current global rank is in the mesh or not). + if _init_backend: + self._setup_world_group_and_device() + self._dim_group_names = self._init_process_groups( + self._layout, + self._rank_map, + self._mesh_dim_names, + backend_override, + ) + + if is_initialized() and get_backend() == "threaded": + # pyrefly: ignore [bad-assignment] + self._thread_id = threading.get_ident() + + if _rank is None: + _rank = get_rank() + + # calculate the coordinates of the current global rank on the mesh + rank_coords = (self.mesh == _rank).nonzero() + if rank_coords.size(0) not in (0, 1): + raise AssertionError( + f"rank_coords.size(0) must be 0 or 1, got {rank_coords.size(0)}" + ) + self._coordinate_on_dim: list[int] | None = ( + rank_coords[0].tolist() if rank_coords.size(0) > 0 else None + ) + + @property + def device_type(self) -> str: + """Returns the device type of the mesh.""" + return self._device_type + + @property + def mesh(self) -> torch.Tensor: + """Returns the tensor representing the layout of devices.""" + full_mesh = self._layout.remap_to_tensor(self._rank_map) + if full_mesh.size(0) == 1: + return full_mesh[0] + my_coords = (full_mesh == get_rank()).nonzero() + if my_coords.size(0) > 0: + return full_mesh[my_coords[0, 0]] + raise RuntimeError( + "In order to get the mesh Tensor of a DeviceMesh it needs to " + "either have all its original dimensions (e.g., no slicing) " + "or it needs to contain the local rank" + ) + + @property + def mesh_dim_names(self) -> tuple[str, ...] | None: + """Returns the names of mesh dimensions.""" + return self._mesh_dim_names + + def _setup_world_group_and_device(self): + default_initialized = is_initialized() + # TODO: think about how to allow pg options to be passed to world group + # or mesh dimension groups + if not default_initialized: + init_process_group() + + world_size = get_world_size() + if self._layout.numel() > world_size: + raise RuntimeError( + f"Mesh should not be bigger than default world size {world_size}, but found {self._layout.numel()} ranks!" + ) + + # ONLY set the device if the current device is not initialized, if user already + # set the device before DeviceMesh init, we respect the user's choice. + device_handle = _get_device_handle(self._device_type) + if device_handle and not device_handle.is_initialized(): + # auto set the cuda/cuda-like device only if user has not set it, if there's LOCAL_RANK + # env variable from launchers, we use it to set the device. + if "LOCAL_RANK" in os.environ: + local_rank = int(os.environ["LOCAL_RANK"]) + logger.info( + "Setting default device for the current process based on LOCAL_RANK=%s", + local_rank, + ) + device_handle.set_device(local_rank) + else: + warnings.warn( + "It seems like you did not set/select the default device for the current process before the DeviceMesh " + "initialization or use a launcher (i.e. torchrun) which populates `LOCAL_RANK` environment variable. " + "It is recommended to set the current device for the process BEFORE the DeviceMesh initialization so that " + "the underlying communicator (i.e. NCCL) can be initialized properly. " + "Given that the current process has no default device selected, DeviceMesh will use a heuristic to set the " + "device_id via `global_rank % num_devices_per_host`, assuming homogeneous hardware cluster. ", + stacklevel=2, + ) + # heuristic to set the current cuda/cuda-like device base on num of gpu devices available in each host + # NOTE: This device selection would only work for homogeneous hardware. + num_devices_per_host = device_handle.device_count() + if ( + world_size > num_devices_per_host + and world_size % num_devices_per_host != 0 + ): + raise RuntimeError( + f"DeviceMesh only support homogeneous hardware, but found " + f"{world_size} ranks and {num_devices_per_host} {self._device_type} devices!" + ) + device_handle.set_device(get_rank() % num_devices_per_host) + + return _get_default_group() + + @staticmethod + def _init_one_process_group( + sub_layout: _MeshLayout, + rank_map: torch.Tensor, + dim_name: str, + backend_override: BackendConfig, + ) -> GroupName | None: + # Generate a 2D global mesh tensor for the current dim for PG creation. + pg_ranks_by_dim = sub_layout.nest().remap_to_tensor(rank_map) + backend, pg_options = backend_override + # We need to explicitly pass in timeout when specified in option, otherwise + # the default timeout will be used to override the timeout set in option. + # TODO: remove this once we have fixed inside c10d level. + timeout = pg_options._timeout if pg_options else None + + # If we have a 2D mesh with mesh_dim_names ("dp", "tp"), the group description + # of the subgroups would be `mesh_dim_dp` and `mesh_name_tp`. + # If the mesh doesn't have a mesh_dim_names, then the group description of the + # subgroup would be `mesh_dim_0` and `mesh_dim_1`. + group_desc = f"mesh_{dim_name}" + + dim_group = None + default_group = _get_default_group() + + # Early return if there is only one sub_layout in the mesh layout. + if sub_layout.numel() == get_world_size() and backend_override == ( + None, + None, + ): + # Append the default pg to the first dim groups only if the default pg is compatible with `self._device_type`. + # Otherwise, create new pg. + ranks = list(range(get_world_size())) + dim_group = ( + new_group( + backend="cpu:gloo,cuda:nccl", + ranks=ranks, + group_desc="mesh_default", + ) + if torch.cuda.is_available() + and get_backend(default_group) == "gloo" + else default_group + ) + return dim_group.group_name # type: ignore[union-attr] + + # If bound_device_id exists, it means the nccl communicator has been eagerly initialized + # so that we can use `split_group` to create subgroups through `ncclCommSplit`. + # In this case, we only need to make one API call (`split_group``) for the subgroup creation + # for each mesh dimension. In a 2 * 4 mesh, we only need to make two API calls per ranks to create + # all the subgroups. + # Otherwise, we need to make more than one API call (`new_group`) for subgroup creations. The + # numbers of API calls are equal to the number of subgroups for each mesh dimension. In a 2 * 4 + # mesh, we need to make two API calls per ranks to create all the subgroups. + if ( + getattr(default_group, "bound_device_id", None) is not None + and torch.cuda.is_available() + and ( + backend is None + or default_group._get_backend(torch.device("cuda")).name() + == backend + ) + ): + dim_group = split_group( + parent_pg=default_group, + timeout=timeout, + pg_options=pg_options, + split_ranks=pg_ranks_by_dim.tolist(), + group_desc=group_desc, + ) + return dim_group.group_name # type: ignore[union-attr] + + # If the subgroup has been already created through `split_group`, we simply loop over `pg_ranks_by_dim` + # and append the `group_name` to the `dim_group_names` list when the current rank is in the subgroup. + # Otherwise, we use `new_group` instead of `split_group` to create subgroups by looping over `pg_ranks_by_dim` + # along with appending information to the `dim_group_names` list whenever necessary. + pg_name = None + for dim_mesh in pg_ranks_by_dim: + subgroup_ranks = dim_mesh.tolist() + dim_group = new_group( + ranks=subgroup_ranks, + timeout=timeout, + backend=backend, + pg_options=pg_options, + group_desc=group_desc, + ) + + # only add to dim_groups if the current rank in the subgroup + if get_rank() in subgroup_ranks: + if pg_name is not None: + raise RuntimeError( + f"Each device mesh dimension should get only one process group, but got {get_rank()} " + f"in {subgroup_ranks}!" + ) + pg_name = dim_group.group_name + return pg_name + + @staticmethod + def _init_process_groups( + layout: _MeshLayout, + rank_map: torch.Tensor, + mesh_dim_names: tuple[str, ...] | None, + backend_override: tuple[BackendConfig, ...], + ) -> list[GroupName]: + # group_name associated with each mesh dimension, each + # mesh dimension should have one sub-group per rank + dim_group_names: list[GroupName | None] = [] + # create sub pgs base on the mesh argument specified + for dim in range(len(layout)): + dim_name = mesh_dim_names[dim] if mesh_dim_names else f"dim_{dim}" + dim_group_names.append( + DeviceMesh._init_one_process_group( + layout[dim], rank_map, dim_name, backend_override[dim] + ) + ) + # Filter out None values. If any are None then they should all be None. + dim_non_none_group_names = [n for n in dim_group_names if n is not None] + assert not dim_non_none_group_names or len(dim_non_none_group_names) == len( + dim_group_names + ) + return dim_non_none_group_names + + def _get_root_mesh(self) -> "DeviceMesh": + return self._root_mesh if self._root_mesh else self + + def __enter__(self) -> "DeviceMesh": + # set this mesh as the current mesh in mesh env + _mesh_resources.mesh_stack.append(self) + return self + + # pyre-fixme[2]: Parameter must be annotated. + def __exit__(self, exc_type, exc_value, exc_traceback) -> None: + # pop this mesh from mesh env + _mesh_resources.mesh_stack.pop() + + def __repr__(self) -> str: + device_mesh_repr = ( + f"({', '.join(f'{k}={v}' for k, v in zip(self._mesh_dim_names, self._layout.top_level_sizes))})" + if self._mesh_dim_names + else f"{self._layout.top_level_sizes}" + ) + device_mesh_repr = f"DeviceMesh({device_mesh_repr}, '{self.device_type}', stride={self._layout.strides}" + # We only print the mesh tensor if the debug mode is turned on. + if os.environ.get("TORCH_DISTRIBUTED_DEBUG", "") == "DETAIL": + device_mesh_repr += f", Mesh: {self.mesh.tolist()}" + return f"{device_mesh_repr})" + + def __hash__(self): + # lazily compute hash + self._hash = getattr(self, "_hash", None) + if not self._hash: + self._hash = hash( + ( + self._flatten_rank_map, + self._layout, + self._device_type, + self._mesh_dim_names, + self._thread_id, + ) + ) + return self._hash + + def __eq__(self, other: object) -> bool: + if self is other: + return True + if not isinstance(other, DeviceMesh): + return False + return ( + self._flatten_rank_map == other._flatten_rank_map + and self._layout == other._layout + and self._device_type == other._device_type + and self._mesh_dim_names == other._mesh_dim_names + and self._thread_id == other._thread_id + ) + + def __getitem__(self, mesh_dim_names: str | tuple[str, ...]) -> "DeviceMesh": + """ + Slice the current DeviceMesh based on the mesh_dim_names given to create a submesh. + The submesh created consists of the dimensions and the communicators indicated by + ``mesh_dim_names`` + + Args: + mesh_dim_names (Union[str, tuple[str, ...]]): the name or the tuple of names of the + mesh dimension of the DeviceMesh to create the submesh for. + Returns: + A :class:`DeviceMesh` object + + The following program runs on each process/rank in an SPMD manner in a world size of 8. + In the first example: + Calling mesh_2d["tp"] on rank 0, 1, 2, 3 returns a 1D submesh of DeviceMesh:([0, 1, 2, 3]). + Calling mesh_2d["tp"] on rank 4, 5, 6, 7 returns a 1D submesh of DeviceMesh:([4, 5, 6, 7]). + Calling mesh_2d["dp"] on rank 0, 4 returns a 1D submesh of DeviceMesh:([0, 4]). + Calling mesh_2d["dp"] on rank 1, 5 returns a 1D submesh of DeviceMesh:([1, 5]). + Calling mesh_2d["dp"] on rank 2, 6 returns a 1D submesh of DeviceMesh:([2, 6]). + Calling mesh_2d["dp"] on rank 3, 7 returns a 1D submesh of DeviceMesh:([3, 7]). + + In the second example: + Calling mesh_3d["dp", "cp"] on rank 0, 1, 4, 5 returns a 2D submesh of DeviceMesh:([[0, 1], [4, 5]]). + Calling mesh_3d["dp", "cp"] on rank 2, 3, 6, 7 returns a 2D submesh of DeviceMesh:([[2, 3], [6, 7]]). + Calling mesh_3d["cp", "dp"] on rank 0, 1, 4, 5 returns a 2D submesh of DeviceMesh:([[0, 4], [1, 5]]). + Calling mesh_3d["cp", "dp"] on rank 2, 3, 6, 7 returns a 2D submesh of DeviceMesh:([[2, 6], [3, 7]]). + + Example:: + + >>> # xdoctest: +SKIP("no rank") + >>> from torch.distributed.device_mesh import DeviceMesh + >>> + >>> # Initialize a 2D device mesh as (2, 4) to represent the topology + >>> # of cross-host(dim 0), and within-host (dim 1). + >>> mesh_2d = init_device_mesh(device_type="cuda", (2,4), mesh_dim_names=("dp", "tp")) + >>> tp_mesh = mesh_2d["tp"] + >>> dp_mesh = mesh_2d["dp"] + >>> + >>> # Initialize a 3D mesh. + >>> mesh_3d = init_device_mesh(device_type="cuda", (2,2,2), mesh_dim_names=("dp", "pp", "cp")) + >>> # The order of the mesh_dim_names provided deteremines the order of dimensions in the submesh. + >>> dp_cp_mesh = mesh_3d["dp", "cp"] + >>> cp_dp_mesh = mesh_3d["cp", "dp"] + """ + if not self._mesh_dim_names: + raise RuntimeError("Cannot slice a DeviceMesh without mesh_dim_names!") + + mesh_dim_names = ( + (mesh_dim_names,) if isinstance(mesh_dim_names, str) else mesh_dim_names + ) + + if mesh_dim_names == self._mesh_dim_names: + return self + else: + sliced_mesh_layout = self._get_slice_mesh_layout(mesh_dim_names) + # When using FakeTensorMode to trace the model, `_create_sub_mesh()` will + # fail as it will require a real tensor to manipulate. + # `unset_fake_temporarily()` will allow us to materialize the tensors + # within `_create_sub_mesh`, which should not affect modling. + # + # Note that this should be orthogonal to torch.compile(). But whether + # we can compile device_mesh `slicing` (no graph break) is not verified + # yet and need a follow-up, + # TODO: compiler + device_mesh slicing. + with torch._subclasses.fake_tensor.unset_fake_temporarily(): + submesh = self._create_sub_mesh(sliced_mesh_layout, mesh_dim_names) + return submesh + + def get_group(self, mesh_dim: int | str | None = None) -> ProcessGroup: + """ + Returns the single ProcessGroup specified by mesh_dim, or, if mesh_dim is not specified and the + DeviceMesh is 1-dimensional, returns the only ProcessGroup in the mesh. + + Args: + mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index + of the mesh dimension. Default is None. + + Returns: + A :class:`ProcessGroup` object. + """ + if not hasattr(self, "_dim_group_names"): + raise RuntimeError("DeviceMesh process groups not initialized!") + + if len(self._layout) > 1 and mesh_dim is None: + raise RuntimeError( + f"Found the DeviceMesh have {len(self._layout)} dimensions", + "Optional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.", + "If you want to get the list of all the ProcessGroups in the DeviceMesh," + "please use `get_all_groups()` instead.", + ) + + # Quick return if the current device_mesh is a 1D mesh. + if len(self._layout) == 1 and mesh_dim is None: + return not_none(_resolve_process_group(self._dim_group_names[0])) + + root_mesh = self._get_root_mesh() + root_to_flatten_mapping = root_mesh._flatten_mapping + if root_to_flatten_mapping and mesh_dim in root_to_flatten_mapping: + dim_group_name = root_to_flatten_mapping[ + mesh_dim # type: ignore[index] + ]._dim_group_names[0] + return not_none(_resolve_process_group(dim_group_name)) + else: + mesh_dim = ( + self._get_mesh_dim_by_name(mesh_dim) + if isinstance(mesh_dim, str) + else mesh_dim + ) + if not isinstance(mesh_dim, int): + raise AssertionError( + f"mesh_dim must be an int, got {type(mesh_dim)}" + ) + return not_none(_resolve_process_group(self._dim_group_names[mesh_dim])) + + def get_all_groups(self) -> list[ProcessGroup]: + """ + Returns a list of ProcessGroups for all mesh dimensions. + + Returns: + A list of :class:`ProcessGroup` object. + """ + return [self.get_group(i) for i in range(len(self._layout))] + + def _create_sub_mesh( + self, + layout: _MeshLayout, + submesh_dim_names: tuple[str, ...], + ) -> "DeviceMesh": + root_mesh = self._get_root_mesh() + slice_dim_group_name = [] + for name in submesh_dim_names: + if name in not_none(self._mesh_dim_names): + slice_dim_group_name.append( + self._dim_group_names[ # type: ignore[has-type] + not_none(self._mesh_dim_names).index(name) + ] + ) + else: + # If device_mesh is not root_mesh, we already throw error in _get_slice_mesh_layout + # Since we will deprecate the slicing of flattened dim_name from root mesh soon, + # we don't want to optimize the code furthermore. + flatten_mesh = self._flatten_mapping[name] + slice_dim_group_name.append( + flatten_mesh._dim_group_names[ # type: ignore[has-type] + not_none(flatten_mesh._mesh_dim_names).index(name) + ] + ) + res_submesh = DeviceMesh( + self._device_type, + _layout=layout, + _rank_map=root_mesh._rank_map, + mesh_dim_names=submesh_dim_names, + _root_mesh=root_mesh, + _init_backend=False, + ) + res_submesh._dim_group_names = slice_dim_group_name + return res_submesh + + def _create_flatten_mesh( + self, + mesh_dim_name: str | None = None, + backend_override: BackendConfig = (None, None), + ) -> "DeviceMesh": + root_mesh = self._get_root_mesh() + + if not mesh_dim_name: + mesh_dim_name = "_".join(not_none(self._mesh_dim_names)) + + # Flatten a 1D device mesh into its original mesh_dim_name will return itself. + if self.ndim == 1 and mesh_dim_name in not_none(self._mesh_dim_names): + return self + + # Check whether the mesh_dim_name for flattened mesh is valid. + invalid_dim_names = not_none(root_mesh._mesh_dim_names) + if mesh_dim_name in invalid_dim_names: + raise ValueError( + f"{mesh_dim_name} already exists for submesh of the {root_mesh}. ", + f"The mesh_dim_names of submesh and flattened mesh are {invalid_dim_names}. " + f"Please specify another valid mesh_dim_name.", + ) + + flattened_mesh_layout = self._layout.coalesce() + if len(flattened_mesh_layout) > 1: + flattened_mesh_layout = flattened_mesh_layout.nest() + # Quick return if the flatten mesh has been created before. + if mesh_dim_name in root_mesh._flatten_mapping: + if ( + flattened_mesh_layout + == root_mesh._flatten_mapping[mesh_dim_name]._layout + ): + return root_mesh._flatten_mapping[mesh_dim_name] + else: + raise ValueError( + f"Flatten mesh with mesh_dim_name {mesh_dim_name} has been created before, " + f"Please specify another valid mesh_dim_name." + ) + + res_flattened_mesh = DeviceMesh( + root_mesh._device_type, + _layout=flattened_mesh_layout, + _rank_map=root_mesh._rank_map, + mesh_dim_names=(mesh_dim_name,), + _root_mesh=root_mesh, + backend_override=(backend_override,), + ) + root_mesh._flatten_mapping[mesh_dim_name] = res_flattened_mesh + + return res_flattened_mesh + + def _get_root_mesh_dim(self) -> int | None: + """ + Returns the index of the mesh dim in the root mesh. + The device_mesh passed in needs to be sliced out from the root mesh + or submesh of the root mesh. + """ + root_mesh = self._get_root_mesh() + child_mesh_dim_names = self._mesh_dim_names + if root_mesh and child_mesh_dim_names: + if len(child_mesh_dim_names) != 1: + raise AssertionError("The submesh can only be a 1D mesh.") + child_mesh_dim_name = child_mesh_dim_names[0] + return root_mesh._get_mesh_dim_by_name(child_mesh_dim_name) + return None + + def _get_mesh_dim_by_name(self, mesh_dim_name: str) -> int: + if self._mesh_dim_names is None or len(self._mesh_dim_names) == 0: + raise KeyError( + "No `mesh_dim_names` found.", + ) + if mesh_dim_name not in self._mesh_dim_names: + raise KeyError( + f"Mesh dimension '{mesh_dim_name}' does not exist.", + f"Available mesh dimensions are: mesh_dim_names={self._mesh_dim_names}", + ) + return not_none(self._mesh_dim_names.index(mesh_dim_name)) + + def _get_slice_mesh_layout( + self, mesh_dim_names: tuple[str, ...] + ) -> _MeshLayout: + """ + Validate whether the mesh_dim_names is valid for slicing the given device_mesh. + If valid, return dim indexes of the slice mesh in the device mesh. + """ + slice_from_root = True + if self != self._get_root_mesh(): + slice_from_root = False + + # The slice mesh_dim_names should consist either the current device_mesh's mesh_dim_names + # or its flattened mesh's mesh_dim_names if it's root_mesh. + flatten_name_to_root_layout = ( + { + key: mesh._layout + for key, mesh in self._get_root_mesh()._flatten_mapping.items() + } + if slice_from_root + else {} + ) + valid_mesh_dim_names = [ + *not_none(self._mesh_dim_names), + *flatten_name_to_root_layout, + ] + + if not all( + mesh_dim_name in valid_mesh_dim_names + for mesh_dim_name in mesh_dim_names + ): + raise KeyError( + f"Invalid mesh_dim_names {mesh_dim_names} specified. " + f"Valid mesh_dim_names are {valid_mesh_dim_names}." + ) + + layout_sliced = [] + for name in mesh_dim_names: + if name in not_none(self._mesh_dim_names): + layout_sliced.append( + self._layout[not_none(self._mesh_dim_names).index(name)] + ) + elif name in flatten_name_to_root_layout: + warnings.warn( + "Slicing a flattened dim from root mesh will be deprecated in PT 2.11. " + "Users need to bookkeep the flattened mesh directly. ", + stacklevel=2, + ) + layout_sliced.append(flatten_name_to_root_layout[name]) + + sliced_sizes = tuple(l.sizes for l in layout_sliced) + sliced_strides = tuple(l.strides for l in layout_sliced) + + # The check below is from DeviceMesh's implementation before adopting CuTe layout for internal + # bookkeeping and it can be removed but we need to define what is the expected behavior. + # TODO: Remove the below check and define the expected behavior. + # Validate the order of the slice mesh dim indices. + # This needs to be in ascending order. + pre_stride = -1 + for stride in reversed(sliced_strides): + # Note that with CuTe layout, we can support slicing flattened non-contiguous mesh dims with no problem. + # But this will make this behavior complicated so we decided to not support it for now. + if not is_int(stride): + raise NotImplementedError( + "Currently, this only allows slicing out a contiguous flattened dim." + ) + if stride < pre_stride: + raise KeyError( + f"Invalid mesh_dim_names {mesh_dim_names} specified. " + "Mesh dim indices should be in ascending order." + ) + pre_stride = stride + + # When users sliced dim_names outside from current mesh, we will check whether + # there is layout overlap. + # TODO: Eventually we will just directly throw error here because + # we will deprecate the slicing of flattened dim_name from root mesh. + layout_sliced = _MeshLayout(sliced_sizes, sliced_strides) + if not layout_sliced.check_non_overlap(): + raise RuntimeError( + f"Slicing overlapping dim_names {mesh_dim_names} is not allowed." + ) + + return layout_sliced + + # TODO: to make this use case by other components public API in the future. + def _get_all_submeshes(self, mesh_dim_name: str) -> list["DeviceMesh"]: + """ + Return all the submeshes of a given mesh dimension of the device mesh. + """ + mesh_dim = self._get_mesh_dim_by_name(mesh_dim_name) + layout = self._layout[mesh_dim] + pg_ranks_by_dim = layout.remap_to_tensor(self._rank_map) + cur_rank = self.get_rank() + res_submeshes = [] + for mesh_1d in pg_ranks_by_dim: + submesh = DeviceMesh( + self._device_type, + mesh_1d, + mesh_dim_names=(mesh_dim_name,), + _init_backend=False, + ) + submesh._dim_group_names = ( # type: ignore[has-type] + [self._dim_group_names[mesh_dim]] # type: ignore[has-type] + if cur_rank in mesh_1d + else [] + ) + res_submeshes.append(submesh) + + return res_submeshes + + @staticmethod + def from_group( + group: ProcessGroup | list[ProcessGroup], + device_type: str, + mesh: Union[torch.Tensor, "ArrayLike"] | None = None, + *, + mesh_dim_names: tuple[str, ...] | None = None, + ) -> "DeviceMesh": + """ + Constructs a :class:`DeviceMesh` with ``device_type`` from an + existing :class:`ProcessGroup` or a list of existing :class:`ProcessGroup`. + + The constructed device mesh has number of dimensions equal to the + number of groups passed. For example, if a single process group is passed in, + the resulted DeviceMesh is a 1D mesh. If a list of 2 process groups is passed in, + the resulted DeviceMesh is a 2D mesh. + + If more than one group is passed, then the ``mesh`` and ``mesh_dim_names`` arguments + are required. The order of the process groups passed in determines the topology of + the mesh. For example, the first process group will be the 0th dimension of the DeviceMesh. + The `mesh` tensor passed in must have the same number of dimensions as the number of process + groups passed in, and the order of the dimensions in the `mesh` tensor must match the order + in the process groups passed in. + + Args: + group (ProcessGroup or list[ProcessGroup]): the existing ProcessGroup + or a list of existing ProcessGroups. + device_type (str): The device type of the mesh. Currently supports: "cpu", + "cuda/cuda-like". Passing in a device type with a GPU index, such as "cuda:0", + is not allowed. + mesh (torch.Tensor or ArrayLike, optional): A multi-dimensional array or an + integer tensor describing the layout of devices, where the IDs are global IDs + of the default process group. Default is None. + mesh_dim_names (tuple[str, ...], optional): A tuple of mesh dimension names to assign + to each dimension of the multi-dimensional array describing the layout of devices. + Its length must match the length of `mesh_shape`. Each string in `mesh_dim_names` + must be unique. Default is None. + + Returns: + DeviceMesh: A :class:`DeviceMesh` object representing the device layout. + """ + + # 1D scenario + if isinstance(group, ProcessGroup): + group_ranks = get_process_group_ranks(group) + if ( + isinstance(mesh, torch.Tensor) and mesh.tolist() != group_ranks + ) or ( + mesh is not None + and not isinstance(mesh, torch.Tensor) + and mesh != group_ranks + ): + raise ValueError( + f"Invalid mesh {str(mesh)} for ProcessGroup with ranks {group_ranks}" + ) + mesh = torch.tensor(group_ranks, device="cpu", dtype=torch.int) + device_mesh = DeviceMesh( + device_type, + mesh, + mesh_dim_names=mesh_dim_names, + _init_backend=False, + ) + device_mesh._dim_group_names = [group.group_name] + return device_mesh + + # nD scenario + groups = list(group) + if len(groups) == 0: + raise ValueError("Expects at least one ProcessGroup to be passed") + if mesh is None: + raise ValueError("Must pass mesh if passing multiple ProcessGroups") + if mesh_dim_names is None: + raise ValueError( + "Must pass mesh_dim_names if passing multiple ProcessGroups" + ) + # When init a DeviceMesh with multiple ProcessGroups directly, we need to make sure + # the mesh tensor is contiguous. Otherwise, the layout we inferred from the mesh tensor + # will have larger span than the actual tensor. This is just internal implementation detail + # and does not affect user facing behavior. + mesh = ( + mesh.detach().to(dtype=torch.int, device="cpu") + if isinstance(mesh, torch.Tensor) + else torch.tensor(mesh, device="cpu", dtype=torch.int) + ) + if mesh.ndim != len(groups): + raise ValueError( + "Expects mesh with ndim equal to number of ProcessGroups but got " + f"mesh {mesh.tolist()} and {len(groups)} ProcessGroups" + ) + device_mesh = DeviceMesh( + device_type, mesh, mesh_dim_names=mesh_dim_names, _init_backend=False + ) + device_mesh._dim_group_names = [group.group_name for group in groups] + return device_mesh + + def size(self, mesh_dim: int | None = None) -> int: + if mesh_dim is not None: + return self._layout[mesh_dim].numel() + return self._layout.numel() + + @property + def ndim(self) -> int: + return len(self._layout) + + @property + def shape(self) -> tuple[int, ...]: + return self._layout.top_level_sizes + + def get_rank(self) -> int: + """ + Returns the current global rank. + """ + return get_rank() + + def get_local_rank(self, mesh_dim: int | str | None = None) -> int: + """ + Returns the local rank of the given mesh_dim of the DeviceMesh. + + Args: + mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index + of the mesh dimension. Default is None. + + Returns: + An integer denotes the local rank. + + The following program runs on each process/rank in an SPMD manner. In this example, we have 2 + hosts with 4 GPUs each. + Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 0, 1, 2, 3 would return 0. + Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 4, 5, 6, 7 would return 1. + Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 0, 4 would return 0. + Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 1, 5 would return 1. + Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 2, 6 would return 2. + Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 3, 7 would return 3. + + Example:: + + >>> # xdoctest: +SKIP("no rank") + >>> from torch.distributed.device_mesh import DeviceMesh + >>> + >>> # Initialize device mesh as (2, 4) to represent the topology + >>> # of cross-host(dim 0), and within-host (dim 1). + >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) + """ + if self.ndim > 1 and mesh_dim is None: + raise RuntimeError( + f"Found the DeviceMesh have {len(self._layout)} dimensions", + "Optional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.", + ) + elif mesh_dim is None: + mesh_dim = 0 + + mesh_dim_group = not_none(self.get_group(mesh_dim)) + if not isinstance(mesh_dim_group, ProcessGroup): + raise AssertionError( + "We expect ProcessGroup before calling `get_rank`!" + ) + return not_none(get_rank(mesh_dim_group)) + + def get_coordinate(self) -> list[int] | None: + """ + Return the relative indices of this rank relative to all + dimensions of the mesh. If this rank is not part of the mesh, return None. + """ + return self._coordinate_on_dim if self._coordinate_on_dim else None + + def _flatten( + self, + mesh_dim_name: str | None = None, + backend_override: None + | str + | C10dBackend.Options + | tuple[str, C10dBackend.Options] = None, + ) -> "DeviceMesh": + """ + Returns a 1D DeviceMesh by flattening the current DeviceMesh. + + If no mesh_dim_name is provided, the default is a string concatenating the mesh_dim_names of the + given submesh with each mesh_dim_name separated by "_". For example, if we have a 3D mesh + DeviceMesh([[[0, 1], [2, 3]], [[4, 5], [6, 7]]], mesh_dim_names=("dp", "cp", "tp")), calling + mesh_3d["dp", "cp"]._flatten() will create a 1D submesh DeviceMesh([0, 2, 4, 6], mesh_dim_names=("dp_cp",)) + on rank 0, 2, 4, 6 and a 1D submesh DeviceMesh([1, 3, 5, 7], mesh_dim_names=("dp_cp",)) on rank 1, 3, 5, 7. + + After the flattened dimension is created, to access the flattened dimension in mesh_3d, one can use the + existing slicing method to obtain the flattened mesh through calling mesh_3d["dp_cp"]. + """ + if not self._mesh_dim_names: + raise RuntimeError( + "Cannot flatten a DeviceMesh without mesh_dim_names!" + ) + + if backend_override is not None: + (backend_override_tuple,) = _normalize_backend_override( + {0: backend_override}, 1 + ) + else: + backend_override_tuple = (None, None) + + return self._create_flatten_mesh(mesh_dim_name, backend_override_tuple) + + def _create_unflatten_mesh( + self, + dim: int, + mesh_sizes: tuple[int, ...], + mesh_dim_names: tuple[str, ...], + backend_override: tuple[ + tuple[str | None, C10dBackend.Options | None], ... + ] = ((None, None),), + ) -> "DeviceMesh": + inner_layout = _MeshLayout(tuple(mesh_sizes), suffix_product(mesh_sizes)) + + if inner_layout.numel() != self._layout[dim].numel(): + raise ValueError( + f"The product of {mesh_sizes=} is {inner_layout.numel()}, " + f"but the original dimension at dim={dim} has size {self._layout[dim].numel()}. " + f"These must be equal for unflatten to work correctly." + ) + + partial_layout = self._layout[dim].composition(inner_layout) + unflattened_layout = self._layout.splice(dim, dim + 1, partial_layout) + unflattened_mesh_dim_names = list(not_none(self.mesh_dim_names)) + unflattened_mesh_dim_names[dim : dim + 1] = list(mesh_dim_names) + + root_mesh = self._get_root_mesh() + res_mesh = DeviceMesh( + self.device_type, + _layout=unflattened_layout, + _rank_map=root_mesh._rank_map, + mesh_dim_names=tuple(unflattened_mesh_dim_names), + _root_mesh=root_mesh, + _init_backend=False, + ) + + # If original mesh has initiated its backend, we need to initialize the backend + # of unflatten dims as well. + # TODO: To make backend init more efficient with cute layout representation and support + # per dim backend init. + if hasattr(self, "_dim_group_names"): + dim_group_names = self._dim_group_names.copy() + dim_group_names[dim : dim + 1] = self._init_process_groups( + partial_layout, + root_mesh._rank_map, + mesh_dim_names, + backend_override, + ) + res_mesh._dim_group_names = dim_group_names + + return res_mesh + + def _unflatten( + self, + dim: int | str, + mesh_sizes: tuple[int, ...], + mesh_dim_names: tuple[str, ...], + backend_override: dict[ + str, str | C10dBackend.Options | tuple[str, C10dBackend.Options] + ] + | None = None, + ) -> "DeviceMesh": + """ + Returns a DeviceMesh by unflatten the current DeviceMesh. + + This api can be used to unflatten a N-D DeviceMesh into N-1+len(mesh_sizes)-D meshes or submeshes. + The dim is the dimension to be unflattened which can be either a string or an integer. + + The mesh_sizes is a tuple which specifies the shape of the mesh unflatten into for the given dim. + The mesh_dim_names is a list of strings which specifies the names of the dimensions of the mesh unflatten into. + Its length must match the length of mesh_sizes. + + For example, if we have a 1D mesh DeviceMesh([0, 1, 2, 3, 4, 5, 6, 7], mesh_dim_names=("world")), + calling mesh_1d._unflatten(0, (2, 2, 4), ["dp", "pp", "tp"]) will create a 3D mesh + DeviceMesh([[[0, 1], [2, 3]], [[4, 5], [6, 7]]], mesh_dim_names=("dp", "cp", "tp")). + + Note that after calling the unflatten, there is no access to the unflattened dimension in mesh_1d, one can only + use the newly unflattened mesh to slice out the unflattened mesh dims. + """ + if isinstance(dim, int) and dim >= self.ndim: + raise ValueError( + f"dim {dim} specified in `_unflatten` is out of range {self.ndim}" + ) + elif isinstance(dim, str) and dim in not_none(self.mesh_dim_names): + raise ValueError( + f"dim {dim} specified in `_unflatten` is not in {self.mesh_dim_names}" + ) + + if len(mesh_sizes) != len(mesh_dim_names): + raise RuntimeError( + "mesh_dim_names must have same length as mesh_sizes in _unflatten!" + ) + + if isinstance(dim, str): + dim = not_none(self.mesh_dim_names).index(dim) + + if backend_override is not None: + backend_override_tuple = tuple( + _normalize_backend_override( + backend_override, # type: ignore[arg-type] + len(mesh_sizes), + mesh_dim_names, + ) + ) + else: + backend_override_tuple = ((None, None),) * len(mesh_dim_names) + + return self._create_unflatten_mesh( + dim, + mesh_sizes, + mesh_dim_names, + backend_override_tuple, + ) + + @staticmethod + def _concatenate(device_mesh_list: list["DeviceMesh"]) -> "DeviceMesh": + concat_dim_names: list[str] = [] + concat_sizes: list[IntTuple] = [] + concat_strides: list[IntTuple] = [] + concat_dim_group_name: list[GroupName] = [] + flatten_rank_map = device_mesh_list[0]._flatten_rank_map + for dm in device_mesh_list: + for i in range(len(dm._layout)): + concat_sizes.append(dm._layout[i].sizes) + concat_strides.append(dm._layout[i].strides) + concat_dim_names.extend(not_none(dm.mesh_dim_names)) + concat_dim_group_name.extend(not_none(dm._dim_group_names)) + # Concatenate device mesh having different root mesh tensors are meaningless + # because the concatenated indices should be indexed by the same root mesh tensor. + if dm._flatten_rank_map != flatten_rank_map: + raise RuntimeError( + "Cannot concatenate DeviceMeshes derived from different device meshs" + ) + concat_mesh_layout = _MeshLayout(tuple(concat_sizes), tuple(concat_strides)) + if not concat_mesh_layout.check_non_overlap(): + raise RuntimeError( + f"Cannot concatenate overlapping meshes: {device_mesh_list}" + ) + res_mesh = DeviceMesh( + device_mesh_list[0].device_type, + _layout=concat_mesh_layout, + _rank_map=device_mesh_list[0]._rank_map, + mesh_dim_names=tuple(concat_dim_names), + _root_mesh=device_mesh_list[0]._get_root_mesh(), + _init_backend=False, + ) + res_mesh._dim_group_names = concat_dim_group_name + return res_mesh + + def _normalize_backend_override( + backend_override: dict[ + int | str, + str | C10dBackend.Options | tuple[str, C10dBackend.Options], + ], + ndim: int, + mesh_dim_names: tuple[str, ...] | None = None, + ) -> Iterator[BackendConfig]: + if mesh_dim_names is None: + mesh_dim_names = () + for dim_idx, dim_name in zip_longest(range(ndim), mesh_dim_names): + if dim_name is not None and dim_name in backend_override: + if dim_idx in backend_override: + raise RuntimeError( + f"Found redundant dim index {dim_idx} and " + f"name {dim_name} in backend_override" + ) + val = backend_override.pop(dim_name) + elif dim_idx in backend_override: + val = backend_override.pop(dim_idx) + else: + yield (None, None) + continue + + if isinstance(val, str): + yield (val, None) + elif isinstance(val, C10dBackend.Options): + yield (None, val) + else: + yield val + + if backend_override: + raise RuntimeError( + f"Found invalid keys in backend_override: got {list(backend_override.keys())}, " + f"expected integers in range [0, {ndim}) or one of {mesh_dim_names}" + ) + + def init_device_mesh( + device_type: str, + mesh_shape: tuple[int, ...], + *, + mesh_dim_names: tuple[str, ...] | None = None, + backend_override: dict[ + int | str, str | C10dBackend.Options | tuple[str, C10dBackend.Options] + ] + | None = None, + ) -> DeviceMesh: + """ + Initializes a `DeviceMesh` based on `device_type`, `mesh_shape`, and `mesh_dim_names` parameters. + + This creates a DeviceMesh with an n-dimensional array layout, where `n` is the length of `mesh_shape`. + If `mesh_dim_names` is provided, each dimension is labeled as `mesh_dim_names[i]`. + + .. note:: + `init_device_mesh` follows SPMD programming model, meaning the same PyTorch Python program + runs on all processes/ranks in the cluster. Ensure `mesh_shape` (the dimensions of the nD array + describing device layout) is identical across all ranks. Inconsistent `mesh_shape` may lead to hanging. + + .. note:: + If no process group is found, init_device_mesh will initialize distributed process group/groups + required for distributed communications behind the scene. + + Args: + device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like", "xpu". + Passing in a device type with a GPU index, such as "cuda:0", is not allowed. + mesh_shape (Tuple[int]): A tuple defining the dimensions of the multi-dimensional array + describing the layout of devices. + mesh_dim_names (tuple[str, ...], optional): A tuple of mesh dimension names to assign to each dimension + of the multi-dimensional array describing the layout of devices. Its length must match the length + of `mesh_shape`. Each string in `mesh_dim_names` must be unique. + backend_override (Dict[int | str, tuple[str, Options] | str | Options], optional): Overrides for some or all of + the ProcessGroups that will be created for each mesh dimension. Each key can be either the index of a + dimension or its name (if mesh_dim_names is provided). Each value can be a tuple containing the name + of the backend and its options, or just one of these two components (in which case the other will be + set to its default value). + + Returns: + DeviceMesh: A :class:`DeviceMesh` object representing the device layout. + + Example:: + + >>> # xdoctest: +SKIP("no rank") + >>> from torch.distributed.device_mesh import init_device_mesh + >>> + >>> mesh_1d = init_device_mesh("cuda", mesh_shape=(8,)) + >>> mesh_2d = init_device_mesh("cuda", mesh_shape=(2, 8), mesh_dim_names=("dp", "tp")) + + """ + if mesh_dim_names is not None: + if len(set(mesh_dim_names)) != len(mesh_dim_names): + raise RuntimeError( + "Each mesh_dim_name must be unique.", + f"Found repeated mesh_dim_name in mesh_dim_names {mesh_dim_names}", + ) + + if len(mesh_shape) != len(mesh_dim_names): + raise RuntimeError( + "mesh_shape and mesh_dim_names should have same length!", + f"Found len(mesh_dim_names): {len(mesh_dim_names)} and len(mesh_shape):{len(mesh_shape)}.", + ) + + if backend_override is not None: + backend_override_tuple = tuple( + _normalize_backend_override( + backend_override, len(mesh_shape), mesh_dim_names + ) + ) + else: + backend_override_tuple = None + + # assume valid device types are all letters + if device_type and not device_type.isalpha(): + raise RuntimeError( + f"Device type with index is not supported but got {device_type}. ", + "If you maintained a 'torch.device' object, it's recommended to pass in 'device.type'.", + ) + + layout = _MeshLayout(tuple(mesh_shape), suffix_product(tuple(mesh_shape))) + # Always initialize the (identity) rank map on CPU, regardless of what the + # external device type has been set to be (e.g. meta) + with torch.device("cpu"): + rank_map = torch.arange(layout.numel(), dtype=torch.int) + device_mesh = DeviceMesh( + device_type=device_type, + _layout=layout, + _rank_map=rank_map, + mesh_dim_names=mesh_dim_names, + backend_override=backend_override_tuple, + ) + + return device_mesh diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/distributed_c10d.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/distributed_c10d.py new file mode 100644 index 0000000000000000000000000000000000000000..f149177fc61a8eccd995aed9fe6354891e9c32e7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/distributed_c10d.py @@ -0,0 +1,6286 @@ +# mypy: allow-untyped-defs +"""Distributed Collective Communication (c10d).""" + +import collections.abc +import contextlib +import copy +import ctypes +import hashlib +import io +import itertools +import logging +import os +import pickle +import sys +import time +import warnings +from collections import namedtuple +from collections.abc import Callable +from datetime import timedelta +from typing import Any, NewType, TYPE_CHECKING +from typing_extensions import deprecated + +import torch +from torch._C import _DistStoreError as DistStoreError +from torch._C._distributed_c10d import ( + _DistributedBackendOptions, + _register_process_group, + _resolve_process_group, + _unregister_all_process_groups, + _unregister_process_group, + AllgatherOptions, + AllreduceCoalescedOptions, + AllreduceOptions, + AllToAllOptions, + BarrierOptions, + BroadcastOptions, + DebugLevel, + GatherOptions, + get_debug_level, + PrefixStore, + ProcessGroup, + ReduceOp, + ReduceOptions, + ReduceScatterOptions, + ScatterOptions, + Store, + Work, +) +from torch._utils_internal import set_pytorch_distributed_envs_from_justknobs +from torch.monitor import _WaitCounter +from torch.overrides import handle_torch_function, has_torch_function +from torch.utils._typing_utils import not_none + +from .c10d_logger import _exception_logger, _time_logger +from .constants import default_pg_nccl_timeout, default_pg_timeout +from .rendezvous import register_rendezvous_handler, rendezvous # noqa: F401 + + +__all__ = [ + "Backend", + "BackendConfig", + "GroupMember", + "P2POp", + "all_gather", + "all_gather_coalesced", + "all_gather_object", + "all_reduce", + "all_reduce_coalesced", + "all_to_all", + "all_to_all_single", + "barrier", + "batch_isend_irecv", + "broadcast", + "send_object_list", + "recv_object_list", + "broadcast_object_list", + "destroy_process_group", + "gather", + "gather_object", + "get_backend_config", + "get_backend", + "get_default_backend_for_device", + "get_rank", + "get_world_size", + "get_pg_count", + "group", + "init_process_group", + "irecv", + "is_gloo_available", + "is_initialized", + "is_mpi_available", + "is_backend_available", + "is_nccl_available", + "is_torchelastic_launched", + "is_ucc_available", + "is_xccl_available", + "isend", + "monitored_barrier", + "new_group", + "new_subgroups", + "new_subgroups_by_enumeration", + "recv", + "reduce", + "reduce_scatter", + "scatter", + "scatter_object_list", + "send", + "supports_complex", + "AllreduceCoalescedOptions", + "AllreduceOptions", + "AllToAllOptions", + "BarrierOptions", + "BroadcastOptions", + "GatherOptions", + "GroupName", + "PrefixStore", + "ProcessGroup", + "ReduceOp", + "ReduceOptions", + "ReduceScatterOptions", + "ScatterOptions", + "Store", + "DebugLevel", + "get_debug_level", + "Work", + "default_pg_timeout", + "get_group_rank", + "get_global_rank", + "get_process_group_ranks", + "reduce_op", + "all_gather_into_tensor", + "reduce_scatter_tensor", + "get_node_local_rank", + "split_group", + "shrink_group", +] + +_MPI_AVAILABLE = True +_NCCL_AVAILABLE = True +_GLOO_AVAILABLE = True +_UCC_AVAILABLE = True +_XCCL_AVAILABLE = True + +_pickler = pickle.Pickler +_unpickler = pickle.Unpickler + +GroupName = NewType("GroupName", str) + + +# Change __module__ of all imported types from torch._C._distributed_c10d that are public +def _export_c_types() -> None: + _public_types_to_change_module = [ + AllreduceCoalescedOptions, + AllreduceOptions, + AllToAllOptions, + BarrierOptions, + BroadcastOptions, + GatherOptions, + PrefixStore, + ProcessGroup, + ReduceOp, + ReduceOptions, + ReduceScatterOptions, + ScatterOptions, + Store, + DebugLevel, + get_debug_level, + Work, + ] + for type in _public_types_to_change_module: + type.__module__ = "torch.distributed.distributed_c10d" + + +_export_c_types() + +try: + from torch._C._distributed_c10d import ProcessGroupMPI + + ProcessGroupMPI.__module__ = "torch.distributed.distributed_c10d" + __all__ += ["ProcessGroupMPI"] +except ImportError: + _MPI_AVAILABLE = False + +try: + from torch._C._distributed_c10d import ProcessGroupNCCL + + ProcessGroupNCCL.__module__ = "torch.distributed.distributed_c10d" + __all__ += ["ProcessGroupNCCL"] +except ImportError: + _NCCL_AVAILABLE = False + +try: + from torch._C._distributed_c10d import _ProcessGroupWrapper, ProcessGroupGloo + + ProcessGroupGloo.__module__ = "torch.distributed.distributed_c10d" + __all__ += ["ProcessGroupGloo"] +except ImportError: + _GLOO_AVAILABLE = False + +try: + from torch._C._distributed_c10d import ProcessGroupUCC + + ProcessGroupUCC.__module__ = "torch.distributed.distributed_c10d" + __all__ += ["ProcessGroupUCC"] +except ImportError: + _UCC_AVAILABLE = False + +try: + from torch._C._distributed_c10d import ProcessGroupXCCL + + ProcessGroupXCCL.__module__ = "torch.distributed.distributed_c10d" + __all__ += ["ProcessGroupXCCL"] +except ImportError: + _XCCL_AVAILABLE = False + +logger = logging.getLogger(__name__) + +PG_WRAPPER_STORE_PREFIX = "pg_wrapper" + + +# Some reduce ops are not supported by complex numbers and will result in an error. +# We currently provide complex support to the distributed API by viewing +# complex tensors as real (torch.view_as_real), meaning that calling +# these unsupported ops will return garbage values rather than error out. +# (e.g. max(2+3i, 3+2i) = 3+3i) +# We'd like calls to unsupported ops to error out accordingly, +# rather than returning garbage values. +def supports_complex(reduceOp: ReduceOp) -> bool: + """Return true if reduce ops is supported. False otherwise.""" + denyList = [ + ReduceOp.MAX, + ReduceOp.MIN, + ReduceOp.PRODUCT, + ReduceOp.BAND, + ReduceOp.BOR, + ReduceOp.BXOR, + ] + return reduceOp not in denyList + + +# TODO refactor into enum/strenum +class Backend(str): # noqa: SLOT000 + """ + An enum-like class for backends. + + Available backends: GLOO, NCCL, UCC, MPI, XCCL, and other registered backends. + + The values of this class are lowercase strings, e.g., ``"gloo"``. They can + be accessed as attributes, e.g., ``Backend.NCCL``. + + This class can be directly called to parse the string, e.g., + ``Backend(backend_str)`` will check if ``backend_str`` is valid, and + return the parsed lowercase string if so. It also accepts uppercase strings, + e.g., ``Backend("GLOO")`` returns ``"gloo"``. + + .. note:: The entry ``Backend.UNDEFINED`` is present but only used as + initial value of some fields. Users should neither use it directly + nor assume its existence. + """ + + UNDEFINED = "undefined" + GLOO = "gloo" + NCCL = "nccl" + UCC = "ucc" + MPI = "mpi" + XCCL = "xccl" + + _BackendPlugin = namedtuple("_BackendPlugin", ["creator_fn", "extended_api"]) + + _plugins: dict[str, _BackendPlugin] = {} + + backend_list = [UNDEFINED, GLOO, NCCL, XCCL, UCC, MPI] + + # 3rd-party devices can register the default backend support here + default_device_backend_map: dict[str, str] = { + "cpu": GLOO, + "cuda": NCCL, + "xpu": XCCL, + "mps": GLOO, + } + + backend_capability: dict[str, list[str]] = { + GLOO: ["cpu", "cuda"], + NCCL: ["cuda"], + XCCL: ["xpu"], + UCC: ["cpu", "cuda"], + MPI: ["cpu", "cuda"], + } + + backend_type_map: dict[str, ProcessGroup.BackendType] = { + UNDEFINED: ProcessGroup.BackendType.UNDEFINED, + GLOO: ProcessGroup.BackendType.GLOO, + NCCL: ProcessGroup.BackendType.NCCL, + XCCL: ProcessGroup.BackendType.XCCL, + UCC: ProcessGroup.BackendType.UCC, + MPI: ProcessGroup.BackendType.MPI, + } + + def __new__(cls, name: str): + """Create and return a new instance of the class.""" + if not isinstance(name, str): + raise ValueError("Backend constructor parameter must be string-ish") + value = getattr(Backend, name.upper(), Backend.UNDEFINED) + + if value == Backend.UNDEFINED: + value = name.lower() + return value + + @classmethod + def register_backend( + cls, + name, + func, + extended_api: bool = False, + devices: str | list[str] | None = None, + ) -> None: + """ + Register a new backend with the given name and instantiating function. + + This class method is used by 3rd party ``ProcessGroup`` extension to + register new backends. + + Args: + name (str): Backend name of the ``ProcessGroup`` extension. It + should match the one in ``init_process_group()``. + func (function): Function handler that instantiates the backend. + The function should be implemented in the backend + extension and takes four arguments, including + ``store``, ``rank``, ``world_size``, and ``timeout``. + extended_api (bool, optional): Whether the backend supports extended argument structure. + Default: ``False``. If set to ``True``, the backend + will get an instance of ``c10d::DistributedBackendOptions``, and + a process group options object as defined by the backend implementation. + device (str or list of str, optional): device type this backend + supports, e.g. "cpu", "cuda", etc. If `None`, + assuming both "cpu" and "cuda" + + .. note:: This support of 3rd party backend is experimental and subject to change. + + """ + # This takes care of CUSTOM Out-of-tree backend types, update in backend_list indicates availability + if not hasattr(Backend, name.upper()): + setattr(Backend, name.upper(), name.lower()) + if name.lower() not in Backend.backend_list: + Backend.backend_list.append(name.lower()) + + if devices is not None: + for device in devices: + if device not in Backend.default_device_backend_map: + Backend.default_device_backend_map[device] = name.lower() + Backend.backend_type_map[name.lower()] = ProcessGroup.BackendType.CUSTOM + + # Update device capability matrix in Backend class + if devices is None: + # This is more of a backward support for groups like `threaded`: + # assume default devices "cpu" and "cuda", but warn + warnings.warn( + f"Device capability of {name} unspecified, assuming `cpu` and " + "`cuda` or `xpu`. Please specify it via the `devices` argument of " + "`register_backend`.", + stacklevel=2, + ) + Backend.backend_capability[name.lower()] = ( + ["cpu", "cuda", "xpu"] if torch.xpu.is_available() else ["cpu", "cuda"] + ) + elif isinstance(devices, str): + # Single device string specified. Simply convert to list. + Backend.backend_capability[name.lower()] = [devices] + else: + Backend.backend_capability[name.lower()] = devices + + Backend._plugins[name.upper()] = Backend._BackendPlugin(func, extended_api) + + +class BackendConfig: + """Backend configuration class.""" + + def __init__(self, backend: Backend): + """Init.""" + self.device_backend_map: dict[str, Backend] = {} + # pyrefly: ignore [bad-assignment] + backend = str(backend) + + if backend == Backend.UNDEFINED: + # Detect the accelerator on the machine. If no accelerator is + # available, it returns CPU. + device_type = torch._C._get_accelerator().type + try: + backend_str = Backend.default_device_backend_map[device_type] + self.device_backend_map[device_type] = Backend(backend_str) + except KeyError: + raise ValueError( + f"We detected accelerator {device_type} on your machine. " + f"But we don't know which communication backend to use for this accelerator. " + f"Please specify the `backend` argument in the `init_process_group` call." + ) from None + elif backend.lower() in Backend.backend_list: + # Cases for when backend is a single string (without device types) + # e.g. "nccl", "gloo", "ucc", "mpi" + supported_devices = Backend.backend_capability[backend.lower()] + backend_val = Backend(backend) + + self.device_backend_map = dict.fromkeys(supported_devices, backend_val) + elif ":" in backend.lower(): + # Backend specified in "device:backend" format + # make sure the backend string is in the correct format + # "{device_type1}:{backend1},{device_type2}:{backend2}" + # e.g. "cpu:gloo,cuda:nccl" + backend_str_error_message = f"""The custom backend string argument is invalid: {backend}. + Custom backend string is an experimental feature where the backend string must be in the format: + ":,:...". e.g. 'cpu:gloo,cuda:nccl'""" + + # parse the backend string and populate the device_backend_map + for device_backend_pair_str in backend.lower().split(","): + device_backend_pair = device_backend_pair_str.split(":") + if len(device_backend_pair) != 2: + raise ValueError( + f"Invalid device:backend pairing: \ + {device_backend_pair_str}. {backend_str_error_message}" + ) + # pyrefly: ignore [bad-assignment] + device, backend = device_backend_pair + if device in self.device_backend_map: + raise ValueError( + f"Duplicate device type {device} \ + in backend string: {backend}. {backend_str_error_message}" + ) + self.device_backend_map[device] = Backend(backend) + else: + # User specified a single backend name whose device capability is + # unknown, assuming it can support the default devices of PyTorch + # (cpu and cuda) + warnings.warn( + f"Device capability of {backend} unknown, assuming `cpu` and " + "`cuda`. You can specify it in `device:backend` format in " + "`init_process_group` call.", + stacklevel=2, + ) + backend_val = Backend(backend) + self.device_backend_map = { + "cpu": backend_val, + "cuda": backend_val, + "xpu": backend_val, + } + + logger.info("Using backend config: %s", self.device_backend_map) + + def __repr__(self): + """Return all the device:backend pairs separated by commas.""" + return ",".join( + f"{device}:{backend}" for device, backend in self.device_backend_map.items() + ) + + def get_device_backend_map(self) -> dict[str, Backend]: + """Return backend map of the device.""" + return self.device_backend_map + + +class _reduce_op: + r""" + Deprecated enum-like class. + + For reduction operations: ``SUM``, ``PRODUCT``, ``MIN``, and ``MAX``. + + :class:`~torch.distributed.ReduceOp` is recommended to use instead. + """ + + def __init__(self) -> None: + # __members__ is a dict storing key-value pairs for enum classes + for k, v in ReduceOp.RedOpType.__members__.items(): + setattr(self, k, v) + self.__members__ = ReduceOp.RedOpType.__members__ + + @deprecated( + "`torch.distributed.reduce_op` is deprecated, " + "please use `torch.distributed.ReduceOp` instead", + category=FutureWarning, + ) + def __getattribute__(self, key): + return object.__getattribute__(self, key) + + +reduce_op = _reduce_op() + + +class P2POp: + """ + A class to build point-to-point operations for ``batch_isend_irecv``. + + This class builds the type of P2P operation, communication buffer, peer rank, + Process Group, and tag. Instances of this class will be passed to + ``batch_isend_irecv`` for point-to-point communications. + + Args: + op (Callable): A function to send data to or receive data from a peer process. + The type of ``op`` is either ``torch.distributed.isend`` or + ``torch.distributed.irecv``. + tensor (Tensor): Tensor to send or receive. + peer (int, optional): Destination or source rank. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + tag (int, optional): Tag to match send with recv. + group_peer (int, optional): Destination or source rank. + """ + + def __init__( + self, + op: Callable, + tensor: torch.Tensor, + peer: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_peer: int | None = None, + ): + """Init.""" + self.op = op + self.tensor = tensor + self.group = _group_or_default_group(group) + self.peer = _canonicalize_group_rank( + self.group, peer, group_peer, return_global=True + ) + self.tag = tag + self.group_peer = _canonicalize_group_rank(self.group, peer, group_peer) + + def __new__( + cls, + op: Callable, + tensor: torch.Tensor, + peer: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_peer: int | None = None, + ): + """Create and return a new instance of the class.""" + _check_op(op) + _check_single_tensor(tensor, "tensor") + + return object.__new__(cls) + + def __repr__(self): + my_group_rank = get_rank(self.group) + op_name = self.op.__name__ + group_name = self.group.group_name if self.group else "default_pg" + if "send" in op_name: + s = my_group_rank + d = self.group_peer + elif "recv" in op_name: + s = self.group_peer + d = my_group_rank + else: + return super().__repr__() + + return f"P2POp({op_name} pg={group_name}, group_src={s}, group_dst={d}, {self.tensor.shape}, {self.tensor.dtype})" + + +class _CollOp: + """ + A class to capture collective operations. + + Args: + op (Callable): A collective function, e.g. ``torch.distributed.all_reduce``. + tensor (Tensor): Tensor to operate on. + dst_tensor (Tensor, optional): Provided when source and destination tensors are not the same. + redop (ReduceOp, optional): reduce operation. + root (int, optional): root of broadcast or reduce. + """ + + def __init__( + self, + op: Callable, + tensor: torch.Tensor, + dst_tensor: torch.Tensor | None = None, + redop: ReduceOp | None = None, + root: int | None = None, + ): + self.op = op + self.tensor = tensor + self.dst_tensor = dst_tensor + self.redop = redop + self.root = root + + +# DO NOT USE THESE FIELDS DIRECTLY. +# Use them through the _world object to make sure the _world override mechanism +_pg_map: dict[ProcessGroup, tuple[str, Store]] = {} +_pg_names: dict[ProcessGroup, GroupName] = {} +_pg_group_ranks: dict[ProcessGroup, dict[int, int]] = {} +# For a pg, it is a map from ProcessGroup to BackendConfig +_pg_backend_config: dict[ProcessGroup, str] = {} +_group_count = 0 +_tags_to_pg: dict[str, list[ProcessGroup]] = {} +_pg_to_tag: dict[ProcessGroup, str] = {} +_backend: str | None = None + + +class _World: + """ + Container class for c10d process group state. + + This is used during registration and lookup of PG state. + + .. warning:: This is an experimental API intended to expose the inner workings + of c10d and is subject to change.. + """ + + def __init__(self) -> None: + self._default_pg = None + self._pg_coalesce_state: dict[ProcessGroup, list[_CollOp]] = {} + + @property + def default_pg(self) -> ProcessGroup | None: + """ + Process group that includes all ranks of the cluster. + + This default ProcessGroup is used by c10d APIs when a ProcessGroup is needed + but None is provided. + """ + return self._default_pg + + @default_pg.setter + def default_pg(self, value) -> None: + self._default_pg = value + + @property + def pg_map(self) -> dict[ProcessGroup, tuple[str, Store]]: + """ + Provide Mapping from ProcessGroup to backend name and store. + + For NCCL and GLOO pg, it is a map from ProcessGroup to (Backend, Store) + For MPI pg, it is a map from ProcessGroup to (Backend, None) + + TODO don't expose the map, expose fine grained ops + """ + global _pg_map + return _pg_map + + @property + def pg_names(self) -> dict[ProcessGroup, GroupName]: + """ + Process group's names, map from ProcessGroup to str. + + TODO don't expose the map, expose fine grained ops + """ + global _pg_names + return _pg_names + + @property + def pg_group_ranks(self) -> dict[ProcessGroup, dict[int, int]]: + """ + Process group's global rank to local rank mapping. + + TODO don't expose the map, expose fine grained ops + """ + global _pg_group_ranks + return _pg_group_ranks + + @property + def pg_backend_config(self) -> dict[ProcessGroup, str]: + """ + Process group's backend config. + + TODO don't expose the map, expose fine grained ops + """ + global _pg_backend_config + return _pg_backend_config + + @property + def group_count(self) -> int: + """ + Process group count for default naming. + + TODO don't expose group_count, use something else instead + """ + global _group_count + return _group_count + + @group_count.setter + def group_count(self, value: int) -> None: + """Use to compute the name of ProcessGroups when using global synchronization.""" + global _group_count + _group_count = value + + @property + def tags_to_pg(self) -> dict[str, list[ProcessGroup]]: + global _tags_to_pg + return _tags_to_pg + + @property + def pg_to_tag(self) -> dict[ProcessGroup, str]: + global _pg_to_tag + return _pg_to_tag + + @property + def pg_coalesce_state(self) -> dict[ProcessGroup, list[_CollOp]]: + return self._pg_coalesce_state + + @property + def pg_config_info(self) -> list[dict[str, Any]]: + """ + Return a list of dict with process groups and backends. + + Along with their unique IDs and configurations (types and ranks). + """ + config_info: list[dict[str, Any]] = [] + default_pg_size = _get_group_size(None) + for pg in self.pg_map: + ranks = self.pg_group_ranks[pg] + config_info.append( + { + "pg_name": self.pg_names[pg], + "pg_desc": pg.group_desc, + "backend_config": self.pg_backend_config[pg], + "ranks": ( + list(ranks.keys()) if len(ranks) != default_pg_size else [] + ), # 'ranks' is an empty list when all ranks are involved in a pg + "group_size": len(ranks), + "group_count": self.group_count, + } + ) + return config_info + + +_world = _World() +"""Holds the singleton instance of ``_World`` used by c10. Experimental extension point to override it""" + + +class _WorldMeta(type): + """ + Meta class of ``group`` and ``GroupMember``. + + Allows them to have the class property ``WORLD``. + """ + + # Points to the default PG once initialized. + @property + def WORLD(cls) -> ProcessGroup | None: + return _world.default_pg + + @WORLD.setter + def WORLD(cls, pg: ProcessGroup | None): + _world.default_pg = pg + + +class group(metaclass=_WorldMeta): + """Group class. Placeholder.""" + + +class GroupMember(metaclass=_WorldMeta): + """Group member class.""" + + NON_GROUP_MEMBER = -100 + + +def _get_default_timeout(backend: Backend) -> timedelta: + # see note on nccl vs other backend timeout (constants.py) + if backend == Backend.NCCL: + if not isinstance(default_pg_nccl_timeout, timedelta): + # TODO moco benchmark on CPU initializes pgnccl backend today, triggered this assert in CI before it was + # changed to be a warning. We should fix the moco model. + warnings.warn( + "Attempted to get default timeout for nccl backend, but NCCL support is not compiled", + stacklevel=2, + ) + return default_pg_timeout + return default_pg_nccl_timeout + else: + return default_pg_timeout + + +def _check_valid_timeout(timeout: Any) -> None: + if not isinstance(timeout, timedelta): + raise TypeError( + f"Expected timeout argument to be of type datetime.timedelta, got {timeout}" + ) + + +# Default process group state +_default_pg_init_method: str | None = None + +STORE_BASED_BARRIER_PREFIX = "store_based_barrier_key" + + +def _get_object_coll_device(group: ProcessGroup | None = None) -> str: + """ + .. note:: This is an internal helper and does not have backward + compatibility, please use with caution. + + Return the device type to use with ``group`` for object collectives or + barrier. + + There are selection rules: + 1. If user specifies exactly one backend in ``init_process_group`` call: + use that backend + 2. Else if user specifies multiple "device:backend" pairs in init_process_group: + If "cpu" is among those pairs, use "cpu" (because the object is in cpu memory); + Otherwise, use the first backend (sort of a random pick). + + Args: + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + + Returns: + str: The device type to use for object collective with ``group``. + + """ + group = group or _get_default_group() + + if not isinstance(group, ProcessGroup): + warnings.warn( + f"You are using a Backend {type(group)} as a ProcessGroup. " + "This usage is deprecated since PyTorch 2.0. Please use a public API " + "of PyTorch Distributed instead.", + stacklevel=2, + ) + # Provide backward compatibility to cases where `group` passed in is + # actually a Backend (like `ProcessGroupGloo`) rather than a + # `ProcessGroup` in PT 2.0 sense + if isinstance(group, ProcessGroupGloo): + # RPC uses Gloo for object collectives + return "cpu" + else: + raise ValueError(f"Expecting a ProcessGroup, but got a {type(group)}.") + + """ + ``group._device_types`` is a property pybind that returns the devices + ("cpu", "cuda", etc) supported by ``group``. Can be multiple if the + ``group`` supports multiple devices. + """ + devices = group._device_types + + if len(devices) == 1: + # User fixed exactly one backend in `init_process_group` + return devices[0].type + elif len(devices) == 0: + # No backend has been registered with this PG (maybe because no + # collective has been run?) We pick cpu as the default and hopefully + # this would lazily init Gloo or other available cpu backend. + return "cpu" + elif torch.device("cpu") in devices: + # There are multiple backends in this PG and cpu is among them. + # cpu is preferred as the object is in cpu memory. No need for device + # copy. + return "cpu" + else: + # No cpu in the backend list. Randomly pick the first backend + return devices[0].type + + +def _get_pg_default_device(group: ProcessGroup | None = None) -> torch.device: + """ + .. note:: This method will be deprecated, it only stays for + backward-compatiblity reason. Alternatives: + + - If you need to find a device for object collectives, please use + `_get_object_coll_device(group)`. + + - If you need to query the device types supported by group, please use + `_device_capability(group)`. + + Return the device type registered with ``group``. + + For example, if `init_process_group("nccl", ...)` was called, the returned + value would be `torch.device("cuda")`. + + Errors out if no device has been registered. + + Args: + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + + Returns: + torch.device: The device type registered with ``group``. + """ + + warnings.warn( + "`_get_pg_default_device` will be deprecated, it only stays for " + "backward-compatiblity reason. If you need to find a device for object " + "collectives, please use `_get_object_coll_device`. If you need to query " + "the device types supported by group, please use " + "`_device_capability(group)`. ", + stacklevel=2, + ) + group = group or _get_default_group() + + if not isinstance(group, ProcessGroup): + # Provide backward compatibility to cases where `group` passed in is + # actually a Backend (like `ProcessGroupGloo`) rather than a + # `ProcessGroup` in PT 2.0 sense + warnings.warn( + f"You are using a Backend {type(group)} as a ProcessGroup. " + "This usage is deprecated since PyTorch 2.0. Please use a public API " + "of PyTorch Distributed instead.", + FutureWarning, + stacklevel=3, + ) + # Most users create Gloo with private API for object collectives + return torch.device("cpu") + + """ + ``group._device_types`` is a property pybind that returns the devices + ("cpu", "cuda", etc) supported by ``group``. Can be multiple if the + ``group`` supports multiple devices. + """ + devices = group._device_types + + if len(devices) == 1: + # User fixed exactly one backend in `init_process_group` + return devices[0] + elif len(devices) == 0: + raise RuntimeError( + "Default device not found, because no backend has been registered " + "with this ProcessGroup." + ) + else: + # There are multiple backends in this PG. + if torch.device("cpu") in devices: + rv = torch.device("cpu") + else: + rv = devices[0] + warnings.warn( + "Multiple backends are registered with this ProcessGroup. We cannot " + f"determine which one is the default. Returning {rv}. " + "Please consider using other APIs.", + stacklevel=2, + ) + return rv + + +def _device_capability(group: ProcessGroup | None = None) -> list[str]: + """ + Return the device type(s) supported by ``group``. + + Args: + group (ProcessGroup, optional): The process group to query. If None, + the default process group will be used. + + Returns: + List[str]: A list of device types supported by ``group``. + """ + group = group or _get_default_group() + return [device.type for device in group._device_types] + + +@_time_logger +def _store_based_barrier( + rank, + store, + group_name: GroupName, + rendezvous_count, + timeout, + logging_interval=timedelta(seconds=10), +) -> None: + """ + Store based barrier for synchronizing processes. + + Barrier based on store which is used for synchronizing processes after + ``init_process_group`` or ``new_group``. Intended to be used only with + those two methods and is not a generic alternative to ``barrier()``. + """ + store_key = f"{STORE_BASED_BARRIER_PREFIX}:{group_name}" + store.add(store_key, 1) + logger.debug("Added key: %s to store for rank: %s", store_key, rank) + + # Now wait for all workers to check in with the store. + world_size = rendezvous_count + worker_count = store.add(store_key, 0) + + last_worker_key = f"{store_key}:last_worker" + if worker_count == world_size: + store.set(last_worker_key, "1") + + # adjust the timeout to be at least 10secs + 1sec per thousand ranks to reduce the odds of timeout + # this value was empirically found while scale testing. + logging_interval = max(logging_interval, timedelta(seconds=10 + world_size / 1000)) + + start = time.time() + while True: + try: + # This will throw an exception after the logging_interval in which we print out + # the status of the group or time out officially, throwing runtime error + store.wait([last_worker_key], logging_interval) + break + except RuntimeError as e: + worker_count = store.add(store_key, 0) + # Print status periodically to keep track. + logger.debug( # noqa: G200 + "Waiting in store based barrier to initialize process group for %s seconds" + "rank: %s, key: %s (world_size=%s, num_workers_joined=%s, timeout=%s error=%s)", + time.time() - start, + rank, + store_key, + world_size, + worker_count, + timeout, + e, + ) + + if timedelta(seconds=(time.time() - start)) > timeout: + raise DistStoreError( # noqa: B904 + "Timed out initializing process group in store based barrier on " + f"rank {rank}, for key: {store_key} (world_size={world_size}, " + f"num_workers_joined={worker_count}, timeout={timeout} error={e})" + ) + + logger.info( + "Rank %s: Completed store-based barrier for key:%s with %s nodes.", + rank, + store_key, + world_size, + ) + + +def _rank_not_in_group(group: ProcessGroup | None) -> bool: + """Check if the current process's rank is not in a given group.""" + if group is None: + return False + return group == GroupMember.NON_GROUP_MEMBER + + +def _warn_not_in_group(op_name) -> None: + global_rank = -1 if GroupMember.WORLD is None else GroupMember.WORLD.rank() + warnings.warn( + f"Running {op_name} on global rank {global_rank} which does not " + "belong to the given group.", + stacklevel=2, + ) + + +def get_group_rank(group: ProcessGroup, global_rank: int) -> int: + """ + Translate a global rank into a group rank. + + ``global_rank`` must be part of ``group`` otherwise this raises RuntimeError. + + Args: + group (ProcessGroup): ProcessGroup to find the relative rank. + global_rank (int): Global rank to query. + + Returns: + Group rank of ``global_rank`` relative to ``group`` + + N.B. calling this function on the default process group returns identity + """ + if group is GroupMember.WORLD: + return global_rank + if group not in _world.pg_group_ranks: + raise ValueError( + f"Group {group} is not registered, please create group with torch.distributed.new_group API" + ) + group_ranks = _world.pg_group_ranks[group] + if global_rank not in group_ranks: + raise ValueError(f"Global rank {global_rank} is not part of group {group}") + + return group_ranks[global_rank] + + +def get_global_rank(group: ProcessGroup, group_rank: int) -> int: + """ + Translate a group rank into a global rank. + + ``group_rank`` must be part of `group` otherwise this raises RuntimeError. + + Args: + group (ProcessGroup): ProcessGroup to find the global rank from. + group_rank (int): Group rank to query. + + Returns: + Global rank of ``group_rank`` relative to ``group`` + + N.B. calling this function on the default process group returns identity + """ + if group is GroupMember.WORLD: + return group_rank + if group not in _world.pg_group_ranks: + raise ValueError( + f"Group {group} is not registered, please create group with torch.distributed.new_group API" + ) + for rank, grp_rank in _world.pg_group_ranks[group].items(): + if grp_rank == group_rank: + return rank + raise ValueError(f"Group rank {group_rank} is not part of group {group}") + + +# TODO: remove this once the ecosystem moves away from it. +@deprecated( + "`torch.distributed.distributed_c10d._get_global_rank` is deprecated, " + "please use `torch.distributed.distributed_c10d.get_global_rank` instead", + category=FutureWarning, +) +def _get_global_rank(group, rank) -> int: + """Use get_global_rank as this method is deprecated.""" + return get_global_rank(group, rank) + + +def get_process_group_ranks(group: ProcessGroup | None) -> list[int]: + """ + Get all ranks associated with ``group``. + + Args: + group (Optional[ProcessGroup]): ProcessGroup to get all ranks from. + If None, the default process group will be used. + + Returns: + List of global ranks ordered by group rank. + """ + return list(_world.pg_group_ranks[group or _get_default_group()].keys()) + + +def _get_group_size(group: ProcessGroup | None) -> int: + """Get a given group's world size.""" + if group is GroupMember.WORLD or group is None: + default_pg = _get_default_group() + return default_pg.size() + return group.size() + + +def _get_group_size_by_name(group_name: GroupName) -> int: + group = _resolve_process_group(group_name) + return group.size() + + +def _resolve_group_name_by_ranks_and_tag(ranks: list[int], tag: str) -> GroupName: + # TODO(yifu): remove this function once ranks + tag is not a supported + # identifier for process group for functional collectives. + group = _find_pg_by_ranks_and_tag(tag, ranks) + if group is None: + raise ValueError("") + return group.group_name + + +def _check_single_tensor(param, param_name: str) -> None: + """Check that the parameter ``param_name`` is a single tensor.""" + if not isinstance(param, torch.Tensor): + raise TypeError( + f"""Invalid function argument. Expected parameter `{param_name}` of type torch.Tensor + but got {type(param)} instead.""" + ) + + +def _check_tensor_list(param, param_name: str) -> None: + """Check that the parameter ``param_name`` is a list of tensors.""" + if not isinstance(param, list): + raise TypeError( + f"""Invalid function argument. Expected parameter `{param_name}` of type List[torch.Tensor] + but got {type(param)} instead.""" + ) + elif not all(isinstance(p, torch.Tensor) for p in param): + raise TypeError( + f"""Invalid function argument. Expected parameter `{param_name}` of type List[torch.Tensor] + but got {type(param)} with elements of type {[type(p) for p in param]}.""" + ) + + +def _group_or_default_group(group: ProcessGroup | None = None) -> ProcessGroup: + if group is None or group is GroupMember.WORLD: + group = _get_default_group() + return group + + +def _canonicalize_group_rank( + group: ProcessGroup, + global_rank: int | None = None, + group_rank: int | None = None, + return_global: bool = False, +) -> int: + """ + Helper method to take _either_ a global rank or a group rank and produce a group rank. + + If 'return_global' is true, produce a global rank instead of a group rank. + """ + + if group_rank is not None: + if global_rank is not None: + raise ValueError("Can't specify both group_rank and global_rank") + if return_global: + return get_global_rank(group, group_rank) + else: + if global_rank is None: + raise ValueError("Must specify global_rank or group_rank") + if return_global: + return global_rank + group_rank = get_group_rank(group, global_rank) + return group_rank + + +def _check_not_self_rank(group: ProcessGroup, rank: int, rank_type: str): + if group.rank() == rank: + raise ValueError( + f"Invalid {rank_type} rank: {rank_type} rank should not be the same as " + "the rank of the current process." + ) + + +def _as_iterable(obj) -> collections.abc.Iterable: + return obj if isinstance(obj, list) else (obj,) + + +def _ensure_all_tensors_same_dtype(*tensors) -> None: + last_dtype = None + # pyrefly: ignore [bad-assignment] + for tensor in itertools.chain.from_iterable(map(_as_iterable, tensors)): + tensor_dtype = tensor.dtype + # Mixing complex and its element type is allowed + if tensor_dtype.is_complex: + tensor_dtype = ( + torch.float32 if tensor_dtype == torch.complex64 else torch.complex128 + ) + + if last_dtype is None: + last_dtype = tensor_dtype + else: + if last_dtype != tensor_dtype: + raise ValueError( + "Invalid usage of tensors with different dtypes" + f"Found {last_dtype} and {tensor.dtype}" + ) + + +def _check_op(op) -> None: + """Check that the ``op`` is either isend or irecv.""" + if op not in [isend, irecv]: + raise ValueError( + "Invalid ``op``. Expected ``op`` " + "to be of type ``torch.distributed.isend`` or " + "``torch.distributed.irecv``." + ) + + +def _check_p2p_op_list(p2p_op_list) -> None: + """ + Check that the ``p2p_op_list`` is a list of P2POp instances. + + Also, check that all ops use the same group. + """ + if not isinstance(p2p_op_list, list) or not all( + isinstance(p2p_op, P2POp) for p2p_op in p2p_op_list + ): + raise ValueError( + "Invalid ``p2p_op_list``. Each op is expected to " + "to be of type ``torch.distributed.P2POp``." + ) + + group = p2p_op_list[0].group + if not all(group == p2p_op.group for p2p_op in p2p_op_list): + raise ValueError("All ops need to use the same group.") + + +def is_mpi_available() -> bool: + """Check if the MPI backend is available.""" + return _MPI_AVAILABLE + + +def is_nccl_available() -> bool: + """Check if the NCCL backend is available.""" + return _NCCL_AVAILABLE + + +def is_gloo_available() -> bool: + """Check if the Gloo backend is available.""" + return _GLOO_AVAILABLE + + +def is_ucc_available() -> bool: + """Check if the UCC backend is available.""" + return _UCC_AVAILABLE + + +def is_xccl_available() -> bool: + """Check if the XCCL backend is available.""" + return _XCCL_AVAILABLE + + +def _check_single_backend_availability(backend_name: str) -> bool: + """ + Helper function to check if a single backend is available. + """ + available_func = getattr( + torch.distributed, f"is_{str(backend_name).lower()}_available", None + ) + if available_func: + return available_func() + return str(backend_name).lower() in Backend.backend_list + + +def is_backend_available(backend: str) -> bool: + """ + Check backend availability. + + Checks if the given backend is available and supports the built-in backends or + third-party backends through function ``Backend.register_backend``. + + Args: + backend (str): Backend name. + Returns: + bool: Returns true if the backend is available otherwise false. + """ + # If the backend has an ``is_backend_available`` function, return the result of that function directly + if ":" in backend.lower(): # composite backend like "cpu:gloo" + backend_config = BackendConfig(Backend(backend)) + device_backend_map = backend_config.get_device_backend_map() + return all( + _check_single_backend_availability(str(backend_name)) + for backend_name in device_backend_map.values() + ) + else: + # Handle simple backend strings like "nccl", "gloo" + return _check_single_backend_availability(backend) + + +def is_initialized() -> bool: + """Check if the default process group has been initialized.""" + return GroupMember.WORLD is not None + + +def is_torchelastic_launched() -> bool: + """ + Check whether this process was launched with ``torch.distributed.elastic`` (aka torchelastic). + + The existence of ``TORCHELASTIC_RUN_ID`` environment + variable is used as a proxy to determine whether the current process + was launched with torchelastic. This is a reasonable proxy since + ``TORCHELASTIC_RUN_ID`` maps to the rendezvous id which is always a + non-null value indicating the job id for peer discovery purposes.. + """ + return os.getenv("TORCHELASTIC_RUN_ID") is not None + + +def _is_barrier_after_init() -> int: + # Environment variable to control whether process group should perform a + # barrier after its init. Default value is 0, i.e. no barrier. If you + # experience issue with this setting, you may set + # `TORCH_DIST_INIT_BARRIER=1` to add the barrier. + return int(os.getenv("TORCH_DIST_INIT_BARRIER", "0")) + + +def _get_default_group() -> ProcessGroup: + """Get the default process group created by init_process_group.""" + if not is_initialized(): + raise ValueError( + "Default process group has not been initialized, " + "please make sure to call init_process_group." + ) + if TYPE_CHECKING: + return not_none(GroupMember.WORLD) + else: + return GroupMember.WORLD + + +def _get_default_store() -> Store: + """Get the default store created by init_process_group.""" + if not is_initialized(): + raise ValueError( + "Default process group has not been initialized, " + "please make sure to call init_process_group." + ) + default_pg = _get_default_group() + _, default_store = _world.pg_map[default_pg] + return default_store + + +def _update_default_pg(pg: ProcessGroup | None) -> None: + _world.default_pg = pg + rank = pg.rank() if pg is not None and pg != GroupMember.NON_GROUP_MEMBER else -1 + torch._C._distributed_c10d._set_global_rank(rank) + + +def get_backend_config(group: ProcessGroup | None = None) -> str: + """ + Return the backend configuration of the given process group. + + Args: + group (ProcessGroup, optional): The process group to work on. The + default is the general main process group. If another specific group + is specified, the calling process must be part of :attr:`group`. + + Returns: + The backend configuration of the given process group as a lower case string. + + """ + pg = group or _get_default_group() + if _rank_not_in_group(pg): + raise ValueError("Invalid process group specified") + backend_config = _world.pg_backend_config.get(pg) + return str(not_none(backend_config)) + + +def get_backend(group: ProcessGroup | None = None) -> Backend: + """ + Return the backend of the given process group. + + Args: + group (ProcessGroup, optional): The process group to work on. The + default is the general main process group. If another specific group + is specified, the calling process must be part of :attr:`group`. + + Returns: + The backend of the given process group as a lower case string. + + """ + pg = group or _get_default_group() + if _rank_not_in_group(pg): + raise ValueError("Invalid process group specified") + + pg_store = _world.pg_map.get(pg, None) + if pg_store is None: + raise ValueError( + f"Process group {pg} is not initialized in the world group map. Please initialize the group first." + ) + + return Backend(not_none(pg_store)[0]) + + +def get_default_backend_for_device(device: str | torch.device) -> str: + """ + Return the default backend for the given device. + + Args: + device (Union[str, torch.device]): The device to get the default backend for. + + Returns: + The default backend for the given device as a lower case string. + + """ + if isinstance(device, torch.device): + device_str = device.type + else: + device_str = torch.device(device).type + + backend = Backend.default_device_backend_map.get(device_str) + if backend is None: + raise ValueError(f"Default backend not registered for device : {device}") + + return backend + + +def _get_process_group_uid(pg: ProcessGroup) -> int: + backend = None + try: + backend = pg._get_backend(torch.device("cuda")) + except RuntimeError: + pass + if is_nccl_available() and isinstance(backend, ProcessGroupNCCL): + return backend.uid + return -1 + + +def _get_pg_config(group: ProcessGroup | None = None) -> dict[str, Any]: + """ + Return the pg configuration of the given process group. + + """ + pg = group or _get_default_group() + return { + "pg_name": _get_process_group_name(pg), + "pg_desc": pg.group_desc, + "backend_config": get_backend_config(pg), + "pg_size": _get_group_size(pg), + "ranks": get_process_group_ranks(pg), + } + + +def _get_all_pg_configs() -> list[dict[str, Any]]: + """ + Return the pg configuration of all the process groups. + + """ + config_info: list[dict[str, Any]] = [_get_pg_config(pg) for pg in _world.pg_map] + return config_info + + +def get_pg_count() -> int: + """ + Return the number of process groups. + + """ + return _world.group_count + + +def get_node_local_rank(fallback_rank: int | None = None) -> int: + """ + Return the local rank of the current process relative to the node. + + Semantically, this is a useful concept for mapping processes to devices. + For example, on a node with 8 accelerator you could use the node local rank to decide + which accelerator device to bind the process to. + + In practice, the actual assignment of node local ranks is handled by the process launcher outside of pytorch, + and communicated via the `LOCAL_RANK` environment variable. + + Torchrun will automatically populate `LOCAL_RANK`, but other launchers may not. If `LOCAL_RANK` is unspecified, + this API will fall back to the provided kwarg 'fallback_rank' if specified, otherwise it will raise an error. The + intent is to allow writing an application that runs either in single or multi device contexts without error. + + """ + if "LOCAL_RANK" in os.environ: + return int(os.environ["LOCAL_RANK"]) + elif fallback_rank is not None: + return int(fallback_rank) + raise RuntimeError( + "LOCAL_RANK is not in the environment. Consider passing fallback_rank to allow `get_node_local_rank` to work, " + "assuming you are not running in a multi-device context and want the code to run locally instead." + ) + + +def _add_ephemeral_timeout_for_all_pgs(timeout: timedelta) -> None: + """ + This API adds an ephemeral timeout extension for all PGs locally + on one rank. The timeout gets reset when the first collective issued + after API called finished. + NOTE: We only support to set timeout for cuda backends for now. + NOTE: While this feature + provides flexibility in specific scenarios, it introduces statefulness + to timeout setting. Therefore, it is advisable to use this API sparingly + and consider alternative approaches, such as directly setting the timeout + or utilizing a barrier collective (one can set any timeout to the barrier), + whenever feasible. + + Args: + timeout (timedelta): The delta of timeout to extend. + + Returns: + None. + """ + for pg in _world.pg_map: + devices = pg._device_types + if torch.device("cuda") in devices: + backend = pg._get_backend(torch.device("cuda")) + if is_nccl_available() and isinstance(backend, ProcessGroupNCCL): + backend._add_ephemeral_timeout(timeout) + + +def _set_pg_timeout(timeout: timedelta, group: ProcessGroup | None = None) -> None: + """ + Set the timeout for the given process group when users want to use a different timeout instead of + default values. + + Args: + timeout (timedelta): Timeout for operations executed against the process group which + users want to set. Default value is 10 minutes for NCCL and 30 minutes for other backends. + This is the duration after which collectives will be aborted asynchronously and the process will crash. + This is done since CUDA execution is async and it is no longer safe to continue executing user code since + failed async NCCL operations might result in subsequent CUDA operations running on corrupted data. + When TORCH_NCCL_BLOCKING_WAIT is set, the process will block and wait for this timeout. + + group (ProcessGroup, optional): The process group to work on. The + default is the general main process group. If another specific group + is specified, the calling process must be part of :attr:`group`. + + Returns: + None + """ + if group is None: + group = _get_default_group() + if _rank_not_in_group(group): + raise ValueError("Invalid process group specified") + if not isinstance(group, ProcessGroup): + raise AssertionError(f"Expected ProcessGroup, got {type(group)}") + devices = group._device_types + backends = set() + if torch.device("cpu") in devices and is_gloo_available(): + backend = group._get_backend(torch.device("cpu")) + if isinstance(backend, ProcessGroupGloo): + backends.add(backend) + if torch.device("cuda") in devices: + backend = group._get_backend(torch.device("cuda")) + if is_nccl_available() and isinstance(backend, ProcessGroupNCCL): + backends.add(backend) # type: ignore[arg-type] + elif is_gloo_available() and isinstance(backend, ProcessGroupGloo): + backends.add(backend) # type: ignore[arg-type] + if len(backends) == 0: + warnings.warn( + "Set timeout is now only supported for either nccl or gloo.", stacklevel=2 + ) + for backend in backends: + backend._set_default_timeout(timeout) + + +@_exception_logger +@_time_logger +def init_process_group( + backend: str | None = None, + init_method: str | None = None, + timeout: timedelta | None = None, + world_size: int = -1, + rank: int = -1, + store: Store | None = None, + group_name: str = "", + pg_options: Any | None = None, + device_id: torch.device | int | None = None, + _ranks: list[int] | None = None, +) -> None: + """ + Initialize the default distributed process group. + + This will also initialize the distributed package. + + There are 2 main ways to initialize a process group: + 1. Specify ``store``, ``rank``, and ``world_size`` explicitly. + 2. Specify ``init_method`` (a URL string) which indicates where/how + to discover peers. Optionally specify ``rank`` and ``world_size``, + or encode all required parameters in the URL and omit them. + + If neither is specified, ``init_method`` is assumed to be "env://". + + + Args: + backend (str or Backend, optional): The backend to use. Depending on + build-time configurations, valid values include ``mpi``, ``gloo``, + ``nccl``, ``ucc``, ``xccl`` or one that is registered by a third-party + plugin. + Since 2.6, if ``backend`` is not provided, c10d will use a backend + registered for the device type indicated by the `device_id` kwarg + (if provided). The known default registrations today are: ``nccl`` + for ``cuda``, ``gloo`` for ``cpu``, ``xccl`` for ``xpu``. + If neither ``backend`` nor ``device_id`` is provided, c10d will + detect the accelerator on the run-time machine and use a backend + registered for that detected accelerator (or ``cpu``). + This field can be given as a lowercase string (e.g., ``"gloo"``), + which can also be accessed via :class:`Backend` attributes (e.g., + ``Backend.GLOO``). + If using multiple processes per machine with ``nccl`` backend, each + process must have exclusive access to every GPU it uses, as sharing + GPUs between processes can result in deadlock or NCCL invalid usage. + ``ucc`` backend is experimental. + Default backend for the device can be queried with + :func:`get_default_backend_for_device`. + init_method (str, optional): URL specifying how to initialize the + process group. Default is "env://" if no + ``init_method`` or ``store`` is specified. + Mutually exclusive with ``store``. + world_size (int, optional): Number of processes participating in + the job. Required if ``store`` is specified. + rank (int, optional): Rank of the current process (it should be a + number between 0 and ``world_size``-1). + Required if ``store`` is specified. + store(Store, optional): Key/value store accessible to all workers, used + to exchange connection/address information. + Mutually exclusive with ``init_method``. + timeout (timedelta, optional): Timeout for operations executed against + the process group. Default value is 10 minutes for NCCL and 30 minutes for other backends. + This is the duration after which collectives will be aborted asynchronously and the process will crash. + This is done since CUDA execution is async and it is no longer safe to continue executing user code since + failed async NCCL operations might result in subsequent CUDA operations running on corrupted data. + When TORCH_NCCL_BLOCKING_WAIT is set, the process will block and wait for this timeout. + + group_name (str, optional, deprecated): Group name. This argument is ignored + pg_options (ProcessGroupOptions, optional): process group options + specifying what additional options need to be passed in during + the construction of specific process groups. As of now, the only + options we support is ``ProcessGroupNCCL.Options`` for the ``nccl`` + backend, ``is_high_priority_stream`` can be specified so that + the nccl backend can pick up high priority cuda streams when + there're compute kernels waiting. For other available options to config nccl, + See https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclconfig-t + device_id (torch.device | int, optional): a single, specific device + this process will work on, allowing for backend-specific + optimizations. Currently this has two effects, only under + NCCL: the communicator is immediately formed (calling + ``ncclCommInit*`` immediately rather than the normal lazy + call) and sub-groups will use ``ncclCommSplit`` when + possible to avoid unnecessary overhead of group creation. If you + want to know NCCL initialization error early, you can also use this + field. If an `int` is provided, the API assumes that the accelerator + type at compile time will be used. + _ranks: The ranks in the process group. If provided, the process + group name will be the hash of all the ranks in the group. + + .. note:: To enable ``backend == Backend.MPI``, PyTorch needs to be built from source + on a system that supports MPI. + + .. note:: Support for multiple backends is experimental. Currently when no backend is + specified, both ``gloo`` and ``nccl`` backends will be created. The ``gloo`` backend + will be used for collectives with CPU tensors and the ``nccl`` backend will be used + for collectives with CUDA tensors. A custom backend can be specified by passing in + a string with format ":,:", e.g. + "cpu:gloo,cuda:custom_backend". + + """ + + global _world + + global _backend + global _default_pg_init_method + + if GroupMember.WORLD is not None: + raise ValueError("trying to initialize the default process group twice!") + + set_pytorch_distributed_envs_from_justknobs() + + # Depending on the import order, some trace_rules functions may be evaluated + # during the import phase. In such a case, these functions may not correctly + # add the distributed related rules due to import circular dependency. + # We need to clear the lru_cache during the runtime to ensure the correctness + # of these trace_rules. + # + # Since this API must be called before all distributed code being compiled, + # clearing the cache here should be safe. + if "torch._dynamo" in sys.modules: + torch._dynamo.trace_rules.clear_lru_cache() + + if not ((store is None) or (init_method is None)): + raise AssertionError("Cannot specify both init_method and store.") + + if store is not None: + if not world_size > 0: + raise AssertionError("world_size must be positive if using store") + if not rank >= 0: + raise AssertionError("rank must be non-negative if using store") + elif init_method is None: + init_method = "env://" + + # Get the compile-time accelerator type. + # None indicates no accelerator support. + acc = torch.accelerator.current_accelerator() + + # Auto complete device id + if isinstance(device_id, int): + if acc is None: + raise ValueError( + "device_id is an int, but no accelerator support is found from the current compilation. " + "Please use a different compiled version that supports your accelerator." + ) + device_id = torch.device(acc.type, device_id) + + # Sanity check device_id + if device_id is not None and device_id.type != "cpu": + # Type + if acc is None or device_id.type != acc.type: + raise ValueError( + f"device_id {device_id} does not match the current compilation's accelerator support: {acc}. " + "Please use a different compiled version that supports your accelerator." + ) + # Index + if device_id.index is None: + raise ValueError("Please use a device_id with index.") + # Range + if device_id.index >= torch.accelerator.device_count(): + raise ValueError( + f"device_id {device_id} is out of range. Please use a device index less than " + f"the number of accelerators available: {torch.accelerator.device_count()}." + ) + + logger.info("Using device: %s", device_id) + + # If user did not provide a backend string but provided a device id, e.g. + # >>> init_process_group(device_id=device) + # we try to figure out the backend name based on the device type. + if backend is None and device_id is not None: + # Note: 3rd-party devices can register default backend through the + # default map below. + backend = Backend.default_device_backend_map.get(device_id.type) + + # If we still cannot figure it out, e.g. + # >>> init_process_group() + # we set it to `undefined` and rely on lazy init. + if backend is None: + backend = "undefined" + + # Convert string into `Backend` type + backend = Backend(backend) + + if timeout is None: + timeout = _get_default_timeout(backend) + + _check_valid_timeout(timeout) + + """ + Group name is not visible to users unless they access + internals of c10d. This means we can ignore the value + they provide as it not exposed in a public way. + """ + if _ranks is None or len(_ranks) == 0: + group_name = _process_group_name([], use_hashed_name=False) + else: + group_name = _process_group_name(_ranks, use_hashed_name=True) + if backend == Backend.MPI: + if world_size != -1 or rank != -1: + warnings.warn( + f"For MPI backend, world_size ({world_size}) and rank ({rank}) " + "are ignored since they are assigned by the " + "MPI runtime.", + stacklevel=2, + ) + + default_pg, _ = _new_process_group_helper( + -1, + -1, + [], + backend, + Store(), # Placeholder value since store cannot be None + group_name, + timeout=timeout, + group_desc="default_pg", + ) + else: + # backward compatible API + if store is None: + if backend == "fake": + from torch.testing._internal.distributed.fake_pg import FakeStore + + store = FakeStore() + else: + rendezvous_iterator = rendezvous( + not_none(init_method), rank, world_size, timeout=timeout + ) + store, rank, world_size = next(rendezvous_iterator) + store.set_timeout(timeout) + + # Use a PrefixStore to avoid accidental overrides of keys used by + # different systems (e.g. RPC) in case the store is multi-tenant. + store = PrefixStore("default_pg", store) + + default_pg, _ = _new_process_group_helper( + world_size, + rank, + [], + backend, + store, + group_name, + backend_options=pg_options, + timeout=timeout, + device_id=device_id, + group_desc="default_pg", + ) + + _update_default_pg(default_pg) + + _world.pg_group_ranks[GroupMember.WORLD] = { # type: ignore[index] + i: i + for i in range(GroupMember.WORLD.size()) # type: ignore[attr-defined] + } + _backend = _world.pg_map[not_none(GroupMember.WORLD)][0] + _default_pg_init_method = init_method + + old_hook = sys.excepthook + excepthook_prefix = f"[rank{get_rank()}]" + + def _distributed_excepthook(*args): + old_stderr = sys.stderr + sys.stderr = buf = io.StringIO() + try: + old_hook(*args) + finally: + sys.stderr = old_stderr + msg = buf.getvalue() + msg = "\n".join( + f"{excepthook_prefix}: {s}" if s != "" else "" for s in msg.split("\n") + ) + sys.stderr.write(msg) + sys.stderr.flush() + + sys.excepthook = _distributed_excepthook + + if _is_barrier_after_init() == 1: + # barrier at the end to ensure that once we return from this method, all + # process groups including global variables (if any) are updated + # correctly on all ranks. + # Update 04/2023: for large-scale runs, this barrier (esp. store-based + # barrier) may be costly and/or unscalable. Also, in a lot of cases, + # these barriers may be unnecessary, as proven by a green CI after + # removal. An environment variable `TORCH_DIST_INIT_BARRIER` has been + # added which enables this barrier only when set to 1. + logger.debug( + "Performing barrier after ProcessGroup initialization since " + "TORCH_DIST_INIT_BARRIER = 1" + ) + if backend == Backend.MPI: + # MPI backend doesn't use store. + barrier() + else: + # Use store based barrier here since barrier() used a bunch of + # default devices and messes up NCCL internal state. + _store_based_barrier(rank, store, group_name, world_size, timeout) + + +def _get_split_source(pg: ProcessGroup): + split_from = None + if pg.bound_device_id: + split_from = pg._get_backend(pg.bound_device_id) + elif pg is _world.default_pg: + try: + # pyrefly: ignore [missing-attribute] + split_from = pg._get_backend(torch.device("cuda")) + except RuntimeError: + # no cuda device associated with this backend + pass + + if not split_from or not split_from.supports_splitting: + return None + + # If necessary, find a backend to split from by peeling process + # group wrappers from our potentially wrapped process group. + while _GLOO_AVAILABLE and isinstance(split_from, _ProcessGroupWrapper): + split_from = split_from.wrapped_pg + + return split_from + + +def _new_process_group_helper( + group_size, + group_rank, + global_ranks_in_group, + backend, + store, + group_name: GroupName, + backend_options=None, + timeout=None, + pg_tag=None, + device_id=None, + group_desc=None, +): + """ + Create a new distributed process group. + + This function must be called by ALL processes in the global group, even if + the calling process is not part of the newly created group. In that case, + this function returns GroupMember.NON_GROUP_MEMBER. + + This function is called with ``global_ranks_in_group == []`` for the default group. + """ + global _world + + if group_name in _world.pg_names.values(): + raise ValueError( + "The specified group name has already been " + "created, please use a different group name" + ) + + if device_id is not None and (device_id.index is None or device_id.type == "cpu"): + raise ValueError( + "init_process_group device_id parameter must be an accelerator with an index" + ) + + # Note: _new_process_group_helper is only called from init_process_group, which always provides a timeout value + _check_valid_timeout(timeout) + + if pg_tag not in [None, ""]: + # creating with the same tag and rank set results in the same underlying PG + existing_group = _find_pg_by_ranks_and_tag(pg_tag, global_ranks_in_group) + if existing_group: + _, prefix_store = _world.pg_map[existing_group] + return existing_group, prefix_store + + group_desc = "undefined" if group_desc is None else group_desc + + # The list of group ranks is empty if we're creating the default group. + is_default_group = len(global_ranks_in_group) == 0 + + # nccl and potentially other backends allow creation of + # communicators based on pre-existing ones, which can save + # initialization time. Due to lazy initialization of + # communicators in some backends, we have to be careful and only + # split when we *know* the default PG has already started communicator initialization. + # We know this if we have bound a device id to the default pg (eager initialized). + if is_initialized() and _get_default_group().bound_device_id: + split_from = _get_split_source(_get_default_group()) + else: + split_from = None + + # If this is a subgroup (which means group_ranks is specified), + # we check if the current process is a member of the new group. + if not is_default_group: + global_rank = _get_default_group().rank() + if global_rank not in global_ranks_in_group: + # If we are using `ncclCommSplit` (or similar split from + # other APIs) to create the communicator, we will need to + # call `ncclCommSplit` on *all* ranks in this new group's + # parent group, even those not in the new group. This is + # a requirement of the NCCL API as otherwise we would get + # out of sync. + if split_from: + split_from.perform_nocolor_split(_get_default_group().bound_device_id) + return GroupMember.NON_GROUP_MEMBER, None + + prefix_store = PrefixStore(f"{group_name}/", store) + # The backend for PG will be set later based on what's inside BackendConfig + # and timeout are set in each backend's option. + pg: ProcessGroup = ProcessGroup( + prefix_store, + group_rank, + group_size, + ) + backend_config = BackendConfig(backend) + # Set the default backend when single backend is passed in. + if "," not in str(backend) and ":" not in str(backend): + if backend not in Backend.backend_type_map: + raise AssertionError(f"Unknown backend type {backend}") + if backend == Backend.UNDEFINED: + # Currently when backend is UNDEFINED, only one backend will be initialized + # we use nccl (if cuda is available) or gloo as default backend + # so we can correctly call getDefaultBackend which in ProcessGroup. + if Backend.NCCL in backend_config.get_device_backend_map().values(): + pg._set_default_backend(ProcessGroup.BackendType.NCCL) + else: + pg._set_default_backend(ProcessGroup.BackendType.GLOO) + else: + pg._set_default_backend(Backend.backend_type_map[backend]) + # In order to correctly call pg._has_hooks(), we should set the default backend + # when multi backend is passed in + else: + if Backend.NCCL in backend_config.device_backend_map.values(): + pg._set_default_backend(ProcessGroup.BackendType.NCCL) + elif Backend._plugins.keys(): + custom_backend = next(iter(Backend._plugins.keys())) + if custom_backend in backend_config.device_backend_map.values(): + pg._set_default_backend(ProcessGroup.BackendType.CUSTOM) + else: + pg._set_default_backend(ProcessGroup.BackendType.GLOO) + + if device_id: + pg.bound_device_id = device_id + backend_class: torch._C._distributed_c10d.Backend + for device, backend_str in backend_config.get_device_backend_map().items(): + # Use the group name as prefix in the default store, such that + # a single store can be reused by multiple groups. + backend_prefix_store = PrefixStore(f"{device}/", prefix_store) + + if backend_str == Backend.MPI: + if not is_mpi_available(): + raise RuntimeError( + "Distributed package doesn't have MPI built in." + " MPI is only included if you build PyTorch from" + " source on a host that has MPI installed." + ) + backend_class = ProcessGroupMPI.create(global_ranks_in_group) + backend_type = ProcessGroup.BackendType.MPI + if not backend_class: + return GroupMember.NON_GROUP_MEMBER, None + # create new process group with accurate rank and size + if pg.rank() == -1 and pg.size() == -1: + pg = ProcessGroup( + backend_prefix_store, + backend_class.rank(), + backend_class.size(), + ) + pg._set_default_backend(backend_type) + elif backend_str == Backend.GLOO: + # TODO: remove this check after lazy initialization is supported + # if pg_options is not None: + # raise RuntimeError("GLOO options not supported") + if not is_gloo_available(): + raise RuntimeError("Distributed package doesn't have Gloo built in") + backend_class = ProcessGroupGloo( + backend_prefix_store, + group_rank, + group_size, + # pyrefly: ignore [bad-argument-type] + timeout=timeout, + ) + backend_class.options.global_ranks_in_group = global_ranks_in_group + backend_class.options.group_name = group_name + backend_type = ProcessGroup.BackendType.GLOO + elif backend_str == Backend.NCCL: + if not is_nccl_available(): + raise RuntimeError("Distributed package doesn't have NCCL built in") + if backend_options is not None: + if not isinstance(backend_options, ProcessGroupNCCL.Options): + raise AssertionError( + "Expected backend_options argument to be of type ProcessGroupNCCL.Options" + ) + if backend_options._timeout != timeout: + warnings.warn( + "backend_options._timeout was specified, " + "but timeout kwarg has a default value that will always override it. ", + stacklevel=2, + ) + else: + # default backend_options for NCCL + backend_options = ProcessGroupNCCL.Options() + backend_options.is_high_priority_stream = False + # pyrefly: ignore [bad-argument-type] + backend_options._timeout = timeout + + if split_from: + backend_options.split_from = split_from + backend_options.split_color = _process_group_color( + global_ranks_in_group + ) + backend_options.global_ranks_in_group = global_ranks_in_group + backend_options.group_name = group_name + backend_class = ProcessGroupNCCL( + backend_prefix_store, group_rank, group_size, backend_options + ) + backend_type = ProcessGroup.BackendType.NCCL + elif backend_str == Backend.UCC and is_ucc_available(): + # TODO: once UCC plugin is fully deprecated, remove + # is_ucc_available() from above elif-condition and raise + # RuntimeError if is_ucc_available() returns false. + + backend_class = ProcessGroupUCC( + backend_prefix_store, + group_rank, + group_size, + # pyrefly: ignore [bad-argument-type] + timeout=timeout, + ) + backend_type = ProcessGroup.BackendType.UCC + elif backend_str == Backend.XCCL: + if not is_xccl_available(): + raise RuntimeError("Distributed package doesn't have XCCL built in") + backend_options = ProcessGroupXCCL.Options() + backend_options.global_ranks_in_group = global_ranks_in_group + backend_options.group_name = group_name + # pyrefly: ignore [bad-argument-type] + backend_options._timeout = timeout + backend_class = ProcessGroupXCCL( + backend_prefix_store, group_rank, group_size, backend_options + ) + backend_type = ProcessGroup.BackendType.XCCL + else: + if backend_str.upper() not in Backend._plugins: + raise AssertionError(f"Unknown c10d backend type {backend_str.upper()}") + + backend_plugin = Backend._plugins[backend_str.upper()] + creator_fn = backend_plugin.creator_fn + extended_api = backend_plugin.extended_api + backend_type = ProcessGroup.BackendType.CUSTOM + + if not extended_api: + backend_class = creator_fn( + backend_prefix_store, group_rank, group_size, timeout + ) + else: + dist_backend_opts = _DistributedBackendOptions() + dist_backend_opts.store = backend_prefix_store + dist_backend_opts.group_rank = group_rank + dist_backend_opts.group_size = group_size + # pyrefly: ignore [bad-argument-type] + dist_backend_opts.timeout = timeout + dist_backend_opts.group_id = group_name + dist_backend_opts.global_ranks_in_group = global_ranks_in_group + + backend_class = creator_fn(dist_backend_opts, backend_options) + + # Set sequence numbers for gloo and nccl backends. + if backend_str == Backend.GLOO: + if not isinstance(backend_class, ProcessGroupGloo): + raise AssertionError( + f"Expected ProcessGroupGloo, got {type(backend_class)}" + ) + backend_class._set_sequence_number_for_group() + elif backend_str == Backend.NCCL: + if not isinstance(backend_class, ProcessGroupNCCL): + raise AssertionError( + f"Expected ProcessGroupNCCL, got {type(backend_class)}" + ) + backend_class._set_sequence_number_for_group() + + # If the type is a subclass of ProcessGroup then return this process group immediately + # TODO: This defaults to the old behavior for PythonProcessGroups which overwrites the + # ProcessGroup instance + if issubclass(type(backend_class), ProcessGroup): + pg = backend_class # type: ignore[assignment] + break + + # Process group wrapper initialization for supported PGs when TORCH_DISTRIBUTED_DEBUG is set + if ( + backend_str in [Backend.GLOO, Backend.NCCL, Backend.UCC] + or backend_str.upper() in Backend._plugins + ): + # In debug mode and if GLOO is available, wrap in a wrapper PG that + # enables enhanced collective checking for debuggability. + if get_debug_level() == DebugLevel.DETAIL: + if not _GLOO_AVAILABLE: + logger.info( + """TORCH_DISTRIBUTED_DEBUG was set to DETAIL, but + GLOO is not available. Build with Gloo to + create a wrapper process group in debug mode + to aid collective desynchronization debugging.""" + ) + else: + backend_class = _create_process_group_wrapper( + wrapped_pg=backend_class, + store_prefix=group_name, + store=backend_prefix_store, + rank=group_rank, + world_size=group_size, + # pyrefly: ignore [bad-argument-type] + timeout=timeout, + ) + + # register only a single backend when all get_device_backend_map values are the same + if len(set(backend_config.get_device_backend_map().values())) == 1: + for device in backend_config.get_device_backend_map(): + pg._register_backend(torch.device(device), backend_type, backend_class) + + # break out of outer loop to not create any more backends + break + + pg._register_backend(torch.device(device), backend_type, backend_class) + + # set group_name and group_dsec to backend + if group_name is None: + raise AssertionError("group_name must not be None") + if group_desc is None: + raise AssertionError("group_desc must not be None") + pg._set_group_name(group_name) + pg._set_group_desc(group_desc) + + if device_id and pg._get_backend(device_id).supports_splitting: + eager_backend = pg._get_backend(device_id) + eager_backend.eager_connect_single_device(device_id) + + # update global state + _world.pg_map[pg] = (backend, prefix_store) + _world.pg_names[pg] = group_name + _register_process_group(group_name, pg) + + _world.pg_backend_config[pg] = str(backend_config) + # "" is the default tag for user PGs + if pg_tag in [None, ""]: + pg_tag = f"ptd:{group_name}" + _world.tags_to_pg.setdefault("", []).append(pg) + else: + pg_tag = f"user:{pg_tag}" + + _world.tags_to_pg.setdefault(pg_tag, []).append(pg) + _world.pg_to_tag[pg] = pg_tag + return pg, prefix_store + + +def destroy_process_group(group: ProcessGroup | None = None): + """ + Destroy a given process group, and deinitialize the distributed package. + + Args: + group (ProcessGroup, optional): The process group to be destroyed, if + group.WORLD is given, all process + groups including the default one will + be destroyed. + """ + global _world + + if group == GroupMember.NON_GROUP_MEMBER: + return + + if group is None: + pg = GroupMember.WORLD + else: + pg = group + + if pg is None: + raise AssertionError("Process group cannot be None") + if _world.pg_map.get(pg, None) is None: + raise ValueError("Invalid process group specified") + + # When users register Python onCompletion hooks, those hooks will run on a + # different thread than the main thread. Today, the ProcessGroup dtor does + # wait for that thread. However, the dtor might finish after the Python + # Interpreter exits. After that grabbing the GIL for the Python hook will crash. + # We can either revive the interpreter when running hooks or keep the main one + # alive until all works and hooks are done. The current implementation does the + # latter. Therefore, we explicitly call _wait_for_pending_works() here to wait + # for the pending hooks to finish. + if type(pg) is ProcessGroup and pg._has_hooks(): + pg._wait_for_pending_works() + + if group is None or group == GroupMember.WORLD: + # shutdown all backends in the order of pg names. shutting down in order because + # ncclCommAbort() was a 'collective' call in some versions of NCCL. + for pg_to_shutdown in sorted( + _world.pg_names, key=lambda x: _world.pg_names[x], reverse=True + ): + pg_to_shutdown.shutdown() + + _update_default_pg(None) + _world.pg_map.clear() + _world.pg_names.clear() + _world.pg_group_ranks.clear() + _world.pg_backend_config.clear() + _world.pg_to_tag.clear() + _world.tags_to_pg.clear() + _world.pg_coalesce_state.clear() + _unregister_all_process_groups() + + # when process group doesn't have an explicit name (only WORLD (default) + # process group can have an explicit name), we use global _world.group_count + # to generate the name. We need to reset the counter on destruction to + # allow consistent value to be generated when we re-create process + # groups after some trainers recover from failure + # + # We only reset this when WORLD is being destroyed because if this + # process group is in good state, we aren't dealing with failures. + _world.group_count = 0 + else: + pg.shutdown() + del _world.pg_map[pg] + del _world.pg_names[pg] + del _world.pg_group_ranks[pg] + del _world.pg_backend_config[pg] + if pg in _world.pg_coalesce_state: + warnings.warn( + "Some coalesced collectives haven't been launched when " + "ProcessGroup is destroyed. They will be cleaned.", + stacklevel=2, + ) + del _world.pg_coalesce_state[pg] + + tag = _world.pg_to_tag.get(pg) + del _world.pg_to_tag[pg] + if tag is not None: + try: + _world.tags_to_pg[tag].remove(pg) + if tag.startswith("ptd:"): + _world.tags_to_pg[""].remove(pg) + except Exception: + pass + _unregister_process_group(pg.group_name) + + +def _abort_process_group(group: ProcessGroup | None = None): + """ + Abort a given process group. If group.WORLD (i.e. `None`) is given, all + process groups including the default one will be aborted. + + Args: + group (ProcessGroup, optional): The process group to be aborted. + + .. note:: this API is experimental and currently only works with the NCCL + backend. + + .. note:: this API should be used with `TORCH_NCCL_ASYNC_ERROR_HANDLING` + turned off (i.e. set to 0). Otherwise, ProcessGroupNCCL's watchdog may + automatically handle errors or timeouts for you including aborting the + ProcessGroup. + """ + global _world + + if group == GroupMember.NON_GROUP_MEMBER: + return + + pg = group or GroupMember.WORLD + + if pg is None: + raise AssertionError("Process group cannot be None") + if _world.pg_map.get(pg, None) is None: + raise ValueError("Invalid process group specified or has been destroyed.") + + try: + backend = pg._get_backend(torch.device("cuda")) + except RuntimeError: + backend = None + + if group is None or group == GroupMember.WORLD: + # Abort all backends within a ncclGroupStart|End semantic. + # This ensures that different NCCL communicators' abort calls won't + # deadlock each other. + # For details, please see: https://github.com/pytorch/pytorch/issues/119797 + if is_nccl_available() and isinstance(backend, ProcessGroupNCCL): + backend._group_start() + for pg_to_abort in sorted( + _world.pg_names, key=lambda x: _world.pg_names[x], reverse=True + ): + pg_to_abort.abort() + if is_nccl_available() and isinstance(backend, ProcessGroupNCCL): + backend._group_end() + + _update_default_pg(None) + _world.pg_map.clear() + _world.pg_names.clear() + _world.pg_group_ranks.clear() + _world.pg_backend_config.clear() + _world.pg_to_tag.clear() + _world.tags_to_pg.clear() + _world.pg_coalesce_state.clear() + _unregister_all_process_groups() + + # when process group doesn't have an explicit name (only WORLD (default) + # process group can have an explicit name), we use global _world.group_count + # to generate the name. We need to reset the counter on destruction to + # allow consistent value to be generated when we re-create process + # groups after some trainers recover from failure + # + # We only reset this when WORLD is being destroyed because if this + # process group is in good state, we aren't dealing with failures. + _world.group_count = 0 + else: + pg.abort() + del _world.pg_map[pg] + del _world.pg_names[pg] + del _world.pg_group_ranks[pg] + del _world.pg_backend_config[pg] + if pg in _world.pg_coalesce_state: + warnings.warn( + "Some coalesced collectives haven't been launched when " + "ProcessGroup is aborted. They will be cleaned.", + stacklevel=2, + ) + del _world.pg_coalesce_state[pg] + + tag = _world.pg_to_tag.get(pg) + del _world.pg_to_tag[pg] + if tag is not None: + try: + _world.tags_to_pg[tag].remove(pg) + if tag.startswith("ptd:"): + _world.tags_to_pg[""].remove(pg) + except Exception: + pass + _unregister_process_group(pg.group_name) + + +def get_rank(group: ProcessGroup | None = None) -> int: + """ + Return the rank of the current process in the provided ``group``, default otherwise. + + Rank is a unique identifier assigned to each process within a distributed + process group. They are always consecutive integers ranging from 0 to + ``world_size``. + + Args: + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + + Returns: + The rank of the process group + -1, if not part of the group + + """ + if _rank_not_in_group(group): + return -1 + + default_pg = _get_default_group() + if group is None or group is GroupMember.WORLD: + return default_pg.rank() + + return get_group_rank(group, default_pg.rank()) + + +def get_world_size(group: ProcessGroup | None = None) -> int: + """ + Return the number of processes in the current process group. + + Args: + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + + Returns: + The world size of the process group + -1, if not part of the group + + """ + if _rank_not_in_group(group): + return -1 + + return _get_group_size(group) + + +def isend( + tensor: torch.Tensor, + dst: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_dst: int | None = None, +) -> Work | None: + """ + Send a tensor asynchronously. + + .. warning:: + Modifying ``tensor`` before the request completes causes undefined + behavior. + + .. warning:: + ``tag`` is not supported with the NCCL backend. + + Unlike send, which is blocking, isend allows src == dst rank, i.e. send to self. + + Args: + tensor (Tensor): Tensor to send. + dst (int): Destination rank on global process group (regardless of ``group`` argument) + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + tag (int, optional): Tag to match send with remote recv + group_dst (int, optional): Destination rank on ``group``. Invalid to specify both ``dst`` and ``group_dst`` + + Returns: + A distributed request object. + None, if not part of the group + + """ + group = _group_or_default_group(group) + group_dst = _canonicalize_group_rank(group, dst, group_dst) + _check_single_tensor(tensor, "tensor") + if _rank_not_in_group(group): + _warn_not_in_group("isend") + return None + + if tensor.is_complex(): + tensor = torch.view_as_real(tensor) + + return group.send([tensor], group_dst, tag) + + +def irecv( + tensor: torch.Tensor, + src: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_src: int | None = None, +) -> Work | None: + """ + Receives a tensor asynchronously. + + .. warning:: + ``tag`` is not supported with the NCCL backend. + + Unlike recv, which is blocking, irecv allows src == dst rank, i.e. recv from self. + + Args: + tensor (Tensor): Tensor to fill with received data. + src (int, optional): Source rank on global process group (regardless of ``group`` argument). + Will receive from any process if unspecified. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + tag (int, optional): Tag to match recv with remote send + group_src (int, optional): Destination rank on ``group``. Invalid to specify both ``src`` and ``group_src``. + + Returns: + A distributed request object. + None, if not part of the group + + """ + _check_single_tensor(tensor, "tensor") + if _rank_not_in_group(group): + _warn_not_in_group("irecv") + return None + + if tensor.is_complex(): + tensor = torch.view_as_real(tensor) + + group = _group_or_default_group(group) + if src is None and group_src is None: + return group.recv_anysource([tensor], tag) + else: + group_src = _canonicalize_group_rank(group, src, group_src) + return group.recv([tensor], group_src, tag) + + +@_exception_logger +def send( + tensor: torch.Tensor, + dst: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_dst: int | None = None, +) -> None: + """ + Send a tensor synchronously. + + .. warning:: + ``tag`` is not supported with the NCCL backend. + + Args: + tensor (Tensor): Tensor to send. + dst (int): Destination rank on global process group (regardless of ``group`` argument). + Destination rank should not be the same as the rank of the current process. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + tag (int, optional): Tag to match send with remote recv + group_dst (int, optional): Destination rank on ``group``. Invalid to specify both ``dst`` and ``group_dst``. + + """ + group = _group_or_default_group(group) + group_dst = _canonicalize_group_rank(group, dst, group_dst) + _check_not_self_rank(group, group_dst, "destination") + work = isend(tensor, group=group, tag=tag, group_dst=group_dst) + if work is not None: + work.wait() + + +@_exception_logger +def recv( + tensor: torch.Tensor, + src: int | None = None, + group: ProcessGroup | None = None, + tag: int = 0, + group_src: int | None = None, +) -> int: + """ + Receives a tensor synchronously. + + .. warning:: + ``tag`` is not supported with the NCCL backend. + + Args: + tensor (Tensor): Tensor to fill with received data. + src (int, optional): Source rank on global process group (regardless of ``group`` argument). + Will receive from any process if unspecified. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + tag (int, optional): Tag to match recv with remote send + group_src (int, optional): Destination rank on ``group``. Invalid to specify both ``src`` and ``group_src``. + Returns: + Sender rank + -1, if not part of the group + + """ + work = irecv(tensor, src=src, group=group, tag=tag, group_src=group_src) + if work is None: + return -1 + work.wait() + if src is None: + if group_src is None: + group_src = work._source_rank() + group = _group_or_default_group(group) + _check_not_self_rank(group, group_src, "source") + src = get_global_rank(group, group_src) + return src + + +class _IllegalWork(Work): + def __getattribute__(self, name): + if name in [ + "is_success", + "exception", + "wait", + "source_rank", + "_source_rank", + "result", + "synchronize", + ]: + raise ValueError(f"Illegal to call {name} on IllegalWork object") + + +class _CoalescingManager: + def __init__(self) -> None: + self.works: list[Work] = [] + + def append(self, work: Work | None = None): + if work: + self.works.append(work) + + def wait(self): + for work in self.works: + work.wait() + + +@contextlib.contextmanager +def _coalescing_manager( + group: ProcessGroup | None = None, + device: torch.device | None = None, + async_ops: bool = False, +): + """ + Context manager used to coalesce collectives or P2P operations when possible. + + Args: + group (`ProcessGroup`, optional): The process group to work on. If None, + the default process group will be used. + device (`torch.device`, optional): Default is None, set to a device if + there isn't a `**_coalesced` implementation by the backend. + async_ops (`bool`, optional): whether the coalesced ops are async ops. + + Examples: + >>> # xdoctest: +SKIP("no rank") + >>> # Synchronous ops + >>> with _coalescing_manager(): + >>> for i in range(num_colls): + >>> dist.all_reduce(tensors[i]) + >>> # Asynchronous ops + >>> with _coalescing_manager(async_ops=True) as cm: + >>> for i in range(num_colls): + >>> dist.all_reduce(tensors[i]) + >>> cm.wait() + + .. warning:: + :func:`_coalescing_manager` currently do not support coalescing + all-reduces with different reduce operators, e.g. `ReduceOp.SUM` mixed + with `ReduceOp.PRODUCT`. + """ + group = group or _get_default_group() + op_list = _world.pg_coalesce_state.setdefault(group, []) + if op_list: + raise ValueError( + "ProcessGroup has non-empty op list at the start of coalescing" + ) + if device: + group._start_coalescing(device) + cm = _CoalescingManager() + yield cm + work = None + op_list = _world.pg_coalesce_state.pop(group) + if op_list: + # Collectives supporting "Fast Path" coalescing are captured. + # See implementation in corresponding collective APIs. + # Currently supported: + # - coalesced `all_reduce` + # - coalesced `all_gather_into_tensor` + # - coalesced `reduce_scatter_tensor` + op0 = op_list[0].op + if op0 is all_reduce: + tensors = [op.tensor for op in op_list] + all_reduce_opts = AllreduceCoalescedOptions() + all_reduce_opts.reduceOp = not_none(op_list[0].redop) + all_reduce_opts.asyncOp = async_ops + work = group.allreduce_coalesced(tensors, all_reduce_opts) + elif op0 is all_gather_into_tensor: + inputs = [] + outputs = [] + for op in op_list: + inputs.append(op.tensor) + outputs.append(not_none(op.dst_tensor)) + all_gather_opts = AllgatherOptions() + all_gather_opts.asyncOp = async_ops + work = group.allgather_into_tensor_coalesced(outputs, inputs) + elif op0 is reduce_scatter_tensor: + inputs = [] + outputs = [] + for op in op_list: + inputs.append(op.tensor) + outputs.append(not_none(op.dst_tensor)) + reduce_opts = ReduceScatterOptions() + reduce_opts.reduceOp = not_none(op_list[0].redop) + reduce_opts.asyncOp = async_ops + work = group.reduce_scatter_tensor_coalesced(outputs, inputs, reduce_opts) + else: + raise AssertionError( + f"Coalescing manager does not support fast-path coalescing of {op0}, " + f"yet {op0} is still recorded in op list. This is an internal error of c10d." + ) + + if device: + # Old style of letting each coll inside the context manager to call into C++ counterpart via python binding + work = group._end_coalescing(device) + + if async_ops: + cm.append(work) + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +class _TimeEstimator: + def __init__(self) -> None: + self.estimated_time: float | None = None + + +@contextlib.contextmanager +def _time_estimator( + group: ProcessGroup | None = None, + device: torch.device | None = None, +): + """ + Context manager used to estimate time of collectives. + Within the context manager, nothing is actually run and the backend just simulates + the collective time only. + + Args: + group (`ProcessGroup`, optional): The process group to work on. If None, + the default process group will be used. + device (`torch.device`, optional): Default is None, set to a device if + there isn't a `**_coalesced` implementation by the backend. + + Examples: + >>> # xdoctest: +SKIP("no rank") + >>> # Synchronous ops + >>> with _time_estimator() as cm: + >>> for i in range(num_colls): + >>> dist.all_reduce(tensors[i]) + >>> # estimate time is stored in cm.estimated_time + + .. warning:: + :func:`_time_estimator` currently only support NCCL backend but it can + easily be extended to other backends. + + Also a NCCL communicator needs to be created because only with a real communicator can we do accurate estimation. + The communicator internally has knowledge about the links it runs on + (e.g. intra-node or inter-node, whether the links are NVLink or PCI-e or IB). + """ + # TODO: We need to also support torch inductor for the time estimator. + group = group or _get_default_group() + device = device or _get_pg_default_device(group) + backend = group._get_backend(device) + if not backend.supports_time_estimate: + raise NotImplementedError( + f"collective time estimator is not supported in the current version of backend {backend}" + ) + backend._start_time_estimate() # type: ignore[attr-defined] + cm = _TimeEstimator() + yield cm + cm.estimated_time = backend._end_time_estimate() # type: ignore[attr-defined] + + +def batch_isend_irecv(p2p_op_list: list[P2POp]) -> list[Work]: + """ + Send or Receive a batch of tensors asynchronously and return a list of requests. + + Process each of the operations in ``p2p_op_list`` and return the corresponding + requests. NCCL, Gloo, and UCC backend are currently supported. + + Args: + p2p_op_list: A list of point-to-point operations(type of each operator is + ``torch.distributed.P2POp``). The order of the isend/irecv in the list + matters and it needs to match with corresponding isend/irecv on the + remote end. + + Returns: + A list of distributed request objects returned by calling the corresponding + op in the op_list. + + Examples: + >>> # xdoctest: +SKIP("no rank") + >>> send_tensor = torch.arange(2, dtype=torch.float32) + 2 * rank + >>> recv_tensor = torch.randn(2, dtype=torch.float32) + >>> send_op = dist.P2POp(dist.isend, send_tensor, (rank + 1) % world_size) + >>> recv_op = dist.P2POp( + ... dist.irecv, recv_tensor, (rank - 1 + world_size) % world_size + ... ) + >>> reqs = batch_isend_irecv([send_op, recv_op]) + >>> for req in reqs: + >>> req.wait() + >>> recv_tensor + tensor([2, 3]) # Rank 0 + tensor([0, 1]) # Rank 1 + + .. note:: Note that when this API is used with the NCCL PG backend, users must set + the current GPU device with `torch.cuda.set_device`, otherwise it will + lead to unexpected hang issues. + + In addition, if this API is the first collective call in the ``group`` + passed to ``dist.P2POp``, all ranks of the ``group`` must participate in + this API call; otherwise, the behavior is undefined. If this API call is + not the first collective call in the ``group``, batched P2P operations + involving only a subset of ranks of the ``group`` are allowed. + """ + _check_p2p_op_list(p2p_op_list) + group = p2p_op_list[0].group + if group is None: + group = _get_default_group() + device = p2p_op_list[0].tensor.device + + def peer_kwarg(op: P2POp) -> dict[str, int]: + key = "group_dst" if op.op is isend else "group_src" + return {key: op.group_peer} + + if type(group) is ProcessGroup and group._get_backend(device).supports_coalescing: + # NCCL style coalescing + with _coalescing_manager(group, device, async_ops=True) as cm: + for p2p_op in p2p_op_list: + p2p_op.op( + p2p_op.tensor, + group=p2p_op.group, + tag=p2p_op.tag, + **peer_kwarg(p2p_op), + ) + + return cm.works + else: + # backend not support coalescing + reqs = [] + for p2p_op in p2p_op_list: + work = p2p_op.op( + p2p_op.tensor, + group=p2p_op.group, + tag=p2p_op.tag, + **peer_kwarg(p2p_op), + ) + if work: + reqs.append(work) + return reqs + + +@_exception_logger +def broadcast( + tensor: torch.Tensor, + src: int | None = None, + group: ProcessGroup | None = None, + async_op: bool = False, + group_src: int | None = None, +): + """ + Broadcasts the tensor to the whole group. + + ``tensor`` must have the same number of elements in all processes + participating in the collective. + + Args: + tensor (Tensor): Data to be sent if ``src`` is the rank of current + process, and tensor to be used to save received data otherwise. + src (int): Source rank on global process group (regardless of ``group`` argument). + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + group_src (int): Source rank on ``group``. Must specify one of ``group_src`` + and ``src`` but not both. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + """ + group = _group_or_default_group(group) + group_src = _canonicalize_group_rank(group, src, group_src, return_global=False) + _check_single_tensor(tensor, "tensor") + if _rank_not_in_group(group): + _warn_not_in_group("broadcast") + return + + opts = BroadcastOptions() + opts.rootRank = group_src + opts.rootTensor = 0 + opts.asyncOp = async_op + if tensor.is_complex(): + tensor = torch.view_as_real(tensor) + work = group.broadcast([tensor], opts) + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def all_reduce(tensor, op=ReduceOp.SUM, group=None, async_op: bool = False): + """ + Reduces the tensor data across all machines in a way that all get the final result. + + After the call ``tensor`` is going to be bitwise identical in all processes. + + Complex tensors are supported. + + Args: + tensor (Tensor): Input and output of the collective. The function + operates in-place. + op (optional): One of the values from + ``torch.distributed.ReduceOp`` + enum. Specifies an operation used for element-wise reductions. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + Examples: + >>> # xdoctest: +SKIP("no rank") + >>> # All tensors below are of torch.int64 type. + >>> # We have 2 process groups, 2 ranks. + >>> device = torch.device(f"cuda:{rank}") + >>> tensor = torch.arange(2, dtype=torch.int64, device=device) + 1 + 2 * rank + >>> tensor + tensor([1, 2], device='cuda:0') # Rank 0 + tensor([3, 4], device='cuda:1') # Rank 1 + >>> dist.all_reduce(tensor, op=ReduceOp.SUM) + >>> tensor + tensor([4, 6], device='cuda:0') # Rank 0 + tensor([4, 6], device='cuda:1') # Rank 1 + + >>> # All tensors below are of torch.cfloat type. + >>> # We have 2 process groups, 2 ranks. + >>> tensor = torch.tensor( + ... [1 + 1j, 2 + 2j], dtype=torch.cfloat, device=device + ... ) + 2 * rank * (1 + 1j) + >>> tensor + tensor([1.+1.j, 2.+2.j], device='cuda:0') # Rank 0 + tensor([3.+3.j, 4.+4.j], device='cuda:1') # Rank 1 + >>> dist.all_reduce(tensor, op=ReduceOp.SUM) + >>> tensor + tensor([4.+4.j, 6.+6.j], device='cuda:0') # Rank 0 + tensor([4.+4.j, 6.+6.j], device='cuda:1') # Rank 1 + + """ + # Dynamo has built-in logic to map legacy distributed ops to functional collectives. + # Let's redirect to a torch function mode that can mimic this logic outside Dynamo + # (e.g., non-strict export implements such a torch function mode). + relevant_args = (tensor,) + if has_torch_function(relevant_args): + return handle_torch_function( + all_reduce, + relevant_args, + tensor, + op=op, + group=group, + async_op=async_op, + ) + + _check_single_tensor(tensor, "tensor") + if _rank_not_in_group(group): + _warn_not_in_group("all_reduce") + return + + if tensor.is_complex(): + if not supports_complex(op): + raise ValueError(f"all_reduce does not support {op} on complex tensors") + tensor = torch.view_as_real(tensor) + + opts = AllreduceOptions() + opts.reduceOp = op + opts.asyncOp = async_op + if group is None: + group = _get_default_group() + + if group in _world.pg_coalesce_state: + # We are in coalescing context, do not issue single operation, just append a collective representation + coll = _CollOp(all_reduce, tensor, None, op, None) + _world.pg_coalesce_state[group].append(coll) + if async_op: + return _IllegalWork() + else: + return None + + work = group.allreduce([tensor], opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +@deprecated( + "`torch.distributed.all_reduce_coalesced` will be deprecated. If you must " + "use it, please revisit our documentation later at " + "https://pytorch.org/docs/main/distributed.html#collective-functions", + category=FutureWarning, +) +def all_reduce_coalesced(tensors, op=ReduceOp.SUM, group=None, async_op: bool = False): + """ + WARNING: at this time individual shape checking is not implemented across nodes. + + For example, if the rank 0 node passes [torch.rand(4), torch.rand(2)] and the + rank 1 node passes [torch.rand(2), torch.rand(2), torch.rand(2)], the allreduce + operation will proceed without complaint and return erroneous outputs. This lack + of shape checking results in significant performance improvements but users of this + function should take extra care to ensure that each node passes in tensors whose + shapes match across nodes. + + Reduces each tensor in tensors (residing on the same device) across all machines + in such a way that all get the final result. + + After the call each tensor in tensors is going to bitwise identical + in all processes. + + Complex tensors are supported. + + Args: + tensors (Union[List[Tensor], Tensor]): Input and output of the collective. + The function operates in-place. + op (Optional[ReduceOp]): One of the values from + ``torch.distributed.ReduceOp`` enum. Specifies an operation used for + element-wise reductions. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (Optional[bool]): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + """ + if isinstance(tensors, torch.Tensor): + tensors = [tensors] + _check_tensor_list(tensors, "tensor") + _ensure_all_tensors_same_dtype(tensors) + if _rank_not_in_group(group): + _warn_not_in_group("all_reduce_coalesced") + return + + if any(t.is_complex() for t in tensors) and not supports_complex(op): + raise ValueError(f"all_reduce does not support {op} on complex tensors") + + tensors = [t if not t.is_complex() else torch.view_as_real(t) for t in tensors] + + opts = AllreduceCoalescedOptions() + opts.reduceOp = op + opts.asyncOp = async_op + group = group or _get_default_group() + work = group.allreduce_coalesced(tensors, opts) + + if async_op: + return work.get_future() + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def reduce( + tensor: torch.Tensor, + dst: int | None = None, + op=ReduceOp.SUM, + group: ProcessGroup | None = None, + async_op: bool = False, + group_dst: int | None = None, +): + """ + Reduces the tensor data across all machines. + + Only the process with rank ``dst`` is going to receive the final result. + + Args: + tensor (Tensor): Input and output of the collective. The function + operates in-place. + dst (int): Destination rank on global process group (regardless of ``group`` argument) + op (optional): One of the values from + ``torch.distributed.ReduceOp`` + enum. Specifies an operation used for element-wise reductions. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + group_dst (int): Destination rank on ``group``. Must specify one of ``group_dst`` + and ``dst`` but not both. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + """ + group = _group_or_default_group(group) + group_dst = _canonicalize_group_rank(group, dst, group_dst, return_global=False) + _check_single_tensor(tensor, "tensor") + if _rank_not_in_group(group): + _warn_not_in_group("reduce") + return + + opts = ReduceOptions() + opts.reduceOp = op + opts.rootRank = group_dst + opts.asyncOp = async_op + work = group.reduce([tensor], opts) + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +def _object_to_tensor(obj, device, group): + with _WaitCounter("pytorch.wait_counter.c10d._object_to_tensor").guard(): + f = io.BytesIO() + _pickler(f).dump(obj) + byte_storage = torch.ByteStorage._from_buffer(f.getvalue()) # type: ignore[attr-defined] + # Do not replace `torch.ByteTensor` or `torch.LongTensor` with torch.tensor and specifying dtype. + # Otherwise, it will cause 100X slowdown. + # See: https://github.com/pytorch/pytorch/issues/65696 + byte_tensor = torch.ByteTensor(byte_storage).to(device) + if get_debug_level() == DebugLevel.DETAIL and is_nccl_available(): + backend = get_backend(group) + if backend == Backend.NCCL: + hash = torch._C._distributed_c10d._hash_tensors([byte_tensor]) + logger.warning( + "_object_to_tensor size: %s hash value: %s", + byte_tensor.numel(), + hash, + ) + local_size = torch.LongTensor([byte_tensor.numel()]).to(device) + return byte_tensor, local_size + + +def _tensor_to_object(tensor, tensor_size, group): + with _WaitCounter("pytorch.wait_counter.c10d._tensor_to_object").guard(): + if get_debug_level() == DebugLevel.DETAIL and is_nccl_available(): + backend = get_backend(group) + if backend == Backend.NCCL: + hash = torch._C._distributed_c10d._hash_tensors([tensor]) + logger.warning( + "_tensor_to_object size: %s hash value: %s", tensor.numel(), hash + ) + tensor = tensor.cpu() + buf = tensor.numpy().tobytes()[:tensor_size] + return _unpickler(io.BytesIO(buf)).load() + + +@_exception_logger +def all_gather_object(object_list, obj, group=None): + """ + Gathers picklable objects from the whole group into a list. + + Similar to :func:`all_gather`, but Python objects can be passed in. + Note that the object must be picklable in order to be gathered. + + Args: + object_list (list[Any]): Output list. It should be correctly sized as the + size of the group for this collective and will contain the output. + obj (Any): Pickable Python object to be broadcast from current process. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + + Returns: + None. If the calling rank is part of this group, the output of the + collective will be populated into the input ``object_list``. If the + calling rank is not part of the group, the passed in ``object_list`` will + be unmodified. + + .. note:: Note that this API differs slightly from the :func:`all_gather` + collective since it does not provide an ``async_op`` handle and thus + will be a blocking call. + + .. note:: For NCCL-based processed groups, internal tensor representations + of objects must be moved to the GPU device before communication takes + place. In this case, the device used is given by + ``torch.cuda.current_device()`` and it is the user's responsibility to + ensure that this is set so that each rank has an individual GPU, via + ``torch.cuda.set_device()``. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`all_gather_object` uses ``pickle`` module implicitly, which is + known to be insecure. It is possible to construct malicious pickle data + which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`all_gather_object` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`all_gather` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> # Assumes world_size of 3. + >>> gather_objects = ["foo", 12, {1: 2}] # any picklable object + >>> output = [None for _ in gather_objects] + >>> dist.all_gather_object(output, gather_objects[dist.get_rank()]) + >>> output + ['foo', 12, {1: 2}] + """ + if _rank_not_in_group(group): + _warn_not_in_group("all_gather_object") + return + + current_device = _get_object_coll_device(group) + input_tensor, local_size = _object_to_tensor(obj, current_device, group) + + # Gather all local sizes. This is so that we can find the max size, and index + # until the correct size when deserializing the tensors. + group_size = get_world_size(group=group) + object_sizes_tensor = torch.zeros( + group_size, dtype=torch.long, device=current_device + ) + object_size_list = [ + object_sizes_tensor[i].unsqueeze(dim=0) for i in range(group_size) + ] + # Allgather tensor sizes + all_gather(object_size_list, local_size, group=group) + max_object_size = int(max(object_size_list).item()) # type: ignore[type-var] + # Resize tensor to max size across all ranks. + input_tensor.resize_(max_object_size) + coalesced_output_tensor = torch.empty( + max_object_size * group_size, dtype=torch.uint8, device=current_device + ) + # Output tensors are nonoverlapping views of coalesced_output_tensor + output_tensors = [ + coalesced_output_tensor[max_object_size * i : max_object_size * (i + 1)] + for i in range(group_size) + ] + all_gather(output_tensors, input_tensor, group=group) + # Deserialize outputs back to object. + for i, tensor in enumerate(output_tensors): + tensor = tensor.type(torch.uint8) + tensor_size = object_size_list[i] + object_list[i] = _tensor_to_object(tensor, tensor_size, group) + + +@_exception_logger +def gather_object( + obj: Any, + object_gather_list: list[Any] | None = None, + dst: int | None = None, + group: ProcessGroup | None = None, + group_dst: int | None = None, +): + """ + Gathers picklable objects from the whole group in a single process. + + Similar to :func:`gather`, but Python objects can be passed in. Note that the + object must be picklable in order to be gathered. + + Args: + obj (Any): Input object. Must be picklable. + object_gather_list (list[Any]): Output list. On the ``dst`` rank, it + should be correctly sized as the size of the group for this + collective and will contain the output. Must be ``None`` on non-dst + ranks. (default is ``None``) + dst (int, optional): Destination rank on global process group (regardless of ``group`` argument). + (If both ``dst`` and ``group_dst`` are None, default is global rank 0) + group: (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + group_dst (int, optional): Destination rank on ``group``. Invalid to specify both ``dst`` and ``group_dst`` + + Returns: + None. On the ``dst`` rank, ``object_gather_list`` will contain the + output of the collective. + + .. note:: Note that this API differs slightly from the gather collective + since it does not provide an async_op handle and thus will be a blocking + call. + + .. note:: For NCCL-based processed groups, internal tensor representations + of objects must be moved to the GPU device before communication takes + place. In this case, the device used is given by + ``torch.cuda.current_device()`` and it is the user's responsibility to + ensure that this is set so that each rank has an individual GPU, via + ``torch.cuda.set_device()``. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`gather_object` uses ``pickle`` module implicitly, which is + known to be insecure. It is possible to construct malicious pickle data + which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`gather_object` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`gather` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> # Assumes world_size of 3. + >>> gather_objects = ["foo", 12, {1: 2}] # any picklable object + >>> output = [None for _ in gather_objects] + >>> dist.gather_object( + ... gather_objects[dist.get_rank()], + ... output if dist.get_rank() == 0 else None, + ... dst=0 + ... ) + >>> # On rank 0 + >>> output + ['foo', 12, {1: 2}] + """ + group = _group_or_default_group(group) + if dst is None and group_dst is None: + dst = 0 + group_dst = _canonicalize_group_rank(group, dst, group_dst, return_global=False) + if _rank_not_in_group(group): + _warn_not_in_group("gather_object") + return + + # Ensure object_gather_list is specified appropriately. + my_group_rank = group.rank() + _validate_output_list_for_rank(my_group_rank, group_dst, object_gather_list) + current_device = _get_object_coll_device(group) + input_tensor, local_size = _object_to_tensor(obj, current_device, group) + + # Gather all local sizes. This is so that we can find the max size, and index + # until the correct size when deserializing the tensors. + group_size = get_world_size(group=group) + object_sizes_tensor = torch.zeros( + group_size, dtype=torch.long, device=current_device + ) + object_size_list = [ + object_sizes_tensor[i].unsqueeze(dim=0) for i in range(group_size) + ] + # Allgather tensor sizes. An all-gather is needed here despite this being a + # gather, since each rank needs to broadcast a tensor of the same (maximal) + # size. + all_gather(object_size_list, local_size, group=group) + max_object_size = int(max(object_size_list).item()) # type: ignore[type-var] + # Resize tensor to max size across all ranks. + input_tensor.resize_(max_object_size) + # Avoid populating output tensors if the result won't be gathered on this rank. + if my_group_rank == group_dst: + coalesced_output_tensor = torch.empty( + max_object_size * group_size, dtype=torch.uint8, device=current_device + ) + # Output tensors are nonoverlapping views of coalesced_output_tensor + output_tensors = [ + coalesced_output_tensor[max_object_size * i : max_object_size * (i + 1)] + for i in range(group_size) + ] + # All ranks call gather with equal-sized tensors. + gather( + input_tensor, + gather_list=output_tensors if my_group_rank == group_dst else None, # type: ignore[possibly-undefined] + group_dst=group_dst, + group=group, + ) + if my_group_rank != group_dst: + return + + if object_gather_list is None: + raise AssertionError("Must provide object_gather_list on dst rank") + # pyrefly: ignore # unbound-name + for i, tensor in enumerate(output_tensors): + tensor = tensor.type(torch.uint8) + tensor_size = object_size_list[i] + object_gather_list[i] = _tensor_to_object(tensor, tensor_size, group) + + +@_exception_logger +def send_object_list( + object_list: list[Any], + dst: int | None = None, + group: ProcessGroup | None = None, + device: torch.device | None = None, + group_dst: int | None = None, + use_batch: bool = False, +): + """ + Sends picklable objects in ``object_list`` synchronously. + + Similar to :func:`send`, but Python objects can be passed in. + Note that all objects in ``object_list`` must be picklable in order to be + sent. + + Args: + object_list (List[Any]): List of input objects to sent. + Each object must be picklable. Receiver must provide lists of equal sizes. + dst (int): Destination rank to send ``object_list`` to. + Destination rank is based on global process group (regardless of ``group`` argument) + group: (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + device (``torch.device``, optional): If not None, the objects are + serialized and converted to tensors which are moved to the + ``device`` before sending. Default is ``None``. + group_dst (int, optional): Destination rank on ``group``. + Must specify one of ``dst`` and ``group_dst`` but not both + use_batch (bool, optional): If True, use batch p2p operations instead of + regular send operations. This avoids initializing 2-rank communicators and + uses existing entire group communicators. See batch_isend_irecv for usage and + assumptions. Default is ``False``. + Returns: + ``None``. + + .. note:: For NCCL-based process groups, internal tensor representations + of objects must be moved to the GPU device before communication takes + place. In this case, the device used is given by + ``torch.cuda.current_device()`` and it is the user's responsibility to + ensure that this is set so that each rank has an individual GPU, via + ``torch.cuda.set_device()``. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`send_object_list` uses ``pickle`` module implicitly, which + is known to be insecure. It is possible to construct malicious pickle + data which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`send_object_list` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`send` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> # Assumes backend is not NCCL + >>> device = torch.device("cpu") + >>> if dist.get_rank() == 0: + >>> # Assumes world_size of 2. + >>> objects = ["foo", 12, {1: 2}] # any picklable object + >>> dist.send_object_list(objects, dst=1, device=device) + >>> else: + >>> objects = [None, None, None] + >>> dist.recv_object_list(objects, src=0, device=device) + >>> objects + ['foo', 12, {1: 2}] + """ + group = _group_or_default_group(group) + group_dst = _canonicalize_group_rank(group, dst, group_dst) + _check_not_self_rank(group, group_dst, "destination") + + if _rank_not_in_group(group): + _warn_not_in_group("send_object_list") + return + + # Current device selection. + # To preserve backwards compatibility, ``device`` is default to ``None`` + # in which case we run current logic of device selection, i.e. + # ``current_device`` is CUDA if backend is NCCL otherwise CPU device. In the + # case it is not ``None`` we move the size and object tensors to be + # sent to this device. + current_device = device or _get_object_coll_device(group) + # Serialize object_list elements to tensors on src rank. + tensor_list, size_list = zip( + *[_object_to_tensor(obj, current_device, group) for obj in object_list] + ) + object_sizes_tensor = torch.cat(size_list) + + # Send object sizes + if use_batch: + batch_isend_irecv( + [P2POp(isend, object_sizes_tensor, group_peer=group_dst, group=group)] + ).pop().wait() + else: + send(object_sizes_tensor, group_dst=group_dst, group=group) + + # Concatenate and send serialized object tensors + # Note: torch.cat will do an extra memory copy to the current device, if the tensor_list + # has only one element, we can skip the copy. + if len(tensor_list) == 1: # type: ignore[possibly-undefined] + object_tensor = tensor_list[0] + else: + object_tensor = torch.cat(tensor_list) + + if use_batch: + batch_isend_irecv( + [P2POp(isend, object_tensor, group_peer=group_dst, group=group)] + ).pop().wait() + else: + send(object_tensor, group_dst=group_dst, group=group) + + +@_exception_logger +def recv_object_list( + object_list: list[Any], + src: int | None = None, + group: ProcessGroup | None = None, + device: torch.device | None = None, + group_src: int | None = None, + use_batch: bool = False, +): + """ + Receives picklable objects in ``object_list`` synchronously. + + Similar to :func:`recv`, but can receive Python objects. + + Args: + object_list (List[Any]): List of objects to receive into. + Must provide a list of sizes equal to the size of the list being sent. + src (int, optional): Source rank from which to recv ``object_list``. + Source rank is based on global process group (regardless of ``group`` argument) + Will receive from any rank if set to None. Default is ``None``. + group: (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + device (``torch.device``, optional): If not None, receives on this device. + Default is ``None``. + group_src (int, optional): Destination rank on ``group``. Invalid to specify both ``src`` and ``group_src``. + use_batch (bool, optional): If True, use batch p2p operations instead of + regular send operations. This avoids initializing 2-rank communicators and + uses existing entire group communicators. See batch_isend_irecv for usage and + assumptions. Default is ``False``. + + Returns: + Sender rank. -1 if rank is not part of the group. If rank is part of the group, + ``object_list`` will contain the sent objects from ``src`` rank. + + .. note:: For NCCL-based process groups, internal tensor representations + of objects must be moved to the GPU device before communication takes + place. In this case, the device used is given by + ``torch.cuda.current_device()`` and it is the user's responsibility to + ensure that this is set so that each rank has an individual GPU, via + ``torch.cuda.set_device()``. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`recv_object_list` uses ``pickle`` module implicitly, which + is known to be insecure. It is possible to construct malicious pickle + data which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`recv_object_list` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`recv` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> # Assumes backend is not NCCL + >>> device = torch.device("cpu") + >>> if dist.get_rank() == 0: + >>> # Assumes world_size of 2. + >>> objects = ["foo", 12, {1: 2}] # any picklable object + >>> dist.send_object_list(objects, dst=1, device=device) + >>> else: + >>> objects = [None, None, None] + >>> dist.recv_object_list(objects, src=0, device=device) + >>> objects + ['foo', 12, {1: 2}] + """ + group = _group_or_default_group(group) + group_src = _canonicalize_group_rank(group, src, group_src) + _check_not_self_rank(group, group_src, "source") + + if _rank_not_in_group(group): + _warn_not_in_group("recv_object_list") + return -1 + + # Current device selection. + # To preserve backwards compatibility, ``device`` is default to ``None`` + # in which case we run current logic of device selection, i.e. + # ``current_device`` is CUDA if backend is NCCL otherwise CPU device. In the + # case it is not ``None`` we move the size and object tensors to be + # received to this device. + current_device = device or _get_object_coll_device(group) + object_sizes_tensor = torch.empty( + len(object_list), dtype=torch.long, device=current_device + ) + + # Receive object sizes + if use_batch: + work = batch_isend_irecv( + [ + P2POp( + irecv, + object_sizes_tensor, + group_peer=group_src, + group=group, + ) + ] + ).pop() + work.wait() + rank_sizes = get_global_rank(group, group_src) + else: + rank_sizes = recv(object_sizes_tensor, group=group, group_src=group_src) + + # Tensor to receive serialized objects into. + object_tensor = torch.empty( # type: ignore[call-overload] + torch.sum(object_sizes_tensor).item(), # type: ignore[arg-type] + dtype=torch.uint8, + device=current_device, + ) + + if use_batch: + work = batch_isend_irecv( + [ + P2POp( + irecv, + object_tensor, + group_peer=group_src, + group=group, + ) + ] + ).pop() + work.wait() + rank_objects = get_global_rank(group, group_src) + else: + rank_objects = recv(object_tensor, group=group, group_src=group_src) + if rank_sizes != rank_objects: + raise AssertionError("Mismatch in return ranks for object sizes and objects.") + # Deserialize objects using their stored sizes. + offset = 0 + for i, obj_size in enumerate(object_sizes_tensor): + obj_view = object_tensor[offset : offset + obj_size] + obj_view = obj_view.type(torch.uint8) + offset += obj_size + object_list[i] = _tensor_to_object(obj_view, obj_size, group) + return rank_objects + + +@_exception_logger +def broadcast_object_list( + object_list: list[Any], + src: int | None = None, + group: ProcessGroup | None = None, + device: torch.device | None = None, + group_src: int | None = None, +): + """ + Broadcasts picklable objects in ``object_list`` to the whole group. + + Similar to :func:`broadcast`, but Python objects can be passed in. + Note that all objects in ``object_list`` must be picklable in order to be + broadcasted. + + Args: + object_list (List[Any]): List of input objects to broadcast. + Each object must be picklable. Only objects on the ``src`` rank will + be broadcast, but each rank must provide lists of equal sizes. + src (int): Source rank from which to broadcast ``object_list``. + Source rank is based on global process group (regardless of ``group`` argument) + group: (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + device (``torch.device``, optional): If not None, the objects are + serialized and converted to tensors which are moved to the + ``device`` before broadcasting. Default is ``None``. + group_src (int): Source rank on ``group``. Must not specify one of ``group_src`` + and ``src`` but not both. + + Returns: + ``None``. If rank is part of the group, ``object_list`` will contain the + broadcasted objects from ``src`` rank. + + .. note:: For NCCL-based process groups, internal tensor representations + of objects must be moved to the GPU device before communication takes + place. In this case, the device used is given by + ``torch.cuda.current_device()`` and it is the user's responsibility to + ensure that this is set so that each rank has an individual GPU, via + ``torch.cuda.set_device()``. + + .. note:: Note that this API differs slightly from the :func:`broadcast` + collective since it does not provide an ``async_op`` handle and thus + will be a blocking call. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`broadcast_object_list` uses ``pickle`` module implicitly, which + is known to be insecure. It is possible to construct malicious pickle + data which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`broadcast_object_list` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`broadcast` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> if dist.get_rank() == 0: + >>> # Assumes world_size of 3. + >>> objects = ["foo", 12, {1: 2}] # any picklable object + >>> else: + >>> objects = [None, None, None] + >>> # Assumes backend is not NCCL + >>> device = torch.device("cpu") + >>> dist.broadcast_object_list(objects, src=0, device=device) + >>> objects + ['foo', 12, {1: 2}] + """ + group = _group_or_default_group(group) + if src is None and group_src is None: + src = 0 + group_src = _canonicalize_group_rank(group, src, group_src, return_global=False) + if _rank_not_in_group(group): + _warn_not_in_group("broadcast_object_list") + return + + # Current device selection. + # To preserve backwards compatibility, ``device`` is default to ``None`` + # in which case we run current logic of device selection, i.e. + # ``current_device`` is CUDA if backend is NCCL otherwise CPU device. In the + # case it is not ``None`` we move the size and object tensors to be + # broadcasted to this device. + current_device = device or _get_object_coll_device(group) + my_group_rank = group.rank() + # Serialize object_list elements to tensors on src rank. + if my_group_rank == group_src: + tensor_list, size_list = zip( + *[_object_to_tensor(obj, current_device, group) for obj in object_list] + ) + object_sizes_tensor = torch.cat(size_list) + else: + object_sizes_tensor = torch.empty( + len(object_list), dtype=torch.long, device=current_device + ) + + # Broadcast object sizes + broadcast(object_sizes_tensor, group_src=group_src, group=group) + + # Concatenate and broadcast serialized object tensors + # Note: torch.cat will do an extra memory copy to the current device, if the tensor_list + # has only one element, we can skip the copy. + if my_group_rank == group_src: + if len(tensor_list) == 1: # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + object_tensor = tensor_list[0] + else: + # pyrefly: ignore [unbound-name] + object_tensor = torch.cat(tensor_list) + else: + object_tensor = torch.empty( # type: ignore[call-overload] + torch.sum(object_sizes_tensor).item(), # type: ignore[arg-type] + dtype=torch.uint8, + device=current_device, + ) + + broadcast(object_tensor, group_src=group_src, group=group) + # Deserialize objects using their stored sizes. + offset = 0 + if my_group_rank != group_src: + for i, obj_size in enumerate(object_sizes_tensor): + obj_view = object_tensor[offset : offset + obj_size] + obj_view = obj_view.type(torch.uint8) + offset += obj_size + object_list[i] = _tensor_to_object(obj_view, obj_size, group) + + +@_exception_logger +def scatter_object_list( + scatter_object_output_list: list[Any], + scatter_object_input_list: list[Any] | None = None, + src: int | None = None, + group: ProcessGroup | None = None, + group_src: int | None = None, +): + """ + Scatters picklable objects in ``scatter_object_input_list`` to the whole group. + + Similar to :func:`scatter`, but Python objects can be passed in. On + each rank, the scattered object will be stored as the first element of + ``scatter_object_output_list``. Note that all objects in + ``scatter_object_input_list`` must be picklable in order to be scattered. + + Args: + scatter_object_output_list (List[Any]): Non-empty list whose first + element will store the object scattered to this rank. + scatter_object_input_list (List[Any], optional): List of input objects to scatter. + Each object must be picklable. Only objects on the ``src`` rank will + be scattered, and the argument can be ``None`` for non-src ranks. + src (int): Source rank from which to scatter ``scatter_object_input_list``. + Source rank is based on global process group (regardless of ``group`` argument). + (If both ``src`` and ``group_src`` are None, default is global rank 0) + group: (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. Default is ``None``. + group_src (int, optional): Source rank on ``group``. Invalid to specify both ``src`` and ``group_src`` + + Returns: + ``None``. If rank is part of the group, ``scatter_object_output_list`` + will have its first element set to the scattered object for this rank. + + .. note:: Note that this API differs slightly from the scatter collective + since it does not provide an ``async_op`` handle and thus will be a + blocking call. + + .. warning:: + Object collectives have a number of serious performance and scalability + limitations. See :ref:`object_collectives` for details. + + .. warning:: + :func:`scatter_object_list` uses ``pickle`` module implicitly, which + is known to be insecure. It is possible to construct malicious pickle + data which will execute arbitrary code during unpickling. Only call this + function with data you trust. + + .. warning:: + Calling :func:`scatter_object_list` with GPU tensors is not well supported + and inefficient as it incurs GPU -> CPU transfer since tensors would be + pickled. Please consider using :func:`scatter` instead. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> if dist.get_rank() == 0: + >>> # Assumes world_size of 3. + >>> objects = ["foo", 12, {1: 2}] # any picklable object + >>> else: + >>> # Can be any list on non-src ranks, elements are not used. + >>> objects = [None, None, None] + >>> output_list = [None] + >>> dist.scatter_object_list(output_list, objects, src=0) + >>> # Rank i gets objects[i]. For example, on rank 2: + >>> output_list + [{1: 2}] + """ + group = _group_or_default_group(group) + if src is None and group_src is None: + src = 0 + group_src = _canonicalize_group_rank(group, src, group_src, return_global=False) + if _rank_not_in_group(group): + _warn_not_in_group("scatter_object_list") + return + + if ( + not isinstance(scatter_object_output_list, list) + or len(scatter_object_output_list) < 1 + ): + raise ValueError( + "Expected argument scatter_object_output_list to be a list of size at least 1." + ) + + my_group_rank = group.rank() + pg_device = _get_object_coll_device(group) + if my_group_rank == group_src: + if scatter_object_input_list is None: + raise ValueError( + "source rank must provide non-None scatter_object_input_list" + ) + tensor_list, tensor_sizes = zip( + *[ + _object_to_tensor(obj, pg_device, group) + for obj in scatter_object_input_list + ] + ) + tensor_list, tensor_sizes = list(tensor_list), list(tensor_sizes) + + # Src rank broadcasts the maximum tensor size. This is because all ranks are + # expected to call into scatter() with equal-sized tensors. + max_tensor_size = max(tensor_sizes) # type: ignore[possibly-undefined] + for tensor in tensor_list: # type: ignore[possibly-undefined] + tensor.resize_(max_tensor_size) + else: + max_tensor_size = torch.tensor([0], dtype=torch.long, device=pg_device) + broadcast(max_tensor_size, group_src=group_src, group=group) + + # Scatter actual serialized objects + # pyrefly: ignore [no-matching-overload] + output_tensor = torch.empty( + max_tensor_size.item(), dtype=torch.uint8, device=pg_device + ) + scatter( + output_tensor, + scatter_list=None if my_group_rank != group_src else tensor_list, # type: ignore[possibly-undefined] + group_src=group_src, + group=group, + ) + + # Scatter per-object sizes to trim tensors when deserializing back to object + obj_tensor_size = torch.tensor([0], dtype=torch.long, device=pg_device) + scatter( + obj_tensor_size, + scatter_list=None if my_group_rank != group_src else tensor_sizes, # type: ignore[possibly-undefined] + group_src=group_src, + group=group, + ) + + # Deserialize back to object + scatter_object_output_list[0] = _tensor_to_object( + output_tensor, obj_tensor_size, group + ) + + +@_exception_logger +def all_gather(tensor_list, tensor, group=None, async_op=False): + """ + Gathers tensors from the whole group in a list. + + Complex and uneven sized tensors are supported. + + Args: + tensor_list (list[Tensor]): Output list. It should contain + correctly-sized tensors to be used for output of the collective. + Uneven sized tensors are supported. + tensor (Tensor): Tensor to be broadcast from current process. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + Examples: + >>> # xdoctest: +SKIP("need process group init") + >>> # All tensors below are of torch.int64 dtype. + >>> # We have 2 process groups, 2 ranks. + >>> device = torch.device(f"cuda:{rank}") + >>> tensor_list = [ + ... torch.zeros(2, dtype=torch.int64, device=device) for _ in range(2) + ... ] + >>> tensor_list + [tensor([0, 0], device='cuda:0'), tensor([0, 0], device='cuda:0')] # Rank 0 + [tensor([0, 0], device='cuda:1'), tensor([0, 0], device='cuda:1')] # Rank 1 + >>> tensor = torch.arange(2, dtype=torch.int64, device=device) + 1 + 2 * rank + >>> tensor + tensor([1, 2], device='cuda:0') # Rank 0 + tensor([3, 4], device='cuda:1') # Rank 1 + >>> dist.all_gather(tensor_list, tensor) + >>> tensor_list + [tensor([1, 2], device='cuda:0'), tensor([3, 4], device='cuda:0')] # Rank 0 + [tensor([1, 2], device='cuda:1'), tensor([3, 4], device='cuda:1')] # Rank 1 + + >>> # All tensors below are of torch.cfloat dtype. + >>> # We have 2 process groups, 2 ranks. + >>> tensor_list = [ + ... torch.zeros(2, dtype=torch.cfloat, device=device) for _ in range(2) + ... ] + >>> tensor_list + [tensor([0.+0.j, 0.+0.j], device='cuda:0'), tensor([0.+0.j, 0.+0.j], device='cuda:0')] # Rank 0 + [tensor([0.+0.j, 0.+0.j], device='cuda:1'), tensor([0.+0.j, 0.+0.j], device='cuda:1')] # Rank 1 + >>> tensor = torch.tensor( + ... [1 + 1j, 2 + 2j], dtype=torch.cfloat, device=device + ... ) + 2 * rank * (1 + 1j) + >>> tensor + tensor([1.+1.j, 2.+2.j], device='cuda:0') # Rank 0 + tensor([3.+3.j, 4.+4.j], device='cuda:1') # Rank 1 + >>> dist.all_gather(tensor_list, tensor) + >>> tensor_list + [tensor([1.+1.j, 2.+2.j], device='cuda:0'), tensor([3.+3.j, 4.+4.j], device='cuda:0')] # Rank 0 + [tensor([1.+1.j, 2.+2.j], device='cuda:1'), tensor([3.+3.j, 4.+4.j], device='cuda:1')] # Rank 1 + + """ + # Dynamo has built-in logic to map legacy distributed ops to functional collectives. + # Let's redirect to a torch function mode that can mimic this logic outside Dynamo + # (e.g., non-strict export implements such a torch function mode). + relevant_args = (tensor,) + if has_torch_function(relevant_args): + return handle_torch_function( + all_gather, + relevant_args, + tensor_list, + tensor, + group=group, + async_op=async_op, + ) + + _check_tensor_list(tensor_list, "tensor_list") + _check_single_tensor(tensor, "tensor") + _ensure_all_tensors_same_dtype(tensor_list, tensor) + if _rank_not_in_group(group): + _warn_not_in_group("all_gather") + return + + tensor_list = [ + t if not t.is_complex() else torch.view_as_real(t) for t in tensor_list + ] + tensor = tensor if not tensor.is_complex() else torch.view_as_real(tensor) + + group = group or _get_default_group() + opts = AllgatherOptions() + opts.asyncOp = async_op + work = group.allgather([tensor_list], [tensor], opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def all_gather_into_tensor(output_tensor, input_tensor, group=None, async_op=False): + """ + Gather tensors from all ranks and put them in a single output tensor. + + This function requires all tensors to be the same size on each process. + + Args: + output_tensor (Tensor): Output tensor to accommodate tensor elements + from all ranks. It must be correctly sized to have one of the + following forms: + (i) a concatenation of all the input tensors along the primary + dimension; for definition of "concatenation", see ``torch.cat()``; + (ii) a stack of all the input tensors along the primary dimension; + for definition of "stack", see ``torch.stack()``. + Examples below may better explain the supported output forms. + input_tensor (Tensor): Tensor to be gathered from current rank. + Different from the ``all_gather`` API, the input tensors in this + API must have the same size across all ranks. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + Examples: + >>> # xdoctest: +SKIP("need process group init") + >>> # All tensors below are of torch.int64 dtype and on CUDA devices. + >>> # We have two ranks. + >>> device = torch.device(f"cuda:{rank}") + >>> tensor_in = torch.arange(2, dtype=torch.int64, device=device) + 1 + 2 * rank + >>> tensor_in + tensor([1, 2], device='cuda:0') # Rank 0 + tensor([3, 4], device='cuda:1') # Rank 1 + >>> # Output in concatenation form + >>> tensor_out = torch.zeros(world_size * 2, dtype=torch.int64, device=device) + >>> dist.all_gather_into_tensor(tensor_out, tensor_in) + >>> tensor_out + tensor([1, 2, 3, 4], device='cuda:0') # Rank 0 + tensor([1, 2, 3, 4], device='cuda:1') # Rank 1 + >>> # Output in stack form + >>> tensor_out2 = torch.zeros(world_size, 2, dtype=torch.int64, device=device) + >>> dist.all_gather_into_tensor(tensor_out2, tensor_in) + >>> tensor_out2 + tensor([[1, 2], + [3, 4]], device='cuda:0') # Rank 0 + tensor([[1, 2], + [3, 4]], device='cuda:1') # Rank 1 + """ + # Dynamo has built-in logic to map legacy distributed ops to functional collectives. + # Let's redirect to a torch function mode that can mimic this logic outside Dynamo + # (e.g., non-strict export implements such a torch function mode). + relevant_args = (input_tensor,) + if has_torch_function(relevant_args): + return handle_torch_function( + all_gather_into_tensor, + relevant_args, + output_tensor, + input_tensor, + group=group, + async_op=async_op, + ) + + _check_single_tensor(input_tensor, "input_tensor") + _check_single_tensor(output_tensor, "output_tensor") + if _rank_not_in_group(group): + _warn_not_in_group("all_gather_into_tensor") + return + + output_tensor = ( + output_tensor + if not output_tensor.is_complex() + else torch.view_as_real(output_tensor) + ) + input_tensor = ( + input_tensor + if not input_tensor.is_complex() + else torch.view_as_real(input_tensor) + ) + + opts = AllgatherOptions() + opts.asyncOp = async_op + + group = group or _get_default_group() + + if group in _world.pg_coalesce_state: + # We are in coalescing context, do not issue single operation, just append a collective representation + coll = _CollOp(all_gather_into_tensor, input_tensor, output_tensor) + _world.pg_coalesce_state[group].append(coll) + if async_op: + return _IllegalWork() + else: + return None + + work = group._allgather_base(output_tensor, input_tensor, opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +@deprecated( + "`torch.distributed._all_gather_base` is a private function and will be deprecated. " + "Please use `torch.distributed.all_gather_into_tensor` instead.", + category=FutureWarning, +) +def _all_gather_base(output_tensor, input_tensor, group=None, async_op: bool = False): + """ + Single tensor all gather. Gathers a single tensor from all ranks, and puts them in a single output tensor. + + Args: + output_tensor (Tensor): Output tensor. It should contain + correctly-sized tensors to be used for output of the collective. + input_tensor (Tensor): Tensor to be broadcast from current process. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + .. warning:: + `_all_gather_base` is a private function. Users should use + `all_gather_into_tensor` instead. + + """ + return all_gather_into_tensor(output_tensor, input_tensor, group, async_op) + + +@_exception_logger +@deprecated( + "`torch.distributed.all_gather_coalesced` will be deprecated. If you must use it, " + "please revisit our documentation later at " + "https://pytorch.org/docs/main/distributed.html#collective-functions", + category=FutureWarning, +) +def all_gather_coalesced( + output_tensor_lists, input_tensor_list, group=None, async_op: bool = False +): + """ + Gathers input tensors from the whole group in a list in a coalesced manner. + + Complex tensors are supported. + + Args: + output_tensor_lists (list[list[Tensor]]): Output list. It should contain + correctly-sized tensors to be used for output of the collective. + input_tensor_list (list[Tensor]): Tensors to be broadcast from + current process. At least one tensor has to be non empty. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + Example: + we have 2 process groups, 2 ranks. + rank 0 passes: + input_tensor_list = [[[1, 1], [1, 1]], [2], [3, 3]] + output_tensor_lists = + [[[[-1, -1], [-1, -1]], [-1], [-1, -1]], + [[[-1, -1], [-1, -1]], [-1], [-1, -1]]] + rank 1 passes: + input_tensor_list = [[[3, 3], [3, 3]], [5], [1, 1]] + output_tensor_lists = + [[[[-1, -1], [-1, -1]], [-1], [-1, -1]], + [[[-1, -1], [-1, -1]], [-1], [-1, -1]]] + both rank 0 and 1 get: + output_tensor_lists = + [[[1, 1], [1, 1]], [2], [3, 3]], + [[3, 3], [3, 3]], [5], [1, 1]]]. + + WARNING: at this time individual shape checking is not implemented across nodes. + For example, if the rank 0 node passes [torch.rand(4), torch.rand(2)] and the + rank 1 node passes [torch.rand(2), torch.rand(2), torch.rand(2)], the + all_gather_coalesced operation will proceed without complaint and return + erroneous outputs. This lack of shape checking results in significant + performance improvements but users of this function should take extra care + to ensure that each node passes in tensors whose shapes match across nodes. + """ + # We only check basic compatibility with C++ params here, C++ code will + # do shape and type checking. + if _rank_not_in_group(group): + _warn_not_in_group("all_gather_coalesced") + return + _check_tensor_list(input_tensor_list, "input_tensor_list") + _ensure_all_tensors_same_dtype(input_tensor_list) + if not isinstance(output_tensor_lists, list): + raise TypeError( + "Invalid function argument: output_tensor_lists should be a list" + ) + for output_tensor_list in output_tensor_lists: + _check_tensor_list(output_tensor_list, "output_tensor_lists") + _ensure_all_tensors_same_dtype(output_tensor_list) + + output_tensor_lists = [ + [t if not t.is_complex() else torch.view_as_real(t) for t in l] + for l in output_tensor_lists + ] + input_tensor_list = [ + t if not t.is_complex() else torch.view_as_real(t) for t in input_tensor_list + ] + + group = group or _get_default_group() + opts = AllgatherOptions() + opts.asyncOp = async_op + work = group.allgather_coalesced(output_tensor_lists, input_tensor_list, opts) + + if async_op: + return work.get_future() + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +def _validate_output_list_for_rank(my_rank: int, dst: int, gather_list): + if dst == my_rank: + if not gather_list: + raise ValueError( + "Argument ``gather_list`` must be specified on destination rank." + ) + elif gather_list: + raise ValueError( + "Argument ``gather_list`` must NOT be specified on non-destination ranks." + ) + + +@_exception_logger +def gather( + tensor: torch.Tensor, + gather_list: list[torch.Tensor] | None = None, + dst: int | None = None, + group: ProcessGroup | None = None, + async_op: bool = False, + group_dst: int | None = None, +): + """ + Gathers a list of tensors in a single process. + + This function requires all tensors to be the same size on each process. + + Args: + tensor (Tensor): Input tensor. + gather_list (list[Tensor], optional): List of appropriately, + same-sized tensors to use for gathered data + (default is None, must be specified on the destination rank) + dst (int, optional): Destination rank on global process group (regardless of ``group`` argument). + (If both ``dst`` and ``group_dst`` are None, default is global rank 0) + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + group_dst (int, optional): Destination rank on ``group``. Invalid to specify both ``dst`` and ``group_dst`` + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + .. note:: Note that all Tensors in gather_list must have the same size. + + Example:: + >>> # xdoctest: +SKIP("no rank") + >>> # We have 2 process groups, 2 ranks. + >>> tensor_size = 2 + >>> device = torch.device(f'cuda:{rank}') + >>> tensor = torch.ones(tensor_size, device=device) + rank + >>> if dist.get_rank() == 0: + >>> gather_list = [torch.zeros_like(tensor, device=device) for i in range(2)] + >>> else: + >>> gather_list = None + >>> dist.gather(tensor, gather_list, dst=0) + >>> # Rank 0 gets gathered data. + >>> gather_list + [tensor([1., 1.], device='cuda:0'), tensor([2., 2.], device='cuda:0')] # Rank 0 + None # Rank 1 + + """ + _check_single_tensor(tensor, "tensor") + + # Parameter ``gather_list`` may be left unspecified on non-dst ranks. + if gather_list: + _check_tensor_list(gather_list, "gather_list") + else: + gather_list = [] + _ensure_all_tensors_same_dtype(tensor, gather_list) + group = _group_or_default_group(group) + if _rank_not_in_group(group): + _warn_not_in_group("gather") + return + if dst is None and group_dst is None: + dst = 0 + group_dst = _canonicalize_group_rank(group, dst, group_dst, return_global=False) + my_group_rank = group.rank() + _validate_output_list_for_rank(my_group_rank, group_dst, gather_list) + output_tensors = [gather_list] if group_dst == my_group_rank else [] + input_tensors = [tensor] + + opts = GatherOptions() + opts.rootRank = group_dst + opts.asyncOp = async_op + work = group.gather(output_tensors, input_tensors, opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def scatter( + tensor: torch.Tensor, + scatter_list: list[torch.Tensor] | None = None, + src: int | None = None, + group: ProcessGroup | None = None, + async_op: bool = False, + group_src: int | None = None, +): + """ + Scatters a list of tensors to all processes in a group. + + Each process will receive exactly one tensor and store its data in the + ``tensor`` argument. + + Complex tensors are supported. + + Args: + tensor (Tensor): Output tensor. + scatter_list (list[Tensor]): List of tensors to scatter (default is + None, must be specified on the source rank) + src (int): Source rank on global process group (regardless of ``group`` argument). + (If both ``src`` and ``group_src`` are None, default is global rank 0) + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + group_src (int, optional): Source rank on ``group``. Invalid to specify both ``src`` and ``group_src`` + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + .. note:: Note that all Tensors in scatter_list must have the same size. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> tensor_size = 2 + >>> device = torch.device(f'cuda:{rank}') + >>> output_tensor = torch.zeros(tensor_size, device=device) + >>> if dist.get_rank() == 0: + >>> # Assumes world_size of 2. + >>> # Only tensors, all of which must be the same size. + >>> t_ones = torch.ones(tensor_size, device=device) + >>> t_fives = torch.ones(tensor_size, device=device) * 5 + >>> scatter_list = [t_ones, t_fives] + >>> else: + >>> scatter_list = None + >>> dist.scatter(output_tensor, scatter_list, src=0) + >>> # Rank i gets scatter_list[i]. + >>> output_tensor + tensor([1., 1.], device='cuda:0') # Rank 0 + tensor([5., 5.], device='cuda:1') # Rank 1 + + """ + _check_single_tensor(tensor, "tensor") + # Parameter ``scatter_list`` may be left unspecified on non-src ranks. + if scatter_list: + _check_tensor_list(scatter_list, "scatter_list") + else: + scatter_list = [] + _ensure_all_tensors_same_dtype(tensor, scatter_list) + group = _group_or_default_group(group) + if src is None and group_src is None: + src = 0 + group_src = _canonicalize_group_rank(group, src, group_src, return_global=False) + if _rank_not_in_group(group): + _warn_not_in_group("scatter") + return + scatter_list = [ + t if not t.is_complex() else torch.view_as_real(t) for t in scatter_list + ] + tensor = tensor if not tensor.is_complex() else torch.view_as_real(tensor) + + my_group_rank = group.rank() + if group_src == my_group_rank: + if not scatter_list: + raise ValueError( + "Argument ``scatter_list`` must be specified on source rank." + ) + input_tensors = [scatter_list] + output_tensors = [tensor] + else: + if scatter_list: + raise ValueError( + "Argument ``scatter_list`` must NOT be specified on non-source ranks." + ) + input_tensors = [] + output_tensors = [tensor] + + opts = ScatterOptions() + opts.rootRank = group_src + opts.asyncOp = async_op + work = group.scatter(output_tensors, input_tensors, opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def reduce_scatter( + output, input_list, op=ReduceOp.SUM, group=None, async_op: bool = False +): + """ + Reduces, then scatters a list of tensors to all processes in a group. + + Args: + output (Tensor): Output tensor. + input_list (list[Tensor]): List of tensors to reduce and scatter. + op (optional): One of the values from + ``torch.distributed.ReduceOp`` + enum. Specifies an operation used for element-wise reductions. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + """ + _check_single_tensor(output, "output") + _check_tensor_list(input_list, "input_list") + _ensure_all_tensors_same_dtype(output, input_list) + if _rank_not_in_group(group): + _warn_not_in_group("reduce_scatter") + return + + opts = ReduceScatterOptions() + opts.reduceOp = op + opts.asyncOp = async_op + + group = group or _get_default_group() + work = group.reduce_scatter([output], [input_list], opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def reduce_scatter_tensor(output, input, op=ReduceOp.SUM, group=None, async_op=False): + """ + Reduces, then scatters a tensor to all ranks in a group. + + Args: + output (Tensor): Output tensor. It should have the same size across all + ranks. + input (Tensor): Input tensor to be reduced and scattered. Its size + should be output tensor size times the world size. The input tensor + can have one of the following shapes: + (i) a concatenation of the output tensors along the primary + dimension, or + (ii) a stack of the output tensors along the primary dimension. + For definition of "concatenation", see ``torch.cat()``. + For definition of "stack", see ``torch.stack()``. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + Examples: + >>> # xdoctest: +SKIP("need process group init") + >>> # All tensors below are of torch.int64 dtype and on CUDA devices. + >>> # We have two ranks. + >>> device = torch.device(f"cuda:{rank}") + >>> tensor_out = torch.zeros(2, dtype=torch.int64, device=device) + >>> # Input in concatenation form + >>> tensor_in = torch.arange(world_size * 2, dtype=torch.int64, device=device) + >>> tensor_in + tensor([0, 1, 2, 3], device='cuda:0') # Rank 0 + tensor([0, 1, 2, 3], device='cuda:1') # Rank 1 + >>> dist.reduce_scatter_tensor(tensor_out, tensor_in) + >>> tensor_out + tensor([0, 2], device='cuda:0') # Rank 0 + tensor([4, 6], device='cuda:1') # Rank 1 + >>> # Input in stack form + >>> tensor_in = torch.reshape(tensor_in, (world_size, 2)) + >>> tensor_in + tensor([[0, 1], + [2, 3]], device='cuda:0') # Rank 0 + tensor([[0, 1], + [2, 3]], device='cuda:1') # Rank 1 + >>> dist.reduce_scatter_tensor(tensor_out, tensor_in) + >>> tensor_out + tensor([0, 2], device='cuda:0') # Rank 0 + tensor([4, 6], device='cuda:1') # Rank 1 + + """ + # Dynamo has built-in logic to map legacy distributed ops to functional collectives. + # Let's redirect to a torch function mode that can mimic this logic outside Dynamo + # (e.g., non-strict export implements such a torch function mode). + relevant_args = (input,) + if has_torch_function(relevant_args): + return handle_torch_function( + reduce_scatter_tensor, + relevant_args, + output, + input, + op=op, + group=group, + async_op=async_op, + ) + + _check_single_tensor(output, "output") + _check_single_tensor(input, "input") + + if _rank_not_in_group(group): + _warn_not_in_group("reduce_scatter_tensor") + return + + opts = ReduceScatterOptions() + opts.reduceOp = op + opts.asyncOp = async_op + + group = group or _get_default_group() + + # Check if we are in coalescing context + # If we are, do not issue single operation, just append a collective representation + if group in _world.pg_coalesce_state: + coll = _CollOp(reduce_scatter_tensor, input, output, op, None) + _world.pg_coalesce_state[group].append(coll) + if async_op: + return _IllegalWork() + else: + return None + + work = group._reduce_scatter_base(output, input, opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@deprecated( + "`torch.distributed._reduce_scatter_base` is a private function and will be deprecated. " + "Please use `torch.distributed.reduce_scatter_tensor` instead.", + category=FutureWarning, +) +def _reduce_scatter_base( + output, input, op=ReduceOp.SUM, group=None, async_op: bool = False +): + """ + Reduces, then scatters a flattened tensor to all processes in a group. + + Args: + output (Tensor): Output tensor. + input (Tensor): Input tensor that is of size output tensor size times world size + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + .. warning:: + `_reduce_scatter_base` is a private function. Users should use + `reduce_scatter_tensor` instead. + + """ + return reduce_scatter_tensor(output, input, op, group, async_op) + + +@_exception_logger +def all_to_all_single( + output, + input, + output_split_sizes=None, + input_split_sizes=None, + group=None, + async_op: bool = False, +): + """ + Split input tensor and then scatter the split list to all processes in a group. + + Later the received tensors are concatenated from all the processes in the group + and returned as a single output tensor. + + Complex tensors are supported. + + Args: + output (Tensor): Gathered concatenated output tensor. + input (Tensor): Input tensor to scatter. + output_split_sizes: (list[Int], optional): Output split sizes for dim 0 + if specified None or empty, dim 0 of ``output`` tensor must divide + equally by ``world_size``. + input_split_sizes: (list[Int], optional): Input split sizes for dim 0 + if specified None or empty, dim 0 of ``input`` tensor must divide + equally by ``world_size``. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + .. warning:: + `all_to_all_single` is experimental and subject to change. + + Examples: + >>> # xdoctest: +SKIP("Undefined rank") + >>> input = torch.arange(4) + rank * 4 + >>> input + tensor([0, 1, 2, 3]) # Rank 0 + tensor([4, 5, 6, 7]) # Rank 1 + tensor([8, 9, 10, 11]) # Rank 2 + tensor([12, 13, 14, 15]) # Rank 3 + >>> output = torch.empty([4], dtype=torch.int64) + >>> dist.all_to_all_single(output, input) + >>> output + tensor([0, 4, 8, 12]) # Rank 0 + tensor([1, 5, 9, 13]) # Rank 1 + tensor([2, 6, 10, 14]) # Rank 2 + tensor([3, 7, 11, 15]) # Rank 3 + + >>> # Essentially, it is similar to following operation: + >>> scatter_list = list(input.chunk(world_size)) + >>> gather_list = list(output.chunk(world_size)) + >>> for i in range(world_size): + >>> dist.scatter(gather_list[i], scatter_list if i == rank else [], src = i) + + >>> # Another example with uneven split + >>> input + tensor([0, 1, 2, 3, 4, 5]) # Rank 0 + tensor([10, 11, 12, 13, 14, 15, 16, 17, 18]) # Rank 1 + tensor([20, 21, 22, 23, 24]) # Rank 2 + tensor([30, 31, 32, 33, 34, 35, 36]) # Rank 3 + >>> input_splits + [2, 2, 1, 1] # Rank 0 + [3, 2, 2, 2] # Rank 1 + [2, 1, 1, 1] # Rank 2 + [2, 2, 2, 1] # Rank 3 + >>> output_splits + [2, 3, 2, 2] # Rank 0 + [2, 2, 1, 2] # Rank 1 + [1, 2, 1, 2] # Rank 2 + [1, 2, 1, 1] # Rank 3 + >>> output = ... + >>> dist.all_to_all_single(output, input, output_splits, input_splits) + >>> output + tensor([ 0, 1, 10, 11, 12, 20, 21, 30, 31]) # Rank 0 + tensor([ 2, 3, 13, 14, 22, 32, 33]) # Rank 1 + tensor([ 4, 15, 16, 23, 34, 35]) # Rank 2 + tensor([ 5, 17, 18, 24, 36]) # Rank 3 + + + >>> # Another example with tensors of torch.cfloat type. + >>> input = torch.tensor( + ... [1 + 1j, 2 + 2j, 3 + 3j, 4 + 4j], dtype=torch.cfloat + ... ) + 4 * rank * (1 + 1j) + >>> input + tensor([1+1j, 2+2j, 3+3j, 4+4j]) # Rank 0 + tensor([5+5j, 6+6j, 7+7j, 8+8j]) # Rank 1 + tensor([9+9j, 10+10j, 11+11j, 12+12j]) # Rank 2 + tensor([13+13j, 14+14j, 15+15j, 16+16j]) # Rank 3 + >>> output = torch.empty([4], dtype=torch.int64) + >>> dist.all_to_all_single(output, input) + >>> output + tensor([1+1j, 5+5j, 9+9j, 13+13j]) # Rank 0 + tensor([2+2j, 6+6j, 10+10j, 14+14j]) # Rank 1 + tensor([3+3j, 7+7j, 11+11j, 15+15j]) # Rank 2 + tensor([4+4j, 8+8j, 12+12j, 16+16j]) # Rank 3 + """ + # Dynamo has built-in logic to map legacy distributed ops to functional collectives. + # Let's redirect to a torch function mode that can mimic this logic outside Dynamo + # (e.g., non-strict export implements such a torch function mode). + relevant_args = (input,) + if has_torch_function(relevant_args): + return handle_torch_function( + all_to_all_single, + relevant_args, + output, + input, + output_split_sizes=output_split_sizes, + input_split_sizes=input_split_sizes, + group=group, + async_op=async_op, + ) + + if _rank_not_in_group(group): + _warn_not_in_group("all_to_all_single") + return + + opts = AllToAllOptions() + opts.asyncOp = async_op + _check_single_tensor(output, "output") + _check_single_tensor(input, "input") + _ensure_all_tensors_same_dtype(output, input) + + if input.is_complex(): + input = torch.view_as_real(input) + if output.is_complex(): + output = torch.view_as_real(output) + + output_split_sizes = [] if output_split_sizes is None else output_split_sizes + input_split_sizes = [] if input_split_sizes is None else input_split_sizes + + group = group or _get_default_group() + work = group.alltoall_base( + output, input, output_split_sizes, input_split_sizes, opts + ) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def all_to_all( + output_tensor_list, input_tensor_list, group=None, async_op: bool = False +): + """ + Scatters list of input tensors to all processes in a group and return gathered list of tensors in output list. + + Complex tensors are supported. + + Args: + output_tensor_list (list[Tensor]): List of tensors to be gathered one + per rank. + input_tensor_list (list[Tensor]): List of tensors to scatter one per rank. + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group. + + .. warning:: + `all_to_all` is experimental and subject to change. + + Examples: + >>> # xdoctest: +SKIP("Undefined rank") + >>> input = torch.arange(4) + rank * 4 + >>> input = list(input.chunk(4)) + >>> input + [tensor([0]), tensor([1]), tensor([2]), tensor([3])] # Rank 0 + [tensor([4]), tensor([5]), tensor([6]), tensor([7])] # Rank 1 + [tensor([8]), tensor([9]), tensor([10]), tensor([11])] # Rank 2 + [tensor([12]), tensor([13]), tensor([14]), tensor([15])] # Rank 3 + >>> output = list(torch.empty([4], dtype=torch.int64).chunk(4)) + >>> dist.all_to_all(output, input) + >>> output + [tensor([0]), tensor([4]), tensor([8]), tensor([12])] # Rank 0 + [tensor([1]), tensor([5]), tensor([9]), tensor([13])] # Rank 1 + [tensor([2]), tensor([6]), tensor([10]), tensor([14])] # Rank 2 + [tensor([3]), tensor([7]), tensor([11]), tensor([15])] # Rank 3 + + >>> # Essentially, it is similar to following operation: + >>> scatter_list = input + >>> gather_list = output + >>> for i in range(world_size): + >>> dist.scatter(gather_list[i], scatter_list if i == rank else [], src=i) + + >>> input + tensor([0, 1, 2, 3, 4, 5]) # Rank 0 + tensor([10, 11, 12, 13, 14, 15, 16, 17, 18]) # Rank 1 + tensor([20, 21, 22, 23, 24]) # Rank 2 + tensor([30, 31, 32, 33, 34, 35, 36]) # Rank 3 + >>> input_splits + [2, 2, 1, 1] # Rank 0 + [3, 2, 2, 2] # Rank 1 + [2, 1, 1, 1] # Rank 2 + [2, 2, 2, 1] # Rank 3 + >>> output_splits + [2, 3, 2, 2] # Rank 0 + [2, 2, 1, 2] # Rank 1 + [1, 2, 1, 2] # Rank 2 + [1, 2, 1, 1] # Rank 3 + >>> input = list(input.split(input_splits)) + >>> input + [tensor([0, 1]), tensor([2, 3]), tensor([4]), tensor([5])] # Rank 0 + [tensor([10, 11, 12]), tensor([13, 14]), tensor([15, 16]), tensor([17, 18])] # Rank 1 + [tensor([20, 21]), tensor([22]), tensor([23]), tensor([24])] # Rank 2 + [tensor([30, 31]), tensor([32, 33]), tensor([34, 35]), tensor([36])] # Rank 3 + >>> output = ... + >>> dist.all_to_all(output, input) + >>> output + [tensor([0, 1]), tensor([10, 11, 12]), tensor([20, 21]), tensor([30, 31])] # Rank 0 + [tensor([2, 3]), tensor([13, 14]), tensor([22]), tensor([32, 33])] # Rank 1 + [tensor([4]), tensor([15, 16]), tensor([23]), tensor([34, 35])] # Rank 2 + [tensor([5]), tensor([17, 18]), tensor([24]), tensor([36])] # Rank 3 + + >>> # Another example with tensors of torch.cfloat type. + >>> input = torch.tensor( + ... [1 + 1j, 2 + 2j, 3 + 3j, 4 + 4j], dtype=torch.cfloat + ... ) + 4 * rank * (1 + 1j) + >>> input = list(input.chunk(4)) + >>> input + [tensor([1+1j]), tensor([2+2j]), tensor([3+3j]), tensor([4+4j])] # Rank 0 + [tensor([5+5j]), tensor([6+6j]), tensor([7+7j]), tensor([8+8j])] # Rank 1 + [tensor([9+9j]), tensor([10+10j]), tensor([11+11j]), tensor([12+12j])] # Rank 2 + [tensor([13+13j]), tensor([14+14j]), tensor([15+15j]), tensor([16+16j])] # Rank 3 + >>> output = list(torch.empty([4], dtype=torch.int64).chunk(4)) + >>> dist.all_to_all(output, input) + >>> output + [tensor([1+1j]), tensor([5+5j]), tensor([9+9j]), tensor([13+13j])] # Rank 0 + [tensor([2+2j]), tensor([6+6j]), tensor([10+10j]), tensor([14+14j])] # Rank 1 + [tensor([3+3j]), tensor([7+7j]), tensor([11+11j]), tensor([15+15j])] # Rank 2 + [tensor([4+4j]), tensor([8+8j]), tensor([12+12j]), tensor([16+16j])] # Rank 3 + + """ + if _rank_not_in_group(group): + _warn_not_in_group("all_to_all") + return + + opts = AllToAllOptions() + opts.asyncOp = async_op + _check_tensor_list(output_tensor_list, "output_tensor_list") + _check_tensor_list(input_tensor_list, "input_tensor_list") + _ensure_all_tensors_same_dtype(output_tensor_list, input_tensor_list) + + input_tensor_list = [ + t if not t.is_complex() else torch.view_as_real(t) for t in input_tensor_list + ] + output_tensor_list = [ + t if not t.is_complex() else torch.view_as_real(t) for t in output_tensor_list + ] + + group = group or _get_default_group() + work = group.alltoall(output_tensor_list, input_tensor_list, opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +@_exception_logger +def barrier( + group: ProcessGroup | None = GroupMember.WORLD, + async_op: bool = False, + device_ids=None, +): + """ + Synchronize all processes. + + This collective blocks processes until the whole group enters this function, + if async_op is False, or if async work handle is called on wait(). + + Args: + group (ProcessGroup, optional): The process group to work on. If None, + the default process group will be used. + async_op (bool, optional): Whether this op should be an async op + device_ids ([int], optional): List of device/GPU ids. Only one id is expected. + + Returns: + Async work handle, if async_op is set to True. + None, if not async_op or if not part of the group + + .. note:: `ProcessGroupNCCL` now blocks the cpu thread till the completion of the barrier collective. + .. note:: `ProcessGroupNCCL` implements barrier as an all_reduce of a 1-element tensor. A device must be chosen + for allocating this tensor. The device choice is made by checking in this order (1) the first device passed to + `device_ids` arg of barrier if not None, (2) the device passed to init_process_group if not None, (3) the device + that was first used with this process group, if another collective with tensor inputs has been performed, (4) + the device index indicated by the global rank mod local device count. + """ + group = group or _get_default_group() + + if _rank_not_in_group(group): + _warn_not_in_group("barrier") + return + + opts = BarrierOptions() + opts.asyncOp = async_op + # Detect the accelerator on the machine. If no accelerator is available, it + # returns CPU. + device = torch._C._get_accelerator() + if isinstance(device_ids, list): + opts.device_ids = device_ids + # use only the first device id + # pyrefly: ignore [read-only] + opts.device = torch.device(device.type, device_ids[0]) + elif getattr(group, "bound_device_id", None) is not None: + # Use device id from `init_process_group(device_id=...)` + opts.device = group.bound_device_id # type: ignore[assignment] + elif device.type == "cpu" or _get_object_coll_device(group) == "cpu": + # pyrefly: ignore [read-only] + opts.device = torch.device("cpu") + else: + # Use the current device set by the user. If user did not set any, this + # may use default device 0, causing issues like hang or all processes + # creating context on device 0. + # pyrefly: ignore [read-only] + opts.device = device + if group.rank() == 0: + warnings.warn( # warn only once + "barrier(): using the device under current context. " + "You can specify `device_id` in `init_process_group` to mute this warning.", + stacklevel=2, + ) + + work = group.barrier(opts=opts) + + if async_op: + return work + elif ( + work is not None + ): # Backward compatible with backends that don't sync at CPP level + work.wait() + # Otherwise, the backend has sync'ed at CPP level + + +def monitored_barrier( + group: ProcessGroup | None = GroupMember.WORLD, + timeout=None, + wait_all_ranks: bool = False, +): + """ + Synchronize processes similar to ``torch.distributed.barrier``, but consider a configurable timeout. + + It is able to report ranks that did not pass this barrier within the provided timeout. + Specifically, for non-zero ranks, will block until a send/recv is processed from rank 0. + Rank 0 will block until all send /recv from other ranks are processed, and will report + failures for ranks that failed to respond in time. Note that if one rank does not reach the + monitored_barrier (for example due to a hang), all other ranks would fail in monitored_barrier. + + This collective will block all processes/ranks in the group, until the + whole group exits the function successfully, making it useful for debugging + and synchronizing. However, it can have a performance impact and should only + be used for debugging or scenarios that require full synchronization points + on the host-side. For debugging purposes, this barrier can be inserted + before the application's collective calls to check if any ranks are + desynchronized. + + .. note:: Note that this collective is only supported with the GLOO backend. + + Args: + group (ProcessGroup, optional): The process group to work on. If + ``None``, the default process group will be used. + timeout (datetime.timedelta, optional): Timeout for monitored_barrier. + If ``None``, the default process group timeout will be used. + wait_all_ranks (bool, optional): Whether to collect all failed ranks or + not. By default, this is ``False`` and ``monitored_barrier`` on rank 0 + will throw on the first failed rank it encounters in order to fail + fast. By setting ``wait_all_ranks=True`` ``monitored_barrier`` will + collect all failed ranks and throw an error containing information + about all failed ranks. + + Returns: + ``None``. + + Example:: + >>> # xdoctest: +SKIP("need process group init") + >>> # Note: Process group initialization omitted on each rank. + >>> import torch.distributed as dist + >>> if dist.get_rank() != 1: + >>> dist.monitored_barrier() # Raises exception indicating that + >>> # rank 1 did not call into monitored_barrier. + >>> # Example with wait_all_ranks=True + >>> if dist.get_rank() == 0: + >>> dist.monitored_barrier(wait_all_ranks=True) # Raises exception + >>> # indicating that ranks 1, 2, ... world_size - 1 did not call into + >>> # monitored_barrier. + """ + # Need to call rank not in group before using the group, otherwise + # "Invalid process group" error is raised. + if _rank_not_in_group(group): + _warn_not_in_group("monitored_barrier") + return + + if get_backend(group) != Backend.GLOO: + raise ValueError("monitored_barrier is only implemented for GLOO backend.") + + if timeout is None: + timeout = _get_default_timeout(get_backend(group)) + elif isinstance(timeout, float): + # TODO(whc) apparently some existing test case for monitored_barrier passes in a timeout in float format? + warnings.warn( + "Please specify timeout arg as a timedelta. " + f"Converting current value of {timeout} assuming it represents seconds", + stacklevel=2, + ) + timeout = timedelta(seconds=timeout) + + _check_valid_timeout(timeout) + + group_to_use = _get_default_group() if group is None else group + return group_to_use.monitored_barrier( # type:ignore[attr-defined] + timeout, wait_all_ranks=wait_all_ranks + ) + + +def _create_process_group_wrapper( + wrapped_pg: torch._C._distributed_c10d.Backend, + store_prefix: str, + store: Store, + rank: int, + world_size: int, + timeout: timedelta = default_pg_timeout, +): + if not _GLOO_AVAILABLE: + raise AssertionError("ProcessGroupWrapper unsupported without GLOO backend.") + + # (whc) this appears to be just for the gloo backend? if so, `default_pg_timeout` is appropriate... + + # Create a separate prefix store for the helper process group. + prefix = f"{PG_WRAPPER_STORE_PREFIX}:{store_prefix}" + store = PrefixStore(prefix, store) + helper_pg = ProcessGroupGloo(store, rank, world_size, timeout=timeout) + # Wrap the underlying pg with ProcessGroupWrapper. + wrapped_pg = _ProcessGroupWrapper(wrapped_pg, helper_pg) + return wrapped_pg + + +# helper function for deterministically hashing a list of ranks to a unique +# string +def _hash_ranks_to_str(ranks: list[int]) -> str: + rank_join: str = "_".join(map(str, ranks)) + # In case there is already a PG with the same rank composition + unique_str = "_".join([rank_join, str(len(_world.pg_names))]) + return hashlib.sha1(bytes(unique_str, "utf-8"), usedforsecurity=False).hexdigest() + + +# Takes a list of ranks and computes an integer color +def _process_group_color(ranks: list[int]) -> int: + # Convert list to tuple to make it hashable + # pyrefly: ignore [bad-assignment] + ranks = tuple(ranks) + hash_value = hash(ranks) + # Split color must be: + # - a non-negative integer; + # - a type compatible with C's int because we are pybinding to the latter. + # Thus, we limit the hash value within c_int's max value. + max_c_int = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) + color = abs(hash_value) % max_c_int + return color + + +def _process_group_name(ranks, use_hashed_name) -> GroupName: + # Create name for a process group. + global _world + if use_hashed_name: + pg_name = GroupName(_hash_ranks_to_str(ranks)) + else: + pg_name = GroupName(str(_world.group_count)) + _world.group_count += 1 + # TODO: why is group count incremented only in the else path? + return pg_name + + +def _get_backend_from_str(backend: str | None = None) -> Backend: + # Default to the same backend as the global process group + # if backend is not specified. + if not backend: + backend = get_backend(_get_default_group()) + return Backend(backend) + + +def _is_safe_to_split() -> bool: + """ + Checks if it is safe to split the any process group in the world. + This is only safe if the default pg has a bound device id, otherwise + users must be aware that a pg is only splittable after the first collective is + issued. + """ + return _get_default_group().bound_device_id is not None + + +@_time_logger +def split_group( + parent_pg: ProcessGroup | None = None, + split_ranks: list | None = None, + timeout: timedelta | None = None, + pg_options: Any | None = None, + group_desc: str | None = None, +) -> ProcessGroup | None: + """ + Create a new process group split from the given parent process group. + + warning:: This is an experimental API. Only the ``NCCL`` and custom plugin backends + are supported. Other backends will raise an error. + Users of this API must guarantee that all ranks in the parent group enter this API call, + and the split of the sub groups is the same across all ranks in the parent group. + + Args: + parent_pg (ProcessGroup, optional): The parent process group. If None, + the default process group will be used. Users need to guarantee that + the parent group is fully initialized (e.g, communicators are initialized) + split_ranks (list[list[int]]): the split ranks, which is a list of list of ranks. + Users need to make sure the validity of the split ranks such that one + split (represented by one inner list of ints) does not overlap with any other split. + Note that the ranks in each split is the group rank (instead of global rank) + in the parent pg. For example, if the parent group has 4 ranks, and split_ranks can be + [[0, 1], [2, 3]]. Note [[0,1]] is also a valid split, in which case ranks 2, 3 would + return a non-group member. + timeout (timedelta, optional): see `init_process_group` for details and default value. + pg_options (ProcessGroupOptions, optional): Additional options need to be passed in during + the construction of specific process groups. i.e.``is_high_priority_stream`` + can be specified so that process group can pick up high priority cuda streams. + group_desc (str, optional): a string to describe the process group. + + Returns: + ProcessGroup if the current rank is within one split/subgroup given by split_ranks, + or None if the current rank is not part of any split_ranks`. + + """ + # check inputs + if split_ranks is None or len(split_ranks) == 0: + raise ValueError("split_ranks cannot be None or empty") + + global _world + default_pg = _get_default_group() + device_id = default_pg.bound_device_id + if not device_id: + raise RuntimeError( + "No device associated with the default pg, not safe to split any process groups" + ) + global_rank = default_pg.rank() + global_world_size = default_pg.size() + + if not parent_pg: + parent_pg = default_pg + if parent_pg not in _world.pg_group_ranks: + raise ValueError(f"Group {parent_pg} is not registered") + + parent_global_to_group_ranks = _world.pg_group_ranks[parent_pg] + parent_group_to_global_ranks = { + group_rank: global_rank + for global_rank, group_rank in parent_global_to_group_ranks.items() + } + + if global_rank not in parent_global_to_group_ranks: + raise ValueError( + f"Global rank {global_rank} is not part of the parent group {parent_pg}" + ) + + parent_group_rank = parent_global_to_group_ranks[global_rank] + parent_backend = parent_pg._get_backend(torch.device("cuda")) + + # if the parent backend does not support splitting, raise error + # currently this API only support NCCL backend + if not parent_backend or not parent_backend.supports_splitting: + raise RuntimeError( + "No backend for the parent process group or its backend does not support splitting" + ) + + # set the group_desc before the color or no_cloor split + if hasattr(parent_backend, "comm_split_count") and group_desc is None: + group_desc = f"{parent_pg.group_desc}:split:{parent_backend.comm_split_count()}" # type: ignore[attr-defined] + + parent_backend_str, _ = _world.pg_map[parent_pg] + # same type of backend as the parent process group + backend = Backend(parent_backend_str) + backend_config = BackendConfig(backend) + + if pg_options is None: + # default pg_options same as the parent process group + # A deep copy is needed because if the option will be modified inside split + # and if we split parent pg multiple times, we will run into device out of bound error. + pg_options = copy.deepcopy(parent_backend.options) + + # this timeout defaulting/validation is used for all the new_groups/new_subgroups variants, + # which may just pass their timeout value (or None) + if timeout is None: + timeout = _get_default_timeout(backend) + _check_valid_timeout(timeout) + + # find my group of ranks and my group local rank in split_ranks + # for ranks which are not in any split PGs, we just pass in this the first split group + # and None will be returned. + my_group = split_ranks[0] + + for split_group in split_ranks: + if len(split_group) == 0: + raise ValueError("the split group cannot be empty") + if len(split_group) > global_world_size: + raise ValueError( + "the split group's size should be less or equal to the world_size set by init_process_group" + ) + if len(split_group) != len(set(split_group)): + raise ValueError("the split group cannot have duplicate ranks") + split_group = sorted(split_group) + if parent_group_rank in split_group: + my_group = split_group + break + + # use_hashed_name is True to ensure that subgroups have unique names. + # This is needed as some backends (e.g. Gloo) use the group name as a + # PrefixStore prefix for initialization of splits. Thus, names have to be + # unique to avoid key collisions. + group_name = _process_group_name(my_group, use_hashed_name=True) + split_pg = parent_pg.split_group( + my_group, + timeout=timeout, + opts=pg_options, + group_name=group_name, + group_desc=group_desc, + ) + if split_pg is None: + return None + + global_ranks_in_my_group = [parent_group_to_global_ranks[rank] for rank in my_group] + split_pg.bound_device_id = device_id # type: ignore[union-attr] + split_backend_class = split_pg._get_backend(torch.device("cuda")) + split_backend_class._set_sequence_number_for_group() + if split_pg.group_name != group_name: + raise AssertionError( + f"group name should be set to {group_name} but got {split_pg.group_name}" + ) + + # update global state + _world.pg_map[split_pg] = (backend, split_pg.get_group_store()) + _world.pg_names[split_pg] = group_name + _register_process_group(group_name, split_pg) + _world.pg_backend_config[split_pg] = str(backend_config) + pg_tag = f"ptd:{group_name}" + _world.tags_to_pg.setdefault(pg_tag, []).append(split_pg) + _world.pg_to_tag[split_pg] = pg_tag + + # Create the global rank to group rank mapping + _world.pg_group_ranks[split_pg] = { + global_rank: group_rank + for group_rank, global_rank in enumerate(global_ranks_in_my_group) + } + + return split_pg + + +@_time_logger +def new_group( + ranks=None, + timeout=None, + backend=None, + pg_options=None, + use_local_synchronization: bool = False, + group_desc=None, + device_id: torch.device | None = None, +): + """ + Create a new distributed group. + + This function requires that all processes in the main group (i.e. all + processes that are part of the distributed job) enter this function, even + if they are not going to be members of the group. Additionally, groups + should be created in the same order in all processes. + + .. warning:: + Safe concurrent usage: + When using multiple process groups with the ``NCCL`` backend, the user + must ensure a globally consistent execution order of collectives across + ranks. + + If multiple threads within a process issue collectives, explicit + synchronization is necessary to ensure consistent ordering. + + When using async variants of torch.distributed communication APIs, + a work object is returned and the communication kernel is + enqueued on a separate CUDA stream, allowing overlap of communication + and computation. Once one or more async ops have been issued on one process + group, they must be synchronized with other cuda streams by calling `work.wait()` + before using another process group. + + See `Using multiple NCCL communicators concurrently + ` + for more details. + + Args: + ranks (list[int]): List of ranks of group members. If ``None``, will be + set to all ranks. Default is ``None``. + timeout (timedelta, optional): see `init_process_group` for details and default value. + backend (str or Backend, optional): The backend to use. Depending on + build-time configurations, valid values are ``gloo`` and ``nccl``. + By default uses the same backend as the global group. This field + should be given as a lowercase string (e.g., ``"gloo"``), which can + also be accessed via :class:`Backend` attributes (e.g., + ``Backend.GLOO``). If ``None`` is passed in, the backend + corresponding to the default process group will be used. Default is + ``None``. + pg_options (ProcessGroupOptions, optional): process group options + specifying what additional options need to be passed in during + the construction of specific process groups. i.e. for the ``nccl`` + backend, ``is_high_priority_stream`` can be specified so that + process group can pick up high priority cuda streams. For other available options to config nccl, + See https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclconfig-tuse_local_synchronization + (bool, optional): perform a group-local barrier at the end of the process group creation. + This is different in that non-member ranks don't need to call into API and don't + join the barrier. + group_desc (str, optional): a string to describe the process group. + device_id (torch.device, optional): a single, specific device + to "bind" this process to, The `new_group` call will try to initialize + a communication backend immediately for the device if this field is given. + + Returns: + A handle of distributed group that can be given to collective calls or + GroupMember.NON_GROUP_MEMBER if the rank is not part of ``ranks``. + + N.B. use_local_synchronization doesn't work with MPI. + + N.B. While use_local_synchronization=True can be significantly faster with larger + clusters and small process groups, care must be taken since it changes cluster behavior + as non-member ranks don't join the group barrier(). + + N.B. use_local_synchronization=True can lead to deadlocks when each rank creates + multiple overlapping process groups. To avoid that, make sure all ranks follow the + same global creation order. + """ + return _new_group_with_tag( + ranks, + timeout, + backend, + pg_options, + None, + use_local_synchronization=use_local_synchronization, + group_desc=group_desc, + device_id=device_id, + ) + + +def _new_group_with_tag( + ranks=None, + timeout=None, + backend=None, + backend_options=None, + pg_tag=None, + use_local_synchronization=False, + group_desc=None, + device_id: torch.device | None = None, +): + """ + Variant of ``new_group`` that exposes tag creation. + + :: N.B. The mechanism is experimental and tied to the functional collectives effort, see + ``torch.distributed._functional_collectives`` for reference on how to use it. + """ + global _world + + default_pg = _get_default_group() + if device_id is None: + device_id = default_pg.bound_device_id + elif default_pg.bound_device_id is not None: + if device_id != default_pg.bound_device_id: + raise AssertionError( + "Mismatched bound device between new pg and the default pg." + ) + default_backend, default_store = _world.pg_map[default_pg] + global_rank = default_pg.rank() + global_world_size = default_pg.size() + + # Default to the same backend as the global process group + # if the backend is not specified. + if not backend: + backend = default_backend + backend = Backend(backend) + + # this timeout defaulting/validation is used for all the new_groups/new_subgroups variants, + # which may just pass their timeout value (or None) + if timeout is None: + timeout = _get_default_timeout(backend) + _check_valid_timeout(timeout) + + if use_local_synchronization: + # MPI backend doesn't have have a way for us to perform a partial sync + if backend == Backend.MPI: + raise ValueError( + "MPI backend doesn't support use_local_synchronization=True" + ) + if ranks is not None and get_rank() not in ranks: + return None + + # checks the input ranks + if ranks is not None: + ranks = sorted(ranks) + group_world_size = len(ranks) + if group_world_size > global_world_size: + raise ValueError( + "the new group's world size should be less or " + "equal to the world size set by " + "init_process_group" + ) + # check ranks' sanity + for rank in ranks: + if rank < 0 or rank >= global_world_size: + raise ValueError( + "The new group's rank should be within " + "the world_size set by init_process_group" + ) + if global_rank in ranks: + group_rank = ranks.index(global_rank) + else: + group_rank = None + else: + ranks = list(range(global_world_size)) + group_world_size = global_world_size + group_rank = global_rank + + group_name = _process_group_name(ranks, use_hashed_name=use_local_synchronization) + + pg, pg_store = _new_process_group_helper( + group_world_size, + group_rank, + ranks, + backend, + default_store, + group_name, + backend_options=backend_options, + timeout=timeout, + pg_tag=pg_tag, + device_id=device_id, + group_desc=group_desc, + ) + + # Create the global rank to group rank mapping + _world.pg_group_ranks[pg] = { + global_rank: group_rank for group_rank, global_rank in enumerate(ranks) + } + + if _is_barrier_after_init() == 1: + # barrier at the end to ensure that once we return from this method, all + # process groups including global variables (if any) are updated + # correctly on all ranks. + # Update 04/2023: for large-scale runs, this barrier (esp. store-based + # barrier) may be costly and/or unscalable. Also, in a lot of cases, + # these barriers may be unnecessary, as proven by a green CI after + # removal. An environment variable `TORCH_DIST_INIT_BARRIER` has been + # added which enables this barrier only when set to 1. + logger.info( + "Performing barrier after ProcessGroup initialization since " + "TORCH_DIST_INIT_BARRIER = 1" + ) + if backend == Backend.MPI: + # MPI doesn't have store. + barrier() + else: + barrier_store = pg_store if use_local_synchronization else default_store + world_size = len(ranks) if use_local_synchronization else get_world_size() + # Use store based barrier here since barrier() used a bunch of + # default devices and messes up NCCL internal state. + _store_based_barrier( + global_rank, barrier_store, group_name, world_size, timeout + ) + + return pg + + +def new_subgroups( + group_size=None, + group=None, + timeout=None, + backend=None, + pg_options=None, + group_desc=None, +): + """ + Create subgroups of equal size. + + By default, it creates intra-machine subgroups, + where each of which contains all the ranks of a machine, based on the assumption + that each machine has the same number of devices. + + This is a convenience API that calls ``new_group`` to generate multiple subgroups. + It requires that all processes in the main group (i.e. all + processes that are part of the distributed job) enter this function, even + if they are not going to be members of the group. + + .. warning:: + If ``group_size`` is passed in, the world size must be divisible by ``group_size``. + If no ``group_size`` is passed in, it believe that you are creating a group based + on CUDA and determining the group size by number of CUDA devices, and if not all + the machines have the same number of devices, the subgroup division will be + different across nodes and can cause unexpected behaviors. Therefore, if you are + creating a subgroup that does not depend on CUDA (such as Gloo on CPU), please + pass in ``group_size`` correctly. + + .. warning:: + See warning `Safe concurrent usage` for `new_group` API for important details about + using multiple process groups concurrently in a safe manner. + + Args: + group_size (int, optional): The size of each subgroup. If ``None``, + the default subgroup size is equal to the number of devices on each machine, + based on the assumption that each machine has exactly the same + number of devices. Default is ``None``. + group (ProcessGroup, optional): The process group to work on. If + ``None``, the default process group will be used. Default is ``None``. + timeout (timedelta, optional): see `init_process_group` for details and default value. + backend (str or Backend, optional): The backend to use. Depending on + build-time configurations, valid values are ``gloo`` and ``nccl``. + By default uses the same backend as the global group. This field + should be given as a lowercase string (e.g., ``"gloo"``), which can + also be accessed via :class:`Backend` attributes (e.g., + ``Backend.GLOO``). If ``None`` is passed in, the backend + corresponding to the default process group will be used. Default is + ``None``. + pg_options (ProcessGroupOptions, optional): process group options + specifying what additional options need to be passed in during + the construction of specific process groups. i.e. for the ``nccl`` + backend, ``is_high_priority_stream`` can be specified so that + process group can pick up high priority cuda streams. + group_desc (str, optional): A string describing the group. Each subgroup will + inherit its group_desc + + Returns: + The subgroup containing the current rank, and all the subgroups used for cleanup. + + Examples: + >>> # Create intra-machine subgroups. + >>> # xdoctest: +SKIP("need process group init") + >>> cur_subgroup, subgroups = dist.new_subgroups() + >>> # Allreduce within the machine. + >>> rank = dist.get_rank() + >>> tensor = torch.ones(1, device=rank) * rank + >>> dist.all_reduce(tensor, group=cur_subgroup) + >>> tensor + tensor([28]) # Assume 8 CUDA devices per machine. 28 is sum(range(8)). + >>> # Cleanup. + >>> for subgroup in subgroups: + >>> dist.destroy_process_group(subgroup) + """ + if group_size is None: + if not torch.cuda.is_available(): + raise ValueError( + "Default group size only takes effect when CUDA is available." + "If your subgroup using a backend that does not depend on CUDA," + "please pass in 'group_size' correctly." + ) + group_size = torch.cuda.device_count() + if group_size <= 0: + raise ValueError(f"The arg 'group_size' ({group_size}) must be positive") + + world_size = get_world_size(group=group) + if world_size < group_size: + raise ValueError( + f"The arg 'group_size' ({group_size}) must not exceed the world size ({world_size})" + ) + if world_size % group_size != 0: + raise ValueError( + f"The world size ({world_size}) must be divisible by '{group_size=}'" + ) + + # TODO: Use itertools.batched(get_process_group_ranks(group=group), group_size) instead when Python 3.12 is supported. + ranks = get_process_group_ranks(group=group) + ranks_per_subgroup_list = [ + ranks[i : i + group_size] for i in range(0, len(ranks), group_size) + ] + return new_subgroups_by_enumeration( + ranks_per_subgroup_list, + timeout=timeout, + backend=backend, + pg_options=pg_options, + group_desc=group_desc, + ) + + +def new_subgroups_by_enumeration( + ranks_per_subgroup_list, + timeout=None, + backend=None, + pg_options=None, + group_desc=None, +): + """ + Create subgroups by dividing the global world. + + The division is specified by a nested list of ranks. The subgroups cannot have + overlap, and some ranks may not have to be in any subgroup. + + This is a convenience API that calls ``new_group`` to generate multiple subgroups. + It requires that all processes in the main group (i.e. all + processes that are part of the distributed job) enter this function, even + if they are not going to be members of the group. + + .. warning:: + See warning `Safe concurrent usage` for `new_group` API for important details about + using multiple process groups concurrently in a safe manner. + + Args: + ranks_per_subgroup_list (list[list[int]]): A nested list of ranks of + group members. + timeout (timedelta, optional): see `init_process_group` for details and default value. + backend (str or Backend, optional): The backend to use. Depending on + build-time configurations, valid values are ``gloo`` and ``nccl``. + By default uses the same backend as the global group. This field + should be given as a lowercase string (e.g., ``"gloo"``), which can + also be accessed via :class:`Backend` attributes (e.g., + ``Backend.GLOO``). If ``None`` is passed in, the backend + corresponding to the default process group will be used. Default is + ``None``. + pg_options (ProcessGroupOptions, optional): process group options + specifying what additional options need to be passed in during + the construction of specific process groups. i.e. for the ``nccl`` + backend, ``is_high_priority_stream`` can be specified so that + process group can pick up high priority cuda streams. + group_desc (str, optional): A string describing the group. Each subgroup will + inherit its group_desc. + + Returns: + The subgroup containing the current rank, and all the subgroups used for cleanup. + + Examples: + >>> # Create two subgroups, where each has 2 processes. + >>> # xdoctest: +SKIP("need process group init") + >>> cur_subgroup, subgroups = dist.new_subgroups(ranks=[[0, 2], [1, 3]]) + >>> rank = dist.get_rank() + >>> tensor = torch.ones(1, device=rank) * rank + >>> dist.all_reduce(tensor, group=cur_subgroup) + >>> tensor + tensor([2]) # Subgroup 0: ranks 0 and 2 + tensor([4]) # Subgroup 1: ranks 1 and 3 + """ + if ranks_per_subgroup_list is None or len(ranks_per_subgroup_list) == 0: + raise ValueError("The arg 'ranks_per_subgroup_list' cannot be empty") + + subgroups = [] + cur_subgroup = None + # Create a mapping from rank to subgroup to check if there is any subgroup overlap. + rank_to_ranks_dict = {} # type: ignore[var-annotated] + for ranks in ranks_per_subgroup_list: + subgroup = new_group( + ranks=ranks, + timeout=timeout, + backend=backend, + pg_options=pg_options, + group_desc=group_desc, + ) + subgroups.append(subgroup) + my_rank = get_rank() + for rank in ranks: + if rank in rank_to_ranks_dict: + raise ValueError( + f"Rank {rank} has appeared in both subgroup {rank_to_ranks_dict[rank]} and {ranks}" + ) + rank_to_ranks_dict[rank] = ranks + if my_rank == rank: + cur_subgroup = subgroup + logger.info("Rank %s is assigned to subgroup %s", rank, ranks) + + return cur_subgroup, subgroups + + +def _find_pg_by_ranks_and_tag(tag: str, ranks: list[int]) -> ProcessGroup | None: + if len(tag) > 0 and not tag.startswith("ptd:") and not tag.startswith("user:"): + tag = f"user:{tag}" + + for group in _world.tags_to_pg.get(tag, []): + if group.size() != len(ranks): + continue + + group_ranks = get_process_group_ranks(group) + good = all(r in group_ranks for r in ranks) + if good: + return group + return None + + +def _find_or_create_pg_by_ranks_and_tag( + tag: str, ranks: list[int], stride: int +) -> ProcessGroup: + if len(ranks) % stride != 0: + raise ValueError( + f"Ranks length ({len(ranks)}) must be divisible by stride ({stride})" + ) + + my_rank = get_rank() + my_ranks = None + + if stride == len(ranks): + my_ranks = ranks.copy() + if my_rank not in my_ranks: + raise AssertionError("rankset doesn't include the current node") + else: + for i in range(0, len(ranks), stride): + rank_set = ranks[i : i + stride] + if my_rank in rank_set: + my_ranks = rank_set + if my_ranks is None: + raise AssertionError("rankset doesn't include the current node") + + my_ranks = sorted(my_ranks) + + pg = _find_pg_by_ranks_and_tag(tag, my_ranks) + if pg is not None: + return pg + if tag == "": + raise ValueError("Cannot automatically create PG with empty tag") + # TODO copy settings and timeout from default PG + return _new_group_with_tag(my_ranks, pg_tag=tag) + + +def _get_group_tag(pg: ProcessGroup) -> str: + """Return the tag associated with ``pg``.""" + tag = _world.pg_to_tag[pg] + tag = tag.removeprefix("user:") + return tag + + +def _get_process_group_name(pg: ProcessGroup) -> str: + return _world.pg_names.get(pg, "None") + + +def _get_process_group_store(pg: ProcessGroup) -> Store: + return _world.pg_map[pg][1] + + +# Shrink flags for process group backends +SHRINK_DEFAULT = 0x00 +SHRINK_ABORT = 0x01 + + +@_time_logger +def shrink_group( + ranks_to_exclude: list[int], + group: ProcessGroup | None = None, + shrink_flags: int = SHRINK_DEFAULT, + pg_options: Any | None = None, +) -> ProcessGroup: + """ + Shrinks a process group by excluding specified ranks. + + Creates and returns a new, smaller process group comprising only the ranks + from the original group that were not in the ``ranks_to_exclude`` list. + + Args: + ranks_to_exclude (List[int]): A list of ranks from the original + ``group`` to exclude from the new group. + group (ProcessGroup, optional): The process group to shrink. If ``None``, + the default process group is used. Defaults to ``None``. + shrink_flags (int, optional): Flags to control the shrinking behavior. + Can be ``SHRINK_DEFAULT`` (default) or ``SHRINK_ABORT``. + ``SHRINK_ABORT`` will attempt to terminate ongoing operations + in the parent communicator before shrinking. + Defaults to ``SHRINK_DEFAULT``. + pg_options (ProcessGroupOptions, optional): Backend-specific options to apply + to the shrunken process group. If provided, the backend will use + these options when creating the new group. If omitted, the new group + inherits defaults from the parent. + + Returns: + ProcessGroup: a new group comprised of the remaining ranks. If the + default group was shrunk, the returned group becomes the new default group. + + Raises: + TypeError: if the group’s backend does not support shrinking. + ValueError: if ``ranks_to_exclude`` is invalid (empty, out of bounds, + duplicates, or excludes all ranks). + RuntimeError: if an excluded rank calls this function or the backend + fails the operation. + + Notes: + - Only non-excluded ranks should call this function; excluded ranks + must not participate in the shrink operation. + - Shrinking the default group destroys all other process groups since + rank reassignment makes them inconsistent. + """ + # Step 1: Validate input parameters with comprehensive error checking + _validate_shrink_inputs(ranks_to_exclude, shrink_flags) + + # Step 2: Get target group and essential properties + target_group_info = _prepare_shrink_target_group(group) + + # Step 3: Validate backend requirements and availability + backend_impl = _validate_shrink_backend_requirements(target_group_info) + + # Step 4: Validate ranks against group and check for duplicates + excluded_ranks_set = _validate_and_process_excluded_ranks( + ranks_to_exclude, target_group_info + ) + + # Step 5: Execute the actual shrink operation (backend-specific) + new_backend = backend_impl.shrink( + sorted(excluded_ranks_set), + shrink_flags, + pg_options if pg_options is not None else None, + ) + + # Step 6: Handle cleanup and creation of new process group + target_group_info["pg_options_override"] = pg_options + return _finalize_shrunk_group(target_group_info, excluded_ranks_set, new_backend) + + +def _validate_shrink_inputs(ranks_to_exclude: list[int], shrink_flags: int) -> None: + """Validate input parameters for shrink_group.""" + if not isinstance(ranks_to_exclude, list): + raise TypeError( + f"ranks_to_exclude must be a list, but got {type(ranks_to_exclude).__name__}. " + f"Example: [1, 3, 5] to exclude ranks 1, 3, and 5." + ) + + if not ranks_to_exclude: + raise ValueError( + "ranks_to_exclude cannot be empty. To shrink a group, you must specify at least " + "one rank to exclude. Example: [failed_rank_id]" + ) + + # Validate shrink_flags with clear explanation of valid values + valid_flags = [SHRINK_DEFAULT, SHRINK_ABORT] + if not isinstance(shrink_flags, int) or shrink_flags not in valid_flags: + raise ValueError( + f"Invalid shrink_flags value: {shrink_flags}. Must be one of: " + f"SHRINK_DEFAULT ({SHRINK_DEFAULT}) or SHRINK_ABORT ({SHRINK_ABORT}). " + f"Use SHRINK_ABORT to abort ongoing operations before shrinking." + ) + + +def _prepare_shrink_target_group(group: ProcessGroup | None) -> dict: + """Prepare and validate the target group for shrinking.""" + target_pg = group if group is not None else _get_default_group() + + # Cache frequently accessed properties to avoid repeated calls + group_size = int(target_pg.size()) + group_info = { + "process_group": target_pg, + "is_default_group": (target_pg == _get_default_group()), + "group_size": group_size, + "current_rank": target_pg.rank(), + "group_name": _get_process_group_name(target_pg), + } + + # Validate that we have a valid process group + if group_size <= 1: + raise ValueError( + f"Cannot shrink a process group with size {group_size}. " + f"Group must have at least 2 ranks to support shrinking." + ) + + return group_info + + +def _validate_shrink_backend_requirements(group_info: dict) -> Any: + """Return the backend implementation for the target group or raise if unsupported.""" + target_pg = group_info["process_group"] + group_name = group_info["group_name"] + + # Get the group's backend directly via ProcessGroup API. Prefer a bound device if present, + # otherwise try CUDA then fall back to CPU. + try: + preferred_device = getattr(target_pg, "bound_device_id", None) + if preferred_device is not None: + backend_impl = target_pg._get_backend(preferred_device) + else: + # Try CUDA first if available, else CPU + try: + backend_impl = target_pg._get_backend(torch.device("cuda")) + except Exception: + backend_impl = target_pg._get_backend(torch.device("cpu")) + except RuntimeError as e: + raise RuntimeError( + f"Cannot access device backend for process group '{group_name}'. " + f"Ensure the process group was initialized with a compatible device backend and devices are available." + ) from e + + try: + supports = bool(backend_impl.supports_shrinking) + except Exception: + supports = False + if not supports: + raise TypeError( + f"Process group backend for '{group_name}' does not support shrinking operations." + ) + + return backend_impl + + +def _validate_and_process_excluded_ranks( + ranks_to_exclude: list[int], group_info: dict +) -> set: + """Validate excluded ranks and convert to set for efficient operations.""" + group_size = group_info["group_size"] + current_rank = group_info["current_rank"] + + # Use set for O(1) duplicate detection and membership testing + excluded_ranks_set = set() + + # Validate each rank with detailed error messages + for i, rank in enumerate(ranks_to_exclude): + if not isinstance(rank, int): + raise TypeError( + f"All elements in ranks_to_exclude must be integers. " + f"Element at index {i} is {type(rank).__name__}: {rank}" + ) + + if not (0 <= rank < group_size): + raise ValueError( + f"Rank {rank} at index {i} is out of bounds for group size {group_size}. " + f"Valid ranks are in range [0, {group_size - 1}]." + ) + + if rank in excluded_ranks_set: + raise ValueError( + f"Duplicate rank {rank} found in ranks_to_exclude at index {i}. " + f"Each rank can only be excluded once." + ) + + excluded_ranks_set.add(rank) + + # Ensure we don't exclude all ranks + if len(excluded_ranks_set) >= group_size: + raise ValueError( + f"Cannot exclude all {group_size} ranks from process group. " + f"At least one rank must remain. Excluding {len(excluded_ranks_set)} ranks." + ) + + # Critical check: current rank should not be in excluded list + if current_rank in excluded_ranks_set: + raise RuntimeError( + f"Current rank {current_rank} is in the exclusion list and should not call shrink_group(). " + f"Only non-excluded ranks should participate in the shrinking operation. " + f"Excluded ranks should terminate their processes instead." + ) + + return excluded_ranks_set + + +def _finalize_shrunk_group( + group_info: dict, excluded_ranks_set: set, new_backend +) -> ProcessGroup: + """Clean up old group and create new shrunk process group.""" + target_pg = group_info["process_group"] + is_default_group = group_info["is_default_group"] + + # Handle default group dependencies - destroy other groups first + if is_default_group: + _destroy_all_other_groups(exclude_group=target_pg) + + # Gather original group metadata before cleanup + original_group_metadata = _extract_group_metadata(target_pg) + + # Calculate remaining ranks efficiently + original_ranks = get_process_group_ranks(target_pg) + remaining_ranks = [ + rank for rank in original_ranks if rank not in excluded_ranks_set + ] + + # Clean up the original group + _cleanup_original_group(target_pg, is_default_group) + + # Create and configure the new process group + new_pg = _create_shrunk_process_group( + new_backend, remaining_ranks, original_group_metadata, is_default_group + ) + + # Register the new group in global state + if is_default_group: + _update_default_pg(new_pg) + + # Update global state with new group information + rank_mapping = { + global_rank: group_rank + for group_rank, global_rank in enumerate(remaining_ranks) + } + _update_process_group_global_state( + pg=new_pg, + backend_name=original_group_metadata["backend_name"], + store=original_group_metadata["store"], + group_name=original_group_metadata["new_group_name"], + backend_config=original_group_metadata["backend_config"], + rank_mapping=rank_mapping, + ) + + return new_pg + + +def _extract_group_metadata(target_pg: ProcessGroup) -> dict: + """Extract metadata from the original group before cleanup.""" + original_backend_name, original_store = _world.pg_map[target_pg] + original_backend_config = _world.pg_backend_config.get(target_pg, "") + original_group_name = _get_process_group_name(target_pg) + + # Extract device binding information before cleanup to avoid accessing destroyed group + bound_device_id = None + if hasattr(target_pg, "bound_device_id"): + bound_device_id = target_pg.bound_device_id + + # Generate new group name for the shrunk group; hash for uniqueness across backends + remaining_ranks = list(get_process_group_ranks(target_pg)) + new_group_name = _process_group_name(remaining_ranks, use_hashed_name=True) + + return { + "backend_name": original_backend_name, + "store": original_store, + "backend_config": original_backend_config, + "original_group_name": original_group_name, + "new_group_name": new_group_name, + "bound_device_id": bound_device_id, # Safe to access after cleanup + } + + +def _cleanup_original_group(target_pg: ProcessGroup, is_default_group: bool) -> None: + """Clean up the original process group safely.""" + try: + destroy_process_group(target_pg) + except Exception: + group_type = "default" if is_default_group else "non-default" + logger.warning( + "Failed to destroy %s group during shrinking", group_type, exc_info=True + ) + + # Ensure global state cleanup even if destroy_process_group fails + _cleanup_process_group_global_state(target_pg) + + +def _create_shrunk_process_group( + new_backend, remaining_ranks: list[int], metadata: dict, is_default_group: bool +) -> ProcessGroup: + """Create and configure the new shrunk process group.""" + # Create new group properties + new_group_rank = new_backend.rank() + new_group_size = new_backend.size() + group_name = metadata["new_group_name"] + + # Generate descriptive group description + if is_default_group: + group_desc = "default:shrunken" + else: + group_desc = f"{metadata['original_group_name']}:shrunk" + + # Create process group with new communicator (clone the parent store like split does) + prefix_store = PrefixStore(f"{group_name}/", metadata["store"].clone()) + new_pg = ProcessGroup(prefix_store, new_group_rank, new_group_size) + + # Configure backend using the device type of the new backend's bound device if available, + # otherwise derive from the original group's bound device or fall back to CPU. + backend_device = metadata.get("bound_device_id") + if backend_device is None: + # Default to CPU if no bound device is present + backend_device = torch.device("cpu") + + # Choose backend enum based on device type + if backend_device.type == "cuda": + backend_type = ProcessGroup.BackendType.NCCL + else: + backend_type = ProcessGroup.BackendType.GLOO + + new_pg._register_backend(backend_device, backend_type, new_backend) + new_pg._set_default_backend(backend_type) + + # Inherit device binding from original group if it was bound + bound_device_id = metadata.get("bound_device_id") + if bound_device_id is not None: + new_pg.bound_device_id = bound_device_id + + # Set group metadata + new_pg._set_group_name(group_name) + new_pg._set_group_desc(group_desc) + + # Persist backend configuration overrides (if provided via shrink_group) + backend_config_override = metadata.get("backend_config") + if backend_config_override is not None: + # Store for introspection/debugging and potential backend hooks + _world.pg_backend_config[new_pg] = backend_config_override + + return new_pg + + +def _destroy_all_other_groups(exclude_group: ProcessGroup | None = None) -> None: + """ + Destroy all process groups except the excluded group and clean up all global state. + + This is necessary when shrinking the default group because global ranks + are reassigned by NCCL, making all existing process groups inconsistent. + + Note: Uses abort for non-collective cleanup since excluded ranks may not + participate in collective operations. Backend cleanup is handled independently per group. + + Args: + exclude_group (ProcessGroup, optional): Process group to exclude from destruction. + If None, destroys all process groups. + """ + # Get list of groups to destroy (avoid modifying dict while iterating) + groups_to_destroy = [] + for pg in list(_world.pg_group_ranks.keys()): + if exclude_group is not None and pg == exclude_group: + continue + groups_to_destroy.append(pg) + + # Warn user about automatic destruction + if groups_to_destroy: + group_names = [_get_process_group_name(pg) for pg in groups_to_destroy] + logger.warning( + "Shrinking default group will destroy %d other process groups: %s. " + "This is necessary because shrinking the default group reassigns global ranks, " + "making existing groups inconsistent.", + len(groups_to_destroy), + ", ".join(group_names), + ) + + # Destroy each group and clean up global state + for pg in groups_to_destroy: + try: + # First call abort_process_group which handles the C++ cleanup non-collectively + _abort_process_group(pg) + except Exception: + # Log but don't fail - some groups might already be destroyed + logger.warning( + "Failed to abort process group %s", + _get_process_group_name(pg), + exc_info=True, + ) + + # Ensure all global state is cleaned up even if _abort_process_group fails + # or doesn't clean up everything + _cleanup_process_group_global_state(pg) + + +def _cleanup_process_group_global_state(pg: ProcessGroup) -> None: + """ + Clean up all global state associated with a process group. + + This function ensures complete cleanup of process group state from all + global dictionaries and registries, even if destroy_process_group fails + or doesn't clean up everything. This is critical when destroying multiple + groups to prevent inconsistent state. + + The cleanup removes the process group from: + - _world.pg_map (backend and store mapping) + - _world.pg_names (group name mapping) + - _world.pg_group_ranks (rank mappings) + - _world.pg_backend_config (backend configuration) + - _world.tags_to_pg and _world.pg_to_tag (tag mappings) + - _world.pg_coalesce_state (coalescing state) + - C++ internal registries via _unregister_process_group + + Args: + pg (ProcessGroup): The process group to clean up. + """ + try: + # Clean up main process group mappings + _world.pg_map.pop(pg, None) + _world.pg_group_ranks.pop(pg, None) + _world.pg_backend_config.pop(pg, None) + + # Clean up process group name mapping + group_name = _world.pg_names.pop(pg, None) + + # Clean up tag mappings + pg_tag = _world.pg_to_tag.pop(pg, None) + if pg_tag is not None and pg_tag in _world.tags_to_pg: + try: + _world.tags_to_pg[pg_tag].remove(pg) + # Remove the tag entry if list is empty + if not _world.tags_to_pg[pg_tag]: + _world.tags_to_pg.pop(pg_tag, None) + except (ValueError, KeyError): + # Process group was already removed from the list + pass + + # Clean up any registered process group names using C++ unregister function + if group_name is not None: + try: + _unregister_process_group(group_name) + except Exception: + # Process group name might not be registered or already unregistered + pass + + # Clean up coalesce state if present + _world.pg_coalesce_state.pop(pg, None) + + except Exception: + # Log cleanup failures but don't propagate - we want to continue with other cleanups + logger.warning( + "Failed to fully clean up global state for process group", exc_info=True + ) + + +def _update_process_group_global_state( + pg: ProcessGroup, + backend_name: str, + store: Store, + group_name: GroupName, + backend_config: str, + rank_mapping: dict[int, int] | None = None, + pg_tag: str | None = None, + user_tag: str | None = None, +) -> None: + """ + Update all global state dictionaries for a process group. + + This helper function consolidates the common pattern of updating multiple + global state dictionaries when creating or modifying process groups. + + Args: + pg (ProcessGroup): The process group to update state for. + backend_name (str): Backend name for pg_map. + store (Store): Store instance for pg_map. + group_name (str): Group name for pg_names and registration. + backend_config (str): Backend configuration string. + rank_mapping (Dict[int, int], optional): Global rank to group rank mapping. + If None, skips updating pg_group_ranks. + pg_tag (str, optional): Process group tag. If None, defaults to f"ptd:{group_name}". + user_tag (str, optional): User-provided tag for special tag handling. + If provided, creates "user:{user_tag}" tag and also adds to default "". + """ + # Update main process group mappings + _world.pg_map[pg] = (backend_name, store) + _world.pg_names[pg] = group_name + _world.pg_backend_config[pg] = backend_config + + # Register the process group name + _register_process_group(group_name, pg) + + # Update rank mapping if provided + if rank_mapping is not None: + _world.pg_group_ranks[pg] = rank_mapping + + # Handle tag management + if pg_tag is None: + pg_tag = f"ptd:{group_name}" + + if user_tag is not None: + # Special handling for user-provided tags + # Add to default "" tag first + _world.tags_to_pg.setdefault("", []).append(pg) + # Then create user-specific tag + user_pg_tag = f"user:{user_tag}" + _world.tags_to_pg.setdefault(user_pg_tag, []).append(pg) + _world.pg_to_tag[pg] = user_pg_tag + else: + # Standard process group tag + _world.tags_to_pg.setdefault(pg_tag, []).append(pg) + _world.pg_to_tag[pg] = pg_tag diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/launch.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..ca27435c9e8e3a9a40e496cac6c8350bb4137835 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/launch.py @@ -0,0 +1,207 @@ +# mypy: allow-untyped-defs +r""" +Module ``torch.distributed.launch``. + +``torch.distributed.launch`` is a module that spawns up multiple distributed +training processes on each of the training nodes. + +.. warning:: + + This module is going to be deprecated in favor of :ref:`torchrun `. + +The utility can be used for single-node distributed training, in which one or +more processes per node will be spawned. The utility can be used for either +CPU training or GPU training. If the utility is used for GPU training, +each distributed process will be operating on a single GPU. This can achieve +well-improved single-node training performance. It can also be used in +multi-node distributed training, by spawning up multiple processes on each node +for well-improved multi-node distributed training performance as well. +This will especially be beneficial for systems with multiple Infiniband +interfaces that have direct-GPU support, since all of them can be utilized for +aggregated communication bandwidth. + +In both cases of single-node distributed training or multi-node distributed +training, this utility will launch the given number of processes per node +(``--nproc-per-node``). If used for GPU training, this number needs to be less +or equal to the number of GPUs on the current system (``nproc_per_node``), +and each process will be operating on a single GPU from *GPU 0 to +GPU (nproc_per_node - 1)*. + +**How to use this module:** + +1. Single-Node multi-process distributed training + +:: + + python -m torch.distributed.launch --nproc-per-node=NUM_GPUS_YOU_HAVE + YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3 and all other + arguments of your training script) + +2. Multi-Node multi-process distributed training: (e.g. two nodes) + + +Node 1: *(IP: 192.168.1.1, and has a free port: 1234)* + +:: + + python -m torch.distributed.launch --nproc-per-node=NUM_GPUS_YOU_HAVE + --nnodes=2 --node-rank=0 --master-addr="192.168.1.1" + --master-port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3 + and all other arguments of your training script) + +Node 2: + +:: + + python -m torch.distributed.launch --nproc-per-node=NUM_GPUS_YOU_HAVE + --nnodes=2 --node-rank=1 --master-addr="192.168.1.1" + --master-port=1234 YOUR_TRAINING_SCRIPT.py (--arg1 --arg2 --arg3 + and all other arguments of your training script) + +3. To look up what optional arguments this module offers: + +:: + + python -m torch.distributed.launch --help + + +**Important Notices:** + +1. This utility and multi-process distributed (single-node or +multi-node) GPU training currently only achieves the best performance using +the NCCL distributed backend. Thus NCCL backend is the recommended backend to +use for GPU training. + +2. In your training program, you must parse the command-line argument: +``--local-rank=LOCAL_PROCESS_RANK``, which will be provided by this module. +If your training program uses GPUs, you should ensure that your code only +runs on the GPU device of LOCAL_PROCESS_RANK. This can be done by: + +Parsing the local_rank argument + +:: + + >>> # xdoctest: +SKIP + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--local-rank", "--local_rank", type=int) + >>> args = parser.parse_args() + +Set your device to local rank using either + +:: + + >>> torch.cuda.set_device(args.local_rank) # before your code runs + +or + +:: + + >>> with torch.cuda.device(args.local_rank): + >>> # your code to run + >>> ... + +.. versionchanged:: 2.0.0 + + The launcher will passes the ``--local-rank=`` argument to your script. + From PyTorch 2.0.0 onwards, the dashed ``--local-rank`` is preferred over the + previously used underscored ``--local_rank``. + + For backward compatibility, it may be necessary for users to handle both + cases in their argument parsing code. This means including both ``"--local-rank"`` + and ``"--local_rank"`` in the argument parser. If only ``"--local_rank"`` is + provided, the launcher will trigger an error: "error: unrecognized arguments: + --local-rank=". For training code that only supports PyTorch 2.0.0+, + including ``"--local-rank"`` should be sufficient. + +3. In your training program, you are supposed to call the following function +at the beginning to start the distributed backend. It is strongly recommended +that ``init_method=env://``. Other init methods (e.g. ``tcp://``) may work, +but ``env://`` is the one that is officially supported by this module. + +:: + + >>> torch.distributed.init_process_group(backend='YOUR BACKEND', + >>> init_method='env://') + +4. In your training program, you can either use regular distributed functions +or use :func:`torch.nn.parallel.DistributedDataParallel` module. If your +training program uses GPUs for training and you would like to use +:func:`torch.nn.parallel.DistributedDataParallel` module, +here is how to configure it. + +:: + + >>> model = torch.nn.parallel.DistributedDataParallel(model, + >>> device_ids=[args.local_rank], + >>> output_device=args.local_rank) + +Please ensure that ``device_ids`` argument is set to be the only GPU device id +that your code will be operating on. This is generally the local rank of the +process. In other words, the ``device_ids`` needs to be ``[args.local_rank]``, +and ``output_device`` needs to be ``args.local_rank`` in order to use this +utility + +5. Another way to pass ``local_rank`` to the subprocesses via environment variable +``LOCAL_RANK``. This behavior is enabled when you launch the script with +``--use-env=True``. You must adjust the subprocess example above to replace +``args.local_rank`` with ``os.environ['LOCAL_RANK']``; the launcher +will not pass ``--local-rank`` when you specify this flag. + +.. warning:: + + ``local_rank`` is NOT globally unique: it is only unique per process + on a machine. Thus, don't use it to decide if you should, e.g., + write to a networked filesystem. See + https://github.com/pytorch/pytorch/issues/12042 for an example of + how things can go wrong if you don't do this correctly. + + + +""" + +from typing_extensions import deprecated as _deprecated + +from torch.distributed.run import get_args_parser, run + + +def parse_args(args): + parser = get_args_parser() + parser.add_argument( + "--use-env", + "--use_env", + default=False, + action="store_true", + help="Use environment variable to pass " + "'local rank'. For legacy reasons, the default value is False. " + "If set to True, the script will not pass " + "--local-rank as argument, and will instead set LOCAL_RANK.", + ) + return parser.parse_args(args) + + +def launch(args): + if args.no_python and not args.use_env: + raise ValueError( + "When using the '--no-python' flag, you must also set the '--use-env' flag." + ) + run(args) + + +@_deprecated( + "The module torch.distributed.launch is deprecated\n" + "and will be removed in future. Use torchrun.\n" + "Note that --use-env is set by default in torchrun.\n" + "If your script expects `--local-rank` argument to be set, please\n" + "change it to read from `os.environ['LOCAL_RANK']` instead. See \n" + "https://pytorch.org/docs/stable/distributed.html#launch-utility for \n" + "further instructions\n", + category=FutureWarning, +) +def main(args=None): + args = parse_args(args) + launch(args) + + +if __name__ == "__main__": + main() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/logging_handlers.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/logging_handlers.py new file mode 100644 index 0000000000000000000000000000000000000000..8d070b48bcf0c49a95d7e7d83ae54d693b744b5a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/logging_handlers.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging + + +__all__: list[str] = [] + +_log_handlers: dict[str, logging.Handler] = { + "default": logging.NullHandler(), +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/remote_device.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/remote_device.py new file mode 100644 index 0000000000000000000000000000000000000000..620ec8dd6a3f156eb9dca1961bb15e1e87a8ea0a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/remote_device.py @@ -0,0 +1,122 @@ +# mypy: allow-untyped-defs + +import torch + + +class _remote_device: + """ + Represents a device on a remote worker. + + Args: + remote_device (str or torch.device): Represents a device on a remote worker. + The string format should be one of the following: + + 1. "/", where the device field can be parsed as torch.device type. + E.g., "trainer0/cpu", "trainer0", "ps0/cuda:0". + In addition, the device field can be optional and the default value is "cpu". + 2. "rank:/", where is the rank of the + process and device can be parsed as torch.device type. + E.g., "rank:0/cpu", "rank:0", "rank:0/cuda:0" + 3. and are optional and formats like "cpu" + and "cuda:1", just represent local devices. + """ + + def __init__(self, remote_device: str | torch.device): + PARSE_ERROR = ( + f"Could not parse remote_device: {remote_device}. The valid format is " + "'/' or 'rank:/' or ''" + ) + self._worker_name = None + self._rank = None + self._device: str | int | torch.device | None = None + + if isinstance(remote_device, torch.device): + self._device = remote_device + elif isinstance(remote_device, str): + fields = remote_device.split("/") + if len(fields) == 2: + # pyrefly: ignore [bad-assignment] + self._worker_name, self._device = fields + elif len(fields) == 1: + # Check if this is a valid device. + if _remote_device._is_valid_local_device(fields[0]): + self._device = fields[0] + else: + # pyrefly: ignore [bad-assignment] + self._worker_name = fields[0] + self._device = "cpu" + else: + raise ValueError(PARSE_ERROR) + else: + raise TypeError(f"Invalid type for remote_device: {type(remote_device)}") + + # Do some basic sanity check (no empty string) + if self._worker_name is not None and not self._worker_name: + raise ValueError(PARSE_ERROR) + + # Validate the device. + self._device = torch.device(self._device) + + # Check for rank based format. + if self._worker_name is not None: + fields = self._worker_name.split(":") + if len(fields) == 2: + # rank:/device format, extract rank + if fields[0] == "rank" and fields[1].isdigit(): + self._rank = int(fields[1]) # type: ignore[assignment] + # pyrefly: ignore [bad-assignment] + self._worker_name = None + else: + raise ValueError(PARSE_ERROR) + elif len(fields) > 2: + raise ValueError(PARSE_ERROR) + + @staticmethod + def _is_valid_local_device(device): + # Check for torch.device + try: + torch.device(device) + return True + except Exception: + return False + + def worker_name(self) -> str | None: + """Return the name of remote worker representing the remote device and ``None`` if no worker name is available.""" + return self._worker_name + + def rank(self) -> int | None: + """ + Returns the rank of remote worker representing the remote device. + Returns ``None`` if no rank is available. + """ + return self._rank + + def device(self) -> torch.device: + """Return the local device on the remote worker.""" + return self._device # type: ignore[return-value] + + def __repr__(self): + if self._device is not None: + if self._worker_name is not None: + return f"{self._worker_name}/{self._device}" + elif self._rank is not None: + return f"rank:{self._rank}/{self._device}" + else: + return str(self._device) + else: + if self._worker_name is not None: + return f"{self._worker_name}" + elif self._rank is not None: + return f"{self._rank}" + else: + raise RuntimeError("Invalid state!") + + def __eq__(self, other): + return isinstance(other, _remote_device) and ( + self._worker_name == other._worker_name + and self._device == other._device + and self._rank == other._rank + ) + + def __hash__(self): + return hash(self._worker_name) ^ hash(self._device) ^ hash(self._rank) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/rendezvous.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/rendezvous.py new file mode 100644 index 0000000000000000000000000000000000000000..3471b3a80c880fe79e09ac82fc263a9d4231ef2d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/rendezvous.py @@ -0,0 +1,293 @@ +# mypy: allow-untyped-defs +try: + from urllib.parse import urlparse, urlunparse +except ImportError as e: + raise ImportError( + "urllib cannot be found, urlparse from python2 is no longer supported." + ) from e + +import numbers +import os +import sys +from collections.abc import Callable, Iterator +from datetime import timedelta + +from torch.distributed import FileStore, Store, TCPStore + +from .constants import default_pg_timeout + + +_rendezvous_handlers: dict[str, Callable[..., Iterator[tuple[Store, int, int]]]] = {} + +__all__ = ["register_rendezvous_handler", "rendezvous"] + + +def register_rendezvous_handler(scheme, handler): + """ + Register a new rendezvous handler. + + Before we can run collective algorithms, participating processes + need to find each other and exchange information to be able to + communicate. We call this process rendezvous. + + The outcome of the rendezvous process is a triplet containing a + shared key/value store, the rank of the process, and the total + number of participating processes. + + If none of the bundled rendezvous methods apply to your execution + environment you can opt to register your own rendezvous handler. + Pick a unique name and use the URL scheme to identify it when + calling the `rendezvous()` function. + + Args: + scheme (str): URL scheme to identify your rendezvous handler. + handler (function): Handler that is invoked when the + `rendezvous()` function is called with a URL that uses + the corresponding scheme. It must be a generator function + that yields the triplet. + """ + global _rendezvous_handlers + if scheme in _rendezvous_handlers: + raise RuntimeError(f"Rendezvous handler for {scheme}:// already registered") + _rendezvous_handlers[scheme] = handler + + +# Query will have format "rank=0&world_size=1" and is +# converted into {"rank": 0, "world_size": 1} +def _query_to_dict(query: str) -> dict[str, str]: + return { + pair[0]: pair[1] + for pair in (pair.split("=") for pair in filter(None, query.split("&"))) + } + + +def _get_use_libuv_from_query_dict(query_dict: dict[str, str]) -> bool: + # libuv is the default backend for TCPStore. To enable the non-libuv backend, + # user can explicitly specify ``use_libuv=0`` in the URL parameter. + if sys.platform == "win32": + # PyTorch is built without libuv support on windows, so default to 0 + return query_dict.get("use_libuv", os.environ.get("USE_LIBUV", "0")) == "1" + return query_dict.get("use_libuv", os.environ.get("USE_LIBUV", "1")) == "1" + + +def _rendezvous_helper(url: str, rank: int, world_size_opt: int | None, **kwargs): + result = urlparse(url) + if world_size_opt is None: + world_size = -1 + if result.scheme == "env": + rank = int(os.environ.get("RANK", rank)) + # If the world_size env variable is not present then it is a dynamic group + world_size = int(os.environ.get("WORLD_SIZE", world_size)) + else: + world_size = world_size_opt + if rank != -1 or world_size != -1 or world_size_opt is None: + query_dict = _query_to_dict(result.query) + if "rank" in query_dict or "world_size" in query_dict: + raise AssertionError( + f"The url: {url} has node-specific arguments(rank, world_size) already." + ) + if rank != -1: + query_dict["rank"] = str(rank) + if world_size != -1 or world_size_opt is None: + query_dict["world_size"] = str(world_size) + result = result._replace( + query=f"{'&'.join([f'{k}={v}' for k, v in query_dict.items()])}" + ) + # pyrefly: ignore [bad-assignment] + url = urlunparse(result) + + if result.scheme not in _rendezvous_handlers: + raise RuntimeError(f"No rendezvous handler for {result.scheme}://") + return _rendezvous_handlers[result.scheme](url, **kwargs) + + +def rendezvous(url: str, rank: int = -1, world_size: int = -1, **kwargs): + if not isinstance(url, (str, bytes)): + raise RuntimeError(f"`url` must be a string. {type(url)}: {url}") + + if not isinstance(rank, numbers.Integral): + raise RuntimeError(f"`rank` must be an integer. {rank}") + + if not isinstance(world_size, numbers.Integral): + raise RuntimeError(f"`world_size` must be an integer. {world_size}") + + # pyrefly: ignore [bad-argument-type] + return _rendezvous_helper(url, rank, world_size, **kwargs) + + +def _create_store_from_options(backend_options, rank): + store, _, _ = next(_rendezvous_helper(backend_options.init_method, rank, None)) + return store + + +def _rendezvous_error(msg): + return ValueError("Error initializing torch.distributed using " + msg) + + +def _file_rendezvous_handler(url: str, **kwargs): + def _error(msg): + return _rendezvous_error("file:// rendezvous: " + msg) + + result = urlparse(url) + path = result.path + if sys.platform == "win32": + import urllib.request + + full_path = result.netloc + result.path + path = urllib.request.url2pathname(full_path) + if path: + # Normalizing an empty string produces ".", which is not expected. + path = os.path.normpath(path) + + if not path: + raise _error("path missing") + query_dict = _query_to_dict(result.query) + if "rank" not in query_dict: + raise _error("rank parameter missing") + if "world_size" not in query_dict: + raise _error("world size parameter missing") + + rank = int(query_dict["rank"]) + world_size = int(query_dict["world_size"]) + store = FileStore(path, world_size) + yield (store, rank, world_size) + + # If this configuration is invalidated, there is nothing we can do about it + raise RuntimeError("Unable to perform rerendezvous using file:// method") + + +def _torchelastic_use_agent_store() -> bool: + return os.environ.get("TORCHELASTIC_USE_AGENT_STORE", None) == str(True) + + +def _create_c10d_store( + hostname, port, rank, world_size, timeout, use_libuv=True +) -> Store: + """ + Smartly creates a c10d Store object on ``rank`` based on whether we need to reuse agent store. + + The TCPStore server is assumed to be hosted + on ``hostname:port``. + + By default, the TCPStore server uses the asynchronous implementation + ``LibUVStoreDaemon`` which utilizes libuv. + + If ``torchelastic_use_agent_store()`` is ``True``, then it is assumed that + the agent leader (node rank 0) hosts the TCPStore server (for which the + endpoint is specified by the given ``hostname:port``). Hence + ALL ranks will create and return a TCPStore client (e.g. ``start_daemon=False``). + + If ``torchelastic_use_agent_store()`` is ``False``, then rank 0 will host + the TCPStore (with multi-tenancy) and it is assumed that rank 0's hostname + and port are correctly passed via ``hostname`` and ``port``. All + non-zero ranks will create and return a TCPStore client. + """ + # check if port is uint16_t + if not 0 <= port < 2**16: + raise ValueError(f"port must have value from 0 to 65535 but was {port}.") + + if _torchelastic_use_agent_store(): + # We create a new TCPStore for every retry so no need to add prefix for each attempt. + return TCPStore( + host_name=hostname, + port=port, + world_size=world_size, + is_master=False, + timeout=timeout, + ) + else: + start_daemon = rank == 0 + return TCPStore( + host_name=hostname, + port=port, + world_size=world_size, + is_master=start_daemon, + timeout=timeout, + multi_tenant=True, + use_libuv=use_libuv, + ) + + +def _tcp_rendezvous_handler( + url: str, timeout: timedelta = default_pg_timeout, **kwargs +): + def _error(msg): + return _rendezvous_error("tcp:// rendezvous: " + msg) + + result = urlparse(url) + if result.port is None: + raise _error("port number missing") + query_dict = _query_to_dict(result.query) + if "rank" not in query_dict: + raise _error("rank parameter missing") + if "world_size" not in query_dict: + raise _error("world size parameter missing") + + rank = int(query_dict["rank"]) + world_size = int(query_dict["world_size"]) + use_libuv = _get_use_libuv_from_query_dict(query_dict) + + if result.hostname is None: + raise AssertionError("hostname cannot be None") + + store = _create_c10d_store( + result.hostname, result.port, rank, world_size, timeout, use_libuv + ) + + yield (store, rank, world_size) + + # If this configuration is invalidated, there is nothing we can do about it + raise RuntimeError("Unable to perform re-rendezvous using tcp:// method") + + +def _env_rendezvous_handler( + url: str, timeout: timedelta = default_pg_timeout, **kwargs +): + def _error(msg): + return _rendezvous_error("env:// rendezvous: " + msg) + + def _env_error(var): + return _error(f"environment variable {var} expected, but not set") + + def _get_env_or_raise(env_var: str) -> str: + env_val = os.environ.get(env_var, None) + if not env_val: + raise _env_error(env_var) + else: + return env_val + + result = urlparse(url) + query_dict = _query_to_dict(result.query) + + rank: int + world_size: int + master_port: int + master_addr: str + + if "rank" in query_dict: + rank = int(query_dict["rank"]) + else: + rank = int(_get_env_or_raise("RANK")) + + if "world_size" in query_dict: + world_size = int(query_dict["world_size"]) + else: + world_size = int(_get_env_or_raise("WORLD_SIZE")) + + master_addr = _get_env_or_raise("MASTER_ADDR") + master_port = int(_get_env_or_raise("MASTER_PORT")) + use_libuv = _get_use_libuv_from_query_dict(query_dict) + + store = _create_c10d_store( + master_addr, master_port, rank, world_size, timeout, use_libuv + ) + + yield (store, rank, world_size) + + # If this configuration is invalidated, there is nothing we can do about it + raise RuntimeError("Unable to perform re-rendezvous using env:// method") + + +register_rendezvous_handler("tcp", _tcp_rendezvous_handler) +register_rendezvous_handler("env", _env_rendezvous_handler) +register_rendezvous_handler("file", _file_rendezvous_handler) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/run.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/run.py new file mode 100644 index 0000000000000000000000000000000000000000..c31dbd82f44af4ba55cf016c57c29174e8b1c923 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/run.py @@ -0,0 +1,995 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Module ``torch.distributed.run``. + +``torch.distributed.run`` is a module that spawns up multiple distributed +training processes on each of the training nodes. + +``torchrun`` is a python +`console script `_ +to the main module +`torch.distributed.run `_ +declared in the ``entry_points`` configuration in +`setup.py `_. +It is equivalent to invoking ``python -m torch.distributed.run``. + +``torchrun`` can be used for single-node distributed training, in which one or +more processes per node will be spawned. It can be used for either +CPU training or GPU training. If it is used for GPU training, +each distributed process will be operating on a single GPU. This can achieve +well-improved single-node training performance. ``torchrun`` can also be used in +multi-node distributed training, by spawning up multiple processes on each node +for well-improved multi-node distributed training performance as well. +This will especially be beneficial for systems with multiple Infiniband +interfaces that have direct-GPU support, since all of them can be utilized for +aggregated communication bandwidth. + +In both cases of single-node distributed training or multi-node distributed +training, ``torchrun`` will launch the given number of processes per node +(``--nproc-per-node``). If used for GPU training, this number needs to be less +or equal to the number of GPUs on the current system (``nproc_per_node``), +and each process will be operating on a single GPU from *GPU 0 to +GPU (nproc_per_node - 1)*. + +.. versionchanged:: 2.0.0 + + ``torchrun`` will pass the ``--local-rank=`` argument to your script. + From PyTorch 2.0.0 onwards, the dashed ``--local-rank`` is preferred over the + previously used underscored ``--local_rank``. + + For backward compatibility, it may be necessary for users to handle both + cases in their argument parsing code. This means including both ``"--local-rank"`` + and ``"--local_rank"`` in the argument parser. If only ``"--local_rank"`` is + provided, ``torchrun`` will trigger an error: "error: unrecognized arguments: + --local-rank=". For training code that only supports PyTorch 2.0.0+, + including ``"--local-rank"`` should be sufficient. + + :: + + >>> # xdoctest: +SKIP + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument("--local-rank", "--local_rank", type=int) + >>> args = parser.parse_args() + +Usage +----- + +Single-node multi-worker +++++++++++++++++++++++++ + +:: + + torchrun + --standalone + --nnodes=1 + --nproc-per-node=$NUM_TRAINERS + YOUR_TRAINING_SCRIPT.py (--arg1 ... train script args...) + +.. note:: ``--nproc-per-node`` may be + ``"gpu"`` (spawn one process per GPU), + ``"cpu"`` (spawn one process per CPU), + ``"xpu"`` (spawn one process per XPU), + ``"auto"`` (equivalent to ``"gpu"`` if CUDA is available, + else equivalent to ``"xpu"`` if XPU is available, + else equivalent to ``"cpu"``), + or an integer specifying the number of processes. + See `torch.distributed.run.determine_local_world_size + `_ + for more details. + +Stacked single-node multi-worker +++++++++++++++++++++++++++++++++ + +To run multiple instances (separate jobs) of single-node, multi-worker on the +same host, we need to make sure that each instance (job) is +setup on different ports to avoid port conflicts (or worse, two jobs being merged +as a single job). To do this you have to run with ``--rdzv-backend=c10d`` +and specify a different port by setting ``--rdzv-endpoint=localhost:$PORT_k``. +For ``--nodes=1``, its often convenient to let ``torchrun`` pick a free random +port automatically instead of manually assigning different ports for each run. + +:: + + torchrun + --rdzv-backend=c10d + --rdzv-endpoint=localhost:0 + --nnodes=1 + --nproc-per-node=$NUM_TRAINERS + YOUR_TRAINING_SCRIPT.py (--arg1 ... train script args...) + + +Fault tolerant (fixed sized number of workers, no elasticity, tolerates 3 failures) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +:: + + torchrun + --nnodes=$NUM_NODES + --nproc-per-node=$NUM_TRAINERS + --max-restarts=3 + --rdzv-id=$JOB_ID + --rdzv-backend=c10d + --rdzv-endpoint=$HOST_NODE_ADDR + YOUR_TRAINING_SCRIPT.py (--arg1 ... train script args...) + +``HOST_NODE_ADDR``, in form [:] (e.g. node1.example.com:29400), specifies the node and +the port on which the C10d rendezvous backend should be instantiated and hosted. It can be any +node in your training cluster, but ideally you should pick a node that has a high bandwidth. + +.. note:: + If no port number is specified ``HOST_NODE_ADDR`` defaults to 29400. + +Elastic (``min=1``, ``max=4``, tolerates up to 3 membership changes or failures) +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +:: + + torchrun + --nnodes=1:4 + --nproc-per-node=$NUM_TRAINERS + --max-restarts=3 + --rdzv-id=$JOB_ID + --rdzv-backend=c10d + --rdzv-endpoint=$HOST_NODE_ADDR + YOUR_TRAINING_SCRIPT.py (--arg1 ... train script args...) + +``HOST_NODE_ADDR``, in form [:] (e.g. node1.example.com:29400), specifies the node and +the port on which the C10d rendezvous backend should be instantiated and hosted. It can be any +node in your training cluster, but ideally you should pick a node that has a high bandwidth. + +.. note:: + If no port number is specified ``HOST_NODE_ADDR`` defaults to 29400. + +Note on rendezvous backend +-------------------------- + +For multi-node training you need to specify: + +1. ``--rdzv-id``: A unique job id (shared by all nodes participating in the job) +2. ``--rdzv-backend``: An implementation of + :py:class:`torch.distributed.elastic.rendezvous.RendezvousHandler` +3. ``--rdzv-endpoint``: The endpoint where the rendezvous backend is running; usually in form + ``host:port``. + +Currently ``c10d`` (recommended), ``etcd-v2``, and ``etcd`` (legacy) rendezvous backends are +supported out of the box. To use ``etcd-v2`` or ``etcd``, setup an etcd server with the ``v2`` api +enabled (e.g. ``--enable-v2``). + +.. warning:: + ``etcd-v2`` and ``etcd`` rendezvous use etcd API v2. You MUST enable the v2 API on the etcd + server. Our tests use etcd v3.4.3. + +.. warning:: + For etcd-based rendezvous we recommend using ``etcd-v2`` over ``etcd`` which is functionally + equivalent, but uses a revised implementation. ``etcd`` is in maintenance mode and will be + removed in a future version. + +Definitions +----------- + +1. ``Node`` - A physical instance or a container; maps to the unit that the job manager works with. + +2. ``Worker`` - A worker in the context of distributed training. + +3. ``WorkerGroup`` - The set of workers that execute the same function (e.g. trainers). + +4. ``LocalWorkerGroup`` - A subset of the workers in the worker group running on the same node. + +5. ``RANK`` - The rank of the worker within a worker group. + +6. ``WORLD_SIZE`` - The total number of workers in a worker group. + +7. ``LOCAL_RANK`` - The rank of the worker within a local worker group. + +8. ``LOCAL_WORLD_SIZE`` - The size of the local worker group. + +9. ``rdzv_id`` - A user-defined id that uniquely identifies the worker group for a job. This id is + used by each node to join as a member of a particular worker group. + +9. ``rdzv_backend`` - The backend of the rendezvous (e.g. ``c10d``). This is typically a strongly + consistent key-value store. + +10. ``rdzv_endpoint`` - The rendezvous backend endpoint; usually in form ``:``. + +A ``Node`` runs ``LOCAL_WORLD_SIZE`` workers which comprise a ``LocalWorkerGroup``. The union of +all ``LocalWorkerGroups`` in the nodes in the job comprise the ``WorkerGroup``. + +Environment Variables +--------------------- + +The following environment variables are made available to you in your script: + +1. ``LOCAL_RANK`` - The local rank. + +2. ``RANK`` - The global rank. + +3. ``GROUP_RANK`` - The rank of the worker group. A number between 0 and ``max_nnodes``. When + running a single worker group per node, this is the rank of the node. + +4. ``ROLE_RANK`` - The rank of the worker across all the workers that have the same role. The role + of the worker is specified in the ``WorkerSpec``. + +5. ``LOCAL_WORLD_SIZE`` - The local world size (e.g. number of workers running locally); equals to + ``--nproc-per-node`` specified on ``torchrun``. + +6. ``WORLD_SIZE`` - The world size (total number of workers in the job). + +7. ``ROLE_WORLD_SIZE`` - The total number of workers that was launched with the same role specified + in ``WorkerSpec``. + +8. ``MASTER_ADDR`` - The FQDN of the host that is running worker with rank 0; used to initialize + the Torch Distributed backend. + +9. ``MASTER_PORT`` - The port on the ``MASTER_ADDR`` that can be used to host the C10d TCP store. + +10. ``TORCHELASTIC_RESTART_COUNT`` - The number of worker group restarts so far. + +11. ``TORCHELASTIC_MAX_RESTARTS`` - The configured maximum number of restarts. + +12. ``TORCHELASTIC_RUN_ID`` - Equal to the rendezvous ``run_id`` (e.g. unique job id). + +13. ``PYTHON_EXEC`` - System executable override. If provided, the python user script will + use the value of ``PYTHON_EXEC`` as executable. The `sys.executable` is used by default. + +Deployment +---------- + +1. (Not needed for the C10d backend) Start the rendezvous backend server and get the endpoint (to be + passed as ``--rdzv-endpoint`` to ``torchrun``) + +2. Single-node multi-worker: Start ``torchrun`` on the host to start the agent process which + creates and monitors a local worker group. + +3. Multi-node multi-worker: Start ``torchrun`` with the same arguments on all the nodes + participating in training. + +When using a job/cluster manager, the entry point command to the multi-node job should be ``torchrun``. + +Failure Modes +------------- + +1. Worker failure: For a training job with ``n`` workers, if ``k<=n`` workers fail all workers + are stopped and restarted up to ``max_restarts``. + +2. Agent failure: An agent failure results in a local worker group failure. It is up to the job + manager to fail the entire job (gang semantics) or attempt to replace the node. Both behaviors + are supported by the agent. + +3. Node failure: Same as agent failure. + +Membership Changes +------------------ + +1. Node departure (scale-down): The agent is notified of the departure, all existing workers are + stopped, a new ``WorkerGroup`` is formed, and all workers are started with a new ``RANK`` and + ``WORLD_SIZE``. + +2. Node arrival (scale-up): The new node is admitted to the job, all existing workers are stopped, + a new ``WorkerGroup`` is formed, and all workers are started with a new ``RANK`` and + ``WORLD_SIZE``. + +Important Notices +----------------- + +1. This utility and multi-process distributed (single-node or + multi-node) GPU training currently only achieves the best performance using + the NCCL distributed backend. Thus NCCL backend is the recommended backend to + use for GPU training. + +2. The environment variables necessary to initialize a Torch process group are provided to you by + this module, no need for you to pass ``RANK`` manually. To initialize a process group in your + training script, simply run: + +:: + + >>> # xdoctest: +SKIP("stub") + >>> import torch.distributed as dist + >>> dist.init_process_group(backend="gloo|nccl") + +3. In your training program, you can either use regular distributed functions + or use :func:`torch.nn.parallel.DistributedDataParallel` module. If your + training program uses GPUs for training and you would like to use + :func:`torch.nn.parallel.DistributedDataParallel` module, + here is how to configure it. + +:: + + local_rank = int(os.environ["LOCAL_RANK"]) + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[local_rank], output_device=local_rank + ) + +Please ensure that ``device_ids`` argument is set to be the only GPU device id +that your code will be operating on. This is generally the local rank of the +process. In other words, the ``device_ids`` needs to be ``[int(os.environ("LOCAL_RANK"))]``, +and ``output_device`` needs to be ``int(os.environ("LOCAL_RANK"))`` in order to use this +utility + + +4. On failures or membership changes ALL surviving workers are killed immediately. Make sure to + checkpoint your progress. The frequency of checkpoints should depend on your job's tolerance + for lost work. + +5. This module only supports homogeneous ``LOCAL_WORLD_SIZE``. That is, it is assumed that all + nodes run the same number of local workers (per role). + +6. ``RANK`` is NOT stable. Between restarts, the local workers on a node can be assigned a + different range of ranks than before. NEVER hard code any assumptions about the stable-ness of + ranks or some correlation between ``RANK`` and ``LOCAL_RANK``. + +7. When using elasticity (``min_size!=max_size``) DO NOT hard code assumptions about + ``WORLD_SIZE`` as the world size can change as nodes are allowed to leave and join. + +8. It is recommended for your script to have the following structure: + +:: + + def main(): + load_checkpoint(checkpoint_path) + initialize() + train() + + + def train(): + for batch in iter(dataset): + train_step(batch) + + if should_checkpoint: + save_checkpoint(checkpoint_path) + +9. (Recommended) On worker errors, this tool will summarize the details of the error + (e.g. time, rank, host, pid, traceback, etc). On each node, the first error (by timestamp) + is heuristically reported as the "Root Cause" error. To get tracebacks as part of this + error summary print out, you must decorate your main entrypoint function in your + training script as shown in the example below. If not decorated, then the summary + will not include the traceback of the exception and will only contain the exitcode. + For details on torchelastic error handling see: https://pytorch.org/docs/stable/elastic/errors.html + +:: + + from torch.distributed.elastic.multiprocessing.errors import record + + + @record + def main(): + # do train + pass + + + if __name__ == "__main__": + main() +""" # noqa: E501 + +import os +import sys +import uuid +from argparse import ArgumentParser, REMAINDER +from collections.abc import Callable +from importlib import metadata + +import torch +from torch.distributed.argparse_util import check_env, env +from torch.distributed.elastic.multiprocessing import DefaultLogsSpecs, LogsSpecs, Std +from torch.distributed.elastic.multiprocessing.errors import record +from torch.distributed.elastic.rendezvous.utils import _parse_rendezvous_config +from torch.distributed.elastic.utils import macros +from torch.distributed.elastic.utils.logging import get_logger +from torch.distributed.launcher.api import elastic_launch, LaunchConfig +from torch.numa.binding import ( + AffinityMode as _AffinityMode, # Signify as private with _ + NumaOptions as _NumaOptions, +) +from torch.utils.backend_registration import _get_custom_mod_func + + +logger = get_logger(__name__) + + +def get_args_parser() -> ArgumentParser: + """Parse the command line options.""" + parser = ArgumentParser(description="Torch Distributed Elastic Training Launcher") + + def comma_separated_list(value): + placeholder = "" + value = value.replace(",,", placeholder) + items = value.split(",") + items = [item.replace(placeholder, ",") for item in items] + return items + + # + # Worker/node size related arguments. + # + + parser.add_argument( + "--nnodes", + action=env, + type=str, + default="1:1", + help="Number of nodes, or the range of nodes in form :.", + ) + parser.add_argument( + "--nproc-per-node", + "--nproc_per_node", + action=env, + type=str, + default="1", + help="Number of workers per node; supported values: [auto, cpu, gpu, xpu, int].", + ) + + # + # Rendezvous related arguments + # + + parser.add_argument( + "--rdzv-backend", + "--rdzv_backend", + action=env, + type=str, + default="static", + help="Rendezvous backend.", + ) + parser.add_argument( + "--rdzv-endpoint", + "--rdzv_endpoint", + action=env, + type=str, + default="", + help="Rendezvous backend endpoint; usually in form :.", + ) + parser.add_argument( + "--rdzv-id", + "--rdzv_id", + action=env, + type=str, + default="none", + help="User-defined group id.", + ) + parser.add_argument( + "--rdzv-conf", + "--rdzv_conf", + action=env, + type=str, + default="", + help="Additional rendezvous configuration (=,=,...).", + ) + parser.add_argument( + "--standalone", + action=check_env, + help="Start a local standalone rendezvous backend that is represented by a C10d TCP store " + "on a free port. Useful when launching single-node, multi-worker job. If specified " + "--rdzv-backend, --rdzv-endpoint, --rdzv-id are auto-assigned and any explicitly set values " + "are ignored.", + ) + + # + # User-code launch related arguments. + # + + parser.add_argument( + "--max-restarts", + "--max_restarts", + action=env, + type=int, + default=0, + help="Maximum number of worker group restarts before failing.", + ) + parser.add_argument( + "--monitor-interval", + "--monitor_interval", + action=env, + type=float, + default=0.1, + help="Interval, in seconds, to monitor the state of workers.", + ) + parser.add_argument( + "--start-method", + "--start_method", + action=env, + type=str, + default="spawn", + choices=["spawn", "fork", "forkserver"], + help="Multiprocessing start method to use when creating workers.", + ) + parser.add_argument( + "--event-log-handler", + "--event_log_handler", + action=env, + type=str, + default="null", + help="name of a registered event logging handler (see: https://docs.pytorch.org/docs/stable/elastic/events.html)", + ) + parser.add_argument( + "--role", + action=env, + type=str, + default="default", + help="User-defined role for the workers.", + ) + parser.add_argument( + "-m", + "--module", + action=check_env, + help="Change each process to interpret the launch script as a Python module, executing " + "with the same behavior as 'python -m'.", + ) + parser.add_argument( + "--no-python", + "--no_python", + action=check_env, + help="Skip prepending the training script with 'python' - just execute it directly. Useful " + "when the script is not a Python script.", + ) + + parser.add_argument( + "--run-path", + "--run_path", + action=check_env, + help="Run the training script with runpy.run_path in the same interpreter." + " Script must be provided as an abs path (e.g. /abs/path/script.py)." + " Takes precedence over --no-python.", + ) + parser.add_argument( + "--log-dir", + "--log_dir", + action=env, + type=str, + default=None, + help="Base directory to use for log files (e.g. /var/log/torch/elastic). The same " + "directory is reused for multiple runs (a unique job-level sub-directory is created with " + "rdzv_id as the prefix).", + ) + parser.add_argument( + "-r", + "--redirects", + action=env, + type=str, + default="0", + help="Redirect std streams into a log file in the log directory (e.g. [-r 3] redirects " + "both stdout+stderr for all workers, [-r 0:1,1:2] redirects stdout for local rank 0 and " + "stderr for local rank 1).", + ) + parser.add_argument( + "-t", + "--tee", + action=env, + type=str, + default="0", + help="Tee std streams into a log file and also to console (see --redirects for format).", + ) + + parser.add_argument( + "--local-ranks-filter", + "--local_ranks_filter", + action=env, + type=str, + default="", + help="Only show logs from specified ranks in console (e.g. [--local_ranks_filter=0,1,2] will " + "only show logs from rank 0, 1 and 2). This will only apply to stdout and stderr, not to" + "log files saved via --redirect or --tee", + ) + + parser.add_argument( + "--duplicate-stdout-filters", + "--duplicate_stdout_filters", + action=env, + type=comma_separated_list, + default=[], + help="Duplicates logs streamed to stdout to another specified file with a list of filters (e.g. " + "[--duplicate_stdout_filters 'apple,orange'] will duplicate log lines matching 'apple' " + "OR 'orange'. An empty filters list won't duplicate any lines. Use double comma to escape a comma) ", + ) + + parser.add_argument( + "--duplicate-stderr-filters", + "--duplicate_stderr_filters", + action=env, + type=comma_separated_list, + default=[], + help="Duplicates logs streamed to stderr to another specified file with a list of filters (e.g. " + "[--duplicate_stdout_filters 'apple,orange'] will duplicate log lines matching 'apple' " + "OR 'orange'. An empty filters list won't duplicate any lines. Use double comma to escape a comma) ", + ) + + # + # Backwards compatible parameters with caffe2.distributed.launch. + # + + parser.add_argument( + "--node-rank", + "--node_rank", + type=int, + action=env, + default=0, + help="Rank of the node for multi-node distributed training.", + ) + parser.add_argument( + "--master-addr", + "--master_addr", + default="127.0.0.1", + type=str, + action=env, + help="Address of the master node (rank 0) that only used for static rendezvous. It should " + "be either the IP address or the hostname of rank 0. For single node multi-proc training " + "the --master-addr can simply be 127.0.0.1; IPv6 should have the pattern " + "`[0:0:0:0:0:0:0:1]`.", + ) + parser.add_argument( + "--master-port", + "--master_port", + default=29500, + type=int, + action=env, + help="Port on the master node (rank 0) to be used for communication during distributed " + "training. It is only used for static rendezvous.", + ) + parser.add_argument( + "--local-addr", + "--local_addr", + default=None, + type=str, + action=env, + help="Address of the local node. If specified, will use the given address for connection. " + "Else, will look up the local node address instead. Else, it will be default to local " + "machine's FQDN.", + ) + + parser.add_argument( + "--logs-specs", + "--logs_specs", + default=None, + type=str, + help="torchrun.logs_specs group entrypoint name, value must be type of LogsSpecs. " + "Can be used to override custom logging behavior.", + ) + + parser.add_argument( + "--numa-binding", + "--numa_binding", + type=str, + choices=[mode.value for mode in _AffinityMode], + default=None, + help=""" + If provided, we will affinitize the worker processes based on NUMA nodes + for better performance. (E.g., preferring to allocate memory locally and run on CPUs on the + same NUMA node.) + + NOTE: This is currently only supported for GPUs, and we assume + that the LOCAL_RANK process corresponds to the GPU with index LOCAL_RANK. If this is not + accurate for your workload, this feature may be a pessimization. + + Available options are: + - node: Processes are bound to cpu cores within a NUMA node. This is a good starting point, + but other options may perform even slightly better in some cases. + - socket: Processes are bound to cpu cores within a socket. + - exclusive: Processes are bound to exclusive sets of cpu cores within a NUMA node. + - core-complex: Processes are bound to cpu cores in a core-complex. + NOTE: The core-complex option might not achieve optimal performance on architectures + featuring a single L3 cache per socket.""", + ) + + parser.add_argument( + "--signals-to-handle", + "--signals_to_handle", + action=env, + type=str, + default="SIGTERM,SIGINT,SIGHUP,SIGQUIT", + help="Comma-separated list of signals to handle and forward to subprocesses. " + "Default: SIGTERM,SIGINT,SIGHUP,SIGQUIT. " + "Common additional signals: SIGUSR1,SIGUSR2 (used in SLURM environments).", + ) + + parser.add_argument( + "--virtual-local-rank", + "--virtual_local_rank", + action=check_env, + help="Enable virtual local rank mode for workers. When enabled, LOCAL_RANK is set to 0 " + "for all workers and CUDA_VISIBLE_DEVICES is adjusted so each worker accesses its " + "assigned GPU at device index 0.", + ) + + # + # Positional arguments. + # + + parser.add_argument( + "training_script", + type=str, + help="Full path to the (single GPU) training program/script to be launched in parallel, " + "followed by all the arguments for the training script.", + ) + + # Rest from the training program. + parser.add_argument("training_script_args", nargs=REMAINDER) + + return parser + + +def parse_args(args): + parser = get_args_parser() + return parser.parse_args(args) + + +def parse_min_max_nnodes(nnodes: str): + arr = nnodes.split(":") + + if len(arr) == 1: + min_nodes = max_nodes = int(arr[0]) + elif len(arr) == 2: + min_nodes = int(arr[0]) + max_nodes = int(arr[1]) + else: + raise RuntimeError(f'nnodes={nnodes} is not in "MIN:MAX" format') # noqa: E231 + + return min_nodes, max_nodes + + +def determine_local_world_size(nproc_per_node: str): + try: + logger.info("Using nproc_per_node=%s.", nproc_per_node) + return int(nproc_per_node) + except ValueError as e: + if nproc_per_node == "cpu": + num_proc = os.cpu_count() + device_type = "cpu" + elif nproc_per_node == "gpu": + if not torch.cuda.is_available(): + raise ValueError("Cuda is not available.") from e + device_type = "gpu" + num_proc = torch.cuda.device_count() + elif nproc_per_node == "xpu": + if not torch.xpu.is_available(): + raise ValueError("Xpu is not available.") from e + device_type = "xpu" + num_proc = torch.xpu.device_count() + elif nproc_per_node == torch._C._get_privateuse1_backend_name(): + if not _get_custom_mod_func("is_available")(): + raise ValueError(f"{nproc_per_node} is not available.") from e + device_type = nproc_per_node + num_proc = _get_custom_mod_func("device_count")() + elif nproc_per_node == "auto": + if torch.accelerator.is_available(): + num_proc = torch.accelerator.device_count() + device_type = torch.accelerator.current_accelerator().type # type: ignore[union-attr] + else: + num_proc = os.cpu_count() + device_type = "cpu" + else: + raise ValueError( + f"Unsupported nproc_per_node value: {nproc_per_node}" + ) from e + + logger.info( + "Using nproc_per_node=%s, setting nproc_per_node to %s since the instance has %s %s", + nproc_per_node, + num_proc, + num_proc, + device_type, + ) + return num_proc + + +def get_rdzv_endpoint(args): + if args.rdzv_backend == "static" and not args.rdzv_endpoint: + return f"{args.master_addr}:{args.master_port}" # noqa: E231 + return args.rdzv_endpoint + + +def get_use_env(args) -> bool: + """ + Retrieve ``use_env`` from the args. + + ``use_env`` is a legacy argument, if ``use_env`` is False, the + ``--node-rank`` argument will be transferred to all worker processes. + ``use_env`` is only used by the ``torch.distributed.launch`` and will + be deprecated in future releases. + """ + if not hasattr(args, "use_env"): + return True + return args.use_env + + +def _get_logs_specs_class(logs_specs_name: str | None) -> type[LogsSpecs]: + """ + Attempts to load `torchrun.logs_spec` entrypoint with key of `logs_specs_name` param. + Provides plugin mechanism to provide custom implementation of LogsSpecs. + + Returns `DefaultLogsSpecs` when logs_spec_name is None. + Raises ValueError when entrypoint for `logs_spec_name` can't be found in entrypoints. + """ + logs_specs_cls = None + if logs_specs_name is not None: + eps = metadata.entry_points() + group = eps.select(group="torchrun.logs_specs") + if group.select(name=logs_specs_name): + logs_specs_cls = group[logs_specs_name].load() + + if logs_specs_cls is None: + raise ValueError( + f"Could not find entrypoint under 'torchrun.logs_specs[{logs_specs_name}]' key" + ) + + logger.info( + "Using logs_spec '%s' mapped to %s", logs_specs_name, str(logs_specs_cls) + ) + else: + logs_specs_cls = DefaultLogsSpecs + + return logs_specs_cls + + +def config_from_args(args) -> tuple[LaunchConfig, Callable | str, list[str]]: + # If ``args`` not passed, defaults to ``sys.argv[:1]`` + min_nodes, max_nodes = parse_min_max_nnodes(args.nnodes) + if not (0 < min_nodes <= max_nodes): + raise AssertionError( + f"min_nodes must be > 0 and <= max_nodes, got min_nodes={min_nodes}, max_nodes={max_nodes}" + ) + if args.max_restarts < 0: + raise AssertionError("max_restarts must be >= 0") + + if ( + hasattr(args, "master_addr") + and args.rdzv_backend != "static" + and not args.rdzv_endpoint + ): + logger.warning( + "master_addr is only used for static rdzv_backend and when rdzv_endpoint " + "is not specified." + ) + + nproc_per_node = determine_local_world_size(args.nproc_per_node) + if "OMP_NUM_THREADS" not in os.environ and nproc_per_node > 1: + omp_num_threads = 1 + logger.warning( + "\n*****************************************\n" + "Setting OMP_NUM_THREADS environment variable for each process to be " + "%s in default, to avoid your system being overloaded, " + "please further tune the variable for optimal performance in " + "your application as needed. \n" + "*****************************************", + omp_num_threads, + ) + # This env variable will be passed down to the subprocesses + os.environ["OMP_NUM_THREADS"] = str(omp_num_threads) + + log_line_prefix_template = os.getenv("TORCHELASTIC_LOG_LINE_PREFIX_TEMPLATE") + + rdzv_configs = _parse_rendezvous_config(args.rdzv_conf) + + if args.rdzv_backend == "static": + rdzv_configs["rank"] = args.node_rank + + rdzv_endpoint = get_rdzv_endpoint(args) + + ranks: set[int] | None = None + if args.local_ranks_filter: + try: + ranks = set(map(int, args.local_ranks_filter.split(","))) + if not ranks: + raise AssertionError("ranks set cannot be empty") + except Exception as e: + raise ValueError( + "--local_ranks_filter must be a comma-separated list of integers e.g. --local_ranks_filter=0,1,2" + ) from e + + logs_specs_cls: type[LogsSpecs] = _get_logs_specs_class(args.logs_specs) + # pyrefly: ignore [bad-instantiation] + logs_specs = logs_specs_cls( + log_dir=args.log_dir, + redirects=Std.from_str(args.redirects), + tee=Std.from_str(args.tee), + local_ranks_filter=ranks, + ) + numa_options = ( + None + if args.numa_binding is None + else _NumaOptions(affinity_mode=_AffinityMode(args.numa_binding)) + ) + + config = LaunchConfig( + min_nodes=min_nodes, + max_nodes=max_nodes, + nproc_per_node=nproc_per_node, + run_id=args.rdzv_id, + role=args.role, + rdzv_endpoint=rdzv_endpoint, + rdzv_backend=args.rdzv_backend, + rdzv_configs=rdzv_configs, + max_restarts=args.max_restarts, + monitor_interval=args.monitor_interval, + start_method=args.start_method, + log_line_prefix_template=log_line_prefix_template, + local_addr=args.local_addr, + logs_specs=logs_specs, + event_log_handler=args.event_log_handler, + numa_options=numa_options, + signals_to_handle=args.signals_to_handle, + duplicate_stdout_filters=args.duplicate_stdout_filters, + duplicate_stderr_filters=args.duplicate_stderr_filters, + virtual_local_rank=args.virtual_local_rank, + ) + + with_python = not args.no_python + cmd: Callable | str + cmd_args = [] + use_env = get_use_env(args) + if args.run_path: + cmd = run_script_path + cmd_args.append(args.training_script) + else: + if with_python: + cmd = os.getenv("PYTHON_EXEC", sys.executable) + cmd_args.append("-u") + if args.module: + cmd_args.append("-m") + cmd_args.append(args.training_script) + else: + if args.module: + raise ValueError( + "Don't use both the '--no-python' flag" + " and the '--module' flag at the same time." + ) + cmd = args.training_script + if not use_env: + cmd_args.append(f"--local-rank={macros.local_rank}") + cmd_args.extend(args.training_script_args) + + return config, cmd, cmd_args + + +def run_script_path(training_script: str, *training_script_args: str): + """ + Run the provided `training_script` from within this interpreter. + + Usage: `script_as_function("/abs/path/to/script.py", "--arg1", "val1")` + """ + import runpy + import sys + + sys.argv = [training_script] + [*training_script_args] + runpy.run_path(sys.argv[0], run_name="__main__") + + +def run(args): + torch.multiprocessing._set_thread_name("pt_elastic") + + if args.standalone: + args.rdzv_backend = "c10d" + args.rdzv_endpoint = "localhost:0" + args.rdzv_id = str(uuid.uuid4()) + logger.info( + "\n**************************************\n" + "Rendezvous info:\n" + "--rdzv-backend=%s " + "--rdzv-endpoint=%s " + "--rdzv-id=%s\n" + "**************************************\n", + args.rdzv_backend, + args.rdzv_endpoint, + args.rdzv_id, + ) + + config, cmd, cmd_args = config_from_args(args) + elastic_launch( + config=config, + entrypoint=cmd, + )(*cmd_args) + + +@record +def main(args=None): + args = parse_args(args) + run(args) + + +if __name__ == "__main__": + main() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..74bd176e5c0633bd874972de238a4c6f1550b056 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributed/utils.py @@ -0,0 +1,381 @@ +# mypy: allow-untyped-defs +import dataclasses +import traceback +from collections import OrderedDict +from collections.abc import Callable, Container +from typing import Any, Optional, overload, TypeVar + +import torch +import torch.distributed as dist +from torch import nn +from torch.nn.utils.rnn import PackedSequence + + +__all__ = [] # type: ignore[var-annotated] + + +def _pack_kwargs(*args: Any, **kwargs: Any) -> tuple[tuple[Any, ...], tuple[str, ...]]: + """ + Turn argument list into separate key list and value list (unpack_kwargs does the opposite). + + Inspiration: https://github.com/facebookresearch/fairscale/blob/eeb6684/fairscale/internal/containers.py#L70 + Usage:: + + kwarg_keys, flat_args = pack_kwargs(1, 2, a=3, b=4) + assert kwarg_keys == ("a", "b") + assert flat_args == (1, 2, 3, 4) + args, kwargs = unpack_kwargs(kwarg_keys, flat_args) + assert args == (1, 2) + assert kwargs == {"a": 3, "b": 4} + Returns: + Tuple[Tuple[Any, ...], Tuple[str, ...]]: The first tuple element gives + gives both positional args and kwarg values, where the positional args + proceed kwarg values and kwarg values are ordered consistently with the + kwarg keys. The second tuple element gives the kwarg keys. + The second tuple element's length is at most the first tuple element's length. + """ + kwarg_keys: list[str] = [] + flat_args: list[Any] = list(args) + for k, v in kwargs.items(): + kwarg_keys.append(k) + flat_args.append(v) + + return tuple(flat_args), tuple(kwarg_keys) + + +def _cast_forward_inputs( + dtype: torch.dtype | None, + *args: Any, + **kwargs: Any, +) -> tuple[Any, Any]: + """ + Cast floating point tensors in ``args`` and ``kwargs`` to ``input_dtype``. + + This respects the existing ``requires_grad`` on the tensors. + """ + if dtype is None: + return args, kwargs + + def cast_fn(x: torch.Tensor) -> torch.Tensor: + if not torch.is_floating_point(x) or x.dtype == dtype: + return x + + return x.to(dtype) + + return (_apply_to_tensors(cast_fn, args), _apply_to_tensors(cast_fn, kwargs)) + + +def _unpack_kwargs( + flat_args: tuple[Any, ...], kwarg_keys: tuple[str, ...] +) -> tuple[tuple[Any, ...], dict[str, Any]]: + """See _pack_kwargs.""" + if len(kwarg_keys) > len(flat_args): + raise AssertionError(f"too many keys {len(kwarg_keys)} vs. {len(flat_args)}") + if len(kwarg_keys) == 0: + return flat_args, {} + args = flat_args[: -len(kwarg_keys)] + kwargs = dict(zip(kwarg_keys, flat_args[-len(kwarg_keys) :])) + return args, kwargs + + +S = TypeVar("S", dict, list, tuple) +T = TypeVar("T", torch.Tensor, PackedSequence) + + +@overload +def _recursive_to( + inputs: S, target_device: torch.device, use_side_stream_for_tensor_copies: bool +) -> list[S]: ... + + +@overload +def _recursive_to( + inputs: T, target_device: torch.device, use_side_stream_for_tensor_copies: bool +) -> tuple[T]: ... + + +def _recursive_to(inputs, target_device, use_side_stream_for_tensor_copies): + r"""Recursively moves input to the target_device.""" + + def to_map(obj): + if isinstance(obj, (torch.Tensor, PackedSequence)): + device = obj.data.device if isinstance(obj, PackedSequence) else obj.device + if device == target_device: + return (obj,) + if not use_side_stream_for_tensor_copies: + return (obj.to(target_device),) + else: + # If the custom module is not registered to torch, stream is not used for acceleration + if device.type == "cpu": + return (obj.to(target_device),) + + from torch.nn.parallel._functions import _get_stream + + # Perform CPU -> target_device copies in a background stream. This code is + # motivated from similar logic in torch/nn/parallel/_functions.py + stream = _get_stream(target_device) + with stream: + output = obj.to(target_device) + # synchronize with the copy stream + with torch.accelerator.device_index(target_device.index): + current_stream = torch.accelerator.current_stream() + # Sync the current stream with the copy stream + current_stream.wait_stream(stream) + # Ensure tensor memory is not reused until work on + # main stream is complete + if isinstance(obj, PackedSequence): + output.data.record_stream(current_stream) # type: ignore[arg-type] + else: + if not isinstance(output, torch.Tensor): + raise AssertionError("output must be a torch.Tensor") + output.record_stream(current_stream) # type: ignore[arg-type] + return (output,) + + from torch.nn.parallel.scatter_gather import _is_namedtuple + + if _is_namedtuple(obj): + # pyrefly: ignore [no-matching-overload] + return [type(obj)(*args) for args in zip(*map(to_map, obj))] + if isinstance(obj, tuple) and len(obj) > 0: + # pyrefly: ignore [no-matching-overload] + return list(zip(*map(to_map, obj))) + if isinstance(obj, list) and len(obj) > 0: + # pyrefly: ignore [no-matching-overload] + return [list(i) for i in zip(*map(to_map, obj))] + if isinstance(obj, dict) and len(obj) > 0: + # pyrefly: ignore [no-matching-overload] + return [type(obj)(i) for i in zip(*map(to_map, obj.items()))] + return [obj] + + # Avoid reference cycle + try: + res = to_map(inputs) + finally: + to_map = None # type: ignore[assignment] + return res + + +def _p_assert(cond: Any, s: str, raise_assertion_error: bool = True) -> None: + """Alternate to ``assert`` when in the backward context to print the error message ``s`` since otherwise, it is swallowed.""" + if not cond: + print(s) + traceback.print_stack() + if raise_assertion_error: + raise AssertionError(s) + + +def _alloc_storage(tensor: torch.Tensor, size: torch.Size) -> None: + """ + Allocate storage for ``tensor`` with the given size. + + Returns: + bool: ``True`` if this method allocated storage and ``False`` if the + storage was already allocated. + """ + with torch.no_grad(): + if not torch.distributed._functional_collectives.is_torchdynamo_compiling(): + already_allocated = tensor._typed_storage()._size() == size.numel() + if not already_allocated: + tensor_storage_size = tensor._typed_storage()._size() + _p_assert( + tensor_storage_size == 0, + "Tensor storage should have been resized to be 0 but got PLACEHOLDEr", + ) + tensor._typed_storage()._resize_(size.numel()) + + +def _free_storage(tensor: torch.Tensor): + """ + Frees the underlying storage of ``tensor``. + + Returns: + bool: ``True`` if the method freed the storage and ``False`` if the + storage was already freed. + """ + with torch.no_grad(): + if not torch.distributed._functional_collectives.is_torchdynamo_compiling(): + already_freed = tensor._typed_storage()._size() == 0 + if not already_freed: + _p_assert( + tensor.storage_offset() == 0, + "Freeing a tensor's storage is unsafe when it is not the sole occupant\n" + f"storage offset: {tensor.storage_offset()}\n" + f"storage size: {tensor._typed_storage()._size()}\n" + f"tensor shape: {tensor.shape}", + ) + tensor._typed_storage()._resize_(0) + + +Q = TypeVar("Q") +R = TypeVar("R", dict, list, tuple, set, OrderedDict, PackedSequence, Any) + + +@overload +def _apply_to_tensors( + fn: Callable[[torch.Tensor], Q], container: torch.Tensor +) -> Q: ... + + +@overload +def _apply_to_tensors(fn: Callable[[torch.Tensor], Any], container: R) -> R: ... + + +def _apply_to_tensors(fn, container): + """Recursively apply to all tensor in different kinds of container types.""" + + def apply(x): + from torch.nn.parallel.scatter_gather import _is_namedtuple + + if isinstance(x, torch.Tensor): + return fn(x) + elif hasattr(x, "__dataclass_fields__"): + dc = dataclasses.replace(x) + changes = { + f.name: apply(getattr(dc, f.name)) for f in dataclasses.fields(dc) + } + return dataclasses.replace(dc, **changes) + elif isinstance(x, OrderedDict): + od = x.__class__() + for key, value in x.items(): + od[key] = apply(value) + return od + elif isinstance(x, PackedSequence): + apply(x.data) + return x + elif isinstance(x, dict): + return {key: apply(value) for key, value in x.items()} + elif _is_namedtuple(x): + res = (apply(el) for el in x) + return type(x)(*res) + elif isinstance(x, (list, tuple, set)): + return type(x)(apply(el) for el in x) + else: + return x + + return apply(container) + + +def _to_kwargs( + inputs: tuple[Any, ...], + kwargs: dict[str, Any] | None, + target_device: torch.device, + use_side_stream_for_tensor_copies: bool, +) -> tuple[tuple[Any, ...], tuple[dict[str, Any], ...]]: + moved_inputs = ( + _recursive_to(inputs, target_device, use_side_stream_for_tensor_copies) + if inputs + else [] + ) + moved_kwargs = ( + _recursive_to(kwargs, target_device, use_side_stream_for_tensor_copies) + if kwargs + else [] + ) + if len(moved_inputs) < len(moved_kwargs): + moved_inputs.extend([() for _ in range(len(moved_kwargs) - len(inputs))]) + elif len(moved_kwargs) < len(moved_inputs): + moved_kwargs.extend([{} for _ in range(len(moved_inputs) - len(moved_kwargs))]) + return tuple(moved_inputs), tuple(moved_kwargs) + + +def _verify_param_shape_across_processes( + process_group: dist.ProcessGroup, + tensors: list[torch.Tensor], + logger: Optional["dist.Logger"] = None, +): + return dist._verify_params_across_processes(process_group, tensors, logger) + + +def _sync_module_states( + module: nn.Module, + process_group: dist.ProcessGroup, + broadcast_bucket_size: int, + src: int, + params_and_buffers_to_ignore: Container[str], + broadcast_buffers: bool = True, +) -> None: + """ + Sync ``module``'s parameters and buffers state. + + Syncs ``module``'s parameters and buffers state so that all ranks contain + the same module state across all ranks. Note that this API assumes that all + parameter shapes are consistent before running the synchronization. This can + be checked with ``_verify_param_shape_across_processes``. + """ + module_states: list[torch.Tensor] = [] + for name, param in module.named_parameters(): + if name not in params_and_buffers_to_ignore: + module_states.append(param.detach()) + + if broadcast_buffers: + for name, buffer in module.named_buffers(): + if name not in params_and_buffers_to_ignore: + module_states.append(buffer.detach()) + + _sync_params_and_buffers(process_group, module_states, broadcast_bucket_size, src) + + +def _sync_params_and_buffers( + process_group: dist.ProcessGroup, + module_states: list[torch.Tensor], + broadcast_bucket_size: int, + src: int, +) -> None: + """Synchronize ``module_states`` (list of tensors) across all processes by broadcasting them from rank 0.""" + if len(module_states) > 0: + dist._broadcast_coalesced( + process_group, module_states, broadcast_bucket_size, src + ) + + +def _replace_by_prefix( + state_dict: dict[str, Any], + old_prefix: str, + new_prefix: str, +) -> None: + """ + Replace all keys that match a given old_prefix with a new_prefix (in-place). + + Usage:: + + state_dict = {"layer.xyz": torch.tensor(1)} + replace_by_prefix_(state_dict, "layer.", "module.layer.") + assert state_dict == {"module.layer.xyz": torch.tensor(1)} + """ + if old_prefix == new_prefix: + raise ValueError("old_prefix and new_prefix must be distinct") + for key in list(state_dict.keys()): + if not key.startswith(old_prefix): + continue + new_key = new_prefix + key[len(old_prefix) :] + state_dict[new_key] = state_dict[key] + del state_dict[key] + + +def _data_ptr_allocated(tensor: torch.Tensor) -> bool: + return tensor.untyped_storage().data_ptr() > 0 + + +def _get_root_modules(modules: list[nn.Module]) -> list[nn.Module]: + """ + Returns the modules in ``modules`` that are root modules (i.e. + parent-less) with respect to the set ``modules``. In other words, these + are the modules in ``modules`` that are the not child of any other + module in ``modules``. + """ + root_modules: list[nn.Module] = [] + module_to_modules: dict[nn.Module, set[nn.Module]] = { + module: set(module.modules()) for module in modules + } + for candidate_module in modules: + is_root_module = True + for module, _modules in module_to_modules.items(): + is_child_module = ( + candidate_module is not module and candidate_module in _modules + ) + if is_child_module: + is_root_module = False + break + if is_root_module: + root_modules.append(candidate_module) + return root_modules diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8e204d12fba8fcc3f6eac76a9cc112814263b293 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/__init__.py @@ -0,0 +1,174 @@ +r""" +The ``distributions`` package contains parameterizable probability distributions +and sampling functions. This allows the construction of stochastic computation +graphs and stochastic gradient estimators for optimization. This package +generally follows the design of the `TensorFlow Distributions`_ package. + +.. _`TensorFlow Distributions`: + https://arxiv.org/abs/1711.10604 + +It is not possible to directly backpropagate through random samples. However, +there are two main methods for creating surrogate functions that can be +backpropagated through. These are the score function estimator/likelihood ratio +estimator/REINFORCE and the pathwise derivative estimator. REINFORCE is commonly +seen as the basis for policy gradient methods in reinforcement learning, and the +pathwise derivative estimator is commonly seen in the reparameterization trick +in variational autoencoders. Whilst the score function only requires the value +of samples :math:`f(x)`, the pathwise derivative requires the derivative +:math:`f'(x)`. The next sections discuss these two in a reinforcement learning +example. For more details see +`Gradient Estimation Using Stochastic Computation Graphs`_ . + +.. _`Gradient Estimation Using Stochastic Computation Graphs`: + https://arxiv.org/abs/1506.05254 + +Score function +^^^^^^^^^^^^^^ + +When the probability density function is differentiable with respect to its +parameters, we only need :meth:`~torch.distributions.Distribution.sample` and +:meth:`~torch.distributions.Distribution.log_prob` to implement REINFORCE: + +.. math:: + + \Delta\theta = \alpha r \frac{\partial\log p(a|\pi^\theta(s))}{\partial\theta} + +where :math:`\theta` are the parameters, :math:`\alpha` is the learning rate, +:math:`r` is the reward and :math:`p(a|\pi^\theta(s))` is the probability of +taking action :math:`a` in state :math:`s` given policy :math:`\pi^\theta`. + +In practice we would sample an action from the output of a network, apply this +action in an environment, and then use ``log_prob`` to construct an equivalent +loss function. Note that we use a negative because optimizers use gradient +descent, whilst the rule above assumes gradient ascent. With a categorical +policy, the code for implementing REINFORCE would be as follows:: + + probs = policy_network(state) + # Note that this is equivalent to what used to be called multinomial + m = Categorical(probs) + action = m.sample() + next_state, reward = env.step(action) + loss = -m.log_prob(action) * reward + loss.backward() + +Pathwise derivative +^^^^^^^^^^^^^^^^^^^ + +The other way to implement these stochastic/policy gradients would be to use the +reparameterization trick from the +:meth:`~torch.distributions.Distribution.rsample` method, where the +parameterized random variable can be constructed via a parameterized +deterministic function of a parameter-free random variable. The reparameterized +sample therefore becomes differentiable. The code for implementing the pathwise +derivative would be as follows:: + + params = policy_network(state) + m = Normal(*params) + # Any distribution with .has_rsample == True could work based on the application + action = m.rsample() + next_state, reward = env.step(action) # Assuming that reward is differentiable + loss = -reward + loss.backward() +""" + +from . import transforms +from .bernoulli import Bernoulli +from .beta import Beta +from .binomial import Binomial +from .categorical import Categorical +from .cauchy import Cauchy +from .chi2 import Chi2 +from .constraint_registry import biject_to, transform_to +from .continuous_bernoulli import ContinuousBernoulli +from .dirichlet import Dirichlet +from .distribution import Distribution +from .exp_family import ExponentialFamily +from .exponential import Exponential +from .fishersnedecor import FisherSnedecor +from .gamma import Gamma +from .generalized_pareto import GeneralizedPareto +from .geometric import Geometric +from .gumbel import Gumbel +from .half_cauchy import HalfCauchy +from .half_normal import HalfNormal +from .independent import Independent +from .inverse_gamma import InverseGamma +from .kl import _add_kl_info, kl_divergence, register_kl +from .kumaraswamy import Kumaraswamy +from .laplace import Laplace +from .lkj_cholesky import LKJCholesky +from .log_normal import LogNormal +from .logistic_normal import LogisticNormal +from .lowrank_multivariate_normal import LowRankMultivariateNormal +from .mixture_same_family import MixtureSameFamily +from .multinomial import Multinomial +from .multivariate_normal import MultivariateNormal +from .negative_binomial import NegativeBinomial +from .normal import Normal +from .one_hot_categorical import OneHotCategorical, OneHotCategoricalStraightThrough +from .pareto import Pareto +from .poisson import Poisson +from .relaxed_bernoulli import RelaxedBernoulli +from .relaxed_categorical import RelaxedOneHotCategorical +from .studentT import StudentT +from .transformed_distribution import TransformedDistribution +from .transforms import * # noqa: F403 +from .uniform import Uniform +from .von_mises import VonMises +from .weibull import Weibull +from .wishart import Wishart + + +_add_kl_info() +del _add_kl_info + +__all__ = [ + "Bernoulli", + "Beta", + "Binomial", + "Categorical", + "Cauchy", + "Chi2", + "ContinuousBernoulli", + "Dirichlet", + "Distribution", + "Exponential", + "ExponentialFamily", + "FisherSnedecor", + "Gamma", + "GeneralizedPareto", + "Geometric", + "Gumbel", + "HalfCauchy", + "HalfNormal", + "Independent", + "InverseGamma", + "Kumaraswamy", + "LKJCholesky", + "Laplace", + "LogNormal", + "LogisticNormal", + "LowRankMultivariateNormal", + "MixtureSameFamily", + "Multinomial", + "MultivariateNormal", + "NegativeBinomial", + "Normal", + "OneHotCategorical", + "OneHotCategoricalStraightThrough", + "Pareto", + "RelaxedBernoulli", + "RelaxedOneHotCategorical", + "StudentT", + "Poisson", + "Uniform", + "VonMises", + "Weibull", + "Wishart", + "TransformedDistribution", + "biject_to", + "kl_divergence", + "register_kl", + "transform_to", +] +__all__.extend(transforms.__all__) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/bernoulli.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..1d613e3c152fb9c10974d2b0f2f58b508cec4255 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/bernoulli.py @@ -0,0 +1,145 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, Number + + +__all__ = ["Bernoulli"] + + +class Bernoulli(ExponentialFamily): + r""" + Creates a Bernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both). + + Samples are binary (0 or 1). They take the value `1` with probability `p` + and `0` with probability `1 - p`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Bernoulli(torch.tensor([0.3])) + >>> m.sample() # 30% chance 1; 70% chance 0 + tensor([ 0.]) + + Args: + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + validate_args (bool, optional): whether to validate arguments, None by default + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.boolean + has_enumerate_support = True + _mean_carrier_measure = 0 + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Bernoulli, _instance) + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(Bernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @property + def mean(self) -> Tensor: + return self.probs + + @property + def mode(self) -> Tensor: + mode = (self.probs >= 0.5).to(self.probs) + mode[self.probs == 0.5] = nan + return mode + + @property + def variance(self) -> Tensor: + return self.probs * (1 - self.probs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.bernoulli(self.probs.expand(shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + return -binary_cross_entropy_with_logits(logits, value, reduction="none") + + def entropy(self): + return binary_cross_entropy_with_logits( + self.logits, self.probs, reduction="none" + ) + + def enumerate_support(self, expand=True): + values = torch.arange(2, dtype=self._param.dtype, device=self._param.device) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values + + @property + def _natural_params(self) -> tuple[Tensor]: + return (torch.logit(self.probs),) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return torch.log1p(torch.exp(x)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/beta.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/beta.py new file mode 100644 index 0000000000000000000000000000000000000000..10ddfee12d2502e2396b252ec1fecf3947e7e191 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/beta.py @@ -0,0 +1,119 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.dirichlet import Dirichlet +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Beta"] + + +class Beta(ExponentialFamily): + r""" + Beta distribution parameterized by :attr:`concentration1` and :attr:`concentration0`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Beta(torch.tensor([0.5]), torch.tensor([0.5])) + >>> m.sample() # Beta distributed with concentration concentration1 and concentration0 + tensor([ 0.1046]) + + Args: + concentration1 (float or Tensor): 1st concentration parameter of the distribution + (often referred to as alpha) + concentration0 (float or Tensor): 2nd concentration parameter of the distribution + (often referred to as beta) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration1": constraints.positive, + "concentration0": constraints.positive, + } + support = constraints.unit_interval + has_rsample = True + + def __init__( + self, + concentration1: Union[Tensor, float], + concentration0: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + if isinstance(concentration1, _Number) and isinstance(concentration0, _Number): + concentration1_concentration0 = torch.tensor( + [float(concentration1), float(concentration0)] + ) + else: + concentration1, concentration0 = broadcast_all( + concentration1, concentration0 + ) + concentration1_concentration0 = torch.stack( + [concentration1, concentration0], -1 + ) + self._dirichlet = Dirichlet( + concentration1_concentration0, validate_args=validate_args + ) + super().__init__(self._dirichlet._batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Beta, _instance) + batch_shape = torch.Size(batch_shape) + new._dirichlet = self._dirichlet.expand(batch_shape) + super(Beta, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.concentration1 / (self.concentration1 + self.concentration0) + + @property + def mode(self) -> Tensor: + return self._dirichlet.mode[..., 0] + + @property + def variance(self) -> Tensor: + total = self.concentration1 + self.concentration0 + return self.concentration1 * self.concentration0 / (total.pow(2) * (total + 1)) + + def rsample(self, sample_shape: _size = ()) -> Tensor: + return self._dirichlet.rsample(sample_shape).select(-1, 0) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + heads_tails = torch.stack([value, 1.0 - value], -1) + return self._dirichlet.log_prob(heads_tails) + + def entropy(self): + return self._dirichlet.entropy() + + @property + def concentration1(self) -> Tensor: + result = self._dirichlet.concentration[..., 0] + if isinstance(result, _Number): + return torch.tensor([result]) + else: + return result + + @property + def concentration0(self) -> Tensor: + result = self._dirichlet.concentration[..., 1] + if isinstance(result, _Number): + return torch.tensor([result]) + else: + return result + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.concentration1, self.concentration0) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return torch.lgamma(x) + torch.lgamma(y) - torch.lgamma(x + y) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/binomial.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/binomial.py new file mode 100644 index 0000000000000000000000000000000000000000..744e047556c85c6b2940aada0dd49f95fec166bc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/binomial.py @@ -0,0 +1,182 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) + + +__all__ = ["Binomial"] + + +def _clamp_by_zero(x): + # works like clamp(x, min=0) but has grad at 0 is 0.5 + return (x.clamp(min=0) + x - x.clamp(max=0)) / 2 + + +class Binomial(Distribution): + r""" + Creates a Binomial distribution parameterized by :attr:`total_count` and + either :attr:`probs` or :attr:`logits` (but not both). :attr:`total_count` must be + broadcastable with :attr:`probs`/:attr:`logits`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Binomial(100, torch.tensor([0 , .2, .8, 1])) + >>> x = m.sample() + tensor([ 0., 22., 71., 100.]) + + >>> m = Binomial(torch.tensor([[5.], [10.]]), torch.tensor([0.5, 0.8])) + >>> x = m.sample() + tensor([[ 4., 5.], + [ 7., 6.]]) + + Args: + total_count (int or Tensor): number of Bernoulli trials + probs (Tensor): Event probabilities + logits (Tensor): Event log-odds + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "total_count": constraints.nonnegative_integer, + "probs": constraints.unit_interval, + "logits": constraints.real, + } + has_enumerate_support = True + + def __init__( + self, + total_count: Union[Tensor, int] = 1, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + ( + self.total_count, + # pyrefly: ignore [read-only] + self.probs, + ) = broadcast_all(total_count, probs) + self.total_count = self.total_count.type_as(self.probs) + else: + assert logits is not None # helps mypy + ( + self.total_count, + # pyrefly: ignore [read-only] + self.logits, + ) = broadcast_all(total_count, logits) + self.total_count = self.total_count.type_as(self.logits) + + self._param = self.probs if probs is not None else self.logits + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Binomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count.expand(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(Binomial, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.integer_interval(0, self.total_count) + + @property + def mean(self) -> Tensor: + return self.total_count * self.probs + + @property + def mode(self) -> Tensor: + return ((self.total_count + 1) * self.probs).floor().clamp(max=self.total_count) + + @property + def variance(self) -> Tensor: + return self.total_count * self.probs * (1 - self.probs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.binomial( + self.total_count.expand(shape), self.probs.expand(shape) + ) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_factorial_n = torch.lgamma(self.total_count + 1) + log_factorial_k = torch.lgamma(value + 1) + log_factorial_nmk = torch.lgamma(self.total_count - value + 1) + # k * log(p) + (n - k) * log(1 - p) = k * (log(p) - log(1 - p)) + n * log(1 - p) + # (case logit < 0) = k * logit - n * log1p(e^logit) + # (case logit > 0) = k * logit - n * (log(p) - log(1 - p)) + n * log(p) + # = k * logit - n * logit - n * log1p(e^-logit) + # (merge two cases) = k * logit - n * max(logit, 0) - n * log1p(e^-|logit|) + normalize_term = ( + self.total_count * _clamp_by_zero(self.logits) + + self.total_count * torch.log1p(torch.exp(-torch.abs(self.logits))) + - log_factorial_n + ) + return ( + value * self.logits - log_factorial_k - log_factorial_nmk - normalize_term + ) + + def entropy(self): + total_count = int(self.total_count.max()) + if not self.total_count.min() == total_count: + raise NotImplementedError( + "Inhomogeneous total count not supported by `entropy`." + ) + + log_prob = self.log_prob(self.enumerate_support(False)) + return -(torch.exp(log_prob) * log_prob).sum(0) + + def enumerate_support(self, expand=True): + total_count = int(self.total_count.max()) + if not self.total_count.min() == total_count: + raise NotImplementedError( + "Inhomogeneous total count not supported by `enumerate_support`." + ) + values = torch.arange( + 1 + total_count, dtype=self._param.dtype, device=self._param.device + ) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/categorical.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..6794382042bf2b2a1e68e104713dea67d949c039 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/categorical.py @@ -0,0 +1,170 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import lazy_property, logits_to_probs, probs_to_logits + + +__all__ = ["Categorical"] + + +class Categorical(Distribution): + r""" + Creates a categorical distribution parameterized by either :attr:`probs` or + :attr:`logits` (but not both). + + .. note:: + It is equivalent to the distribution that :func:`torch.multinomial` + samples from. + + Samples are integers from :math:`\{0, \ldots, K-1\}` where `K` is ``probs.size(-1)``. + + If `probs` is 1-dimensional with length-`K`, each element is the relative probability + of sampling the class at that index. + + If `probs` is N-dimensional, the first N-1 dimensions are treated as a batch of + relative probability vectors. + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + See also: :func:`torch.multinomial` + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Categorical(torch.tensor([ 0.25, 0.25, 0.25, 0.25 ])) + >>> m.sample() # equal probability of 0, 1, 2, 3 + tensor(3) + + Args: + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + has_enumerate_support = True + + def __init__( + self, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + if probs.dim() < 1: + raise ValueError("`probs` parameter must be at least one-dimensional.") + # pyrefly: ignore [read-only] + self.probs = probs / probs.sum(-1, keepdim=True) + else: + assert logits is not None # helps mypy + if logits.dim() < 1: + raise ValueError("`logits` parameter must be at least one-dimensional.") + # Normalize + # pyrefly: ignore [read-only] + self.logits = logits - logits.logsumexp(dim=-1, keepdim=True) + self._param = self.probs if probs is not None else self.logits + self._num_events = self._param.size()[-1] + batch_shape = ( + self._param.size()[:-1] if self._param.ndimension() > 1 else torch.Size() + ) + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Categorical, _instance) + batch_shape = torch.Size(batch_shape) + param_shape = batch_shape + torch.Size((self._num_events,)) + if "probs" in self.__dict__: + new.probs = self.probs.expand(param_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(param_shape) + new._param = new.logits + new._num_events = self._num_events + super(Categorical, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.integer_interval(0, self._num_events - 1) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), + nan, + dtype=self.probs.dtype, + device=self.probs.device, + ) + + @property + def mode(self) -> Tensor: + return self.probs.argmax(dim=-1) + + @property + def variance(self) -> Tensor: + return torch.full( + self._extended_shape(), + nan, + dtype=self.probs.dtype, + device=self.probs.device, + ) + + def sample(self, sample_shape=torch.Size()): + if not isinstance(sample_shape, torch.Size): + sample_shape = torch.Size(sample_shape) + probs_2d = self.probs.reshape(-1, self._num_events) + samples_2d = torch.multinomial(probs_2d, sample_shape.numel(), True).T + return samples_2d.reshape(self._extended_shape(sample_shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value = value.long().unsqueeze(-1) + value, log_pmf = torch.broadcast_tensors(value, self.logits) + value = value[..., :1] + return log_pmf.gather(-1, value).squeeze(-1) + + def entropy(self): + min_real = torch.finfo(self.logits.dtype).min + logits = torch.clamp(self.logits, min=min_real) + p_log_p = logits * self.probs + return -p_log_p.sum(-1) + + def enumerate_support(self, expand=True): + num_events = self._num_events + values = torch.arange(num_events, dtype=torch.long, device=self._param.device) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/cauchy.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/cauchy.py new file mode 100644 index 0000000000000000000000000000000000000000..646aa5f4217a23b1dc8fcad8feee50142822213d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/cauchy.py @@ -0,0 +1,100 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Cauchy"] + + +class Cauchy(Distribution): + r""" + Samples from a Cauchy (Lorentz) distribution. The distribution of the ratio of + independent normally distributed random variables with means `0` follows a + Cauchy distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Cauchy(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Cauchy distribution with loc=0 and scale=1 + tensor([ 2.3214]) + + Args: + loc (float or Tensor): mode or median of the distribution. + scale (float or Tensor): half width at half maximum. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Cauchy, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Cauchy, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), nan, dtype=self.loc.dtype, device=self.loc.device + ) + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return torch.full( + self._extended_shape(), inf, dtype=self.loc.dtype, device=self.loc.device + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = self.loc.new(shape).cauchy_() + return self.loc + eps * self.scale + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return ( + -math.log(math.pi) + - self.scale.log() + - (((value - self.loc) / self.scale) ** 2).log1p() + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return torch.atan((value - self.loc) / self.scale) / math.pi + 0.5 + + def icdf(self, value): + return torch.tan(math.pi * (value - 0.5)) * self.scale + self.loc + + def entropy(self): + return math.log(4 * math.pi) + self.scale.log() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/chi2.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/chi2.py new file mode 100644 index 0000000000000000000000000000000000000000..7a654306aad967459a46e191816c79381da163ee --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/chi2.py @@ -0,0 +1,43 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.gamma import Gamma + + +__all__ = ["Chi2"] + + +class Chi2(Gamma): + r""" + Creates a Chi-squared distribution parameterized by shape parameter :attr:`df`. + This is exactly equivalent to ``Gamma(alpha=0.5*df, beta=0.5)`` + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Chi2(torch.tensor([1.0])) + >>> m.sample() # Chi2 distributed with shape df=1 + tensor([ 0.1046]) + + Args: + df (float or Tensor): shape parameter of the distribution + """ + + arg_constraints = {"df": constraints.positive} + + def __init__( + self, + df: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + super().__init__(0.5 * df, 0.5, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Chi2, _instance) + return super().expand(batch_shape, new) + + @property + def df(self) -> Tensor: + return self.concentration * 2 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraint_registry.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraint_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..17b5928b2afd399cc18d7d7064638aa53d76a9d7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraint_registry.py @@ -0,0 +1,291 @@ +# mypy: allow-untyped-defs +r""" +PyTorch provides two global :class:`ConstraintRegistry` objects that link +:class:`~torch.distributions.constraints.Constraint` objects to +:class:`~torch.distributions.transforms.Transform` objects. These objects both +input constraints and return transforms, but they have different guarantees on +bijectivity. + +1. ``biject_to(constraint)`` looks up a bijective + :class:`~torch.distributions.transforms.Transform` from ``constraints.real`` + to the given ``constraint``. The returned transform is guaranteed to have + ``.bijective = True`` and should implement ``.log_abs_det_jacobian()``. +2. ``transform_to(constraint)`` looks up a not-necessarily bijective + :class:`~torch.distributions.transforms.Transform` from ``constraints.real`` + to the given ``constraint``. The returned transform is not guaranteed to + implement ``.log_abs_det_jacobian()``. + +The ``transform_to()`` registry is useful for performing unconstrained +optimization on constrained parameters of probability distributions, which are +indicated by each distribution's ``.arg_constraints`` dict. These transforms often +overparameterize a space in order to avoid rotation; they are thus more +suitable for coordinate-wise optimization algorithms like Adam:: + + loc = torch.zeros(100, requires_grad=True) + unconstrained = torch.zeros(100, requires_grad=True) + scale = transform_to(Normal.arg_constraints["scale"])(unconstrained) + loss = -Normal(loc, scale).log_prob(data).sum() + +The ``biject_to()`` registry is useful for Hamiltonian Monte Carlo, where +samples from a probability distribution with constrained ``.support`` are +propagated in an unconstrained space, and algorithms are typically rotation +invariant.:: + + dist = Exponential(rate) + unconstrained = torch.zeros(100, requires_grad=True) + sample = biject_to(dist.support)(unconstrained) + potential_energy = -dist.log_prob(sample).sum() + +.. note:: + + An example where ``transform_to`` and ``biject_to`` differ is + ``constraints.simplex``: ``transform_to(constraints.simplex)`` returns a + :class:`~torch.distributions.transforms.SoftmaxTransform` that simply + exponentiates and normalizes its inputs; this is a cheap and mostly + coordinate-wise operation appropriate for algorithms like SVI. In + contrast, ``biject_to(constraints.simplex)`` returns a + :class:`~torch.distributions.transforms.StickBreakingTransform` that + bijects its input down to a one-fewer-dimensional space; this a more + expensive less numerically stable transform but is needed for algorithms + like HMC. + +The ``biject_to`` and ``transform_to`` objects can be extended by user-defined +constraints and transforms using their ``.register()`` method either as a +function on singleton constraints:: + + transform_to.register(my_constraint, my_transform) + +or as a decorator on parameterized constraints:: + + @transform_to.register(MyConstraintClass) + def my_factory(constraint): + assert isinstance(constraint, MyConstraintClass) + return MyTransform(constraint.param1, constraint.param2) + +You can create your own registry by creating a new :class:`ConstraintRegistry` +object. +""" + +from torch.distributions import constraints, transforms +from torch.types import _Number + + +__all__ = [ + "ConstraintRegistry", + "biject_to", + "transform_to", +] + + +class ConstraintRegistry: + """ + Registry to link constraints to transforms. + """ + + def __init__(self): + self._registry = {} + super().__init__() + + def register(self, constraint, factory=None): + """ + Registers a :class:`~torch.distributions.constraints.Constraint` + subclass in this registry. Usage:: + + @my_registry.register(MyConstraintClass) + def construct_transform(constraint): + assert isinstance(constraint, MyConstraint) + return MyTransform(constraint.arg_constraints) + + Args: + constraint (subclass of :class:`~torch.distributions.constraints.Constraint`): + A subclass of :class:`~torch.distributions.constraints.Constraint`, or + a singleton object of the desired class. + factory (Callable): A callable that inputs a constraint object and returns + a :class:`~torch.distributions.transforms.Transform` object. + """ + # Support use as decorator. + if factory is None: + return lambda factory: self.register(constraint, factory) + + # Support calling on singleton instances. + if isinstance(constraint, constraints.Constraint): + constraint = type(constraint) + + if not isinstance(constraint, type) or not issubclass( + constraint, constraints.Constraint + ): + raise TypeError( + f"Expected constraint to be either a Constraint subclass or instance, but got {constraint}" + ) + + self._registry[constraint] = factory + return factory + + def __call__(self, constraint): + """ + Looks up a transform to constrained space, given a constraint object. + Usage:: + + constraint = Normal.arg_constraints["scale"] + scale = transform_to(constraint)(torch.zeros(1)) # constrained + u = transform_to(constraint).inv(scale) # unconstrained + + Args: + constraint (:class:`~torch.distributions.constraints.Constraint`): + A constraint object. + + Returns: + A :class:`~torch.distributions.transforms.Transform` object. + + Raises: + `NotImplementedError` if no transform has been registered. + """ + # Look up by Constraint subclass. + try: + factory = self._registry[type(constraint)] + except KeyError: + raise NotImplementedError( + f"Cannot transform {type(constraint).__name__} constraints" + ) from None + return factory(constraint) + + +biject_to = ConstraintRegistry() +transform_to = ConstraintRegistry() + + +################################################################################ +# Registration Table +################################################################################ + + +@biject_to.register(constraints.real) +@transform_to.register(constraints.real) +def _transform_to_real(constraint): + return transforms.identity_transform + + +@biject_to.register(constraints.independent) +def _biject_to_independent(constraint): + base_transform = biject_to(constraint.base_constraint) + return transforms.IndependentTransform( + base_transform, constraint.reinterpreted_batch_ndims + ) + + +@transform_to.register(constraints.independent) +def _transform_to_independent(constraint): + base_transform = transform_to(constraint.base_constraint) + return transforms.IndependentTransform( + base_transform, constraint.reinterpreted_batch_ndims + ) + + +@biject_to.register(constraints.positive) +@biject_to.register(constraints.nonnegative) +@transform_to.register(constraints.positive) +@transform_to.register(constraints.nonnegative) +def _transform_to_positive(constraint): + return transforms.ExpTransform() + + +@biject_to.register(constraints.greater_than) +@biject_to.register(constraints.greater_than_eq) +@transform_to.register(constraints.greater_than) +@transform_to.register(constraints.greater_than_eq) +def _transform_to_greater_than(constraint): + return transforms.ComposeTransform( + [ + transforms.ExpTransform(), + transforms.AffineTransform(constraint.lower_bound, 1), + ] + ) + + +@biject_to.register(constraints.less_than) +@transform_to.register(constraints.less_than) +def _transform_to_less_than(constraint): + return transforms.ComposeTransform( + [ + transforms.ExpTransform(), + transforms.AffineTransform(constraint.upper_bound, -1), + ] + ) + + +@biject_to.register(constraints.interval) +@biject_to.register(constraints.half_open_interval) +@transform_to.register(constraints.interval) +@transform_to.register(constraints.half_open_interval) +def _transform_to_interval(constraint): + # Handle the special case of the unit interval. + lower_is_0 = ( + isinstance(constraint.lower_bound, _Number) and constraint.lower_bound == 0 + ) + upper_is_1 = ( + isinstance(constraint.upper_bound, _Number) and constraint.upper_bound == 1 + ) + if lower_is_0 and upper_is_1: + return transforms.SigmoidTransform() + + loc = constraint.lower_bound + scale = constraint.upper_bound - constraint.lower_bound + return transforms.ComposeTransform( + [transforms.SigmoidTransform(), transforms.AffineTransform(loc, scale)] + ) + + +@biject_to.register(constraints.simplex) +def _biject_to_simplex(constraint): + return transforms.StickBreakingTransform() + + +@transform_to.register(constraints.simplex) +def _transform_to_simplex(constraint): + return transforms.SoftmaxTransform() + + +# TODO define a bijection for LowerCholeskyTransform +@transform_to.register(constraints.lower_cholesky) +def _transform_to_lower_cholesky(constraint): + return transforms.LowerCholeskyTransform() + + +@transform_to.register(constraints.positive_definite) +@transform_to.register(constraints.positive_semidefinite) +def _transform_to_positive_definite(constraint): + return transforms.PositiveDefiniteTransform() + + +@biject_to.register(constraints.corr_cholesky) +@transform_to.register(constraints.corr_cholesky) +def _transform_to_corr_cholesky(constraint): + return transforms.CorrCholeskyTransform() + + +@biject_to.register(constraints.cat) +def _biject_to_cat(constraint): + return transforms.CatTransform( + [biject_to(c) for c in constraint.cseq], constraint.dim, constraint.lengths + ) + + +@transform_to.register(constraints.cat) +def _transform_to_cat(constraint): + return transforms.CatTransform( + [transform_to(c) for c in constraint.cseq], constraint.dim, constraint.lengths + ) + + +@biject_to.register(constraints.stack) +def _biject_to_stack(constraint): + return transforms.StackTransform( + [biject_to(c) for c in constraint.cseq], constraint.dim + ) + + +@transform_to.register(constraints.stack) +def _transform_to_stack(constraint): + return transforms.StackTransform( + [transform_to(c) for c in constraint.cseq], constraint.dim + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraints.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..3e1dec4a0a19e79d1bb5b3c4c01bdf1185283041 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/constraints.py @@ -0,0 +1,738 @@ +# mypy: allow-untyped-defs + +from collections.abc import Callable +from typing import Any, Optional + + +r""" +The following constraints are implemented: + +- ``constraints.boolean`` +- ``constraints.cat`` +- ``constraints.corr_cholesky`` +- ``constraints.dependent`` +- ``constraints.greater_than(lower_bound)`` +- ``constraints.greater_than_eq(lower_bound)`` +- ``constraints.independent(constraint, reinterpreted_batch_ndims)`` +- ``constraints.integer_interval(lower_bound, upper_bound)`` +- ``constraints.interval(lower_bound, upper_bound)`` +- ``constraints.less_than(upper_bound)`` +- ``constraints.lower_cholesky`` +- ``constraints.lower_triangular`` +- ``constraints.MixtureSameFamilyConstraint(base_constraint)`` +- ``constraints.multinomial`` +- ``constraints.nonnegative`` +- ``constraints.nonnegative_integer`` +- ``constraints.one_hot`` +- ``constraints.positive_integer`` +- ``constraints.positive`` +- ``constraints.positive_semidefinite`` +- ``constraints.positive_definite`` +- ``constraints.real_vector`` +- ``constraints.real`` +- ``constraints.simplex`` +- ``constraints.symmetric`` +- ``constraints.stack`` +- ``constraints.square`` +- ``constraints.symmetric`` +- ``constraints.unit_interval`` +""" + +import torch + + +__all__ = [ + "Constraint", + "boolean", + "cat", + "corr_cholesky", + "dependent", + "dependent_property", + "greater_than", + "greater_than_eq", + "independent", + "integer_interval", + "interval", + "half_open_interval", + "is_dependent", + "less_than", + "lower_cholesky", + "lower_triangular", + "MixtureSameFamilyConstraint", + "multinomial", + "nonnegative", + "nonnegative_integer", + "one_hot", + "positive", + "positive_semidefinite", + "positive_definite", + "positive_integer", + "real", + "real_vector", + "simplex", + "square", + "stack", + "symmetric", + "unit_interval", +] + + +class Constraint: + """ + Abstract base class for constraints. + + A constraint object represents a region over which a variable is valid, + e.g. within which a variable can be optimized. + + Attributes: + is_discrete (bool): Whether constrained space is discrete. + Defaults to False. + event_dim (int): Number of rightmost dimensions that together define + an event. The :meth:`check` method will remove this many dimensions + when computing validity. + """ + + is_discrete = False # Default to continuous. + event_dim = 0 # Default to univariate. + + def check(self, value): + """ + Returns a byte tensor of ``sample_shape + batch_shape`` indicating + whether each event in value satisfies this constraint. + """ + raise NotImplementedError + + def __repr__(self): + return self.__class__.__name__[1:] + "()" + + +class _Dependent(Constraint): + """ + Placeholder for variables whose support depends on other variables. + These variables obey no simple coordinate-wise constraints. + + Args: + is_discrete (bool): Optional value of ``.is_discrete`` in case this + can be computed statically. If not provided, access to the + ``.is_discrete`` attribute will raise a NotImplementedError. + event_dim (int): Optional value of ``.event_dim`` in case this + can be computed statically. If not provided, access to the + ``.event_dim`` attribute will raise a NotImplementedError. + """ + + def __init__(self, *, is_discrete=NotImplemented, event_dim=NotImplemented): + self._is_discrete = is_discrete + self._event_dim = event_dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + if self._is_discrete is NotImplemented: + raise NotImplementedError(".is_discrete cannot be determined statically") + return self._is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + if self._event_dim is NotImplemented: + raise NotImplementedError(".event_dim cannot be determined statically") + return self._event_dim + + def __call__(self, *, is_discrete=NotImplemented, event_dim=NotImplemented): + """ + Support for syntax to customize static attributes:: + + constraints.dependent(is_discrete=True, event_dim=1) + """ + if is_discrete is NotImplemented: + is_discrete = self._is_discrete + if event_dim is NotImplemented: + event_dim = self._event_dim + return _Dependent(is_discrete=is_discrete, event_dim=event_dim) + + def check(self, x): + raise ValueError("Cannot determine validity of dependent constraint") + + +def is_dependent(constraint): + """ + Checks if ``constraint`` is a ``_Dependent`` object. + + Args: + constraint : A ``Constraint`` object. + + Returns: + ``bool``: True if ``constraint`` can be refined to the type ``_Dependent``, False otherwise. + + Examples: + >>> import torch + >>> from torch.distributions import Bernoulli + >>> from torch.distributions.constraints import is_dependent + + >>> dist = Bernoulli(probs=torch.tensor([0.6], requires_grad=True)) + >>> constraint1 = dist.arg_constraints["probs"] + >>> constraint2 = dist.arg_constraints["logits"] + + >>> for constraint in [constraint1, constraint2]: + >>> if is_dependent(constraint): + >>> continue + """ + return isinstance(constraint, _Dependent) + + +class _DependentProperty(property, _Dependent): + """ + Decorator that extends @property to act like a `Dependent` constraint when + called on a class and act like a property when called on an object. + + Example:: + + class Uniform(Distribution): + def __init__(self, low, high): + self.low = low + self.high = high + + @constraints.dependent_property(is_discrete=False, event_dim=0) + def support(self): + return constraints.interval(self.low, self.high) + + Args: + fn (Callable): The function to be decorated. + is_discrete (bool): Optional value of ``.is_discrete`` in case this + can be computed statically. If not provided, access to the + ``.is_discrete`` attribute will raise a NotImplementedError. + event_dim (int): Optional value of ``.event_dim`` in case this + can be computed statically. If not provided, access to the + ``.event_dim`` attribute will raise a NotImplementedError. + """ + + def __init__( + self, + fn: Optional[Callable[..., Any]] = None, + *, + is_discrete: Optional[bool] = NotImplemented, + event_dim: Optional[int] = NotImplemented, + ) -> None: + super().__init__(fn) + self._is_discrete = is_discrete + self._event_dim = event_dim + + def __call__(self, fn: Callable[..., Any]) -> "_DependentProperty": # type: ignore[override] + """ + Support for syntax to customize static attributes:: + + @constraints.dependent_property(is_discrete=True, event_dim=1) + def support(self): ... + """ + return _DependentProperty( + fn, is_discrete=self._is_discrete, event_dim=self._event_dim + ) + + +class _IndependentConstraint(Constraint): + """ + Wraps a constraint by aggregating over ``reinterpreted_batch_ndims``-many + dims in :meth:`check`, so that an event is valid only if all its + independent entries are valid. + """ + + def __init__(self, base_constraint, reinterpreted_batch_ndims): + assert isinstance(base_constraint, Constraint) + assert isinstance(reinterpreted_batch_ndims, int) + assert reinterpreted_batch_ndims >= 0 + self.base_constraint = base_constraint + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return self.base_constraint.is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + return self.base_constraint.event_dim + self.reinterpreted_batch_ndims + + def check(self, value): + result = self.base_constraint.check(value) + if result.dim() < self.reinterpreted_batch_ndims: + expected = self.base_constraint.event_dim + self.reinterpreted_batch_ndims + raise ValueError( + f"Expected value.dim() >= {expected} but got {value.dim()}" + ) + result = result.reshape( + result.shape[: result.dim() - self.reinterpreted_batch_ndims] + (-1,) + ) + result = result.all(-1) + return result + + def __repr__(self): + return f"{self.__class__.__name__[1:]}({repr(self.base_constraint)}, {self.reinterpreted_batch_ndims})" + + +class MixtureSameFamilyConstraint(Constraint): + """ + Constraint for the :class:`~torch.distribution.MixtureSameFamily` + distribution that adds back the rightmost batch dimension before + performing the validity check with the component distribution + constraint. + + Args: + base_constraint: The ``Constraint`` object of + the component distribution of + the :class:`~torch.distribution.MixtureSameFamily` distribution. + """ + + def __init__(self, base_constraint): + assert isinstance(base_constraint, Constraint) + self.base_constraint = base_constraint + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return self.base_constraint.is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + return self.base_constraint.event_dim + + def check(self, value): + """ + Check validity of ``value`` as a possible outcome of sampling + the :class:`~torch.distribution.MixtureSameFamily` distribution. + """ + unsqueezed_value = value.unsqueeze(-1 - self.event_dim) + result = self.base_constraint.check(unsqueezed_value) + if value.dim() < self.event_dim: + raise ValueError( + f"Expected value.dim() >= {self.event_dim} but got {value.dim()}" + ) + num_dim_to_keep = value.dim() - self.event_dim + result = result.reshape(result.shape[:num_dim_to_keep] + (-1,)) + result = result.all(-1) + return result + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self.base_constraint)})" + + +class _Boolean(Constraint): + """ + Constrain to the two values `{0, 1}`. + """ + + is_discrete = True + + def check(self, value): + return (value == 0) | (value == 1) + + +class _OneHot(Constraint): + """ + Constrain to one-hot vectors. + """ + + is_discrete = True + event_dim = 1 + + def check(self, value): + is_boolean = (value == 0) | (value == 1) + is_normalized = value.sum(-1).eq(1) + return is_boolean.all(-1) & is_normalized + + +class _IntegerInterval(Constraint): + """ + Constrain to an integer interval `[lower_bound, upper_bound]`. + """ + + is_discrete = True + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return ( + (value % 1 == 0) & (self.lower_bound <= value) & (value <= self.upper_bound) + ) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _IntegerLessThan(Constraint): + """ + Constrain to an integer interval `(-inf, upper_bound]`. + """ + + is_discrete = True + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (value % 1 == 0) & (value <= self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(upper_bound={self.upper_bound})" + return fmt_string + + +class _IntegerGreaterThan(Constraint): + """ + Constrain to an integer interval `[lower_bound, inf)`. + """ + + is_discrete = True + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return (value % 1 == 0) & (value >= self.lower_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _Real(Constraint): + """ + Trivially constrain to the extended real line `[-inf, inf]`. + """ + + def check(self, value): + return value == value # False for NANs. + + +class _GreaterThan(Constraint): + """ + Constrain to a real half line `(lower_bound, inf]`. + """ + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return self.lower_bound < value + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _GreaterThanEq(Constraint): + """ + Constrain to a real half line `[lower_bound, inf)`. + """ + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return self.lower_bound <= value + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _LessThan(Constraint): + """ + Constrain to a real half line `[-inf, upper_bound)`. + """ + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return value < self.upper_bound + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(upper_bound={self.upper_bound})" + return fmt_string + + +class _Interval(Constraint): + """ + Constrain to a real interval `[lower_bound, upper_bound]`. + """ + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (self.lower_bound <= value) & (value <= self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _HalfOpenInterval(Constraint): + """ + Constrain to a real interval `[lower_bound, upper_bound)`. + """ + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (self.lower_bound <= value) & (value < self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _Simplex(Constraint): + """ + Constrain to the unit simplex in the innermost (rightmost) dimension. + Specifically: `x >= 0` and `x.sum(-1) == 1`. + """ + + event_dim = 1 + + def check(self, value): + return torch.all(value >= 0, dim=-1) & ((value.sum(-1) - 1).abs() < 1e-6) + + +class _Multinomial(Constraint): + """ + Constrain to nonnegative integer values summing to at most an upper bound. + + Note due to limitations of the Multinomial distribution, this currently + checks the weaker condition ``value.sum(-1) <= upper_bound``. In the future + this may be strengthened to ``value.sum(-1) == upper_bound``. + """ + + is_discrete = True + event_dim = 1 + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + + def check(self, x): + return (x >= 0).all(dim=-1) & (x.sum(dim=-1) <= self.upper_bound) + + +class _LowerTriangular(Constraint): + """ + Constrain to lower-triangular square matrices. + """ + + event_dim = 2 + + def check(self, value): + value_tril = value.tril() + return (value_tril == value).view(value.shape[:-2] + (-1,)).min(-1)[0] + + +class _LowerCholesky(Constraint): + """ + Constrain to lower-triangular square matrices with positive diagonals. + """ + + event_dim = 2 + + def check(self, value): + value_tril = value.tril() + lower_triangular = ( + (value_tril == value).view(value.shape[:-2] + (-1,)).min(-1)[0] + ) + + positive_diagonal = (value.diagonal(dim1=-2, dim2=-1) > 0).min(-1)[0] + return lower_triangular & positive_diagonal + + +class _CorrCholesky(Constraint): + """ + Constrain to lower-triangular square matrices with positive diagonals and each + row vector being of unit length. + """ + + event_dim = 2 + + def check(self, value): + tol = ( + torch.finfo(value.dtype).eps * value.size(-1) * 10 + ) # 10 is an adjustable fudge factor + row_norm = torch.linalg.norm(value.detach(), dim=-1) + unit_row_norm = (row_norm - 1.0).abs().le(tol).all(dim=-1) + return _LowerCholesky().check(value) & unit_row_norm + + +class _Square(Constraint): + """ + Constrain to square matrices. + """ + + event_dim = 2 + + def check(self, value): + return torch.full( + size=value.shape[:-2], + fill_value=(value.shape[-2] == value.shape[-1]), + dtype=torch.bool, + device=value.device, + ) + + +class _Symmetric(_Square): + """ + Constrain to Symmetric square matrices. + """ + + def check(self, value): + square_check = super().check(value) + if not square_check.all(): + return square_check + return torch.isclose(value, value.mT, atol=1e-6).all(-2).all(-1) + + +class _PositiveSemidefinite(_Symmetric): + """ + Constrain to positive-semidefinite matrices. + """ + + def check(self, value): + sym_check = super().check(value) + if not sym_check.all(): + return sym_check + return torch.linalg.eigvalsh(value).ge(0).all(-1) + + +class _PositiveDefinite(_Symmetric): + """ + Constrain to positive-definite matrices. + """ + + def check(self, value): + sym_check = super().check(value) + if not sym_check.all(): + return sym_check + return torch.linalg.cholesky_ex(value).info.eq(0) + + +class _Cat(Constraint): + """ + Constraint functor that applies a sequence of constraints + `cseq` at the submatrices at dimension `dim`, + each of size `lengths[dim]`, in a way compatible with :func:`torch.cat`. + """ + + def __init__(self, cseq, dim=0, lengths=None): + assert all(isinstance(c, Constraint) for c in cseq) + self.cseq = list(cseq) + if lengths is None: + lengths = [1] * len(self.cseq) + self.lengths = list(lengths) + assert len(self.lengths) == len(self.cseq) + self.dim = dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return any(c.is_discrete for c in self.cseq) + + @property + def event_dim(self) -> int: # type: ignore[override] + return max(c.event_dim for c in self.cseq) + + def check(self, value): + assert -value.dim() <= self.dim < value.dim() + checks = [] + start = 0 + for constr, length in zip(self.cseq, self.lengths): + v = value.narrow(self.dim, start, length) + checks.append(constr.check(v)) + start = start + length # avoid += for jit compat + return torch.cat(checks, self.dim) + + +class _Stack(Constraint): + """ + Constraint functor that applies a sequence of constraints + `cseq` at the submatrices at dimension `dim`, + in a way compatible with :func:`torch.stack`. + """ + + def __init__(self, cseq, dim=0): + assert all(isinstance(c, Constraint) for c in cseq) + self.cseq = list(cseq) + self.dim = dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return any(c.is_discrete for c in self.cseq) + + @property + def event_dim(self) -> int: # type: ignore[override] + dim = max(c.event_dim for c in self.cseq) + if self.dim + dim < 0: + dim += 1 + return dim + + def check(self, value): + assert -value.dim() <= self.dim < value.dim() + vs = [value.select(self.dim, i) for i in range(value.size(self.dim))] + return torch.stack( + [constr.check(v) for v, constr in zip(vs, self.cseq)], self.dim + ) + + +# Public interface. +dependent = _Dependent() +dependent_property = _DependentProperty +independent = _IndependentConstraint +boolean = _Boolean() +one_hot = _OneHot() +nonnegative_integer = _IntegerGreaterThan(0) +positive_integer = _IntegerGreaterThan(1) +integer_interval = _IntegerInterval +real = _Real() +real_vector = independent(real, 1) +positive = _GreaterThan(0.0) +nonnegative = _GreaterThanEq(0.0) +greater_than = _GreaterThan +greater_than_eq = _GreaterThanEq +less_than = _LessThan +multinomial = _Multinomial +unit_interval = _Interval(0.0, 1.0) +interval = _Interval +half_open_interval = _HalfOpenInterval +simplex = _Simplex() +lower_triangular = _LowerTriangular() +lower_cholesky = _LowerCholesky() +corr_cholesky = _CorrCholesky() +square = _Square() +symmetric = _Symmetric() +positive_semidefinite = _PositiveSemidefinite() +positive_definite = _PositiveDefinite() +cat = _Cat +stack = _Stack diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/continuous_bernoulli.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/continuous_bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..40e7e4e944b5641ebccbb7a0f42176be3ed48c29 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/continuous_bernoulli.py @@ -0,0 +1,250 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import ( + broadcast_all, + clamp_probs, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, _size, Number + + +__all__ = ["ContinuousBernoulli"] + + +class ContinuousBernoulli(ExponentialFamily): + r""" + Creates a continuous Bernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both). + + The distribution is supported in [0, 1] and parameterized by 'probs' (in + (0,1)) or 'logits' (real-valued). Note that, unlike the Bernoulli, 'probs' + does not correspond to a probability and 'logits' does not correspond to + log-odds, but the same names are used due to the similarity with the + Bernoulli. See [1] for more details. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = ContinuousBernoulli(torch.tensor([0.3])) + >>> m.sample() + tensor([ 0.2538]) + + Args: + probs (Number, Tensor): (0,1) valued parameters + logits (Number, Tensor): real valued parameters whose sigmoid matches 'probs' + + [1] The continuous Bernoulli: fixing a pervasive error in variational + autoencoders, Loaiza-Ganem G and Cunningham JP, NeurIPS 2019. + https://arxiv.org/abs/1907.06845 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.unit_interval + _mean_carrier_measure = 0 + has_rsample = True + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + lims: tuple[float, float] = (0.499, 0.501), + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + # validate 'probs' here if necessary as it is later clamped for numerical stability + # close to 0 and 1, later on; otherwise the clamped 'probs' would always pass + if validate_args is not None: + if not self.arg_constraints["probs"].check(self.probs).all(): + raise ValueError("The parameter probs has invalid values") + # pyrefly: ignore [read-only] + self.probs = clamp_probs(self.probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + self._lims = lims + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(ContinuousBernoulli, _instance) + new._lims = self._lims + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(ContinuousBernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + def _outside_unstable_region(self): + return torch.max( + torch.le(self.probs, self._lims[0]), torch.gt(self.probs, self._lims[1]) + ) + + def _cut_probs(self): + return torch.where( + self._outside_unstable_region(), + self.probs, + self._lims[0] * torch.ones_like(self.probs), + ) + + def _cont_bern_log_norm(self): + """computes the log normalizing constant as a function of the 'probs' parameter""" + cut_probs = self._cut_probs() + cut_probs_below_half = torch.where( + torch.le(cut_probs, 0.5), cut_probs, torch.zeros_like(cut_probs) + ) + cut_probs_above_half = torch.where( + torch.ge(cut_probs, 0.5), cut_probs, torch.ones_like(cut_probs) + ) + log_norm = torch.log( + torch.abs(torch.log1p(-cut_probs) - torch.log(cut_probs)) + ) - torch.where( + torch.le(cut_probs, 0.5), + torch.log1p(-2.0 * cut_probs_below_half), + torch.log(2.0 * cut_probs_above_half - 1.0), + ) + x = torch.pow(self.probs - 0.5, 2) + taylor = math.log(2.0) + (4.0 / 3.0 + 104.0 / 45.0 * x) * x + return torch.where(self._outside_unstable_region(), log_norm, taylor) + + @property + def mean(self) -> Tensor: + cut_probs = self._cut_probs() + mus = cut_probs / (2.0 * cut_probs - 1.0) + 1.0 / ( + torch.log1p(-cut_probs) - torch.log(cut_probs) + ) + x = self.probs - 0.5 + taylor = 0.5 + (1.0 / 3.0 + 16.0 / 45.0 * torch.pow(x, 2)) * x + return torch.where(self._outside_unstable_region(), mus, taylor) + + @property + def stddev(self) -> Tensor: + return torch.sqrt(self.variance) + + @property + def variance(self) -> Tensor: + cut_probs = self._cut_probs() + vars = cut_probs * (cut_probs - 1.0) / torch.pow( + 1.0 - 2.0 * cut_probs, 2 + ) + 1.0 / torch.pow(torch.log1p(-cut_probs) - torch.log(cut_probs), 2) + x = torch.pow(self.probs - 0.5, 2) + taylor = 1.0 / 12.0 - (1.0 / 15.0 - 128.0 / 945.0 * x) * x + return torch.where(self._outside_unstable_region(), vars, taylor) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return clamp_probs(logits_to_probs(self.logits, is_binary=True)) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + with torch.no_grad(): + return self.icdf(u) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + return self.icdf(u) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + return ( + -binary_cross_entropy_with_logits(logits, value, reduction="none") + + self._cont_bern_log_norm() + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + cut_probs = self._cut_probs() + cdfs = ( + torch.pow(cut_probs, value) * torch.pow(1.0 - cut_probs, 1.0 - value) + + cut_probs + - 1.0 + ) / (2.0 * cut_probs - 1.0) + unbounded_cdfs = torch.where(self._outside_unstable_region(), cdfs, value) + return torch.where( + torch.le(value, 0.0), + torch.zeros_like(value), + torch.where(torch.ge(value, 1.0), torch.ones_like(value), unbounded_cdfs), + ) + + def icdf(self, value): + cut_probs = self._cut_probs() + return torch.where( + self._outside_unstable_region(), + ( + torch.log1p(-cut_probs + value * (2.0 * cut_probs - 1.0)) + - torch.log1p(-cut_probs) + ) + / (torch.log(cut_probs) - torch.log1p(-cut_probs)), + value, + ) + + def entropy(self): + log_probs0 = torch.log1p(-self.probs) + log_probs1 = torch.log(self.probs) + return ( + self.mean * (log_probs0 - log_probs1) + - self._cont_bern_log_norm() + - log_probs0 + ) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (self.logits,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + """computes the log normalizing constant as a function of the natural parameter""" + out_unst_reg = torch.max( + torch.le(x, self._lims[0] - 0.5), torch.gt(x, self._lims[1] - 0.5) + ) + cut_nat_params = torch.where( + out_unst_reg, x, (self._lims[0] - 0.5) * torch.ones_like(x) + ) + log_norm = torch.log( + torch.abs(torch.special.expm1(cut_nat_params)) + ) - torch.log(torch.abs(cut_nat_params)) + taylor = 0.5 * x + torch.pow(x, 2) / 24.0 - torch.pow(x, 4) / 2880.0 + return torch.where(out_unst_reg, log_norm, taylor) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/dirichlet.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/dirichlet.py new file mode 100644 index 0000000000000000000000000000000000000000..33605be16a636f399c04940e9f3cc644979e0dc0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/dirichlet.py @@ -0,0 +1,138 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.types import _size + + +__all__ = ["Dirichlet"] + + +# This helper is exposed for testing. +def _Dirichlet_backward(x, concentration, grad_output): + total = concentration.sum(-1, True).expand_as(concentration) + grad = torch._dirichlet_grad(x, concentration, total) + return grad * (grad_output - (x * grad_output).sum(-1, True)) + + +class _Dirichlet(Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, concentration): + x = torch._sample_dirichlet(concentration) + ctx.save_for_backward(x, concentration) + return x + + @staticmethod + @once_differentiable + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + x, concentration = ctx.saved_tensors + return _Dirichlet_backward(x, concentration, grad_output) + + +class Dirichlet(ExponentialFamily): + r""" + Creates a Dirichlet distribution parameterized by concentration :attr:`concentration`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Dirichlet(torch.tensor([0.5, 0.5])) + >>> m.sample() # Dirichlet distributed with concentration [0.5, 0.5] + tensor([ 0.1046, 0.8954]) + + Args: + concentration (Tensor): concentration parameter of the distribution + (often referred to as alpha) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration": constraints.independent(constraints.positive, 1) + } + support = constraints.simplex + has_rsample = True + + def __init__( + self, + concentration: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + if concentration.dim() < 1: + raise ValueError( + "`concentration` parameter must be at least one-dimensional." + ) + self.concentration = concentration + batch_shape, event_shape = concentration.shape[:-1], concentration.shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Dirichlet, _instance) + batch_shape = torch.Size(batch_shape) + new.concentration = self.concentration.expand(batch_shape + self.event_shape) + super(Dirichlet, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = ()) -> Tensor: + shape = self._extended_shape(sample_shape) + concentration = self.concentration.expand(shape) + return _Dirichlet.apply(concentration) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return ( + torch.xlogy(self.concentration - 1.0, value).sum(-1) + + torch.lgamma(self.concentration.sum(-1)) + - torch.lgamma(self.concentration).sum(-1) + ) + + @property + def mean(self) -> Tensor: + return self.concentration / self.concentration.sum(-1, True) + + @property + def mode(self) -> Tensor: + concentrationm1 = (self.concentration - 1).clamp(min=0.0) + mode = concentrationm1 / concentrationm1.sum(-1, True) + mask = (self.concentration < 1).all(dim=-1) + mode[mask] = torch.nn.functional.one_hot( + mode[mask].argmax(dim=-1), concentrationm1.shape[-1] + ).to(mode) + return mode + + @property + def variance(self) -> Tensor: + con0 = self.concentration.sum(-1, True) + return ( + self.concentration + * (con0 - self.concentration) + / (con0.pow(2) * (con0 + 1)) + ) + + def entropy(self): + k = self.concentration.size(-1) + a0 = self.concentration.sum(-1) + return ( + torch.lgamma(self.concentration).sum(-1) + - torch.lgamma(a0) + - (k - a0) * torch.digamma(a0) + - ((self.concentration - 1.0) * torch.digamma(self.concentration)).sum(-1) + ) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (self.concentration,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return x.lgamma().sum(-1) - torch.lgamma(x.sum(-1)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/distribution.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..2c78dbe429318f753c3fb7ea68a5adeefcc2bf10 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/distribution.py @@ -0,0 +1,348 @@ +# mypy: allow-untyped-defs +import warnings +from typing import Optional +from typing_extensions import deprecated + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.utils import lazy_property +from torch.types import _size + + +__all__ = ["Distribution"] + + +class Distribution: + r""" + Distribution is the abstract base class for probability distributions. + + Args: + batch_shape (torch.Size): The shape over which parameters are batched. + event_shape (torch.Size): The shape of a single sample (without batching). + validate_args (bool, optional): Whether to validate arguments. Default: None. + """ + + has_rsample = False + has_enumerate_support = False + _validate_args = __debug__ + + @staticmethod + def set_default_validate_args(value: bool) -> None: + """ + Sets whether validation is enabled or disabled. + + The default behavior mimics Python's ``assert`` statement: validation + is on by default, but is disabled if Python is run in optimized mode + (via ``python -O``). Validation may be expensive, so you may want to + disable it once a model is working. + + Args: + value (bool): Whether to enable validation. + """ + if value not in [True, False]: + raise ValueError + Distribution._validate_args = value + + def __init__( + self, + batch_shape: torch.Size = torch.Size(), + event_shape: torch.Size = torch.Size(), + validate_args: Optional[bool] = None, + ) -> None: + self._batch_shape = batch_shape + self._event_shape = event_shape + if validate_args is not None: + self._validate_args = validate_args + if self._validate_args: + try: + arg_constraints = self.arg_constraints + except NotImplementedError: + arg_constraints = {} + warnings.warn( + f"{self.__class__} does not define `arg_constraints`. " + + "Please set `arg_constraints = {}` or initialize the distribution " + + "with `validate_args=False` to turn off validation.", + stacklevel=2, + ) + for param, constraint in arg_constraints.items(): + if constraints.is_dependent(constraint): + continue # skip constraints that cannot be checked + if param not in self.__dict__ and isinstance( + getattr(type(self), param), lazy_property + ): + continue # skip checking lazily-constructed args + value = getattr(self, param) + valid = constraint.check(value) + if not torch._is_all_true(valid): + raise ValueError( + f"Expected parameter {param} " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"of distribution {repr(self)} " + f"to satisfy the constraint {repr(constraint)}, " + f"but found invalid values:\n{value}" + ) + super().__init__() + + def expand(self, batch_shape: _size, _instance=None): + """ + Returns a new distribution instance (or populates an existing instance + provided by a derived class) with batch dimensions expanded to + `batch_shape`. This method calls :class:`~torch.Tensor.expand` on + the distribution's parameters. As such, this does not allocate new + memory for the expanded distribution instance. Additionally, + this does not repeat any args checking or parameter broadcasting in + `__init__.py`, when an instance is first created. + + Args: + batch_shape (torch.Size): the desired expanded size. + _instance: new instance provided by subclasses that + need to override `.expand`. + + Returns: + New distribution instance with batch dimensions expanded to + `batch_size`. + """ + raise NotImplementedError + + @property + def batch_shape(self) -> torch.Size: + """ + Returns the shape over which parameters are batched. + """ + return self._batch_shape + + @property + def event_shape(self) -> torch.Size: + """ + Returns the shape of a single sample (without batching). + """ + return self._event_shape + + @property + def arg_constraints(self) -> dict[str, constraints.Constraint]: + """ + Returns a dictionary from argument names to + :class:`~torch.distributions.constraints.Constraint` objects that + should be satisfied by each argument of this distribution. Args that + are not tensors need not appear in this dict. + """ + raise NotImplementedError + + @property + def support(self) -> Optional[constraints.Constraint]: + """ + Returns a :class:`~torch.distributions.constraints.Constraint` object + representing this distribution's support. + """ + raise NotImplementedError + + @property + def mean(self) -> Tensor: + """ + Returns the mean of the distribution. + """ + raise NotImplementedError + + @property + def mode(self) -> Tensor: + """ + Returns the mode of the distribution. + """ + raise NotImplementedError(f"{self.__class__} does not implement mode") + + @property + def variance(self) -> Tensor: + """ + Returns the variance of the distribution. + """ + raise NotImplementedError + + @property + def stddev(self) -> Tensor: + """ + Returns the standard deviation of the distribution. + """ + return self.variance.sqrt() + + def sample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped sample or sample_shape shaped batch of + samples if the distribution parameters are batched. + """ + with torch.no_grad(): + return self.rsample(sample_shape) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped reparameterized sample or sample_shape + shaped batch of reparameterized samples if the distribution parameters + are batched. + """ + raise NotImplementedError + + @deprecated( + "`sample_n(n)` will be deprecated. Use `sample((n,))` instead.", + category=FutureWarning, + ) + def sample_n(self, n: int) -> Tensor: + """ + Generates n samples or n batches of samples if the distribution + parameters are batched. + """ + return self.sample(torch.Size((n,))) + + def log_prob(self, value: Tensor) -> Tensor: + """ + Returns the log of the probability density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def cdf(self, value: Tensor) -> Tensor: + """ + Returns the cumulative density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def icdf(self, value: Tensor) -> Tensor: + """ + Returns the inverse cumulative density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def enumerate_support(self, expand: bool = True) -> Tensor: + """ + Returns tensor containing all values supported by a discrete + distribution. The result will enumerate over dimension 0, so the shape + of the result will be `(cardinality,) + batch_shape + event_shape` + (where `event_shape = ()` for univariate distributions). + + Note that this enumerates over all batched tensors in lock-step + `[[0, 0], [1, 1], ...]`. With `expand=False`, enumeration happens + along dim 0, but with the remaining batch dimensions being + singleton dimensions, `[[0], [1], ..`. + + To iterate over the full Cartesian product use + `itertools.product(m.enumerate_support())`. + + Args: + expand (bool): whether to expand the support over the + batch dims to match the distribution's `batch_shape`. + + Returns: + Tensor iterating over dimension 0. + """ + raise NotImplementedError + + def entropy(self) -> Tensor: + """ + Returns entropy of distribution, batched over batch_shape. + + Returns: + Tensor of shape batch_shape. + """ + raise NotImplementedError + + def perplexity(self) -> Tensor: + """ + Returns perplexity of distribution, batched over batch_shape. + + Returns: + Tensor of shape batch_shape. + """ + return torch.exp(self.entropy()) + + def _extended_shape(self, sample_shape: _size = torch.Size()) -> torch.Size: + """ + Returns the size of the sample returned by the distribution, given + a `sample_shape`. Note, that the batch and event shapes of a distribution + instance are fixed at the time of construction. If this is empty, the + returned shape is upcast to (1,). + + Args: + sample_shape (torch.Size): the size of the sample to be drawn. + """ + if not isinstance(sample_shape, torch.Size): + sample_shape = torch.Size(sample_shape) + return torch.Size(sample_shape + self._batch_shape + self._event_shape) + + def _validate_sample(self, value: Tensor) -> None: + """ + Argument validation for distribution methods such as `log_prob`, + `cdf` and `icdf`. The rightmost dimensions of a value to be + scored via these methods must agree with the distribution's batch + and event shapes. + + Args: + value (Tensor): the tensor whose log probability is to be + computed by the `log_prob` method. + Raises + ValueError: when the rightmost dimensions of `value` do not match the + distribution's batch and event shapes. + """ + if not isinstance(value, torch.Tensor): + raise ValueError("The value argument to log_prob must be a Tensor") + + event_dim_start = len(value.size()) - len(self._event_shape) + if value.size()[event_dim_start:] != self._event_shape: + raise ValueError( + f"The right-most size of value must match event_shape: {value.size()} vs {self._event_shape}." + ) + + actual_shape = value.size() + expected_shape = self._batch_shape + self._event_shape + for i, j in zip(reversed(actual_shape), reversed(expected_shape)): + if i != 1 and j != 1 and i != j: + raise ValueError( + f"Value is not broadcastable with batch_shape+event_shape: {actual_shape} vs {expected_shape}." + ) + try: + support = self.support + except NotImplementedError: + warnings.warn( + f"{self.__class__} does not define `support` to enable " + + "sample validation. Please initialize the distribution with " + + "`validate_args=False` to turn off validation.", + stacklevel=2, + ) + return + assert support is not None + valid = support.check(value) + if not torch._is_all_true(valid): + raise ValueError( + "Expected value argument " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"to be within the support ({repr(support)}) " + f"of the distribution {repr(self)}, " + f"but found invalid values:\n{value}" + ) + + def _get_checked_instance(self, cls, _instance=None): + if _instance is None and type(self).__init__ != cls.__init__: + raise NotImplementedError( + f"Subclass {self.__class__.__name__} of {cls.__name__} that defines a custom __init__ method " + "must also define a custom .expand() method." + ) + return self.__new__(type(self)) if _instance is None else _instance + + def __repr__(self) -> str: + param_names = [k for k, _ in self.arg_constraints.items() if k in self.__dict__] + args_string = ", ".join( + [ + f"{p}: {self.__dict__[p] if self.__dict__[p].numel() == 1 else self.__dict__[p].size()}" + for p in param_names + ] + ) + return self.__class__.__name__ + "(" + args_string + ")" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exp_family.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exp_family.py new file mode 100644 index 0000000000000000000000000000000000000000..6a00cfc7fb8a044572119196f81e8bd9bc457f33 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exp_family.py @@ -0,0 +1,67 @@ +# mypy: allow-untyped-defs +from typing import Union + +import torch +from torch import Tensor +from torch.distributions.distribution import Distribution + + +__all__ = ["ExponentialFamily"] + + +class ExponentialFamily(Distribution): + r""" + ExponentialFamily is the abstract base class for probability distributions belonging to an + exponential family, whose probability mass/density function has the form is defined below + + .. math:: + + p_{F}(x; \theta) = \exp(\langle t(x), \theta\rangle - F(\theta) + k(x)) + + where :math:`\theta` denotes the natural parameters, :math:`t(x)` denotes the sufficient statistic, + :math:`F(\theta)` is the log normalizer function for a given family and :math:`k(x)` is the carrier + measure. + + Note: + This class is an intermediary between the `Distribution` class and distributions which belong + to an exponential family mainly to check the correctness of the `.entropy()` and analytic KL + divergence methods. We use this class to compute the entropy and KL divergence using the AD + framework and Bregman divergences (courtesy of: Frank Nielsen and Richard Nock, Entropies and + Cross-entropies of Exponential Families). + """ + + @property + def _natural_params(self) -> tuple[Tensor, ...]: + """ + Abstract method for natural parameters. Returns a tuple of Tensors based + on the distribution + """ + raise NotImplementedError + + def _log_normalizer(self, *natural_params): + """ + Abstract method for log normalizer function. Returns a log normalizer based on + the distribution and input + """ + raise NotImplementedError + + @property + def _mean_carrier_measure(self) -> float: + """ + Abstract method for expected carrier measure, which is required for computing + entropy. + """ + raise NotImplementedError + + def entropy(self): + """ + Method to compute the entropy using Bregman divergence of the log normalizer. + """ + result: Union[Tensor, float] = -self._mean_carrier_measure + nparams = [p.detach().requires_grad_() for p in self._natural_params] + lg_normal = self._log_normalizer(*nparams) + gradients = torch.autograd.grad(lg_normal.sum(), nparams, create_graph=True) + result += lg_normal + for np, g in zip(nparams, gradients): + result -= (np * g).reshape(self._batch_shape + (-1,)).sum(-1) + return result diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exponential.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exponential.py new file mode 100644 index 0000000000000000000000000000000000000000..86744472e309a7e7637484e895a132264076d36c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/exponential.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Exponential"] + + +class Exponential(ExponentialFamily): + r""" + Creates a Exponential distribution parameterized by :attr:`rate`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Exponential(torch.tensor([1.0])) + >>> m.sample() # Exponential distributed with rate=1 + tensor([ 0.1046]) + + Args: + rate (float or Tensor): rate = 1 / scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"rate": constraints.positive} + support = constraints.nonnegative + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.rate.reciprocal() + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.rate) + + @property + def stddev(self) -> Tensor: + return self.rate.reciprocal() + + @property + def variance(self) -> Tensor: + return self.rate.pow(-2) + + def __init__( + self, + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + (self.rate,) = broadcast_all(rate) + batch_shape = torch.Size() if isinstance(rate, _Number) else self.rate.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Exponential, _instance) + batch_shape = torch.Size(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Exponential, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + return self.rate.new(shape).exponential_() / self.rate + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return self.rate.log() - self.rate * value + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 1 - torch.exp(-self.rate * value) + + def icdf(self, value): + return -torch.log1p(-value) / self.rate + + def entropy(self): + return 1.0 - torch.log(self.rate) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (-self.rate,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return -torch.log(-x) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/fishersnedecor.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/fishersnedecor.py new file mode 100644 index 0000000000000000000000000000000000000000..dbcd7fd9cce4996ef8cdaef152716a9bb8319ed0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/fishersnedecor.py @@ -0,0 +1,108 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.gamma import Gamma +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["FisherSnedecor"] + + +class FisherSnedecor(Distribution): + r""" + Creates a Fisher-Snedecor distribution parameterized by :attr:`df1` and :attr:`df2`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = FisherSnedecor(torch.tensor([1.0]), torch.tensor([2.0])) + >>> m.sample() # Fisher-Snedecor-distributed with df1=1 and df2=2 + tensor([ 0.2453]) + + Args: + df1 (float or Tensor): degrees of freedom parameter 1 + df2 (float or Tensor): degrees of freedom parameter 2 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"df1": constraints.positive, "df2": constraints.positive} + support = constraints.positive + has_rsample = True + + def __init__( + self, + df1: Union[Tensor, float], + df2: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.df1, self.df2 = broadcast_all(df1, df2) + self._gamma1 = Gamma(self.df1 * 0.5, self.df1) + self._gamma2 = Gamma(self.df2 * 0.5, self.df2) + + if isinstance(df1, _Number) and isinstance(df2, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.df1.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(FisherSnedecor, _instance) + batch_shape = torch.Size(batch_shape) + new.df1 = self.df1.expand(batch_shape) + new.df2 = self.df2.expand(batch_shape) + new._gamma1 = self._gamma1.expand(batch_shape) + new._gamma2 = self._gamma2.expand(batch_shape) + super(FisherSnedecor, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + df2 = self.df2.clone(memory_format=torch.contiguous_format) + df2[df2 <= 2] = nan + return df2 / (df2 - 2) + + @property + def mode(self) -> Tensor: + mode = (self.df1 - 2) / self.df1 * self.df2 / (self.df2 + 2) + mode[self.df1 <= 2] = nan + return mode + + @property + def variance(self) -> Tensor: + df2 = self.df2.clone(memory_format=torch.contiguous_format) + df2[df2 <= 4] = nan + return ( + 2 + * df2.pow(2) + * (self.df1 + df2 - 2) + / (self.df1 * (df2 - 2).pow(2) * (df2 - 4)) + ) + + def rsample(self, sample_shape: _size = torch.Size(())) -> Tensor: + shape = self._extended_shape(sample_shape) + # X1 ~ Gamma(df1 / 2, 1 / df1), X2 ~ Gamma(df2 / 2, 1 / df2) + # Y = df2 * df1 * X1 / (df1 * df2 * X2) = X1 / X2 ~ F(df1, df2) + X1 = self._gamma1.rsample(sample_shape).view(shape) + X2 = self._gamma2.rsample(sample_shape).view(shape) + tiny = torch.finfo(X2.dtype).tiny + X2.clamp_(min=tiny) + Y = X1 / X2 + Y.clamp_(min=tiny) + return Y + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + ct1 = self.df1 * 0.5 + ct2 = self.df2 * 0.5 + ct3 = self.df1 / self.df2 + t1 = (ct1 + ct2).lgamma() - ct1.lgamma() - ct2.lgamma() + t2 = ct1 * ct3.log() + (ct1 - 1) * torch.log(value) + t3 = (ct1 + ct2) * torch.log1p(ct3 * value) + return t1 + t2 - t3 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gamma.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gamma.py new file mode 100644 index 0000000000000000000000000000000000000000..aded5ca7b0f775ec8cd8e838193e60cc5ed96e9c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gamma.py @@ -0,0 +1,120 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Gamma"] + + +def _standard_gamma(concentration): + return torch._standard_gamma(concentration) + + +class Gamma(ExponentialFamily): + r""" + Creates a Gamma distribution parameterized by shape :attr:`concentration` and :attr:`rate`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Gamma(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # Gamma distributed with concentration=1 and rate=1 + tensor([ 0.1046]) + + Args: + concentration (float or Tensor): shape parameter of the distribution + (often referred to as alpha) + rate (float or Tensor): rate parameter of the distribution + (often referred to as beta), rate = 1 / scale + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration": constraints.positive, + "rate": constraints.positive, + } + support = constraints.nonnegative + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.concentration / self.rate + + @property + def mode(self) -> Tensor: + return ((self.concentration - 1) / self.rate).clamp(min=0) + + @property + def variance(self) -> Tensor: + return self.concentration / self.rate.pow(2) + + def __init__( + self, + concentration: Union[Tensor, float], + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.concentration, self.rate = broadcast_all(concentration, rate) + if isinstance(concentration, _Number) and isinstance(rate, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.concentration.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Gamma, _instance) + batch_shape = torch.Size(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Gamma, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + value = _standard_gamma(self.concentration.expand(shape)) / self.rate.expand( + shape + ) + value.detach().clamp_( + min=torch.finfo(value.dtype).tiny + ) # do not record in autograd graph + return value + + def log_prob(self, value): + value = torch.as_tensor(value, dtype=self.rate.dtype, device=self.rate.device) + if self._validate_args: + self._validate_sample(value) + return ( + torch.xlogy(self.concentration, self.rate) + + torch.xlogy(self.concentration - 1, value) + - self.rate * value + - torch.lgamma(self.concentration) + ) + + def entropy(self): + return ( + self.concentration + - torch.log(self.rate) + + torch.lgamma(self.concentration) + + (1.0 - self.concentration) * torch.digamma(self.concentration) + ) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.concentration - 1, -self.rate) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return torch.lgamma(x + 1) + (x + 1) * torch.log(-y.reciprocal()) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return torch.special.gammainc(self.concentration, self.rate * value) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/generalized_pareto.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/generalized_pareto.py new file mode 100644 index 0000000000000000000000000000000000000000..c23a575f75400857ab1fd6087955e350493736e0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/generalized_pareto.py @@ -0,0 +1,153 @@ +# mypy: allow-untyped-defs +import math +from numbers import Number, Real + +import torch +from torch import inf, nan +from torch.distributions import constraints, Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["GeneralizedPareto"] + + +class GeneralizedPareto(Distribution): + r""" + Creates a Generalized Pareto distribution parameterized by :attr:`loc`, :attr:`scale`, and :attr:`concentration`. + + The Generalized Pareto distribution is a family of continuous probability distributions on the real line. + Special cases include Exponential (when :attr:`loc` = 0, :attr:`concentration` = 0), Pareto (when :attr:`concentration` > 0, + :attr:`loc` = :attr:`scale` / :attr:`concentration`), and Uniform (when :attr:`concentration` = -1). + + This distribution is often used to model the tails of other distributions. This implementation is based on the + implementation in TensorFlow Probability. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = GeneralizedPareto(torch.tensor([0.1]), torch.tensor([2.0]), torch.tensor([0.4])) + >>> m.sample() # sample from a Generalized Pareto distribution with loc=0.1, scale=2.0, and concentration=0.4 + tensor([ 1.5623]) + + Args: + loc (float or Tensor): Location parameter of the distribution + scale (float or Tensor): Scale parameter of the distribution + concentration (float or Tensor): Concentration parameter of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real, + "scale": constraints.positive, + "concentration": constraints.real, + } + has_rsample = True + + def __init__(self, loc, scale, concentration, validate_args=None): + self.loc, self.scale, self.concentration = broadcast_all( + loc, scale, concentration + ) + if ( + isinstance(loc, Number) + and isinstance(scale, Number) + and isinstance(concentration, Number) + ): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(GeneralizedPareto, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + super(GeneralizedPareto, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.icdf(u) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + z = self._z(value) + eq_zero = torch.isclose(self.concentration, torch.tensor(0.0)) + safe_conc = torch.where( + eq_zero, torch.ones_like(self.concentration), self.concentration + ) + y = 1 / safe_conc + torch.ones_like(z) + where_nonzero = torch.where(y == 0, y, y * torch.log1p(safe_conc * z)) + log_scale = ( + math.log(self.scale) if isinstance(self.scale, Real) else self.scale.log() + ) + return -log_scale - torch.where(eq_zero, z, where_nonzero) + + def log_survival_function(self, value): + if self._validate_args: + self._validate_sample(value) + z = self._z(value) + eq_zero = torch.isclose(self.concentration, torch.tensor(0.0)) + safe_conc = torch.where( + eq_zero, torch.ones_like(self.concentration), self.concentration + ) + where_nonzero = -torch.log1p(safe_conc * z) / safe_conc + return torch.where(eq_zero, -z, where_nonzero) + + def log_cdf(self, value): + return torch.log1p(-torch.exp(self.log_survival_function(value))) + + def cdf(self, value): + return torch.exp(self.log_cdf(value)) + + def icdf(self, value): + loc = self.loc + scale = self.scale + concentration = self.concentration + eq_zero = torch.isclose(concentration, torch.zeros_like(concentration)) + safe_conc = torch.where(eq_zero, torch.ones_like(concentration), concentration) + logu = torch.log1p(-value) + where_nonzero = loc + scale / safe_conc * torch.expm1(-safe_conc * logu) + where_zero = loc - scale * logu + return torch.where(eq_zero, where_zero, where_nonzero) + + def _z(self, x): + return (x - self.loc) / self.scale + + @property + def mean(self): + concentration = self.concentration + valid = concentration < 1 + safe_conc = torch.where(valid, concentration, 0.5) + result = self.loc + self.scale / (1 - safe_conc) + return torch.where(valid, result, nan) + + @property + def variance(self): + concentration = self.concentration + valid = concentration < 0.5 + safe_conc = torch.where(valid, concentration, 0.25) + # pyrefly: ignore [unsupported-operation] + result = self.scale**2 / ((1 - safe_conc) ** 2 * (1 - 2 * safe_conc)) + return torch.where(valid, result, nan) + + def entropy(self): + ans = torch.log(self.scale) + self.concentration + 1 + return torch.broadcast_to(ans, self._batch_shape) + + @property + def mode(self): + return self.loc + + @constraints.dependent_property(is_discrete=False, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + lower = self.loc + upper = torch.where( + self.concentration < 0, lower - self.scale / self.concentration, inf + ) + return constraints.interval(lower, upper) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/geometric.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/geometric.py new file mode 100644 index 0000000000000000000000000000000000000000..7e59a0cefe5d60f180a1d8be7dba1afc693060d9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/geometric.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, Number + + +__all__ = ["Geometric"] + + +class Geometric(Distribution): + r""" + Creates a Geometric distribution parameterized by :attr:`probs`, + where :attr:`probs` is the probability of success of Bernoulli trials. + + .. math:: + + P(X=k) = (1-p)^{k} p, k = 0, 1, ... + + .. note:: + :func:`torch.distributions.geometric.Geometric` :math:`(k+1)`-th trial is the first success + hence draws samples in :math:`\{0, 1, \ldots\}`, whereas + :func:`torch.Tensor.geometric_` `k`-th trial is the first success hence draws samples in :math:`\{1, 2, \ldots\}`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Geometric(torch.tensor([0.3])) + >>> m.sample() # underlying Bernoulli has 30% chance 1; 70% chance 0 + tensor([ 2.]) + + Args: + probs (Number, Tensor): the probability of sampling `1`. Must be in range (0, 1] + logits (Number, Tensor): the log-odds of sampling `1`. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.nonnegative_integer + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + probs_or_logits = probs if probs is not None else logits + if isinstance(probs_or_logits, _Number): + batch_shape = torch.Size() + else: + assert probs_or_logits is not None # helps mypy + batch_shape = probs_or_logits.size() + super().__init__(batch_shape, validate_args=validate_args) + if self._validate_args and probs is not None: + # Add an extra check beyond unit_interval + value = self.probs + valid = value > 0 + if not valid.all(): + invalid_value = value.data[~valid] + raise ValueError( + "Expected parameter probs " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"of distribution {repr(self)} " + f"to be positive but found invalid values:\n{invalid_value}" + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Geometric, _instance) + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + super(Geometric, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return 1.0 / self.probs - 1.0 + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.probs) + + @property + def variance(self) -> Tensor: + return (1.0 / self.probs - 1.0) / self.probs + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + tiny = torch.finfo(self.probs.dtype).tiny + with torch.no_grad(): + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .uniform_() + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + u = u.clamp(min=tiny) + else: + u = self.probs.new(shape).uniform_(tiny, 1) + return (u.log() / (-self.probs).log1p()).floor() + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value, probs = broadcast_all(value, self.probs) + probs = probs.clone(memory_format=torch.contiguous_format) + probs[(probs == 1) & (value == 0)] = 0 + return value * (-probs).log1p() + self.probs.log() + + def entropy(self): + return ( + binary_cross_entropy_with_logits(self.logits, self.probs, reduction="none") + / self.probs + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gumbel.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gumbel.py new file mode 100644 index 0000000000000000000000000000000000000000..c85edafe1e5845eb5dca4ce00c90fd05de2f9128 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/gumbel.py @@ -0,0 +1,92 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, ExpTransform +from torch.distributions.uniform import Uniform +from torch.distributions.utils import broadcast_all, euler_constant +from torch.types import _Number + + +__all__ = ["Gumbel"] + + +class Gumbel(TransformedDistribution): + r""" + Samples from a Gumbel Distribution. + + Examples:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Gumbel(torch.tensor([1.0]), torch.tensor([2.0])) + >>> m.sample() # sample from Gumbel distribution with loc=1, scale=2 + tensor([ 1.0124]) + + Args: + loc (float or Tensor): Location parameter of the distribution + scale (float or Tensor): Scale parameter of the distribution + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.real + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + finfo = torch.finfo(self.loc.dtype) + if isinstance(loc, _Number) and isinstance(scale, _Number): + base_dist = Uniform(finfo.tiny, 1 - finfo.eps, validate_args=validate_args) + else: + base_dist = Uniform( + torch.full_like(self.loc, finfo.tiny), + torch.full_like(self.loc, 1 - finfo.eps), + validate_args=validate_args, + ) + transforms = [ + ExpTransform().inv, + AffineTransform(loc=0, scale=-torch.ones_like(self.scale)), + ExpTransform().inv, + AffineTransform(loc=loc, scale=-self.scale), + ] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Gumbel, _instance) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + # Explicitly defining the log probability function for Gumbel due to precision issues + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + y = (self.loc - value) / self.scale + return (y - y.exp()) - self.scale.log() + + @property + def mean(self) -> Tensor: + return self.loc + self.scale * euler_constant + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def stddev(self) -> Tensor: + return (math.pi / math.sqrt(6)) * self.scale + + @property + def variance(self) -> Tensor: + return self.stddev.pow(2) + + def entropy(self): + return self.scale.log() + (1 + euler_constant) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_cauchy.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_cauchy.py new file mode 100644 index 0000000000000000000000000000000000000000..cb52d3ffaa1b9034a099a7d588f0b184d59d1476 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_cauchy.py @@ -0,0 +1,93 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, Tensor +from torch.distributions import constraints +from torch.distributions.cauchy import Cauchy +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AbsTransform + + +__all__ = ["HalfCauchy"] + + +class HalfCauchy(TransformedDistribution): + r""" + Creates a half-Cauchy distribution parameterized by `scale` where:: + + X ~ Cauchy(0, scale) + Y = |X| ~ HalfCauchy(scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = HalfCauchy(torch.tensor([1.0])) + >>> m.sample() # half-cauchy distributed with scale=1 + tensor([ 2.3214]) + + Args: + scale (float or Tensor): scale of the full Cauchy distribution + """ + + arg_constraints = {"scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.nonnegative + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Cauchy + + def __init__( + self, + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Cauchy(0, scale, validate_args=False) + super().__init__(base_dist, AbsTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(HalfCauchy, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), + math.inf, + dtype=self.scale.dtype, + device=self.scale.device, + ) + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.scale) + + @property + def variance(self) -> Tensor: + return self.base_dist.variance + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value = torch.as_tensor( + value, dtype=self.base_dist.scale.dtype, device=self.base_dist.scale.device + ) + log_prob = self.base_dist.log_prob(value) + math.log(2) + log_prob = torch.where(value >= 0, log_prob, -inf) + return log_prob + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 2 * self.base_dist.cdf(value) - 1 + + def icdf(self, prob): + return self.base_dist.icdf((prob + 1) / 2) + + def entropy(self): + return self.base_dist.entropy() - math.log(2) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..81eb7f8e78550eb2a0b4ba799d3136fe4b817b27 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/half_normal.py @@ -0,0 +1,85 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, Tensor +from torch.distributions import constraints +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AbsTransform + + +__all__ = ["HalfNormal"] + + +class HalfNormal(TransformedDistribution): + r""" + Creates a half-normal distribution parameterized by `scale` where:: + + X ~ Normal(0, scale) + Y = |X| ~ HalfNormal(scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = HalfNormal(torch.tensor([1.0])) + >>> m.sample() # half-normal distributed with scale=1 + tensor([ 0.1046]) + + Args: + scale (float or Tensor): scale of the full Normal distribution + """ + + arg_constraints = {"scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.nonnegative + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Normal + + def __init__( + self, + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(0, scale, validate_args=False) + super().__init__(base_dist, AbsTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(HalfNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return self.scale * math.sqrt(2 / math.pi) + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.scale) + + @property + def variance(self) -> Tensor: + return self.scale.pow(2) * (1 - 2 / math.pi) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_prob = self.base_dist.log_prob(value) + math.log(2) + log_prob = torch.where(value >= 0, log_prob, -inf) + return log_prob + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 2 * self.base_dist.cdf(value) - 1 + + def icdf(self, prob): + return self.base_dist.icdf((prob + 1) / 2) + + def entropy(self): + return self.base_dist.entropy() - math.log(2) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/independent.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/independent.py new file mode 100644 index 0000000000000000000000000000000000000000..8d24df02ae5ea0d432e2b81b3c3362f107c186c8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/independent.py @@ -0,0 +1,138 @@ +# mypy: allow-untyped-defs +from typing import Generic, Optional, TypeVar + +import torch +from torch import Size, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _sum_rightmost +from torch.types import _size + + +__all__ = ["Independent"] + + +D = TypeVar("D", bound=Distribution) + + +class Independent(Distribution, Generic[D]): + r""" + Reinterprets some of the batch dims of a distribution as event dims. + + This is mainly useful for changing the shape of the result of + :meth:`log_prob`. For example to create a diagonal Normal distribution with + the same shape as a Multivariate Normal distribution (so they are + interchangeable), you can:: + + >>> from torch.distributions.multivariate_normal import MultivariateNormal + >>> from torch.distributions.normal import Normal + >>> loc = torch.zeros(3) + >>> scale = torch.ones(3) + >>> mvn = MultivariateNormal(loc, scale_tril=torch.diag(scale)) + >>> [mvn.batch_shape, mvn.event_shape] + [torch.Size([]), torch.Size([3])] + >>> normal = Normal(loc, scale) + >>> [normal.batch_shape, normal.event_shape] + [torch.Size([3]), torch.Size([])] + >>> diagn = Independent(normal, 1) + >>> [diagn.batch_shape, diagn.event_shape] + [torch.Size([]), torch.Size([3])] + + Args: + base_distribution (torch.distributions.distribution.Distribution): a + base distribution + reinterpreted_batch_ndims (int): the number of batch dims to + reinterpret as event dims + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + base_dist: D + + def __init__( + self, + base_distribution: D, + reinterpreted_batch_ndims: int, + validate_args: Optional[bool] = None, + ) -> None: + if reinterpreted_batch_ndims > len(base_distribution.batch_shape): + raise ValueError( + "Expected reinterpreted_batch_ndims <= len(base_distribution.batch_shape), " + f"actual {reinterpreted_batch_ndims} vs {len(base_distribution.batch_shape)}" + ) + shape: Size = base_distribution.batch_shape + base_distribution.event_shape + event_dim: int = reinterpreted_batch_ndims + len(base_distribution.event_shape) + batch_shape = shape[: len(shape) - event_dim] + event_shape = shape[len(shape) - event_dim :] + self.base_dist = base_distribution + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Independent, _instance) + batch_shape = torch.Size(batch_shape) + new.base_dist = self.base_dist.expand( + batch_shape + self.event_shape[: self.reinterpreted_batch_ndims] + ) + new.reinterpreted_batch_ndims = self.reinterpreted_batch_ndims + super(Independent, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @property + def has_rsample(self) -> bool: # type: ignore[override] + return self.base_dist.has_rsample + + @property + def has_enumerate_support(self) -> bool: # type: ignore[override] + if self.reinterpreted_batch_ndims > 0: + return False + return self.base_dist.has_enumerate_support + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def support(self): + result = self.base_dist.support + if self.reinterpreted_batch_ndims: + result = constraints.independent(result, self.reinterpreted_batch_ndims) + return result + + @property + def mean(self) -> Tensor: + return self.base_dist.mean + + @property + def mode(self) -> Tensor: + return self.base_dist.mode + + @property + def variance(self) -> Tensor: + return self.base_dist.variance + + def sample(self, sample_shape=torch.Size()) -> Tensor: + return self.base_dist.sample(sample_shape) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + return self.base_dist.rsample(sample_shape) + + def log_prob(self, value): + log_prob = self.base_dist.log_prob(value) + return _sum_rightmost(log_prob, self.reinterpreted_batch_ndims) + + def entropy(self): + entropy = self.base_dist.entropy() + return _sum_rightmost(entropy, self.reinterpreted_batch_ndims) + + def enumerate_support(self, expand=True): + if self.reinterpreted_batch_ndims > 0: + raise NotImplementedError( + "Enumeration over cartesian product is not implemented" + ) + return self.base_dist.enumerate_support(expand=expand) + + def __repr__(self): + return ( + self.__class__.__name__ + + f"({self.base_dist}, {self.reinterpreted_batch_ndims})" + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/inverse_gamma.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/inverse_gamma.py new file mode 100644 index 0000000000000000000000000000000000000000..fad48b1252fd05f07eaa0c255f884a9bee468fb7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/inverse_gamma.py @@ -0,0 +1,93 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.gamma import Gamma +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import PowerTransform + + +__all__ = ["InverseGamma"] + + +class InverseGamma(TransformedDistribution): + r""" + Creates an inverse gamma distribution parameterized by :attr:`concentration` and :attr:`rate` + where:: + + X ~ Gamma(concentration, rate) + Y = 1 / X ~ InverseGamma(concentration, rate) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterinistic") + >>> m = InverseGamma(torch.tensor([2.0]), torch.tensor([3.0])) + >>> m.sample() + tensor([ 1.2953]) + + Args: + concentration (float or Tensor): shape parameter of the distribution + (often referred to as alpha) + rate (float or Tensor): rate = 1 / scale of the distribution + (often referred to as beta) + """ + + arg_constraints = { + "concentration": constraints.positive, + "rate": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.positive + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Gamma + + def __init__( + self, + concentration: Union[Tensor, float], + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Gamma(concentration, rate, validate_args=validate_args) + neg_one = -base_dist.rate.new_ones(()) + super().__init__( + base_dist, PowerTransform(neg_one), validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(InverseGamma, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def concentration(self) -> Tensor: + return self.base_dist.concentration + + @property + def rate(self) -> Tensor: + return self.base_dist.rate + + @property + def mean(self) -> Tensor: + result = self.rate / (self.concentration - 1) + return torch.where(self.concentration > 1, result, torch.inf) + + @property + def mode(self) -> Tensor: + return self.rate / (self.concentration + 1) + + @property + def variance(self) -> Tensor: + result = self.rate.square() / ( + (self.concentration - 1).square() * (self.concentration - 2) + ) + return torch.where(self.concentration > 2, result, torch.inf) + + def entropy(self): + return ( + self.concentration + + self.rate.log() + + self.concentration.lgamma() + - (1 + self.concentration) * self.concentration.digamma() + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kl.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kl.py new file mode 100644 index 0000000000000000000000000000000000000000..1b312dfbd189e5c53cb1ac9b18673cbd457d2d49 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kl.py @@ -0,0 +1,973 @@ +# mypy: allow-untyped-defs +import math +import warnings +from collections.abc import Callable +from functools import total_ordering + +import torch +from torch import inf, Tensor + +from .bernoulli import Bernoulli +from .beta import Beta +from .binomial import Binomial +from .categorical import Categorical +from .cauchy import Cauchy +from .continuous_bernoulli import ContinuousBernoulli +from .dirichlet import Dirichlet +from .distribution import Distribution +from .exp_family import ExponentialFamily +from .exponential import Exponential +from .gamma import Gamma +from .geometric import Geometric +from .gumbel import Gumbel +from .half_normal import HalfNormal +from .independent import Independent +from .laplace import Laplace +from .lowrank_multivariate_normal import ( + _batch_lowrank_logdet, + _batch_lowrank_mahalanobis, + LowRankMultivariateNormal, +) +from .multivariate_normal import _batch_mahalanobis, MultivariateNormal +from .normal import Normal +from .one_hot_categorical import OneHotCategorical +from .pareto import Pareto +from .poisson import Poisson +from .transformed_distribution import TransformedDistribution +from .uniform import Uniform +from .utils import _sum_rightmost, euler_constant as _euler_gamma + + +_KL_REGISTRY: dict[ + tuple[type, type], Callable +] = {} # Source of truth mapping a few general (type, type) pairs to functions. +_KL_MEMOIZE: dict[ + tuple[type, type], Callable +] = {} # Memoized version mapping many specific (type, type) pairs to functions. + +__all__ = ["register_kl", "kl_divergence"] + + +def register_kl(type_p, type_q): + """ + Decorator to register a pairwise function with :meth:`kl_divergence`. + Usage:: + + @register_kl(Normal, Normal) + def kl_normal_normal(p, q): + # insert implementation here + + Lookup returns the most specific (type,type) match ordered by subclass. If + the match is ambiguous, a `RuntimeWarning` is raised. For example to + resolve the ambiguous situation:: + + @register_kl(BaseP, DerivedQ) + def kl_version1(p, q): ... + @register_kl(DerivedP, BaseQ) + def kl_version2(p, q): ... + + you should register a third most-specific implementation, e.g.:: + + register_kl(DerivedP, DerivedQ)(kl_version1) # Break the tie. + + Args: + type_p (type): A subclass of :class:`~torch.distributions.Distribution`. + type_q (type): A subclass of :class:`~torch.distributions.Distribution`. + """ + if not isinstance(type_p, type) and issubclass(type_p, Distribution): + raise TypeError( + f"Expected type_p to be a Distribution subclass but got {type_p}" + ) + if not isinstance(type_q, type) and issubclass(type_q, Distribution): + raise TypeError( + f"Expected type_q to be a Distribution subclass but got {type_q}" + ) + + def decorator(fun): + _KL_REGISTRY[type_p, type_q] = fun + _KL_MEMOIZE.clear() # reset since lookup order may have changed + return fun + + return decorator + + +@total_ordering +class _Match: + __slots__ = ["types"] + + def __init__(self, *types): + self.types = types + + def __eq__(self, other): + return self.types == other.types + + def __le__(self, other): + for x, y in zip(self.types, other.types): + if not issubclass(x, y): + return False + if x is not y: + break + return True + + +def _dispatch_kl(type_p, type_q): + """ + Find the most specific approximate match, assuming single inheritance. + """ + matches = [ + (super_p, super_q) + for super_p, super_q in _KL_REGISTRY + if issubclass(type_p, super_p) and issubclass(type_q, super_q) + ] + if not matches: + return NotImplemented + # Check that the left- and right- lexicographic orders agree. + # mypy isn't smart enough to know that _Match implements __lt__ + # see: https://github.com/python/typing/issues/760#issuecomment-710670503 + left_p, left_q = min(_Match(*m) for m in matches).types # type: ignore[type-var] + right_q, right_p = min(_Match(*reversed(m)) for m in matches).types # type: ignore[type-var] + left_fun = _KL_REGISTRY[left_p, left_q] + right_fun = _KL_REGISTRY[right_p, right_q] + if left_fun is not right_fun: + warnings.warn( + f"Ambiguous kl_divergence({type_p.__name__}, {type_q.__name__}). " + f"Please register_kl({left_p.__name__}, {right_q.__name__})", + RuntimeWarning, + stacklevel=2, + ) + return left_fun + + +def _infinite_like(tensor): + """ + Helper function for obtaining infinite KL Divergence throughout + """ + return torch.full_like(tensor, inf) + + +def _x_log_x(tensor): + """ + Utility function for calculating x log x + """ + return torch.special.xlogy(tensor, tensor) # produces correct result for x=0 + + +def _batch_trace_XXT(bmat): + """ + Utility function for calculating the trace of XX^{T} with X having arbitrary trailing batch dimensions + """ + n = bmat.size(-1) + m = bmat.size(-2) + flat_trace = bmat.reshape(-1, m * n).pow(2).sum(-1) + return flat_trace.reshape(bmat.shape[:-2]) + + +def kl_divergence(p: Distribution, q: Distribution) -> Tensor: + r""" + Compute Kullback-Leibler divergence :math:`KL(p \| q)` between two distributions. + + .. math:: + + KL(p \| q) = \int p(x) \log\frac {p(x)} {q(x)} \,dx + + Args: + p (Distribution): A :class:`~torch.distributions.Distribution` object. + q (Distribution): A :class:`~torch.distributions.Distribution` object. + + Returns: + Tensor: A batch of KL divergences of shape `batch_shape`. + + Raises: + NotImplementedError: If the distribution types have not been registered via + :meth:`register_kl`. + """ + try: + fun = _KL_MEMOIZE[type(p), type(q)] + except KeyError: + fun = _dispatch_kl(type(p), type(q)) + _KL_MEMOIZE[type(p), type(q)] = fun + if fun is NotImplemented: + raise NotImplementedError( + f"No KL(p || q) is implemented for p type {p.__class__.__name__} and q type {q.__class__.__name__}" + ) + return fun(p, q) + + +################################################################################ +# KL Divergence Implementations +################################################################################ + +# Same distributions + + +@register_kl(Bernoulli, Bernoulli) +def _kl_bernoulli_bernoulli(p, q): + t1 = p.probs * ( + torch.nn.functional.softplus(-q.logits) + - torch.nn.functional.softplus(-p.logits) + ) + t1[q.probs == 0] = inf + t1[p.probs == 0] = 0 + t2 = (1 - p.probs) * ( + torch.nn.functional.softplus(q.logits) - torch.nn.functional.softplus(p.logits) + ) + t2[q.probs == 1] = inf + t2[p.probs == 1] = 0 + return t1 + t2 + + +@register_kl(Beta, Beta) +def _kl_beta_beta(p, q): + sum_params_p = p.concentration1 + p.concentration0 + sum_params_q = q.concentration1 + q.concentration0 + t1 = q.concentration1.lgamma() + q.concentration0.lgamma() + (sum_params_p).lgamma() + t2 = p.concentration1.lgamma() + p.concentration0.lgamma() + (sum_params_q).lgamma() + t3 = (p.concentration1 - q.concentration1) * torch.digamma(p.concentration1) + t4 = (p.concentration0 - q.concentration0) * torch.digamma(p.concentration0) + t5 = (sum_params_q - sum_params_p) * torch.digamma(sum_params_p) + return t1 - t2 + t3 + t4 + t5 + + +@register_kl(Binomial, Binomial) +def _kl_binomial_binomial(p, q): + # from https://math.stackexchange.com/questions/2214993/ + # kullback-leibler-divergence-for-binomial-distributions-p-and-q + if (p.total_count < q.total_count).any(): + raise NotImplementedError( + "KL between Binomials where q.total_count > p.total_count is not implemented" + ) + kl = p.total_count * ( + p.probs * (p.logits - q.logits) + (-p.probs).log1p() - (-q.probs).log1p() + ) + inf_idxs = p.total_count > q.total_count + kl[inf_idxs] = _infinite_like(kl[inf_idxs]) + return kl + + +@register_kl(Categorical, Categorical) +def _kl_categorical_categorical(p, q): + t = p.probs * (p.logits - q.logits) + t[(q.probs == 0).expand_as(t)] = inf + t[(p.probs == 0).expand_as(t)] = 0 + return t.sum(-1) + + +@register_kl(ContinuousBernoulli, ContinuousBernoulli) +def _kl_continuous_bernoulli_continuous_bernoulli(p, q): + t1 = p.mean * (p.logits - q.logits) + t2 = p._cont_bern_log_norm() + torch.log1p(-p.probs) + t3 = -q._cont_bern_log_norm() - torch.log1p(-q.probs) + return t1 + t2 + t3 + + +@register_kl(Dirichlet, Dirichlet) +def _kl_dirichlet_dirichlet(p, q): + # From http://bariskurt.com/kullback-leibler-divergence-between-two-dirichlet-and-beta-distributions/ + sum_p_concentration = p.concentration.sum(-1) + sum_q_concentration = q.concentration.sum(-1) + t1 = sum_p_concentration.lgamma() - sum_q_concentration.lgamma() + t2 = (p.concentration.lgamma() - q.concentration.lgamma()).sum(-1) + t3 = p.concentration - q.concentration + t4 = p.concentration.digamma() - sum_p_concentration.digamma().unsqueeze(-1) + return t1 - t2 + (t3 * t4).sum(-1) + + +@register_kl(Exponential, Exponential) +def _kl_exponential_exponential(p, q): + rate_ratio = q.rate / p.rate + t1 = -rate_ratio.log() + return t1 + rate_ratio - 1 + + +@register_kl(ExponentialFamily, ExponentialFamily) +def _kl_expfamily_expfamily(p, q): + if type(p) is not type(q): + raise NotImplementedError( + "The cross KL-divergence between different exponential families cannot \ + be computed using Bregman divergences" + ) + p_nparams = [np.detach().requires_grad_() for np in p._natural_params] + q_nparams = q._natural_params + lg_normal = p._log_normalizer(*p_nparams) + gradients = torch.autograd.grad(lg_normal.sum(), p_nparams, create_graph=True) + result = q._log_normalizer(*q_nparams) - lg_normal + for pnp, qnp, g in zip(p_nparams, q_nparams, gradients): + term = (qnp - pnp) * g + result -= _sum_rightmost(term, len(q.event_shape)) + return result + + +@register_kl(Gamma, Gamma) +def _kl_gamma_gamma(p, q): + t1 = q.concentration * (p.rate / q.rate).log() + t2 = torch.lgamma(q.concentration) - torch.lgamma(p.concentration) + t3 = (p.concentration - q.concentration) * torch.digamma(p.concentration) + t4 = (q.rate - p.rate) * (p.concentration / p.rate) + return t1 + t2 + t3 + t4 + + +@register_kl(Gumbel, Gumbel) +def _kl_gumbel_gumbel(p, q): + ct1 = p.scale / q.scale + ct2 = q.loc / q.scale + ct3 = p.loc / q.scale + t1 = -ct1.log() - ct2 + ct3 + t2 = ct1 * _euler_gamma + t3 = torch.exp(ct2 + (1 + ct1).lgamma() - ct3) + return t1 + t2 + t3 - (1 + _euler_gamma) + + +@register_kl(Geometric, Geometric) +def _kl_geometric_geometric(p, q): + return -p.entropy() - torch.log1p(-q.probs) / p.probs - q.logits + + +@register_kl(HalfNormal, HalfNormal) +def _kl_halfnormal_halfnormal(p, q): + return _kl_normal_normal(p.base_dist, q.base_dist) + + +@register_kl(Laplace, Laplace) +def _kl_laplace_laplace(p, q): + # From http://www.mast.queensu.ca/~communications/Papers/gil-msc11.pdf + scale_ratio = p.scale / q.scale + loc_abs_diff = (p.loc - q.loc).abs() + t1 = -scale_ratio.log() + t2 = loc_abs_diff / q.scale + t3 = scale_ratio * torch.exp(-loc_abs_diff / p.scale) + return t1 + t2 + t3 - 1 + + +@register_kl(LowRankMultivariateNormal, LowRankMultivariateNormal) +def _kl_lowrankmultivariatenormal_lowrankmultivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two Low Rank Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = _batch_lowrank_logdet( + q._unbroadcasted_cov_factor, q._unbroadcasted_cov_diag, q._capacitance_tril + ) - _batch_lowrank_logdet( + p._unbroadcasted_cov_factor, p._unbroadcasted_cov_diag, p._capacitance_tril + ) + term3 = _batch_lowrank_mahalanobis( + q._unbroadcasted_cov_factor, + q._unbroadcasted_cov_diag, + q.loc - p.loc, + q._capacitance_tril, + ) + # Expands term2 according to + # inv(qcov) @ pcov = [inv(qD) - inv(qD) @ qW @ inv(qC) @ qW.T @ inv(qD)] @ (pW @ pW.T + pD) + # = [inv(qD) - A.T @ A] @ (pD + pW @ pW.T) + qWt_qDinv = q._unbroadcasted_cov_factor.mT / q._unbroadcasted_cov_diag.unsqueeze(-2) + A = torch.linalg.solve_triangular(q._capacitance_tril, qWt_qDinv, upper=False) + term21 = (p._unbroadcasted_cov_diag / q._unbroadcasted_cov_diag).sum(-1) + term22 = _batch_trace_XXT( + p._unbroadcasted_cov_factor * q._unbroadcasted_cov_diag.rsqrt().unsqueeze(-1) + ) + term23 = _batch_trace_XXT(A * p._unbroadcasted_cov_diag.sqrt().unsqueeze(-2)) + term24 = _batch_trace_XXT(A.matmul(p._unbroadcasted_cov_factor)) + term2 = term21 + term22 - term23 - term24 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(MultivariateNormal, LowRankMultivariateNormal) +def _kl_multivariatenormal_lowrankmultivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two (Low Rank) Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = _batch_lowrank_logdet( + q._unbroadcasted_cov_factor, q._unbroadcasted_cov_diag, q._capacitance_tril + ) - 2 * p._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + term3 = _batch_lowrank_mahalanobis( + q._unbroadcasted_cov_factor, + q._unbroadcasted_cov_diag, + q.loc - p.loc, + q._capacitance_tril, + ) + # Expands term2 according to + # inv(qcov) @ pcov = [inv(qD) - inv(qD) @ qW @ inv(qC) @ qW.T @ inv(qD)] @ p_tril @ p_tril.T + # = [inv(qD) - A.T @ A] @ p_tril @ p_tril.T + qWt_qDinv = q._unbroadcasted_cov_factor.mT / q._unbroadcasted_cov_diag.unsqueeze(-2) + A = torch.linalg.solve_triangular(q._capacitance_tril, qWt_qDinv, upper=False) + term21 = _batch_trace_XXT( + p._unbroadcasted_scale_tril * q._unbroadcasted_cov_diag.rsqrt().unsqueeze(-1) + ) + term22 = _batch_trace_XXT(A.matmul(p._unbroadcasted_scale_tril)) + term2 = term21 - term22 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(LowRankMultivariateNormal, MultivariateNormal) +def _kl_lowrankmultivariatenormal_multivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two (Low Rank) Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = 2 * q._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum( + -1 + ) - _batch_lowrank_logdet( + p._unbroadcasted_cov_factor, p._unbroadcasted_cov_diag, p._capacitance_tril + ) + term3 = _batch_mahalanobis(q._unbroadcasted_scale_tril, (q.loc - p.loc)) + # Expands term2 according to + # inv(qcov) @ pcov = inv(q_tril @ q_tril.T) @ (pW @ pW.T + pD) + combined_batch_shape = torch._C._infer_size( + q._unbroadcasted_scale_tril.shape[:-2], p._unbroadcasted_cov_factor.shape[:-2] + ) + n = p.event_shape[0] + q_scale_tril = q._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + p_cov_factor = p._unbroadcasted_cov_factor.expand( + combined_batch_shape + (n, p.cov_factor.size(-1)) + ) + p_cov_diag = torch.diag_embed(p._unbroadcasted_cov_diag.sqrt()).expand( + combined_batch_shape + (n, n) + ) + term21 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_cov_factor, upper=False) + ) + term22 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_cov_diag, upper=False) + ) + term2 = term21 + term22 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(MultivariateNormal, MultivariateNormal) +def _kl_multivariatenormal_multivariatenormal(p, q): + # From https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Kullback%E2%80%93Leibler_divergence + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two Multivariate Normals with\ + different event shapes cannot be computed" + ) + + half_term1 = q._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum( + -1 + ) - p._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + combined_batch_shape = torch._C._infer_size( + q._unbroadcasted_scale_tril.shape[:-2], p._unbroadcasted_scale_tril.shape[:-2] + ) + n = p.event_shape[0] + q_scale_tril = q._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + p_scale_tril = p._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + term2 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_scale_tril, upper=False) + ) + term3 = _batch_mahalanobis(q._unbroadcasted_scale_tril, (q.loc - p.loc)) + return half_term1 + 0.5 * (term2 + term3 - n) + + +@register_kl(Normal, Normal) +def _kl_normal_normal(p, q): + var_ratio = (p.scale / q.scale).pow(2) + t1 = ((p.loc - q.loc) / q.scale).pow(2) + return 0.5 * (var_ratio + t1 - 1 - var_ratio.log()) + + +@register_kl(OneHotCategorical, OneHotCategorical) +def _kl_onehotcategorical_onehotcategorical(p, q): + return _kl_categorical_categorical(p._categorical, q._categorical) + + +@register_kl(Pareto, Pareto) +def _kl_pareto_pareto(p, q): + # From http://www.mast.queensu.ca/~communications/Papers/gil-msc11.pdf + scale_ratio = p.scale / q.scale + alpha_ratio = q.alpha / p.alpha + t1 = q.alpha * scale_ratio.log() + t2 = -alpha_ratio.log() + result = t1 + t2 + alpha_ratio - 1 + result[p.support.lower_bound < q.support.lower_bound] = inf + return result + + +@register_kl(Poisson, Poisson) +def _kl_poisson_poisson(p, q): + return p.rate * (p.rate.log() - q.rate.log()) - (p.rate - q.rate) + + +@register_kl(TransformedDistribution, TransformedDistribution) +def _kl_transformed_transformed(p, q): + if p.transforms != q.transforms: + raise NotImplementedError + if p.event_shape != q.event_shape: + raise NotImplementedError + return kl_divergence(p.base_dist, q.base_dist) + + +@register_kl(Uniform, Uniform) +def _kl_uniform_uniform(p, q): + result = ((q.high - q.low) / (p.high - p.low)).log() + result[(q.low > p.low) | (q.high < p.high)] = inf + return result + + +# Different distributions +@register_kl(Bernoulli, Poisson) +def _kl_bernoulli_poisson(p, q): + return -p.entropy() - (p.probs * q.rate.log() - q.rate) + + +@register_kl(Beta, ContinuousBernoulli) +def _kl_beta_continuous_bernoulli(p, q): + return ( + -p.entropy() + - p.mean * q.logits + - torch.log1p(-q.probs) + - q._cont_bern_log_norm() + ) + + +@register_kl(Beta, Pareto) +def _kl_beta_infinity(p, q): + return _infinite_like(p.concentration1) + + +@register_kl(Beta, Exponential) +def _kl_beta_exponential(p, q): + return ( + -p.entropy() + - q.rate.log() + + q.rate * (p.concentration1 / (p.concentration1 + p.concentration0)) + ) + + +@register_kl(Beta, Gamma) +def _kl_beta_gamma(p, q): + t1 = -p.entropy() + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = (q.concentration - 1) * ( + p.concentration1.digamma() - (p.concentration1 + p.concentration0).digamma() + ) + t4 = q.rate * p.concentration1 / (p.concentration1 + p.concentration0) + return t1 + t2 - t3 + t4 + + +# TODO: Add Beta-Laplace KL Divergence + + +@register_kl(Beta, Normal) +def _kl_beta_normal(p, q): + E_beta = p.concentration1 / (p.concentration1 + p.concentration0) + var_normal = q.scale.pow(2) + t1 = -p.entropy() + t2 = 0.5 * (var_normal * 2 * math.pi).log() + t3 = ( + E_beta * (1 - E_beta) / (p.concentration1 + p.concentration0 + 1) + + E_beta.pow(2) + ) * 0.5 + t4 = q.loc * E_beta + t5 = q.loc.pow(2) * 0.5 + return t1 + t2 + (t3 - t4 + t5) / var_normal + + +@register_kl(Beta, Uniform) +def _kl_beta_uniform(p, q): + result = -p.entropy() + (q.high - q.low).log() + result[(q.low > p.support.lower_bound) | (q.high < p.support.upper_bound)] = inf + return result + + +# Note that the KL between a ContinuousBernoulli and Beta has no closed form + + +@register_kl(ContinuousBernoulli, Pareto) +def _kl_continuous_bernoulli_infinity(p, q): + return _infinite_like(p.probs) + + +@register_kl(ContinuousBernoulli, Exponential) +def _kl_continuous_bernoulli_exponential(p, q): + return -p.entropy() - torch.log(q.rate) + q.rate * p.mean + + +# Note that the KL between a ContinuousBernoulli and Gamma has no closed form +# TODO: Add ContinuousBernoulli-Laplace KL Divergence + + +@register_kl(ContinuousBernoulli, Normal) +def _kl_continuous_bernoulli_normal(p, q): + t1 = -p.entropy() + t2 = 0.5 * (math.log(2.0 * math.pi) + torch.square(q.loc / q.scale)) + torch.log( + q.scale + ) + t3 = (p.variance + torch.square(p.mean) - 2.0 * q.loc * p.mean) / ( + 2.0 * torch.square(q.scale) + ) + return t1 + t2 + t3 + + +@register_kl(ContinuousBernoulli, Uniform) +def _kl_continuous_bernoulli_uniform(p, q): + result = -p.entropy() + (q.high - q.low).log() + return torch.where( + torch.max( + torch.ge(q.low, p.support.lower_bound), + torch.le(q.high, p.support.upper_bound), + ), + torch.ones_like(result) * inf, + result, + ) + + +@register_kl(Exponential, Beta) +@register_kl(Exponential, ContinuousBernoulli) +@register_kl(Exponential, Pareto) +@register_kl(Exponential, Uniform) +def _kl_exponential_infinity(p, q): + return _infinite_like(p.rate) + + +@register_kl(Exponential, Gamma) +def _kl_exponential_gamma(p, q): + ratio = q.rate / p.rate + t1 = -q.concentration * torch.log(ratio) + return ( + t1 + + ratio + + q.concentration.lgamma() + + q.concentration * _euler_gamma + - (1 + _euler_gamma) + ) + + +@register_kl(Exponential, Gumbel) +def _kl_exponential_gumbel(p, q): + scale_rate_prod = p.rate * q.scale + loc_scale_ratio = q.loc / q.scale + t1 = scale_rate_prod.log() - 1 + t2 = torch.exp(loc_scale_ratio) * scale_rate_prod / (scale_rate_prod + 1) + t3 = scale_rate_prod.reciprocal() + return t1 - loc_scale_ratio + t2 + t3 + + +# TODO: Add Exponential-Laplace KL Divergence + + +@register_kl(Exponential, Normal) +def _kl_exponential_normal(p, q): + var_normal = q.scale.pow(2) + rate_sqr = p.rate.pow(2) + t1 = 0.5 * torch.log(rate_sqr * var_normal * 2 * math.pi) + t2 = rate_sqr.reciprocal() + t3 = q.loc / p.rate + t4 = q.loc.pow(2) * 0.5 + return t1 - 1 + (t2 - t3 + t4) / var_normal + + +@register_kl(Gamma, Beta) +@register_kl(Gamma, ContinuousBernoulli) +@register_kl(Gamma, Pareto) +@register_kl(Gamma, Uniform) +def _kl_gamma_infinity(p, q): + return _infinite_like(p.concentration) + + +@register_kl(Gamma, Exponential) +def _kl_gamma_exponential(p, q): + return -p.entropy() - q.rate.log() + q.rate * p.concentration / p.rate + + +@register_kl(Gamma, Gumbel) +def _kl_gamma_gumbel(p, q): + beta_scale_prod = p.rate * q.scale + loc_scale_ratio = q.loc / q.scale + t1 = ( + (p.concentration - 1) * p.concentration.digamma() + - p.concentration.lgamma() + - p.concentration + ) + t2 = beta_scale_prod.log() + p.concentration / beta_scale_prod + t3 = ( + torch.exp(loc_scale_ratio) + * (1 + beta_scale_prod.reciprocal()).pow(-p.concentration) + - loc_scale_ratio + ) + return t1 + t2 + t3 + + +# TODO: Add Gamma-Laplace KL Divergence + + +@register_kl(Gamma, Normal) +def _kl_gamma_normal(p, q): + var_normal = q.scale.pow(2) + beta_sqr = p.rate.pow(2) + t1 = ( + 0.5 * torch.log(beta_sqr * var_normal * 2 * math.pi) + - p.concentration + - p.concentration.lgamma() + ) + t2 = 0.5 * (p.concentration.pow(2) + p.concentration) / beta_sqr + t3 = q.loc * p.concentration / p.rate + t4 = 0.5 * q.loc.pow(2) + return ( + t1 + + (p.concentration - 1) * p.concentration.digamma() + + (t2 - t3 + t4) / var_normal + ) + + +@register_kl(Gumbel, Beta) +@register_kl(Gumbel, ContinuousBernoulli) +@register_kl(Gumbel, Exponential) +@register_kl(Gumbel, Gamma) +@register_kl(Gumbel, Pareto) +@register_kl(Gumbel, Uniform) +def _kl_gumbel_infinity(p, q): + return _infinite_like(p.loc) + + +# TODO: Add Gumbel-Laplace KL Divergence + + +@register_kl(Gumbel, Normal) +def _kl_gumbel_normal(p, q): + param_ratio = p.scale / q.scale + t1 = (param_ratio / math.sqrt(2 * math.pi)).log() + t2 = (math.pi * param_ratio * 0.5).pow(2) / 3 + t3 = ((p.loc + p.scale * _euler_gamma - q.loc) / q.scale).pow(2) * 0.5 + return -t1 + t2 + t3 - (_euler_gamma + 1) + + +@register_kl(Laplace, Beta) +@register_kl(Laplace, ContinuousBernoulli) +@register_kl(Laplace, Exponential) +@register_kl(Laplace, Gamma) +@register_kl(Laplace, Pareto) +@register_kl(Laplace, Uniform) +def _kl_laplace_infinity(p, q): + return _infinite_like(p.loc) + + +@register_kl(Laplace, Normal) +def _kl_laplace_normal(p, q): + var_normal = q.scale.pow(2) + scale_sqr_var_ratio = p.scale.pow(2) / var_normal + t1 = 0.5 * torch.log(2 * scale_sqr_var_ratio / math.pi) + t2 = 0.5 * p.loc.pow(2) + t3 = p.loc * q.loc + t4 = 0.5 * q.loc.pow(2) + return -t1 + scale_sqr_var_ratio + (t2 - t3 + t4) / var_normal - 1 + + +@register_kl(Normal, Beta) +@register_kl(Normal, ContinuousBernoulli) +@register_kl(Normal, Exponential) +@register_kl(Normal, Gamma) +@register_kl(Normal, Pareto) +@register_kl(Normal, Uniform) +def _kl_normal_infinity(p, q): + return _infinite_like(p.loc) + + +@register_kl(Normal, Gumbel) +def _kl_normal_gumbel(p, q): + mean_scale_ratio = p.loc / q.scale + var_scale_sqr_ratio = (p.scale / q.scale).pow(2) + loc_scale_ratio = q.loc / q.scale + t1 = var_scale_sqr_ratio.log() * 0.5 + t2 = mean_scale_ratio - loc_scale_ratio + t3 = torch.exp(-mean_scale_ratio + 0.5 * var_scale_sqr_ratio + loc_scale_ratio) + return -t1 + t2 + t3 - (0.5 * (1 + math.log(2 * math.pi))) + + +@register_kl(Normal, Laplace) +def _kl_normal_laplace(p, q): + loc_diff = p.loc - q.loc + scale_ratio = p.scale / q.scale + loc_diff_scale_ratio = loc_diff / p.scale + t1 = torch.log(scale_ratio) + t2 = ( + math.sqrt(2 / math.pi) * p.scale * torch.exp(-0.5 * loc_diff_scale_ratio.pow(2)) + ) + t3 = loc_diff * torch.erf(math.sqrt(0.5) * loc_diff_scale_ratio) + return -t1 + (t2 + t3) / q.scale - (0.5 * (1 + math.log(0.5 * math.pi))) + + +@register_kl(Pareto, Beta) +@register_kl(Pareto, ContinuousBernoulli) +@register_kl(Pareto, Uniform) +def _kl_pareto_infinity(p, q): + return _infinite_like(p.scale) + + +@register_kl(Pareto, Exponential) +def _kl_pareto_exponential(p, q): + scale_rate_prod = p.scale * q.rate + t1 = (p.alpha / scale_rate_prod).log() + t2 = p.alpha.reciprocal() + t3 = p.alpha * scale_rate_prod / (p.alpha - 1) + result = t1 - t2 + t3 - 1 + result[p.alpha <= 1] = inf + return result + + +@register_kl(Pareto, Gamma) +def _kl_pareto_gamma(p, q): + common_term = p.scale.log() + p.alpha.reciprocal() + t1 = p.alpha.log() - common_term + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = (1 - q.concentration) * common_term + t4 = q.rate * p.alpha * p.scale / (p.alpha - 1) + result = t1 + t2 + t3 + t4 - 1 + result[p.alpha <= 1] = inf + return result + + +# TODO: Add Pareto-Laplace KL Divergence + + +@register_kl(Pareto, Normal) +def _kl_pareto_normal(p, q): + var_normal = 2 * q.scale.pow(2) + common_term = p.scale / (p.alpha - 1) + t1 = (math.sqrt(2 * math.pi) * q.scale * p.alpha / p.scale).log() + t2 = p.alpha.reciprocal() + t3 = p.alpha * common_term.pow(2) / (p.alpha - 2) + t4 = (p.alpha * common_term - q.loc).pow(2) + result = t1 - t2 + (t3 + t4) / var_normal - 1 + result[p.alpha <= 2] = inf + return result + + +@register_kl(Poisson, Bernoulli) +@register_kl(Poisson, Binomial) +def _kl_poisson_infinity(p, q): + return _infinite_like(p.rate) + + +@register_kl(Uniform, Beta) +def _kl_uniform_beta(p, q): + common_term = p.high - p.low + t1 = torch.log(common_term) + t2 = ( + (q.concentration1 - 1) + * (_x_log_x(p.high) - _x_log_x(p.low) - common_term) + / common_term + ) + t3 = ( + (q.concentration0 - 1) + * (_x_log_x(1 - p.high) - _x_log_x(1 - p.low) + common_term) + / common_term + ) + t4 = ( + q.concentration1.lgamma() + + q.concentration0.lgamma() + - (q.concentration1 + q.concentration0).lgamma() + ) + result = t3 + t4 - t1 - t2 + result[(p.high > q.support.upper_bound) | (p.low < q.support.lower_bound)] = inf + return result + + +@register_kl(Uniform, ContinuousBernoulli) +def _kl_uniform_continuous_bernoulli(p, q): + result = ( + -p.entropy() + - p.mean * q.logits + - torch.log1p(-q.probs) + - q._cont_bern_log_norm() + ) + return torch.where( + torch.max( + torch.ge(p.high, q.support.upper_bound), + torch.le(p.low, q.support.lower_bound), + ), + torch.ones_like(result) * inf, + result, + ) + + +@register_kl(Uniform, Exponential) +def _kl_uniform_exponetial(p, q): + result = q.rate * (p.high + p.low) / 2 - ((p.high - p.low) * q.rate).log() + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Uniform, Gamma) +def _kl_uniform_gamma(p, q): + common_term = p.high - p.low + t1 = common_term.log() + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = ( + (1 - q.concentration) + * (_x_log_x(p.high) - _x_log_x(p.low) - common_term) + / common_term + ) + t4 = q.rate * (p.high + p.low) / 2 + result = -t1 + t2 + t3 + t4 + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Uniform, Gumbel) +def _kl_uniform_gumbel(p, q): + common_term = q.scale / (p.high - p.low) + high_loc_diff = (p.high - q.loc) / q.scale + low_loc_diff = (p.low - q.loc) / q.scale + t1 = common_term.log() + 0.5 * (high_loc_diff + low_loc_diff) + t2 = common_term * (torch.exp(-high_loc_diff) - torch.exp(-low_loc_diff)) + return t1 - t2 + + +# TODO: Uniform-Laplace KL Divergence + + +@register_kl(Uniform, Normal) +def _kl_uniform_normal(p, q): + common_term = p.high - p.low + t1 = (math.sqrt(math.pi * 2) * q.scale / common_term).log() + t2 = (common_term).pow(2) / 12 + t3 = ((p.high + p.low - 2 * q.loc) / 2).pow(2) + return t1 + 0.5 * (t2 + t3) / q.scale.pow(2) + + +@register_kl(Uniform, Pareto) +def _kl_uniform_pareto(p, q): + support_uniform = p.high - p.low + t1 = (q.alpha * q.scale.pow(q.alpha) * (support_uniform)).log() + t2 = (_x_log_x(p.high) - _x_log_x(p.low) - support_uniform) / support_uniform + result = t2 * (q.alpha + 1) - t1 + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Independent, Independent) +def _kl_independent_independent(p, q): + if p.reinterpreted_batch_ndims != q.reinterpreted_batch_ndims: + raise NotImplementedError + result = kl_divergence(p.base_dist, q.base_dist) + return _sum_rightmost(result, p.reinterpreted_batch_ndims) + + +@register_kl(Cauchy, Cauchy) +def _kl_cauchy_cauchy(p, q): + # From https://arxiv.org/abs/1905.10965 + t1 = ((p.scale + q.scale).pow(2) + (p.loc - q.loc).pow(2)).log() + t2 = (4 * p.scale * q.scale).log() + return t1 - t2 + + +def _add_kl_info(): + """Appends a list of implemented KL functions to the doc for kl_divergence.""" + rows = [ + "KL divergence is currently implemented for the following distribution pairs:" + ] + for p, q in sorted( + _KL_REGISTRY, key=lambda p_q: (p_q[0].__name__, p_q[1].__name__) + ): + rows.append( + f"* :class:`~torch.distributions.{p.__name__}` and :class:`~torch.distributions.{q.__name__}`" + ) + kl_info = "\n\t".join(rows) + if kl_divergence.__doc__: + kl_divergence.__doc__ += kl_info diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kumaraswamy.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kumaraswamy.py new file mode 100644 index 0000000000000000000000000000000000000000..0b5bb7e34f5f9836d5a02d4661624e22cf8d9009 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/kumaraswamy.py @@ -0,0 +1,108 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, PowerTransform +from torch.distributions.uniform import Uniform +from torch.distributions.utils import broadcast_all, euler_constant + + +__all__ = ["Kumaraswamy"] + + +def _moments(a, b, n): + """ + Computes nth moment of Kumaraswamy using using torch.lgamma + """ + arg1 = 1 + n / a + log_value = torch.lgamma(arg1) + torch.lgamma(b) - torch.lgamma(arg1 + b) + return b * torch.exp(log_value) + + +class Kumaraswamy(TransformedDistribution): + r""" + Samples from a Kumaraswamy distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Kumaraswamy(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Kumaraswamy distribution with concentration alpha=1 and beta=1 + tensor([ 0.1729]) + + Args: + concentration1 (float or Tensor): 1st concentration parameter of the distribution + (often referred to as alpha) + concentration0 (float or Tensor): 2nd concentration parameter of the distribution + (often referred to as beta) + """ + + arg_constraints = { + "concentration1": constraints.positive, + "concentration0": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.unit_interval + has_rsample = True + + def __init__( + self, + concentration1: Union[Tensor, float], + concentration0: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.concentration1, self.concentration0 = broadcast_all( + concentration1, concentration0 + ) + base_dist = Uniform( + torch.full_like(self.concentration0, 0), + torch.full_like(self.concentration0, 1), + validate_args=validate_args, + ) + transforms = [ + PowerTransform(exponent=self.concentration0.reciprocal()), + AffineTransform(loc=1.0, scale=-1.0), + PowerTransform(exponent=self.concentration1.reciprocal()), + ] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Kumaraswamy, _instance) + new.concentration1 = self.concentration1.expand(batch_shape) + new.concentration0 = self.concentration0.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + @property + def mean(self) -> Tensor: + return _moments(self.concentration1, self.concentration0, 1) + + @property + def mode(self) -> Tensor: + # Evaluate in log-space for numerical stability. + log_mode = ( + self.concentration0.reciprocal() * (-self.concentration0).log1p() + - (-self.concentration0 * self.concentration1).log1p() + ) + log_mode[(self.concentration0 < 1) | (self.concentration1 < 1)] = nan + return log_mode.exp() + + @property + def variance(self) -> Tensor: + return _moments(self.concentration1, self.concentration0, 2) - torch.pow( + self.mean, 2 + ) + + def entropy(self): + t1 = 1 - self.concentration1.reciprocal() + t0 = 1 - self.concentration0.reciprocal() + H0 = torch.digamma(self.concentration0 + 1) + euler_constant + return ( + t0 + + t1 * H0 + - torch.log(self.concentration1) + - torch.log(self.concentration0) + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/laplace.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/laplace.py new file mode 100644 index 0000000000000000000000000000000000000000..bb108ce117eb27cb57359270f21748f0080f6dc6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/laplace.py @@ -0,0 +1,105 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Laplace"] + + +class Laplace(Distribution): + r""" + Creates a Laplace distribution parameterized by :attr:`loc` and :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Laplace(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # Laplace distributed with loc=0, scale=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of the distribution + scale (float or Tensor): scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return 2 * self.scale.pow(2) + + @property + def stddev(self) -> Tensor: + return (2**0.5) * self.scale + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Laplace, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Laplace, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + finfo = torch.finfo(self.loc.dtype) + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .uniform_() + u = torch.rand(shape, dtype=self.loc.dtype, device=self.loc.device) * 2 - 1 + return self.loc - self.scale * u.sign() * torch.log1p( + -u.abs().clamp(min=finfo.tiny) + ) + u = self.loc.new(shape).uniform_(finfo.eps - 1, 1) + # TODO: If we ever implement tensor.nextafter, below is what we want ideally. + # u = self.loc.new(shape).uniform_(self.loc.nextafter(-.5, 0), .5) + return self.loc - self.scale * u.sign() * torch.log1p(-u.abs()) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return -torch.log(2 * self.scale) - torch.abs(value - self.loc) / self.scale + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 0.5 - 0.5 * (value - self.loc).sign() * torch.expm1( + -(value - self.loc).abs() / self.scale + ) + + def icdf(self, value): + term = value - 0.5 + return self.loc - self.scale * (term).sign() * torch.log1p(-2 * term.abs()) + + def entropy(self): + return 1 + torch.log(2 * self.scale) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lkj_cholesky.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lkj_cholesky.py new file mode 100644 index 0000000000000000000000000000000000000000..92035b84767e4e4b1636b9bf1d0b8d4bffea1f03 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lkj_cholesky.py @@ -0,0 +1,153 @@ +# mypy: allow-untyped-defs +""" +This closely follows the implementation in NumPyro (https://github.com/pyro-ppl/numpyro). + +Original copyright notice: + +# Copyright: Contributors to the Pyro project. +# SPDX-License-Identifier: Apache-2.0 +""" + +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import Beta, constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["LKJCholesky"] + + +class LKJCholesky(Distribution): + r""" + LKJ distribution for lower Cholesky factor of correlation matrices. + The distribution is controlled by ``concentration`` parameter :math:`\eta` + to make the probability of the correlation matrix :math:`M` generated from + a Cholesky factor proportional to :math:`\det(M)^{\eta - 1}`. Because of that, + when ``concentration == 1``, we have a uniform distribution over Cholesky + factors of correlation matrices:: + + L ~ LKJCholesky(dim, concentration) + X = L @ L' ~ LKJCorr(dim, concentration) + + Note that this distribution samples the + Cholesky factor of correlation matrices and not the correlation matrices + themselves and thereby differs slightly from the derivations in [1] for + the `LKJCorr` distribution. For sampling, this uses the Onion method from + [1] Section 3. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> l = LKJCholesky(3, 0.5) + >>> l.sample() # l @ l.T is a sample of a correlation 3x3 matrix + tensor([[ 1.0000, 0.0000, 0.0000], + [ 0.3516, 0.9361, 0.0000], + [-0.1899, 0.4748, 0.8593]]) + + Args: + dimension (dim): dimension of the matrices + concentration (float or Tensor): concentration/shape parameter of the + distribution (often referred to as eta) + + **References** + + [1] `Generating random correlation matrices based on vines and extended onion method` (2009), + Daniel Lewandowski, Dorota Kurowicka, Harry Joe. + Journal of Multivariate Analysis. 100. 10.1016/j.jmva.2009.04.008 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"concentration": constraints.positive} + support = constraints.corr_cholesky + + def __init__( + self, + dim: int, + concentration: Union[Tensor, float] = 1.0, + validate_args: Optional[bool] = None, + ) -> None: + if dim < 2: + raise ValueError( + f"Expected dim to be an integer greater than or equal to 2. Found dim={dim}." + ) + self.dim = dim + (self.concentration,) = broadcast_all(concentration) + batch_shape = self.concentration.size() + event_shape = torch.Size((dim, dim)) + # This is used to draw vectorized samples from the beta distribution in Sec. 3.2 of [1]. + marginal_conc = self.concentration + 0.5 * (self.dim - 2) + offset = torch.arange( + self.dim - 1, + dtype=self.concentration.dtype, + device=self.concentration.device, + ) + offset = torch.cat([offset.new_zeros((1,)), offset]) + beta_conc1 = offset + 0.5 + beta_conc0 = marginal_conc.unsqueeze(-1) - 0.5 * offset + self._beta = Beta(beta_conc1, beta_conc0) + super().__init__(batch_shape, event_shape, validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LKJCholesky, _instance) + batch_shape = torch.Size(batch_shape) + new.dim = self.dim + new.concentration = self.concentration.expand(batch_shape) + new._beta = self._beta.expand(batch_shape + (self.dim,)) + super(LKJCholesky, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + # This uses the Onion method, but there are a few differences from [1] Sec. 3.2: + # - This vectorizes the for loop and also works for heterogeneous eta. + # - Same algorithm generalizes to n=1. + # - The procedure is simplified since we are sampling the cholesky factor of + # the correlation matrix instead of the correlation matrix itself. As such, + # we only need to generate `w`. + y = self._beta.sample(sample_shape).unsqueeze(-1) + u_normal = torch.randn( + self._extended_shape(sample_shape), dtype=y.dtype, device=y.device + ).tril(-1) + u_hypersphere = u_normal / u_normal.norm(dim=-1, keepdim=True) + # Replace NaNs in first row + u_hypersphere[..., 0, :].fill_(0.0) + w = torch.sqrt(y) * u_hypersphere + # Fill diagonal elements; clamp for numerical stability + eps = torch.finfo(w.dtype).tiny + diag_elems = torch.clamp(1 - torch.sum(w**2, dim=-1), min=eps).sqrt() + w += torch.diag_embed(diag_elems) + return w + + def log_prob(self, value): + # See: https://mc-stan.org/docs/2_25/functions-reference/cholesky-lkj-correlation-distribution.html + # The probability of a correlation matrix is proportional to + # determinant ** (concentration - 1) = prod(L_ii ^ 2(concentration - 1)) + # Additionally, the Jacobian of the transformation from Cholesky factor to + # correlation matrix is: + # prod(L_ii ^ (D - i)) + # So the probability of a Cholesky factor is proportional to + # prod(L_ii ^ (2 * concentration - 2 + D - i)) = prod(L_ii ^ order_i) + # with order_i = 2 * concentration - 2 + D - i + if self._validate_args: + self._validate_sample(value) + diag_elems = value.diagonal(dim1=-1, dim2=-2)[..., 1:] + order = torch.arange(2, self.dim + 1, device=self.concentration.device) + order = 2 * (self.concentration - 1).unsqueeze(-1) + self.dim - order + unnormalized_log_pdf = torch.sum(order * diag_elems.log(), dim=-1) + # Compute normalization constant (page 1999 of [1]) + dm1 = self.dim - 1 + alpha = self.concentration + 0.5 * dm1 + denominator = torch.lgamma(alpha) * dm1 + numerator = torch.mvlgamma(alpha - 0.5, dm1) + # pi_constant in [1] is D * (D - 1) / 4 * log(pi) + # pi_constant in multigammaln is (D - 1) * (D - 2) / 4 * log(pi) + # hence, we need to add a pi_constant = (D - 1) * log(pi) / 2 + pi_constant = 0.5 * dm1 * math.log(math.pi) + normalize_term = pi_constant + numerator - denominator + return unnormalized_log_pdf - normalize_term diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/log_normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/log_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..a283cdea520007d48deace275f898fa821d04e90 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/log_normal.py @@ -0,0 +1,76 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import ExpTransform + + +__all__ = ["LogNormal"] + + +class LogNormal(TransformedDistribution): + r""" + Creates a log-normal distribution parameterized by + :attr:`loc` and :attr:`scale` where:: + + X ~ Normal(loc, scale) + Y = exp(X) ~ LogNormal(loc, scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LogNormal(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # log-normal distributed with mean=0 and stddev=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of log of distribution + scale (float or Tensor): standard deviation of log of the distribution + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.positive + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Normal + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(loc, scale, validate_args=validate_args) + super().__init__(base_dist, ExpTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def loc(self) -> Tensor: + return self.base_dist.loc + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return (self.loc + self.scale.pow(2) / 2).exp() + + @property + def mode(self) -> Tensor: + return (self.loc - self.scale.square()).exp() + + @property + def variance(self) -> Tensor: + scale_sq = self.scale.pow(2) + return scale_sq.expm1() * (2 * self.loc + scale_sq).exp() + + def entropy(self): + return self.base_dist.entropy() + self.loc diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/logistic_normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/logistic_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..8d26a7790631a4a8e9cb5bfbbb3a12b0e9ed4c05 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/logistic_normal.py @@ -0,0 +1,68 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints, Independent +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import StickBreakingTransform + + +__all__ = ["LogisticNormal"] + + +class LogisticNormal(TransformedDistribution): + r""" + Creates a logistic-normal distribution parameterized by :attr:`loc` and :attr:`scale` + that define the base `Normal` distribution transformed with the + `StickBreakingTransform` such that:: + + X ~ LogisticNormal(loc, scale) + Y = log(X / (1 - X.cumsum(-1)))[..., :-1] ~ Normal(loc, scale) + + Args: + loc (float or Tensor): mean of the base distribution + scale (float or Tensor): standard deviation of the base distribution + + Example:: + + >>> # logistic-normal distributed with mean=(0, 0, 0) and stddev=(1, 1, 1) + >>> # of the base Normal distribution + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LogisticNormal(torch.tensor([0.0] * 3), torch.tensor([1.0] * 3)) + >>> m.sample() + tensor([ 0.7653, 0.0341, 0.0579, 0.1427]) + + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.simplex + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Independent[Normal] + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(loc, scale, validate_args=validate_args) + if not base_dist.batch_shape: + base_dist = base_dist.expand([1]) + super().__init__( + base_dist, StickBreakingTransform(), validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogisticNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def loc(self) -> Tensor: + return self.base_dist.base_dist.loc + + @property + def scale(self) -> Tensor: + return self.base_dist.base_dist.scale diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lowrank_multivariate_normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lowrank_multivariate_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..3fe3fdc1bbc9cd86d322d2761e3beff7c16ce13b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/lowrank_multivariate_normal.py @@ -0,0 +1,252 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.multivariate_normal import _batch_mahalanobis, _batch_mv +from torch.distributions.utils import _standard_normal, lazy_property +from torch.types import _size + + +__all__ = ["LowRankMultivariateNormal"] + + +def _batch_capacitance_tril(W, D): + r""" + Computes Cholesky of :math:`I + W.T @ inv(D) @ W` for a batch of matrices :math:`W` + and a batch of vectors :math:`D`. + """ + m = W.size(-1) + Wt_Dinv = W.mT / D.unsqueeze(-2) + K = torch.matmul(Wt_Dinv, W).contiguous() + K.view(-1, m * m)[:, :: m + 1] += 1 # add identity matrix to K + return torch.linalg.cholesky(K) + + +def _batch_lowrank_logdet(W, D, capacitance_tril): + r""" + Uses "matrix determinant lemma":: + log|W @ W.T + D| = log|C| + log|D|, + where :math:`C` is the capacitance matrix :math:`I + W.T @ inv(D) @ W`, to compute + the log determinant. + """ + return 2 * capacitance_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + D.log().sum( + -1 + ) + + +def _batch_lowrank_mahalanobis(W, D, x, capacitance_tril): + r""" + Uses "Woodbury matrix identity":: + inv(W @ W.T + D) = inv(D) - inv(D) @ W @ inv(C) @ W.T @ inv(D), + where :math:`C` is the capacitance matrix :math:`I + W.T @ inv(D) @ W`, to compute the squared + Mahalanobis distance :math:`x.T @ inv(W @ W.T + D) @ x`. + """ + Wt_Dinv = W.mT / D.unsqueeze(-2) + Wt_Dinv_x = _batch_mv(Wt_Dinv, x) + mahalanobis_term1 = (x.pow(2) / D).sum(-1) + mahalanobis_term2 = _batch_mahalanobis(capacitance_tril, Wt_Dinv_x) + return mahalanobis_term1 - mahalanobis_term2 + + +class LowRankMultivariateNormal(Distribution): + r""" + Creates a multivariate normal distribution with covariance matrix having a low-rank form + parameterized by :attr:`cov_factor` and :attr:`cov_diag`:: + + covariance_matrix = cov_factor @ cov_factor.T + cov_diag + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LowRankMultivariateNormal( + ... torch.zeros(2), torch.tensor([[1.0], [0.0]]), torch.ones(2) + ... ) + >>> m.sample() # normally distributed with mean=`[0,0]`, cov_factor=`[[1],[0]]`, cov_diag=`[1,1]` + tensor([-0.2102, -0.5429]) + + Args: + loc (Tensor): mean of the distribution with shape `batch_shape + event_shape` + cov_factor (Tensor): factor part of low-rank form of covariance matrix with shape + `batch_shape + event_shape + (rank,)` + cov_diag (Tensor): diagonal part of low-rank form of covariance matrix with shape + `batch_shape + event_shape` + + Note: + The computation for determinant and inverse of covariance matrix is avoided when + `cov_factor.shape[1] << cov_factor.shape[0]` thanks to `Woodbury matrix identity + `_ and + `matrix determinant lemma `_. + Thanks to these formulas, we just need to compute the determinant and inverse of + the small size "capacitance" matrix:: + + capacitance = I + cov_factor.T @ inv(cov_diag) @ cov_factor + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real_vector, + "cov_factor": constraints.independent(constraints.real, 2), + "cov_diag": constraints.independent(constraints.positive, 1), + } + support = constraints.real_vector + has_rsample = True + + def __init__( + self, + loc: Tensor, + cov_factor: Tensor, + cov_diag: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + if loc.dim() < 1: + raise ValueError("loc must be at least one-dimensional.") + event_shape = loc.shape[-1:] + if cov_factor.dim() < 2: + raise ValueError( + "cov_factor must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + if cov_factor.shape[-2:-1] != event_shape: + raise ValueError( + f"cov_factor must be a batch of matrices with shape {event_shape[0]} x m" + ) + if cov_diag.shape[-1:] != event_shape: + raise ValueError( + f"cov_diag must be a batch of vectors with shape {event_shape}" + ) + + loc_ = loc.unsqueeze(-1) + cov_diag_ = cov_diag.unsqueeze(-1) + try: + loc_, self.cov_factor, cov_diag_ = torch.broadcast_tensors( + loc_, cov_factor, cov_diag_ + ) + except RuntimeError as e: + raise ValueError( + f"Incompatible batch shapes: loc {loc.shape}, cov_factor {cov_factor.shape}, cov_diag {cov_diag.shape}" + ) from e + self.loc = loc_[..., 0] + self.cov_diag = cov_diag_[..., 0] + batch_shape = self.loc.shape[:-1] + + self._unbroadcasted_cov_factor = cov_factor + self._unbroadcasted_cov_diag = cov_diag + self._capacitance_tril = _batch_capacitance_tril(cov_factor, cov_diag) + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LowRankMultivariateNormal, _instance) + batch_shape = torch.Size(batch_shape) + loc_shape = batch_shape + self.event_shape + new.loc = self.loc.expand(loc_shape) + new.cov_diag = self.cov_diag.expand(loc_shape) + new.cov_factor = self.cov_factor.expand(loc_shape + self.cov_factor.shape[-1:]) + new._unbroadcasted_cov_factor = self._unbroadcasted_cov_factor + new._unbroadcasted_cov_diag = self._unbroadcasted_cov_diag + new._capacitance_tril = self._capacitance_tril + super(LowRankMultivariateNormal, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @lazy_property + def variance(self) -> Tensor: # type: ignore[override] + return ( + self._unbroadcasted_cov_factor.pow(2).sum(-1) + self._unbroadcasted_cov_diag + ).expand(self._batch_shape + self._event_shape) + + @lazy_property + def scale_tril(self) -> Tensor: + # The following identity is used to increase the numerically computation stability + # for Cholesky decomposition (see http://www.gaussianprocess.org/gpml/, Section 3.4.3): + # W @ W.T + D = D1/2 @ (I + D-1/2 @ W @ W.T @ D-1/2) @ D1/2 + # The matrix "I + D-1/2 @ W @ W.T @ D-1/2" has eigenvalues bounded from below by 1, + # hence it is well-conditioned and safe to take Cholesky decomposition. + n = self._event_shape[0] + cov_diag_sqrt_unsqueeze = self._unbroadcasted_cov_diag.sqrt().unsqueeze(-1) + Dinvsqrt_W = self._unbroadcasted_cov_factor / cov_diag_sqrt_unsqueeze + K = torch.matmul(Dinvsqrt_W, Dinvsqrt_W.mT).contiguous() + K.view(-1, n * n)[:, :: n + 1] += 1 # add identity matrix to K + scale_tril = cov_diag_sqrt_unsqueeze * torch.linalg.cholesky(K) + return scale_tril.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + covariance_matrix = torch.matmul( + self._unbroadcasted_cov_factor, self._unbroadcasted_cov_factor.mT + ) + torch.diag_embed(self._unbroadcasted_cov_diag) + return covariance_matrix.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def precision_matrix(self) -> Tensor: + # We use "Woodbury matrix identity" to take advantage of low rank form:: + # inv(W @ W.T + D) = inv(D) - inv(D) @ W @ inv(C) @ W.T @ inv(D) + # where :math:`C` is the capacitance matrix. + Wt_Dinv = ( + self._unbroadcasted_cov_factor.mT + / self._unbroadcasted_cov_diag.unsqueeze(-2) + ) + A = torch.linalg.solve_triangular(self._capacitance_tril, Wt_Dinv, upper=False) + precision_matrix = ( + torch.diag_embed(self._unbroadcasted_cov_diag.reciprocal()) - A.mT @ A + ) + return precision_matrix.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + W_shape = shape[:-1] + self.cov_factor.shape[-1:] + eps_W = _standard_normal(W_shape, dtype=self.loc.dtype, device=self.loc.device) + eps_D = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return ( + self.loc + + _batch_mv(self._unbroadcasted_cov_factor, eps_W) + + self._unbroadcasted_cov_diag.sqrt() * eps_D + ) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + diff = value - self.loc + M = _batch_lowrank_mahalanobis( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + diff, + self._capacitance_tril, + ) + log_det = _batch_lowrank_logdet( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + self._capacitance_tril, + ) + return -0.5 * (self._event_shape[0] * math.log(2 * math.pi) + log_det + M) + + def entropy(self): + log_det = _batch_lowrank_logdet( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + self._capacitance_tril, + ) + H = 0.5 * (self._event_shape[0] * (1.0 + math.log(2 * math.pi)) + log_det) + if len(self._batch_shape) == 0: + return H + else: + return H.expand(self._batch_shape) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/mixture_same_family.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/mixture_same_family.py new file mode 100644 index 0000000000000000000000000000000000000000..d379858c6b8044370e77afda53d8b1bace305353 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/mixture_same_family.py @@ -0,0 +1,221 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import Categorical, constraints +from torch.distributions.constraints import MixtureSameFamilyConstraint +from torch.distributions.distribution import Distribution + + +__all__ = ["MixtureSameFamily"] + + +class MixtureSameFamily(Distribution): + r""" + The `MixtureSameFamily` distribution implements a (batch of) mixture + distribution where all component are from different parameterizations of + the same distribution type. It is parameterized by a `Categorical` + "selecting distribution" (over `k` component) and a component + distribution, i.e., a `Distribution` with a rightmost batch shape + (equal to `[k]`) which indexes each (batch of) component. + + Examples:: + + >>> # xdoctest: +SKIP("undefined vars") + >>> # Construct Gaussian Mixture Model in 1D consisting of 5 equally + >>> # weighted normal distributions + >>> mix = D.Categorical(torch.ones(5,)) + >>> comp = D.Normal(torch.randn(5,), torch.rand(5,)) + >>> gmm = MixtureSameFamily(mix, comp) + + >>> # Construct Gaussian Mixture Model in 2D consisting of 5 equally + >>> # weighted bivariate normal distributions + >>> mix = D.Categorical(torch.ones(5,)) + >>> comp = D.Independent(D.Normal( + ... torch.randn(5,2), torch.rand(5,2)), 1) + >>> gmm = MixtureSameFamily(mix, comp) + + >>> # Construct a batch of 3 Gaussian Mixture Models in 2D each + >>> # consisting of 5 random weighted bivariate normal distributions + >>> mix = D.Categorical(torch.rand(3,5)) + >>> comp = D.Independent(D.Normal( + ... torch.randn(3,5,2), torch.rand(3,5,2)), 1) + >>> gmm = MixtureSameFamily(mix, comp) + + Args: + mixture_distribution: `torch.distributions.Categorical`-like + instance. Manages the probability of selecting component. + The number of categories must match the rightmost batch + dimension of the `component_distribution`. Must have either + scalar `batch_shape` or `batch_shape` matching + `component_distribution.batch_shape[:-1]` + component_distribution: `torch.distributions.Distribution`-like + instance. Right-most batch dimension indexes component. + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + has_rsample = False + + def __init__( + self, + mixture_distribution: Categorical, + component_distribution: Distribution, + validate_args: Optional[bool] = None, + ) -> None: + self._mixture_distribution = mixture_distribution + self._component_distribution = component_distribution + + if not isinstance(self._mixture_distribution, Categorical): + raise ValueError( + " The Mixture distribution needs to be an " + " instance of torch.distributions.Categorical" + ) + + if not isinstance(self._component_distribution, Distribution): + raise ValueError( + "The Component distribution need to be an " + "instance of torch.distributions.Distribution" + ) + + # Check that batch size matches + mdbs = self._mixture_distribution.batch_shape + cdbs = self._component_distribution.batch_shape[:-1] + for size1, size2 in zip(reversed(mdbs), reversed(cdbs)): + if size1 != 1 and size2 != 1 and size1 != size2: + raise ValueError( + f"`mixture_distribution.batch_shape` ({mdbs}) is not " + "compatible with `component_distribution." + f"batch_shape`({cdbs})" + ) + + # Check that the number of mixture component matches + km = self._mixture_distribution.logits.shape[-1] + kc = self._component_distribution.batch_shape[-1] + if km is not None and kc is not None and km != kc: + raise ValueError( + f"`mixture_distribution component` ({km}) does not" + " equal `component_distribution.batch_shape[-1]`" + f" ({kc})" + ) + self._num_component = km + + event_shape = self._component_distribution.event_shape + self._event_ndims = len(event_shape) + super().__init__( + batch_shape=cdbs, event_shape=event_shape, validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + batch_shape = torch.Size(batch_shape) + batch_shape_comp = batch_shape + (self._num_component,) + new = self._get_checked_instance(MixtureSameFamily, _instance) + new._component_distribution = self._component_distribution.expand( + batch_shape_comp + ) + new._mixture_distribution = self._mixture_distribution.expand(batch_shape) + new._num_component = self._num_component + new._event_ndims = self._event_ndims + event_shape = new._component_distribution.event_shape + super(MixtureSameFamily, new).__init__( + batch_shape=batch_shape, event_shape=event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def support(self): + return MixtureSameFamilyConstraint(self._component_distribution.support) + + @property + def mixture_distribution(self) -> Categorical: + return self._mixture_distribution + + @property + def component_distribution(self) -> Distribution: + return self._component_distribution + + @property + def mean(self) -> Tensor: + probs = self._pad_mixture_dimensions(self.mixture_distribution.probs) + return torch.sum( + probs * self.component_distribution.mean, dim=-1 - self._event_ndims + ) # [B, E] + + @property + def variance(self) -> Tensor: + # Law of total variance: Var(Y) = E[Var(Y|X)] + Var(E[Y|X]) + probs = self._pad_mixture_dimensions(self.mixture_distribution.probs) + mean_cond_var = torch.sum( + probs * self.component_distribution.variance, dim=-1 - self._event_ndims + ) + var_cond_mean = torch.sum( + probs * (self.component_distribution.mean - self._pad(self.mean)).pow(2.0), + dim=-1 - self._event_ndims, + ) + return mean_cond_var + var_cond_mean + + def cdf(self, x): + x = self._pad(x) + cdf_x = self.component_distribution.cdf(x) + mix_prob = self.mixture_distribution.probs + + return torch.sum(cdf_x * mix_prob, dim=-1) + + def log_prob(self, x): + if self._validate_args: + self._validate_sample(x) + x = self._pad(x) + log_prob_x = self.component_distribution.log_prob(x) # [S, B, k] + log_mix_prob = torch.log_softmax( + self.mixture_distribution.logits, dim=-1 + ) # [B, k] + return torch.logsumexp(log_prob_x + log_mix_prob, dim=-1) # [S, B] + + def sample(self, sample_shape=torch.Size()): + with torch.no_grad(): + sample_len = len(sample_shape) + batch_len = len(self.batch_shape) + gather_dim = sample_len + batch_len + es = self.event_shape + + # mixture samples [n, B] + mix_sample = self.mixture_distribution.sample(sample_shape) + mix_shape = mix_sample.shape + + # component samples [n, B, k, E] + comp_samples = self.component_distribution.sample(sample_shape) + + # Gather along the k dimension + mix_sample_r = mix_sample.reshape( + mix_shape + torch.Size([1] * (len(es) + 1)) + ) + mix_sample_r = mix_sample_r.repeat( + torch.Size([1] * len(mix_shape)) + torch.Size([1]) + es + ) + + samples = torch.gather(comp_samples, gather_dim, mix_sample_r) + return samples.squeeze(gather_dim) + + def _pad(self, x): + return x.unsqueeze(-1 - self._event_ndims) + + def _pad_mixture_dimensions(self, x): + dist_batch_ndims = len(self.batch_shape) + cat_batch_ndims = len(self.mixture_distribution.batch_shape) + pad_ndims = 0 if cat_batch_ndims == 1 else dist_batch_ndims - cat_batch_ndims + xs = x.shape + x = x.reshape( + xs[:-1] + + torch.Size(pad_ndims * [1]) + + xs[-1:] + + torch.Size(self._event_ndims * [1]) + ) + return x + + def __repr__(self): + args_string = ( + f"\n {self.mixture_distribution},\n {self.component_distribution}" + ) + return "MixtureSameFamily" + "(" + args_string + ")" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multinomial.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multinomial.py new file mode 100644 index 0000000000000000000000000000000000000000..3009150d105a80a09ee66c63aa468ddd4dc7aaee --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multinomial.py @@ -0,0 +1,148 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import inf, Tensor +from torch.distributions import Categorical, constraints +from torch.distributions.binomial import Binomial +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["Multinomial"] + + +class Multinomial(Distribution): + r""" + Creates a Multinomial distribution parameterized by :attr:`total_count` and + either :attr:`probs` or :attr:`logits` (but not both). The innermost dimension of + :attr:`probs` indexes over categories. All other dimensions index over batches. + + Note that :attr:`total_count` need not be specified if only :meth:`log_prob` is + called (see example below) + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + - :meth:`sample` requires a single shared `total_count` for all + parameters and samples. + - :meth:`log_prob` allows different `total_count` for each parameter and + sample. + + Example:: + + >>> # xdoctest: +SKIP("FIXME: found invalid values") + >>> m = Multinomial(100, torch.tensor([ 1., 1., 1., 1.])) + >>> x = m.sample() # equal probability of 0, 1, 2, 3 + tensor([ 21., 24., 30., 25.]) + + >>> Multinomial(probs=torch.tensor([1., 1., 1., 1.])).log_prob(x) + tensor([-4.1338]) + + Args: + total_count (int): number of trials + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + total_count: int + + @property + def mean(self) -> Tensor: + return self.probs * self.total_count + + @property + def variance(self) -> Tensor: + return self.total_count * self.probs * (1 - self.probs) + + def __init__( + self, + total_count: int = 1, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if not isinstance(total_count, int): + raise NotImplementedError("inhomogeneous total_count is not supported") + self.total_count = total_count + self._categorical = Categorical(probs=probs, logits=logits) + self._binomial = Binomial(total_count=total_count, probs=self.probs) + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Multinomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count + new._categorical = self._categorical.expand(batch_shape) + super(Multinomial, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=1) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.multinomial(self.total_count) + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + def sample(self, sample_shape=torch.Size()): + sample_shape = torch.Size(sample_shape) + samples = self._categorical.sample( + torch.Size((self.total_count,)) + sample_shape + ) + # samples.shape is (total_count, sample_shape, batch_shape), need to change it to + # (sample_shape, batch_shape, total_count) + shifted_idx = list(range(samples.dim())) + shifted_idx.append(shifted_idx.pop(0)) + samples = samples.permute(*shifted_idx) + counts = samples.new(self._extended_shape(sample_shape)).zero_() + counts.scatter_add_(-1, samples, torch.ones_like(samples)) + return counts.type_as(self.probs) + + def entropy(self): + n = torch.tensor(self.total_count) + + cat_entropy = self._categorical.entropy() + term1 = n * cat_entropy - torch.lgamma(n + 1) + + support = self._binomial.enumerate_support(expand=False)[1:] + binomial_probs = torch.exp(self._binomial.log_prob(support)) + weights = torch.lgamma(support + 1) + term2 = (binomial_probs * weights).sum([0, -1]) + + return term1 + term2 + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + logits = logits.clone(memory_format=torch.contiguous_format) + log_factorial_n = torch.lgamma(value.sum(-1) + 1) + log_factorial_xs = torch.lgamma(value + 1).sum(-1) + logits[(value == 0) & (logits == -inf)] = 0 + log_powers = (logits * value).sum(-1) + return log_factorial_n - log_factorial_xs + log_powers diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multivariate_normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multivariate_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2c8cc2797c5bf85e91ea3535698c06ecaf3f84 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/multivariate_normal.py @@ -0,0 +1,273 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _standard_normal, lazy_property +from torch.types import _size + + +__all__ = ["MultivariateNormal"] + + +def _batch_mv(bmat, bvec): + r""" + Performs a batched matrix-vector product, with compatible but different batch shapes. + + This function takes as input `bmat`, containing :math:`n \times n` matrices, and + `bvec`, containing length :math:`n` vectors. + + Both `bmat` and `bvec` may have any number of leading dimensions, which correspond + to a batch shape. They are not necessarily assumed to have the same batch shape, + just ones which can be broadcasted. + """ + return torch.matmul(bmat, bvec.unsqueeze(-1)).squeeze(-1) + + +def _batch_mahalanobis(bL, bx): + r""" + Computes the squared Mahalanobis distance :math:`\mathbf{x}^\top\mathbf{M}^{-1}\mathbf{x}` + for a factored :math:`\mathbf{M} = \mathbf{L}\mathbf{L}^\top`. + + Accepts batches for both bL and bx. They are not necessarily assumed to have the same batch + shape, but `bL` one should be able to broadcasted to `bx` one. + """ + n = bx.size(-1) + bx_batch_shape = bx.shape[:-1] + + # Assume that bL.shape = (i, 1, n, n), bx.shape = (..., i, j, n), + # we are going to make bx have shape (..., 1, j, i, 1, n) to apply batched tri.solve + bx_batch_dims = len(bx_batch_shape) + bL_batch_dims = bL.dim() - 2 + outer_batch_dims = bx_batch_dims - bL_batch_dims + old_batch_dims = outer_batch_dims + bL_batch_dims + new_batch_dims = outer_batch_dims + 2 * bL_batch_dims + # Reshape bx with the shape (..., 1, i, j, 1, n) + bx_new_shape = bx.shape[:outer_batch_dims] + for sL, sx in zip(bL.shape[:-2], bx.shape[outer_batch_dims:-1]): + bx_new_shape += (sx // sL, sL) + bx_new_shape += (n,) + bx = bx.reshape(bx_new_shape) + # Permute bx to make it have shape (..., 1, j, i, 1, n) + permute_dims = ( + list(range(outer_batch_dims)) + + list(range(outer_batch_dims, new_batch_dims, 2)) + + list(range(outer_batch_dims + 1, new_batch_dims, 2)) + + [new_batch_dims] + ) + bx = bx.permute(permute_dims) + + flat_L = bL.reshape(-1, n, n) # shape = b x n x n + flat_x = bx.reshape(-1, flat_L.size(0), n) # shape = c x b x n + flat_x_swap = flat_x.permute(1, 2, 0) # shape = b x n x c + M_swap = ( + torch.linalg.solve_triangular(flat_L, flat_x_swap, upper=False).pow(2).sum(-2) + ) # shape = b x c + M = M_swap.t() # shape = c x b + + # Now we revert the above reshape and permute operators. + permuted_M = M.reshape(bx.shape[:-1]) # shape = (..., 1, j, i, 1) + permute_inv_dims = list(range(outer_batch_dims)) + for i in range(bL_batch_dims): + permute_inv_dims += [outer_batch_dims + i, old_batch_dims + i] + reshaped_M = permuted_M.permute(permute_inv_dims) # shape = (..., 1, i, j, 1) + return reshaped_M.reshape(bx_batch_shape) + + +def _precision_to_scale_tril(P): + # Ref: https://nbviewer.jupyter.org/gist/fehiepsi/5ef8e09e61604f10607380467eb82006#Precision-to-scale_tril + Lf = torch.linalg.cholesky(torch.flip(P, (-2, -1))) + L_inv = torch.transpose(torch.flip(Lf, (-2, -1)), -2, -1) + Id = torch.eye(P.shape[-1], dtype=P.dtype, device=P.device) + L = torch.linalg.solve_triangular(L_inv, Id, upper=False) + return L + + +class MultivariateNormal(Distribution): + r""" + Creates a multivariate normal (also called Gaussian) distribution + parameterized by a mean vector and a covariance matrix. + + The multivariate normal distribution can be parameterized either + in terms of a positive definite covariance matrix :math:`\mathbf{\Sigma}` + or a positive definite precision matrix :math:`\mathbf{\Sigma}^{-1}` + or a lower-triangular matrix :math:`\mathbf{L}` with positive-valued + diagonal entries, such that + :math:`\mathbf{\Sigma} = \mathbf{L}\mathbf{L}^\top`. This triangular matrix + can be obtained via e.g. Cholesky decomposition of the covariance. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = MultivariateNormal(torch.zeros(2), torch.eye(2)) + >>> m.sample() # normally distributed with mean=`[0,0]` and covariance_matrix=`I` + tensor([-0.2102, -0.5429]) + + Args: + loc (Tensor): mean of the distribution + covariance_matrix (Tensor): positive-definite covariance matrix + precision_matrix (Tensor): positive-definite precision matrix + scale_tril (Tensor): lower-triangular factor of covariance, with positive-valued diagonal + + Note: + Only one of :attr:`covariance_matrix` or :attr:`precision_matrix` or + :attr:`scale_tril` can be specified. + + Using :attr:`scale_tril` will be more efficient: all computations internally + are based on :attr:`scale_tril`. If :attr:`covariance_matrix` or + :attr:`precision_matrix` is passed instead, it is only used to compute + the corresponding lower triangular matrices using a Cholesky decomposition. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real_vector, + "covariance_matrix": constraints.positive_definite, + "precision_matrix": constraints.positive_definite, + "scale_tril": constraints.lower_cholesky, + } + support = constraints.real_vector + has_rsample = True + + def __init__( + self, + loc: Tensor, + covariance_matrix: Optional[Tensor] = None, + precision_matrix: Optional[Tensor] = None, + scale_tril: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if loc.dim() < 1: + raise ValueError("loc must be at least one-dimensional.") + if (covariance_matrix is not None) + (scale_tril is not None) + ( + precision_matrix is not None + ) != 1: + raise ValueError( + "Exactly one of covariance_matrix or precision_matrix or scale_tril may be specified." + ) + + if scale_tril is not None: + if scale_tril.dim() < 2: + raise ValueError( + "scale_tril matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes(scale_tril.shape[:-2], loc.shape[:-1]) + # pyrefly: ignore [read-only] + self.scale_tril = scale_tril.expand(batch_shape + (-1, -1)) + elif covariance_matrix is not None: + if covariance_matrix.dim() < 2: + raise ValueError( + "covariance_matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes( + covariance_matrix.shape[:-2], loc.shape[:-1] + ) + # pyrefly: ignore [read-only] + self.covariance_matrix = covariance_matrix.expand(batch_shape + (-1, -1)) + else: + assert precision_matrix is not None # helps mypy + if precision_matrix.dim() < 2: + raise ValueError( + "precision_matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes( + precision_matrix.shape[:-2], loc.shape[:-1] + ) + # pyrefly: ignore [read-only] + self.precision_matrix = precision_matrix.expand(batch_shape + (-1, -1)) + self.loc = loc.expand(batch_shape + (-1,)) + + event_shape = self.loc.shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + if scale_tril is not None: + self._unbroadcasted_scale_tril = scale_tril + elif covariance_matrix is not None: + self._unbroadcasted_scale_tril = torch.linalg.cholesky(covariance_matrix) + else: # precision_matrix is not None + self._unbroadcasted_scale_tril = _precision_to_scale_tril(precision_matrix) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(MultivariateNormal, _instance) + batch_shape = torch.Size(batch_shape) + loc_shape = batch_shape + self.event_shape + cov_shape = batch_shape + self.event_shape + self.event_shape + new.loc = self.loc.expand(loc_shape) + new._unbroadcasted_scale_tril = self._unbroadcasted_scale_tril + if "covariance_matrix" in self.__dict__: + new.covariance_matrix = self.covariance_matrix.expand(cov_shape) + if "scale_tril" in self.__dict__: + new.scale_tril = self.scale_tril.expand(cov_shape) + if "precision_matrix" in self.__dict__: + new.precision_matrix = self.precision_matrix.expand(cov_shape) + super(MultivariateNormal, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @lazy_property + def scale_tril(self) -> Tensor: + return self._unbroadcasted_scale_tril.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + return torch.matmul( + self._unbroadcasted_scale_tril, self._unbroadcasted_scale_tril.mT + ).expand(self._batch_shape + self._event_shape + self._event_shape) + + @lazy_property + def precision_matrix(self) -> Tensor: + return torch.cholesky_inverse(self._unbroadcasted_scale_tril).expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return ( + self._unbroadcasted_scale_tril.pow(2) + .sum(-1) + .expand(self._batch_shape + self._event_shape) + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.loc + _batch_mv(self._unbroadcasted_scale_tril, eps) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + diff = value - self.loc + M = _batch_mahalanobis(self._unbroadcasted_scale_tril, diff) + half_log_det = ( + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + ) + return -0.5 * (self._event_shape[0] * math.log(2 * math.pi) + M) - half_log_det + + def entropy(self): + half_log_det = ( + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + ) + H = 0.5 * self._event_shape[0] * (1.0 + math.log(2 * math.pi)) + half_log_det + if len(self._batch_shape) == 0: + return H + else: + return H.expand(self._batch_shape) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/negative_binomial.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/negative_binomial.py new file mode 100644 index 0000000000000000000000000000000000000000..07207605de2634d4212b0cade6a74bf6752dd889 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/negative_binomial.py @@ -0,0 +1,150 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.gamma import Gamma +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) + + +__all__ = ["NegativeBinomial"] + + +class NegativeBinomial(Distribution): + r""" + Creates a Negative Binomial distribution, i.e. distribution + of the number of successful independent and identical Bernoulli trials + before :attr:`total_count` failures are achieved. The probability + of success of each Bernoulli trial is :attr:`probs`. + + Args: + total_count (float or Tensor): non-negative number of negative Bernoulli + trials to stop, although the distribution is still valid for real + valued count + probs (Tensor): Event probabilities of success in the half open interval [0, 1) + logits (Tensor): Event log-odds for probabilities of success + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "total_count": constraints.greater_than_eq(0), + "probs": constraints.half_open_interval(0.0, 1.0), + "logits": constraints.real, + } + support = constraints.nonnegative_integer + + def __init__( + self, + total_count: Union[Tensor, float], + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + ( + self.total_count, + # pyrefly: ignore [read-only] + self.probs, + ) = broadcast_all(total_count, probs) + self.total_count = self.total_count.type_as(self.probs) + else: + assert logits is not None # helps mypy + ( + self.total_count, + # pyrefly: ignore [read-only] + self.logits, + ) = broadcast_all(total_count, logits) + self.total_count = self.total_count.type_as(self.logits) + + self._param = self.probs if probs is not None else self.logits + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(NegativeBinomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count.expand(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(NegativeBinomial, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @property + def mean(self) -> Tensor: + return self.total_count * torch.exp(self.logits) + + @property + def mode(self) -> Tensor: + return ((self.total_count - 1) * self.logits.exp()).floor().clamp(min=0.0) + + @property + def variance(self) -> Tensor: + return self.mean / torch.sigmoid(-self.logits) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + @lazy_property + def _gamma(self) -> Gamma: + # Note we avoid validating because self.total_count can be zero. + return Gamma( + concentration=self.total_count, + rate=torch.exp(-self.logits), + validate_args=False, + ) + + def sample(self, sample_shape=torch.Size()): + with torch.no_grad(): + rate = self._gamma.sample(sample_shape=sample_shape) + return torch.poisson(rate) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + + log_unnormalized_prob = self.total_count * F.logsigmoid( + -self.logits + ) + value * F.logsigmoid(self.logits) + + log_normalization = ( + -torch.lgamma(self.total_count + value) + + torch.lgamma(1.0 + value) + + torch.lgamma(self.total_count) + ) + # The case self.total_count == 0 and value == 0 has probability 1 but + # lgamma(0) is infinite. Handle this case separately using a function + # that does not modify tensors in place to allow Jit compilation. + log_normalization = log_normalization.masked_fill( + self.total_count + value == 0.0, 0.0 + ) + + return log_unnormalized_prob - log_normalization diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/normal.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/normal.py new file mode 100644 index 0000000000000000000000000000000000000000..153e3d65a6bbb5a0611ed8304548ebda706ba461 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/normal.py @@ -0,0 +1,124 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import _standard_normal, broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Normal"] + + +class Normal(ExponentialFamily): + r""" + Creates a normal (also called Gaussian) distribution parameterized by + :attr:`loc` and :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Normal(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # normally distributed with loc=0 and scale=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of the distribution (often referred to as mu) + scale (float or Tensor): standard deviation of the distribution + (often referred to as sigma) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def stddev(self) -> Tensor: + return self.scale + + @property + def variance(self) -> Tensor: + return self.stddev.pow(2) + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Normal, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Normal, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.normal(self.loc.expand(shape), self.scale.expand(shape)) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.loc + eps * self.scale + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + # compute the variance + # pyrefly: ignore [unsupported-operation] + var = self.scale**2 + log_scale = ( + math.log(self.scale) + if isinstance(self.scale, _Number) + else self.scale.log() + ) + return ( + -((value - self.loc) ** 2) / (2 * var) + - log_scale + - math.log(math.sqrt(2 * math.pi)) + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 0.5 * ( + 1 + torch.erf((value - self.loc) * self.scale.reciprocal() / math.sqrt(2)) + ) + + def icdf(self, value): + return self.loc + self.scale * torch.erfinv(2 * value - 1) * math.sqrt(2) + + def entropy(self): + return 0.5 + 0.5 * math.log(2 * math.pi) + torch.log(self.scale) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.loc / self.scale.pow(2), -0.5 * self.scale.pow(2).reciprocal()) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return -0.25 * x.pow(2) / y + 0.5 * torch.log(-math.pi / y) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/one_hot_categorical.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/one_hot_categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..6cbcdbc34f02e9a3b21a7230d0b26eedadff5b45 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/one_hot_categorical.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.categorical import Categorical +from torch.distributions.distribution import Distribution +from torch.types import _size + + +__all__ = ["OneHotCategorical", "OneHotCategoricalStraightThrough"] + + +class OneHotCategorical(Distribution): + r""" + Creates a one-hot categorical distribution parameterized by :attr:`probs` or + :attr:`logits`. + + Samples are one-hot coded vectors of size ``probs.size(-1)``. + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + See also: :func:`torch.distributions.Categorical` for specifications of + :attr:`probs` and :attr:`logits`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = OneHotCategorical(torch.tensor([ 0.25, 0.25, 0.25, 0.25 ])) + >>> m.sample() # equal probability of 0, 1, 2, 3 + tensor([ 0., 0., 0., 1.]) + + Args: + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + support = constraints.one_hot + has_enumerate_support = True + + def __init__( + self, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + self._categorical = Categorical(probs, logits) + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(OneHotCategorical, _instance) + batch_shape = torch.Size(batch_shape) + new._categorical = self._categorical.expand(batch_shape) + super(OneHotCategorical, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @property + def _param(self) -> Tensor: + return self._categorical._param + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def mean(self) -> Tensor: + return self._categorical.probs + + @property + def mode(self) -> Tensor: + probs = self._categorical.probs + mode = probs.argmax(dim=-1) + return torch.nn.functional.one_hot(mode, num_classes=probs.shape[-1]).to(probs) + + @property + def variance(self) -> Tensor: + return self._categorical.probs * (1 - self._categorical.probs) + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + def sample(self, sample_shape=torch.Size()): + sample_shape = torch.Size(sample_shape) + probs = self._categorical.probs + num_events = self._categorical._num_events + indices = self._categorical.sample(sample_shape) + return torch.nn.functional.one_hot(indices, num_events).to(probs) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + indices = value.max(-1)[1] + return self._categorical.log_prob(indices) + + def entropy(self): + return self._categorical.entropy() + + def enumerate_support(self, expand=True): + n = self.event_shape[0] + values = torch.eye(n, dtype=self._param.dtype, device=self._param.device) + values = values.view((n,) + (1,) * len(self.batch_shape) + (n,)) + if expand: + values = values.expand((n,) + self.batch_shape + (n,)) + return values + + +class OneHotCategoricalStraightThrough(OneHotCategorical): + r""" + Creates a reparameterizable :class:`OneHotCategorical` distribution based on the straight- + through gradient estimator from [1]. + + [1] Estimating or Propagating Gradients Through Stochastic Neurons for Conditional Computation + (Bengio et al., 2013) + """ + + has_rsample = True + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + samples = self.sample(sample_shape) + probs = self._categorical.probs # cached via @lazy_property + return samples + (probs - probs.detach()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/pareto.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/pareto.py new file mode 100644 index 0000000000000000000000000000000000000000..70556001d3d8327c269a53c7a7619ac90fd8998c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/pareto.py @@ -0,0 +1,74 @@ +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exponential import Exponential +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, ExpTransform +from torch.distributions.utils import broadcast_all +from torch.types import _size + + +__all__ = ["Pareto"] + + +class Pareto(TransformedDistribution): + r""" + Samples from a Pareto Type 1 distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Pareto(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Pareto distribution with scale=1 and alpha=1 + tensor([ 1.5623]) + + Args: + scale (float or Tensor): Scale parameter of the distribution + alpha (float or Tensor): Shape parameter of the distribution + """ + + arg_constraints = {"alpha": constraints.positive, "scale": constraints.positive} + + def __init__( + self, + scale: Union[Tensor, float], + alpha: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.scale, self.alpha = broadcast_all(scale, alpha) + base_dist = Exponential(self.alpha, validate_args=validate_args) + transforms = [ExpTransform(), AffineTransform(loc=0, scale=self.scale)] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand( + self, batch_shape: _size, _instance: Optional["Pareto"] = None + ) -> "Pareto": + new = self._get_checked_instance(Pareto, _instance) + new.scale = self.scale.expand(batch_shape) + new.alpha = self.alpha.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + @property + def mean(self) -> Tensor: + # mean is inf for alpha <= 1 + a = self.alpha.clamp(min=1) + return a * self.scale / (a - 1) + + @property + def mode(self) -> Tensor: + return self.scale + + @property + def variance(self) -> Tensor: + # var is inf for alpha <= 2 + a = self.alpha.clamp(min=2) + return self.scale.pow(2) * a / ((a - 1).pow(2) * (a - 2)) + + @constraints.dependent_property(is_discrete=False, event_dim=0) + def support(self) -> constraints.Constraint: + return constraints.greater_than_eq(self.scale) + + def entropy(self) -> Tensor: + return (self.scale / self.alpha).log() + (1 + self.alpha.reciprocal()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/poisson.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/poisson.py new file mode 100644 index 0000000000000000000000000000000000000000..e1c8ad693bd571cc5dff0571f0084a2a6786a6bc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/poisson.py @@ -0,0 +1,88 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, Number + + +__all__ = ["Poisson"] + + +class Poisson(ExponentialFamily): + r""" + Creates a Poisson distribution parameterized by :attr:`rate`, the rate parameter. + + Samples are nonnegative integers, with a pmf given by + + .. math:: + \mathrm{rate}^k \frac{e^{-\mathrm{rate}}}{k!} + + Example:: + + >>> # xdoctest: +SKIP("poisson_cpu not implemented for 'Long'") + >>> m = Poisson(torch.tensor([4])) + >>> m.sample() + tensor([ 3.]) + + Args: + rate (Number, Tensor): the rate parameter + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"rate": constraints.nonnegative} + support = constraints.nonnegative_integer + + @property + def mean(self) -> Tensor: + return self.rate + + @property + def mode(self) -> Tensor: + return self.rate.floor() + + @property + def variance(self) -> Tensor: + return self.rate + + def __init__( + self, + rate: Union[Tensor, Number], + validate_args: Optional[bool] = None, + ) -> None: + (self.rate,) = broadcast_all(rate) + if isinstance(rate, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.rate.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Poisson, _instance) + batch_shape = torch.Size(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Poisson, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.poisson(self.rate.expand(shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + rate, value = broadcast_all(self.rate, value) + return value.xlogy(rate) - rate - (value + 1).lgamma() + + @property + def _natural_params(self) -> tuple[Tensor]: + return (torch.log(self.rate),) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return torch.exp(x) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_bernoulli.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8a8bdfa4b77f80f9d51ccfe70f564a2510f9e6 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_bernoulli.py @@ -0,0 +1,174 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import SigmoidTransform +from torch.distributions.utils import ( + broadcast_all, + clamp_probs, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.types import _Number, _size, Number + + +__all__ = ["LogitRelaxedBernoulli", "RelaxedBernoulli"] + + +class LogitRelaxedBernoulli(Distribution): + r""" + Creates a LogitRelaxedBernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both), which is the logit of a RelaxedBernoulli + distribution. + + Samples are logits of values in (0, 1). See [1] for more details. + + Args: + temperature (Tensor): relaxation temperature + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + + [1] The Concrete Distribution: A Continuous Relaxation of Discrete Random + Variables (Maddison et al., 2017) + + [2] Categorical Reparametrization with Gumbel-Softmax + (Jang et al., 2017) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.real + + def __init__( + self, + temperature: Tensor, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + self.temperature = temperature + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogitRelaxedBernoulli, _instance) + batch_shape = torch.Size(batch_shape) + new.temperature = self.temperature + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(LogitRelaxedBernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + probs = clamp_probs(self.probs.expand(shape)) + uniforms = clamp_probs( + torch.rand(shape, dtype=probs.dtype, device=probs.device) + ) + return ( + uniforms.log() - (-uniforms).log1p() + probs.log() - (-probs).log1p() + ) / self.temperature + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + diff = logits - value.mul(self.temperature) + return self.temperature.log() + diff - 2 * diff.exp().log1p() + + +class RelaxedBernoulli(TransformedDistribution): + r""" + Creates a RelaxedBernoulli distribution, parametrized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits` + (but not both). This is a relaxed version of the `Bernoulli` distribution, + so the values are in (0, 1), and has reparametrizable samples. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = RelaxedBernoulli(torch.tensor([2.2]), + ... torch.tensor([0.1, 0.2, 0.3, 0.99])) + >>> m.sample() + tensor([ 0.2951, 0.3442, 0.8918, 0.9021]) + + Args: + temperature (Tensor): relaxation temperature + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + """ + + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + # pyrefly: ignore [bad-override] + support = constraints.unit_interval + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: LogitRelaxedBernoulli + + def __init__( + self, + temperature: Tensor, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + base_dist = LogitRelaxedBernoulli(temperature, probs, logits) + super().__init__(base_dist, SigmoidTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(RelaxedBernoulli, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def temperature(self) -> Tensor: + return self.base_dist.temperature + + @property + def logits(self) -> Tensor: + return self.base_dist.logits + + @property + def probs(self) -> Tensor: + return self.base_dist.probs diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_categorical.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..b9049014bd1b853aa59e807081160827e8be40ea --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/relaxed_categorical.py @@ -0,0 +1,163 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.categorical import Categorical +from torch.distributions.distribution import Distribution +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import ExpTransform +from torch.distributions.utils import broadcast_all, clamp_probs +from torch.types import _size + + +__all__ = ["ExpRelaxedCategorical", "RelaxedOneHotCategorical"] + + +class ExpRelaxedCategorical(Distribution): + r""" + Creates a ExpRelaxedCategorical parameterized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits` (but not both). + Returns the log of a point in the simplex. Based on the interface to + :class:`OneHotCategorical`. + + Implementation based on [1]. + + See also: :func:`torch.distributions.OneHotCategorical` + + Args: + temperature (Tensor): relaxation temperature + probs (Tensor): event probabilities + logits (Tensor): unnormalized log probability for each event + + [1] The Concrete Distribution: A Continuous Relaxation of Discrete Random Variables + (Maddison et al., 2017) + + [2] Categorical Reparametrization with Gumbel-Softmax + (Jang et al., 2017) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + support = ( + constraints.real_vector + ) # The true support is actually a submanifold of this. + has_rsample = True + + def __init__( + self, + temperature: Tensor, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + self._categorical = Categorical(probs, logits) + self.temperature = temperature + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(ExpRelaxedCategorical, _instance) + batch_shape = torch.Size(batch_shape) + new.temperature = self.temperature + new._categorical = self._categorical.expand(batch_shape) + super(ExpRelaxedCategorical, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + uniforms = clamp_probs( + torch.rand(shape, dtype=self.logits.dtype, device=self.logits.device) + ) + gumbels = -((-(uniforms.log())).log()) + scores = (self.logits + gumbels) / self.temperature + return scores - scores.logsumexp(dim=-1, keepdim=True) + + def log_prob(self, value): + K = self._categorical._num_events + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + log_scale = torch.full_like( + self.temperature, float(K) + ).lgamma() - self.temperature.log().mul(-(K - 1)) + score = logits - value.mul(self.temperature) + score = (score - score.logsumexp(dim=-1, keepdim=True)).sum(-1) + return score + log_scale + + +class RelaxedOneHotCategorical(TransformedDistribution): + r""" + Creates a RelaxedOneHotCategorical distribution parametrized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits`. + This is a relaxed version of the :class:`OneHotCategorical` distribution, so + its samples are on simplex, and are reparametrizable. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = RelaxedOneHotCategorical(torch.tensor([2.2]), + ... torch.tensor([0.1, 0.2, 0.3, 0.4])) + >>> m.sample() + tensor([ 0.1294, 0.2324, 0.3859, 0.2523]) + + Args: + temperature (Tensor): relaxation temperature + probs (Tensor): event probabilities + logits (Tensor): unnormalized log probability for each event + """ + + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + # pyrefly: ignore [bad-override] + support = constraints.simplex + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: ExpRelaxedCategorical + + def __init__( + self, + temperature: Tensor, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + base_dist = ExpRelaxedCategorical( + temperature, probs, logits, validate_args=validate_args + ) + super().__init__(base_dist, ExpTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(RelaxedOneHotCategorical, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def temperature(self) -> Tensor: + return self.base_dist.temperature + + @property + def logits(self) -> Tensor: + return self.base_dist.logits + + @property + def probs(self) -> Tensor: + return self.base_dist.probs diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/studentT.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/studentT.py new file mode 100644 index 0000000000000000000000000000000000000000..10fb244774071e8e36cc97a016922a976636eb8e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/studentT.py @@ -0,0 +1,128 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, nan, Tensor +from torch.distributions import Chi2, constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _standard_normal, broadcast_all +from torch.types import _size + + +__all__ = ["StudentT"] + + +class StudentT(Distribution): + r""" + Creates a Student's t-distribution parameterized by degree of + freedom :attr:`df`, mean :attr:`loc` and scale :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = StudentT(torch.tensor([2.0])) + >>> m.sample() # Student's t-distributed with degrees of freedom=2 + tensor([ 0.1046]) + + Args: + df (float or Tensor): degrees of freedom + loc (float or Tensor): mean of the distribution + scale (float or Tensor): scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "df": constraints.positive, + "loc": constraints.real, + "scale": constraints.positive, + } + support = constraints.real + has_rsample = True + + @property + def mean(self) -> Tensor: + m = self.loc.clone(memory_format=torch.contiguous_format) + m[self.df <= 1] = nan + return m + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + m = self.df.clone(memory_format=torch.contiguous_format) + m[self.df > 2] = ( + self.scale[self.df > 2].pow(2) + * self.df[self.df > 2] + / (self.df[self.df > 2] - 2) + ) + m[(self.df <= 2) & (self.df > 1)] = inf + m[self.df <= 1] = nan + return m + + def __init__( + self, + df: Union[Tensor, float], + loc: Union[Tensor, float] = 0.0, + scale: Union[Tensor, float] = 1.0, + validate_args: Optional[bool] = None, + ) -> None: + self.df, self.loc, self.scale = broadcast_all(df, loc, scale) + self._chi2 = Chi2(self.df) + batch_shape = self.df.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(StudentT, _instance) + batch_shape = torch.Size(batch_shape) + new.df = self.df.expand(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + new._chi2 = self._chi2.expand(batch_shape) + super(StudentT, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + # NOTE: This does not agree with scipy implementation as much as other distributions. + # (see https://github.com/fritzo/notebooks/blob/master/debug-student-t.ipynb). Using DoubleTensor + # parameters seems to help. + + # X ~ Normal(0, 1) + # Z ~ Chi2(df) + # Y = X / sqrt(Z / df) ~ StudentT(df) + shape = self._extended_shape(sample_shape) + X = _standard_normal(shape, dtype=self.df.dtype, device=self.df.device) + Z = self._chi2.rsample(sample_shape) + Y = X * torch.rsqrt(Z / self.df) + return self.loc + self.scale * Y + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + y = (value - self.loc) / self.scale + Z = ( + self.scale.log() + + 0.5 * self.df.log() + + 0.5 * math.log(math.pi) + + torch.lgamma(0.5 * self.df) + - torch.lgamma(0.5 * (self.df + 1.0)) + ) + return -0.5 * (self.df + 1.0) * torch.log1p(y**2.0 / self.df) - Z + + def entropy(self): + lbeta = ( + torch.lgamma(0.5 * self.df) + + math.lgamma(0.5) + - torch.lgamma(0.5 * (self.df + 1)) + ) + return ( + self.scale.log() + + 0.5 + * (self.df + 1) + * (torch.digamma(0.5 * (self.df + 1)) - torch.digamma(0.5 * self.df)) + + 0.5 * self.df.log() + + lbeta + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transformed_distribution.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transformed_distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..991951c52823c38ebed5ebcd451a01295a221fd7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transformed_distribution.py @@ -0,0 +1,224 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.independent import Independent +from torch.distributions.transforms import ComposeTransform, Transform +from torch.distributions.utils import _sum_rightmost +from torch.types import _size + + +__all__ = ["TransformedDistribution"] + + +class TransformedDistribution(Distribution): + r""" + Extension of the Distribution class, which applies a sequence of Transforms + to a base distribution. Let f be the composition of transforms applied:: + + X ~ BaseDistribution + Y = f(X) ~ TransformedDistribution(BaseDistribution, f) + log p(Y) = log p(X) + log |det (dX/dY)| + + Note that the ``.event_shape`` of a :class:`TransformedDistribution` is the + maximum shape of its base distribution and its transforms, since transforms + can introduce correlations among events. + + An example for the usage of :class:`TransformedDistribution` would be:: + + # Building a Logistic Distribution + # X ~ Uniform(0, 1) + # f = a + b * logit(X) + # Y ~ f(X) ~ Logistic(a, b) + base_distribution = Uniform(0, 1) + transforms = [SigmoidTransform().inv, AffineTransform(loc=a, scale=b)] + logistic = TransformedDistribution(base_distribution, transforms) + + For more examples, please look at the implementations of + :class:`~torch.distributions.gumbel.Gumbel`, + :class:`~torch.distributions.half_cauchy.HalfCauchy`, + :class:`~torch.distributions.half_normal.HalfNormal`, + :class:`~torch.distributions.log_normal.LogNormal`, + :class:`~torch.distributions.pareto.Pareto`, + :class:`~torch.distributions.weibull.Weibull`, + :class:`~torch.distributions.relaxed_bernoulli.RelaxedBernoulli` and + :class:`~torch.distributions.relaxed_categorical.RelaxedOneHotCategorical` + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + + def __init__( + self, + base_distribution: Distribution, + transforms: Union[Transform, list[Transform]], + validate_args: Optional[bool] = None, + ) -> None: + if isinstance(transforms, Transform): + self.transforms = [ + transforms, + ] + elif isinstance(transforms, list): + if not all(isinstance(t, Transform) for t in transforms): + raise ValueError( + "transforms must be a Transform or a list of Transforms" + ) + self.transforms = transforms + else: + raise ValueError( + f"transforms must be a Transform or list, but was {transforms}" + ) + + # Reshape base_distribution according to transforms. + base_shape = base_distribution.batch_shape + base_distribution.event_shape + base_event_dim = len(base_distribution.event_shape) + transform = ComposeTransform(self.transforms) + if len(base_shape) < transform.domain.event_dim: + raise ValueError( + f"base_distribution needs to have shape with size at least {transform.domain.event_dim}, but got {base_shape}." + ) + forward_shape = transform.forward_shape(base_shape) + expanded_base_shape = transform.inverse_shape(forward_shape) + if base_shape != expanded_base_shape: + base_batch_shape = expanded_base_shape[ + : len(expanded_base_shape) - base_event_dim + ] + base_distribution = base_distribution.expand(base_batch_shape) + reinterpreted_batch_ndims = transform.domain.event_dim - base_event_dim + if reinterpreted_batch_ndims > 0: + base_distribution = Independent( + base_distribution, reinterpreted_batch_ndims + ) + self.base_dist = base_distribution + + # Compute shapes. + transform_change_in_event_dim = ( + transform.codomain.event_dim - transform.domain.event_dim + ) + event_dim = max( + transform.codomain.event_dim, # the transform is coupled + base_event_dim + transform_change_in_event_dim, # the base dist is coupled + ) + assert len(forward_shape) >= event_dim + cut = len(forward_shape) - event_dim + batch_shape = forward_shape[:cut] + event_shape = forward_shape[cut:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(TransformedDistribution, _instance) + batch_shape = torch.Size(batch_shape) + shape = batch_shape + self.event_shape + for t in reversed(self.transforms): + shape = t.inverse_shape(shape) + base_batch_shape = shape[: len(shape) - len(self.base_dist.event_shape)] + new.base_dist = self.base_dist.expand(base_batch_shape) + new.transforms = self.transforms + super(TransformedDistribution, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def support(self): + if not self.transforms: + return self.base_dist.support + support = self.transforms[-1].codomain + if len(self.event_shape) > support.event_dim: + support = constraints.independent( + support, len(self.event_shape) - support.event_dim + ) + return support + + @property + def has_rsample(self) -> bool: # type: ignore[override] + return self.base_dist.has_rsample + + def sample(self, sample_shape=torch.Size()): + """ + Generates a sample_shape shaped sample or sample_shape shaped batch of + samples if the distribution parameters are batched. Samples first from + base distribution and applies `transform()` for every transform in the + list. + """ + with torch.no_grad(): + x = self.base_dist.sample(sample_shape) + for transform in self.transforms: + x = transform(x) + return x + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped reparameterized sample or sample_shape + shaped batch of reparameterized samples if the distribution parameters + are batched. Samples first from base distribution and applies + `transform()` for every transform in the list. + """ + x = self.base_dist.rsample(sample_shape) + for transform in self.transforms: + x = transform(x) + return x + + def log_prob(self, value): + """ + Scores the sample by inverting the transform(s) and computing the score + using the score of the base distribution and the log abs det jacobian. + """ + if self._validate_args: + self._validate_sample(value) + event_dim = len(self.event_shape) + log_prob: Union[Tensor, float] = 0.0 + y = value + for transform in reversed(self.transforms): + x = transform.inv(y) + event_dim += transform.domain.event_dim - transform.codomain.event_dim + log_prob = log_prob - _sum_rightmost( + transform.log_abs_det_jacobian(x, y), + event_dim - transform.domain.event_dim, + ) + y = x + + log_prob = log_prob + _sum_rightmost( + self.base_dist.log_prob(y), event_dim - len(self.base_dist.event_shape) + ) + return log_prob + + def _monotonize_cdf(self, value): + """ + This conditionally flips ``value -> 1-value`` to ensure :meth:`cdf` is + monotone increasing. + """ + sign = 1 + for transform in self.transforms: + sign = sign * transform.sign + if isinstance(sign, int) and sign == 1: + return value + return sign * (value - 0.5) + 0.5 + + def cdf(self, value): + """ + Computes the cumulative distribution function by inverting the + transform(s) and computing the score of the base distribution. + """ + for transform in self.transforms[::-1]: + value = transform.inv(value) + if self._validate_args: + self.base_dist._validate_sample(value) + value = self.base_dist.cdf(value) + value = self._monotonize_cdf(value) + return value + + def icdf(self, value): + """ + Computes the inverse cumulative distribution function using + transform(s) and computing the score of the base distribution. + """ + value = self._monotonize_cdf(value) + value = self.base_dist.icdf(value) + for transform in self.transforms: + value = transform(value) + return value diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transforms.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..2fe39d81377a6d8909ca9433a4b2b7ea84e2fa37 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/transforms.py @@ -0,0 +1,1302 @@ +# mypy: allow-untyped-defs +import functools +import math +import operator +import weakref +from collections.abc import Sequence +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + _sum_rightmost, + broadcast_all, + lazy_property, + tril_matrix_to_vec, + vec_to_tril_matrix, +) +from torch.nn.functional import pad, softplus +from torch.types import _Number + + +__all__ = [ + "AbsTransform", + "AffineTransform", + "CatTransform", + "ComposeTransform", + "CorrCholeskyTransform", + "CumulativeDistributionTransform", + "ExpTransform", + "IndependentTransform", + "LowerCholeskyTransform", + "PositiveDefiniteTransform", + "PowerTransform", + "ReshapeTransform", + "SigmoidTransform", + "SoftplusTransform", + "TanhTransform", + "SoftmaxTransform", + "StackTransform", + "StickBreakingTransform", + "Transform", + "identity_transform", +] + + +class Transform: + """ + Abstract class for invertable transformations with computable log + det jacobians. They are primarily used in + :class:`torch.distributions.TransformedDistribution`. + + Caching is useful for transforms whose inverses are either expensive or + numerically unstable. Note that care must be taken with memoized values + since the autograd graph may be reversed. For example while the following + works with or without caching:: + + y = t(x) + t.log_abs_det_jacobian(x, y).backward() # x will receive gradients. + + However the following will error when caching due to dependency reversal:: + + y = t(x) + z = t.inv(y) + grad(z.sum(), [y]) # error because z is x + + Derived classes should implement one or both of :meth:`_call` or + :meth:`_inverse`. Derived classes that set `bijective=True` should also + implement :meth:`log_abs_det_jacobian`. + + Args: + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. + + Attributes: + domain (:class:`~torch.distributions.constraints.Constraint`): + The constraint representing valid inputs to this transform. + codomain (:class:`~torch.distributions.constraints.Constraint`): + The constraint representing valid outputs to this transform + which are inputs to the inverse transform. + bijective (bool): Whether this transform is bijective. A transform + ``t`` is bijective iff ``t.inv(t(x)) == x`` and + ``t(t.inv(y)) == y`` for every ``x`` in the domain and ``y`` in + the codomain. Transforms that are not bijective should at least + maintain the weaker pseudoinverse properties + ``t(t.inv(t(x)) == t(x)`` and ``t.inv(t(t.inv(y))) == t.inv(y)``. + sign (int or Tensor): For bijective univariate transforms, this + should be +1 or -1 depending on whether transform is monotone + increasing or decreasing. + """ + + bijective = False + domain: constraints.Constraint + codomain: constraints.Constraint + + def __init__(self, cache_size: int = 0) -> None: + self._cache_size = cache_size + self._inv: Optional[weakref.ReferenceType[Transform]] = None + if cache_size == 0: + pass # default behavior + elif cache_size == 1: + self._cached_x_y = None, None + else: + raise ValueError("cache_size must be 0 or 1") + super().__init__() + + def __getstate__(self): + state = self.__dict__.copy() + state["_inv"] = None + return state + + @property + def event_dim(self) -> int: + if self.domain.event_dim == self.codomain.event_dim: + return self.domain.event_dim + raise ValueError("Please use either .domain.event_dim or .codomain.event_dim") + + @property + def inv(self) -> "Transform": + """ + Returns the inverse :class:`Transform` of this transform. + This should satisfy ``t.inv.inv is t``. + """ + inv = None + if self._inv is not None: + inv = self._inv() + if inv is None: + inv = _InverseTransform(self) + self._inv = weakref.ref(inv) + return inv + + @property + def sign(self) -> int: + """ + Returns the sign of the determinant of the Jacobian, if applicable. + In general this only makes sense for bijective transforms. + """ + raise NotImplementedError + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + if type(self).__init__ is Transform.__init__: + return type(self)(cache_size=cache_size) + raise NotImplementedError(f"{type(self)}.with_cache is not implemented") + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + # Necessary for Python2 + return not self.__eq__(other) + + def __call__(self, x): + """ + Computes the transform `x => y`. + """ + if self._cache_size == 0: + return self._call(x) + x_old, y_old = self._cached_x_y + if x is x_old: + return y_old + y = self._call(x) + self._cached_x_y = x, y + return y + + def _inv_call(self, y): + """ + Inverts the transform `y => x`. + """ + if self._cache_size == 0: + return self._inverse(y) + x_old, y_old = self._cached_x_y + if y is y_old: + return x_old + x = self._inverse(y) + self._cached_x_y = x, y + return x + + def _call(self, x): + """ + Abstract method to compute forward transformation. + """ + raise NotImplementedError + + def _inverse(self, y): + """ + Abstract method to compute inverse transformation. + """ + raise NotImplementedError + + def log_abs_det_jacobian(self, x, y): + """ + Computes the log det jacobian `log |dy/dx|` given input and output. + """ + raise NotImplementedError + + def __repr__(self): + return self.__class__.__name__ + "()" + + def forward_shape(self, shape): + """ + Infers the shape of the forward computation, given the input shape. + Defaults to preserving shape. + """ + return shape + + def inverse_shape(self, shape): + """ + Infers the shapes of the inverse computation, given the output shape. + Defaults to preserving shape. + """ + return shape + + +class _InverseTransform(Transform): + """ + Inverts a single :class:`Transform`. + This class is private; please instead use the ``Transform.inv`` property. + """ + + def __init__(self, transform: Transform) -> None: + super().__init__(cache_size=transform._cache_size) + self._inv: Transform = transform # type: ignore[assignment] + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + assert self._inv is not None + return self._inv.codomain + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + assert self._inv is not None + return self._inv.domain + + @property + def bijective(self) -> bool: # type: ignore[override] + assert self._inv is not None + return self._inv.bijective + + @property + def sign(self) -> int: + assert self._inv is not None + return self._inv.sign + + @property + def inv(self) -> Transform: + return self._inv + + def with_cache(self, cache_size=1): + assert self._inv is not None + return self.inv.with_cache(cache_size).inv + + def __eq__(self, other): + if not isinstance(other, _InverseTransform): + return False + assert self._inv is not None + return self._inv == other._inv + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self._inv)})" + + def __call__(self, x): + assert self._inv is not None + return self._inv._inv_call(x) + + def log_abs_det_jacobian(self, x, y): + assert self._inv is not None + return -self._inv.log_abs_det_jacobian(y, x) + + def forward_shape(self, shape): + return self._inv.inverse_shape(shape) + + def inverse_shape(self, shape): + return self._inv.forward_shape(shape) + + +class ComposeTransform(Transform): + """ + Composes multiple transforms in a chain. + The transforms being composed are responsible for caching. + + Args: + parts (list of :class:`Transform`): A list of transforms to compose. + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. + """ + + def __init__(self, parts: list[Transform], cache_size: int = 0) -> None: + if cache_size: + parts = [part.with_cache(cache_size) for part in parts] + super().__init__(cache_size=cache_size) + self.parts = parts + + def __eq__(self, other): + if not isinstance(other, ComposeTransform): + return False + return self.parts == other.parts + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + if not self.parts: + return constraints.real + domain = self.parts[0].domain + # Adjust event_dim to be maximum among all parts. + event_dim = self.parts[-1].codomain.event_dim + for part in reversed(self.parts): + event_dim += part.domain.event_dim - part.codomain.event_dim + event_dim = max(event_dim, part.domain.event_dim) + assert event_dim >= domain.event_dim + if event_dim > domain.event_dim: + domain = constraints.independent(domain, event_dim - domain.event_dim) + return domain + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + if not self.parts: + return constraints.real + codomain = self.parts[-1].codomain + # Adjust event_dim to be maximum among all parts. + event_dim = self.parts[0].domain.event_dim + for part in self.parts: + event_dim += part.codomain.event_dim - part.domain.event_dim + event_dim = max(event_dim, part.codomain.event_dim) + assert event_dim >= codomain.event_dim + if event_dim > codomain.event_dim: + codomain = constraints.independent(codomain, event_dim - codomain.event_dim) + return codomain + + @lazy_property + def bijective(self) -> bool: # type: ignore[override] + return all(p.bijective for p in self.parts) + + @lazy_property + def sign(self) -> int: # type: ignore[override] + sign = 1 + for p in self.parts: + sign = sign * p.sign + return sign + + @property + def inv(self) -> Transform: + inv = None + if self._inv is not None: + inv = self._inv() + if inv is None: + inv = ComposeTransform([p.inv for p in reversed(self.parts)]) + self._inv = weakref.ref(inv) + inv._inv = weakref.ref(self) + return inv + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return ComposeTransform(self.parts, cache_size=cache_size) + + def __call__(self, x): + for part in self.parts: + x = part(x) + return x + + def log_abs_det_jacobian(self, x, y): + if not self.parts: + return torch.zeros_like(x) + + # Compute intermediates. This will be free if parts[:-1] are all cached. + xs = [x] + for part in self.parts[:-1]: + xs.append(part(xs[-1])) + xs.append(y) + + terms = [] + event_dim = self.domain.event_dim + for part, x, y in zip(self.parts, xs[:-1], xs[1:]): + terms.append( + _sum_rightmost( + part.log_abs_det_jacobian(x, y), event_dim - part.domain.event_dim + ) + ) + event_dim += part.codomain.event_dim - part.domain.event_dim + return functools.reduce(operator.add, terms) + + def forward_shape(self, shape): + for part in self.parts: + shape = part.forward_shape(shape) + return shape + + def inverse_shape(self, shape): + for part in reversed(self.parts): + shape = part.inverse_shape(shape) + return shape + + def __repr__(self): + fmt_string = self.__class__.__name__ + "(\n " + fmt_string += ",\n ".join([p.__repr__() for p in self.parts]) + fmt_string += "\n)" + return fmt_string + + +identity_transform = ComposeTransform([]) + + +class IndependentTransform(Transform): + """ + Wrapper around another transform to treat + ``reinterpreted_batch_ndims``-many extra of the right most dimensions as + dependent. This has no effect on the forward or backward transforms, but + does sum out ``reinterpreted_batch_ndims``-many of the rightmost dimensions + in :meth:`log_abs_det_jacobian`. + + Args: + base_transform (:class:`Transform`): A base transform. + reinterpreted_batch_ndims (int): The number of extra rightmost + dimensions to treat as dependent. + """ + + def __init__( + self, + base_transform: Transform, + reinterpreted_batch_ndims: int, + cache_size: int = 0, + ) -> None: + super().__init__(cache_size=cache_size) + self.base_transform = base_transform.with_cache(cache_size) + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return IndependentTransform( + self.base_transform, self.reinterpreted_batch_ndims, cache_size=cache_size + ) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.independent( + self.base_transform.domain, self.reinterpreted_batch_ndims + ) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.independent( + self.base_transform.codomain, self.reinterpreted_batch_ndims + ) + + @property + def bijective(self) -> bool: # type: ignore[override] + return self.base_transform.bijective + + @property + def sign(self) -> int: + return self.base_transform.sign + + def _call(self, x): + if x.dim() < self.domain.event_dim: + raise ValueError("Too few dimensions on input") + return self.base_transform(x) + + def _inverse(self, y): + if y.dim() < self.codomain.event_dim: + raise ValueError("Too few dimensions on input") + return self.base_transform.inv(y) + + def log_abs_det_jacobian(self, x, y): + result = self.base_transform.log_abs_det_jacobian(x, y) + result = _sum_rightmost(result, self.reinterpreted_batch_ndims) + return result + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self.base_transform)}, {self.reinterpreted_batch_ndims})" + + def forward_shape(self, shape): + return self.base_transform.forward_shape(shape) + + def inverse_shape(self, shape): + return self.base_transform.inverse_shape(shape) + + +class ReshapeTransform(Transform): + """ + Unit Jacobian transform to reshape the rightmost part of a tensor. + + Note that ``in_shape`` and ``out_shape`` must have the same number of + elements, just as for :meth:`torch.Tensor.reshape`. + + Arguments: + in_shape (torch.Size): The input event shape. + out_shape (torch.Size): The output event shape. + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. (Default 0.) + """ + + bijective = True + + def __init__( + self, + in_shape: torch.Size, + out_shape: torch.Size, + cache_size: int = 0, + ) -> None: + self.in_shape = torch.Size(in_shape) + self.out_shape = torch.Size(out_shape) + if self.in_shape.numel() != self.out_shape.numel(): + raise ValueError("in_shape, out_shape have different numbers of elements") + super().__init__(cache_size=cache_size) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.independent(constraints.real, len(self.in_shape)) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.independent(constraints.real, len(self.out_shape)) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return ReshapeTransform(self.in_shape, self.out_shape, cache_size=cache_size) + + def _call(self, x): + batch_shape = x.shape[: x.dim() - len(self.in_shape)] + return x.reshape(batch_shape + self.out_shape) + + def _inverse(self, y): + batch_shape = y.shape[: y.dim() - len(self.out_shape)] + return y.reshape(batch_shape + self.in_shape) + + def log_abs_det_jacobian(self, x, y): + batch_shape = x.shape[: x.dim() - len(self.in_shape)] + return x.new_zeros(batch_shape) + + def forward_shape(self, shape): + if len(shape) < len(self.in_shape): + raise ValueError("Too few dimensions on input") + cut = len(shape) - len(self.in_shape) + if shape[cut:] != self.in_shape: + raise ValueError( + f"Shape mismatch: expected {shape[cut:]} but got {self.in_shape}" + ) + return shape[:cut] + self.out_shape + + def inverse_shape(self, shape): + if len(shape) < len(self.out_shape): + raise ValueError("Too few dimensions on input") + cut = len(shape) - len(self.out_shape) + if shape[cut:] != self.out_shape: + raise ValueError( + f"Shape mismatch: expected {shape[cut:]} but got {self.out_shape}" + ) + return shape[:cut] + self.in_shape + + +class ExpTransform(Transform): + r""" + Transform via the mapping :math:`y = \exp(x)`. + """ + + domain = constraints.real + codomain = constraints.positive + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, ExpTransform) + + def _call(self, x): + return x.exp() + + def _inverse(self, y): + return y.log() + + def log_abs_det_jacobian(self, x, y): + return x + + +class PowerTransform(Transform): + r""" + Transform via the mapping :math:`y = x^{\text{exponent}}`. + """ + + domain = constraints.positive + codomain = constraints.positive + bijective = True + + def __init__(self, exponent: Tensor, cache_size: int = 0) -> None: + super().__init__(cache_size=cache_size) + (self.exponent,) = broadcast_all(exponent) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return PowerTransform(self.exponent, cache_size=cache_size) + + @lazy_property + def sign(self) -> int: # type: ignore[override] + return self.exponent.sign() # type: ignore[return-value] + + def __eq__(self, other): + if not isinstance(other, PowerTransform): + return False + return self.exponent.eq(other.exponent).all().item() + + def _call(self, x): + return x.pow(self.exponent) + + def _inverse(self, y): + return y.pow(1 / self.exponent) + + def log_abs_det_jacobian(self, x, y): + return (self.exponent * y / x).abs().log() + + def forward_shape(self, shape): + return torch.broadcast_shapes(shape, getattr(self.exponent, "shape", ())) + + def inverse_shape(self, shape): + return torch.broadcast_shapes(shape, getattr(self.exponent, "shape", ())) + + +def _clipped_sigmoid(x): + finfo = torch.finfo(x.dtype) + return torch.clamp(torch.sigmoid(x), min=finfo.tiny, max=1.0 - finfo.eps) + + +class SigmoidTransform(Transform): + r""" + Transform via the mapping :math:`y = \frac{1}{1 + \exp(-x)}` and :math:`x = \text{logit}(y)`. + """ + + domain = constraints.real + codomain = constraints.unit_interval + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, SigmoidTransform) + + def _call(self, x): + return _clipped_sigmoid(x) + + def _inverse(self, y): + finfo = torch.finfo(y.dtype) + y = y.clamp(min=finfo.tiny, max=1.0 - finfo.eps) + return y.log() - (-y).log1p() + + def log_abs_det_jacobian(self, x, y): + return -F.softplus(-x) - F.softplus(x) + + +class SoftplusTransform(Transform): + r""" + Transform via the mapping :math:`\text{Softplus}(x) = \log(1 + \exp(x))`. + The implementation reverts to the linear function when :math:`x > 20`. + """ + + domain = constraints.real + codomain = constraints.positive + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, SoftplusTransform) + + def _call(self, x): + return softplus(x) + + def _inverse(self, y): + return (-y).expm1().neg().log() + y + + def log_abs_det_jacobian(self, x, y): + return -softplus(-x) + + +class TanhTransform(Transform): + r""" + Transform via the mapping :math:`y = \tanh(x)`. + + It is equivalent to + + .. code-block:: python + + ComposeTransform( + [ + AffineTransform(0.0, 2.0), + SigmoidTransform(), + AffineTransform(-1.0, 2.0), + ] + ) + + However this might not be numerically stable, thus it is recommended to use `TanhTransform` + instead. + + Note that one should use `cache_size=1` when it comes to `NaN/Inf` values. + + """ + + domain = constraints.real + codomain = constraints.interval(-1.0, 1.0) + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, TanhTransform) + + def _call(self, x): + return x.tanh() + + def _inverse(self, y): + # We do not clamp to the boundary here as it may degrade the performance of certain algorithms. + # one should use `cache_size=1` instead + return torch.atanh(y) + + def log_abs_det_jacobian(self, x, y): + # We use a formula that is more numerically stable, see details in the following link + # https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/bijectors/tanh.py#L69-L80 + return 2.0 * (math.log(2.0) - x - softplus(-2.0 * x)) + + +class AbsTransform(Transform): + r"""Transform via the mapping :math:`y = |x|`.""" + + domain = constraints.real + codomain = constraints.positive + + def __eq__(self, other): + return isinstance(other, AbsTransform) + + def _call(self, x): + return x.abs() + + def _inverse(self, y): + return y + + +class AffineTransform(Transform): + r""" + Transform via the pointwise affine mapping :math:`y = \text{loc} + \text{scale} \times x`. + + Args: + loc (Tensor or float): Location parameter. + scale (Tensor or float): Scale parameter. + event_dim (int): Optional size of `event_shape`. This should be zero + for univariate random variables, 1 for distributions over vectors, + 2 for distributions over matrices, etc. + """ + + bijective = True + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + event_dim: int = 0, + cache_size: int = 0, + ) -> None: + super().__init__(cache_size=cache_size) + self.loc = loc + self.scale = scale + self._event_dim = event_dim + + @property + def event_dim(self) -> int: + return self._event_dim + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + if self.event_dim == 0: + return constraints.real + return constraints.independent(constraints.real, self.event_dim) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + if self.event_dim == 0: + return constraints.real + return constraints.independent(constraints.real, self.event_dim) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return AffineTransform( + self.loc, self.scale, self.event_dim, cache_size=cache_size + ) + + def __eq__(self, other): + if not isinstance(other, AffineTransform): + return False + + if isinstance(self.loc, _Number) and isinstance(other.loc, _Number): + if self.loc != other.loc: + return False + else: + if not (self.loc == other.loc).all().item(): # type: ignore[union-attr] + return False + + if isinstance(self.scale, _Number) and isinstance(other.scale, _Number): + if self.scale != other.scale: + return False + else: + if not (self.scale == other.scale).all().item(): # type: ignore[union-attr] + return False + + return True + + @property + def sign(self) -> Union[Tensor, int]: # type: ignore[override] + if isinstance(self.scale, _Number): + return 1 if float(self.scale) > 0 else -1 if float(self.scale) < 0 else 0 + return self.scale.sign() + + def _call(self, x): + return self.loc + self.scale * x + + def _inverse(self, y): + return (y - self.loc) / self.scale + + def log_abs_det_jacobian(self, x, y): + shape = x.shape + scale = self.scale + if isinstance(scale, _Number): + result = torch.full_like(x, math.log(abs(scale))) + else: + result = torch.abs(scale).log() + if self.event_dim: + result_size = result.size()[: -self.event_dim] + (-1,) + result = result.view(result_size).sum(-1) + shape = shape[: -self.event_dim] + return result.expand(shape) + + def forward_shape(self, shape): + return torch.broadcast_shapes( + shape, getattr(self.loc, "shape", ()), getattr(self.scale, "shape", ()) + ) + + def inverse_shape(self, shape): + return torch.broadcast_shapes( + shape, getattr(self.loc, "shape", ()), getattr(self.scale, "shape", ()) + ) + + +class CorrCholeskyTransform(Transform): + r""" + Transforms an unconstrained real vector :math:`x` with length :math:`D*(D-1)/2` into the + Cholesky factor of a D-dimension correlation matrix. This Cholesky factor is a lower + triangular matrix with positive diagonals and unit Euclidean norm for each row. + The transform is processed as follows: + + 1. First we convert x into a lower triangular matrix in row order. + 2. For each row :math:`X_i` of the lower triangular part, we apply a *signed* version of + class :class:`StickBreakingTransform` to transform :math:`X_i` into a + unit Euclidean length vector using the following steps: + - Scales into the interval :math:`(-1, 1)` domain: :math:`r_i = \tanh(X_i)`. + - Transforms into an unsigned domain: :math:`z_i = r_i^2`. + - Applies :math:`s_i = StickBreakingTransform(z_i)`. + - Transforms back into signed domain: :math:`y_i = sign(r_i) * \sqrt{s_i}`. + """ + + domain = constraints.real_vector + codomain = constraints.corr_cholesky + bijective = True + + def _call(self, x): + x = torch.tanh(x) + eps = torch.finfo(x.dtype).eps + x = x.clamp(min=-1 + eps, max=1 - eps) + r = vec_to_tril_matrix(x, diag=-1) + # apply stick-breaking on the squared values + # Note that y = sign(r) * sqrt(z * z1m_cumprod) + # = (sign(r) * sqrt(z)) * sqrt(z1m_cumprod) = r * sqrt(z1m_cumprod) + # pyrefly: ignore [unsupported-operation] + z = r**2 + z1m_cumprod_sqrt = (1 - z).sqrt().cumprod(-1) + # Diagonal elements must be 1. + r = r + torch.eye(r.shape[-1], dtype=r.dtype, device=r.device) + y = r * pad(z1m_cumprod_sqrt[..., :-1], [1, 0], value=1) + return y + + def _inverse(self, y): + # inverse stick-breaking + # See: https://mc-stan.org/docs/2_18/reference-manual/cholesky-factors-of-correlation-matrices-1.html + y_cumsum = 1 - torch.cumsum(y * y, dim=-1) + y_cumsum_shifted = pad(y_cumsum[..., :-1], [1, 0], value=1) + y_vec = tril_matrix_to_vec(y, diag=-1) + y_cumsum_vec = tril_matrix_to_vec(y_cumsum_shifted, diag=-1) + t = y_vec / (y_cumsum_vec).sqrt() + # inverse of tanh + x = (t.log1p() - t.neg().log1p()) / 2 + return x + + def log_abs_det_jacobian(self, x, y, intermediates=None): + # Because domain and codomain are two spaces with different dimensions, determinant of + # Jacobian is not well-defined. We return `log_abs_det_jacobian` of `x` and the + # flattened lower triangular part of `y`. + + # See: https://mc-stan.org/docs/2_18/reference-manual/cholesky-factors-of-correlation-matrices-1.html + y1m_cumsum = 1 - (y * y).cumsum(dim=-1) + # by taking diagonal=-2, we don't need to shift z_cumprod to the right + # also works for 2 x 2 matrix + y1m_cumsum_tril = tril_matrix_to_vec(y1m_cumsum, diag=-2) + stick_breaking_logdet = 0.5 * (y1m_cumsum_tril).log().sum(-1) + tanh_logdet = -2 * (x + softplus(-2 * x) - math.log(2.0)).sum(dim=-1) + return stick_breaking_logdet + tanh_logdet + + def forward_shape(self, shape): + # Reshape from (..., N) to (..., D, D). + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + N = shape[-1] + D = round((0.25 + 2 * N) ** 0.5 + 0.5) + if D * (D - 1) // 2 != N: + raise ValueError("Input is not a flattened lower-diagonal number") + return shape[:-1] + (D, D) + + def inverse_shape(self, shape): + # Reshape from (..., D, D) to (..., N). + if len(shape) < 2: + raise ValueError("Too few dimensions on input") + if shape[-2] != shape[-1]: + raise ValueError("Input is not square") + D = shape[-1] + N = D * (D - 1) // 2 + return shape[:-2] + (N,) + + +class SoftmaxTransform(Transform): + r""" + Transform from unconstrained space to the simplex via :math:`y = \exp(x)` then + normalizing. + + This is not bijective and cannot be used for HMC. However this acts mostly + coordinate-wise (except for the final normalization), and thus is + appropriate for coordinate-wise optimization algorithms. + """ + + domain = constraints.real_vector + codomain = constraints.simplex + + def __eq__(self, other): + return isinstance(other, SoftmaxTransform) + + def _call(self, x): + logprobs = x + probs = (logprobs - logprobs.max(-1, True)[0]).exp() + return probs / probs.sum(-1, True) + + def _inverse(self, y): + probs = y + return probs.log() + + def forward_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape + + def inverse_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape + + +class StickBreakingTransform(Transform): + """ + Transform from unconstrained space to the simplex of one additional + dimension via a stick-breaking process. + + This transform arises as an iterated sigmoid transform in a stick-breaking + construction of the `Dirichlet` distribution: the first logit is + transformed via sigmoid to the first probability and the probability of + everything else, and then the process recurses. + + This is bijective and appropriate for use in HMC; however it mixes + coordinates together and is less appropriate for optimization. + """ + + domain = constraints.real_vector + codomain = constraints.simplex + bijective = True + + def __eq__(self, other): + return isinstance(other, StickBreakingTransform) + + def _call(self, x): + offset = x.shape[-1] + 1 - x.new_ones(x.shape[-1]).cumsum(-1) + z = _clipped_sigmoid(x - offset.log()) + z_cumprod = (1 - z).cumprod(-1) + y = pad(z, [0, 1], value=1) * pad(z_cumprod, [1, 0], value=1) + return y + + def _inverse(self, y): + y_crop = y[..., :-1] + offset = y.shape[-1] - y.new_ones(y_crop.shape[-1]).cumsum(-1) + sf = 1 - y_crop.cumsum(-1) + # we clamp to make sure that sf is positive which sometimes does not + # happen when y[-1] ~ 0 or y[:-1].sum() ~ 1 + sf = torch.clamp(sf, min=torch.finfo(y.dtype).tiny) + x = y_crop.log() - sf.log() + offset.log() + return x + + def log_abs_det_jacobian(self, x, y): + offset = x.shape[-1] + 1 - x.new_ones(x.shape[-1]).cumsum(-1) + x = x - offset.log() + # use the identity 1 - sigmoid(x) = exp(-x) * sigmoid(x) + detJ = (-x + F.logsigmoid(x) + y[..., :-1].log()).sum(-1) + return detJ + + def forward_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape[:-1] + (shape[-1] + 1,) + + def inverse_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape[:-1] + (shape[-1] - 1,) + + +class LowerCholeskyTransform(Transform): + """ + Transform from unconstrained matrices to lower-triangular matrices with + nonnegative diagonal entries. + + This is useful for parameterizing positive definite matrices in terms of + their Cholesky factorization. + """ + + domain = constraints.independent(constraints.real, 2) + codomain = constraints.lower_cholesky + + def __eq__(self, other): + return isinstance(other, LowerCholeskyTransform) + + def _call(self, x): + return x.tril(-1) + x.diagonal(dim1=-2, dim2=-1).exp().diag_embed() + + def _inverse(self, y): + return y.tril(-1) + y.diagonal(dim1=-2, dim2=-1).log().diag_embed() + + +class PositiveDefiniteTransform(Transform): + """ + Transform from unconstrained matrices to positive-definite matrices. + """ + + domain = constraints.independent(constraints.real, 2) + codomain = constraints.positive_definite + + def __eq__(self, other): + return isinstance(other, PositiveDefiniteTransform) + + def _call(self, x): + x = LowerCholeskyTransform()(x) + return x @ x.mT + + def _inverse(self, y): + y = torch.linalg.cholesky(y) + return LowerCholeskyTransform().inv(y) + + +class CatTransform(Transform): + """ + Transform functor that applies a sequence of transforms `tseq` + component-wise to each submatrix at `dim`, of length `lengths[dim]`, + in a way compatible with :func:`torch.cat`. + + Example:: + + x0 = torch.cat([torch.range(1, 10), torch.range(1, 10)], dim=0) + x = torch.cat([x0, x0], dim=0) + t0 = CatTransform([ExpTransform(), identity_transform], dim=0, lengths=[10, 10]) + t = CatTransform([t0, t0], dim=0, lengths=[20, 20]) + y = t(x) + """ + + transforms: list[Transform] + + def __init__( + self, + tseq: Sequence[Transform], + dim: int = 0, + lengths: Optional[Sequence[int]] = None, + cache_size: int = 0, + ) -> None: + assert all(isinstance(t, Transform) for t in tseq) + if cache_size: + tseq = [t.with_cache(cache_size) for t in tseq] + super().__init__(cache_size=cache_size) + self.transforms = list(tseq) + if lengths is None: + lengths = [1] * len(self.transforms) + self.lengths = list(lengths) + assert len(self.lengths) == len(self.transforms) + self.dim = dim + + @lazy_property + def event_dim(self) -> int: # type: ignore[override] + return max(t.event_dim for t in self.transforms) + + @lazy_property + def length(self) -> int: + return sum(self.lengths) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return CatTransform(self.transforms, self.dim, self.lengths, cache_size) + + def _call(self, x): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == self.length + yslices = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + xslice = x.narrow(self.dim, start, length) + yslices.append(trans(xslice)) + start = start + length # avoid += for jit compat + return torch.cat(yslices, dim=self.dim) + + def _inverse(self, y): + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == self.length + xslices = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + yslice = y.narrow(self.dim, start, length) + xslices.append(trans.inv(yslice)) + start = start + length # avoid += for jit compat + return torch.cat(xslices, dim=self.dim) + + def log_abs_det_jacobian(self, x, y): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == self.length + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == self.length + logdetjacs = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + xslice = x.narrow(self.dim, start, length) + yslice = y.narrow(self.dim, start, length) + logdetjac = trans.log_abs_det_jacobian(xslice, yslice) + if trans.event_dim < self.event_dim: + logdetjac = _sum_rightmost(logdetjac, self.event_dim - trans.event_dim) + logdetjacs.append(logdetjac) + start = start + length # avoid += for jit compat + # Decide whether to concatenate or sum. + dim = self.dim + if dim >= 0: + dim = dim - x.dim() + dim = dim + self.event_dim + if dim < 0: + return torch.cat(logdetjacs, dim=dim) + else: + return sum(logdetjacs) + + @property + def bijective(self) -> bool: # type: ignore[override] + return all(t.bijective for t in self.transforms) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.cat( + [t.domain for t in self.transforms], self.dim, self.lengths + ) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.cat( + [t.codomain for t in self.transforms], self.dim, self.lengths + ) + + +class StackTransform(Transform): + """ + Transform functor that applies a sequence of transforms `tseq` + component-wise to each submatrix at `dim` + in a way compatible with :func:`torch.stack`. + + Example:: + + x = torch.stack([torch.range(1, 10), torch.range(1, 10)], dim=1) + t = StackTransform([ExpTransform(), identity_transform], dim=1) + y = t(x) + """ + + transforms: list[Transform] + + def __init__( + self, tseq: Sequence[Transform], dim: int = 0, cache_size: int = 0 + ) -> None: + assert all(isinstance(t, Transform) for t in tseq) + if cache_size: + tseq = [t.with_cache(cache_size) for t in tseq] + super().__init__(cache_size=cache_size) + self.transforms = list(tseq) + self.dim = dim + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return StackTransform(self.transforms, self.dim, cache_size) + + def _slice(self, z): + return [z.select(self.dim, i) for i in range(z.size(self.dim))] + + def _call(self, x): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == len(self.transforms) + yslices = [] + for xslice, trans in zip(self._slice(x), self.transforms): + yslices.append(trans(xslice)) + return torch.stack(yslices, dim=self.dim) + + def _inverse(self, y): + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == len(self.transforms) + xslices = [] + for yslice, trans in zip(self._slice(y), self.transforms): + xslices.append(trans.inv(yslice)) + return torch.stack(xslices, dim=self.dim) + + def log_abs_det_jacobian(self, x, y): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == len(self.transforms) + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == len(self.transforms) + logdetjacs = [] + yslices = self._slice(y) + xslices = self._slice(x) + for xslice, yslice, trans in zip(xslices, yslices, self.transforms): + logdetjacs.append(trans.log_abs_det_jacobian(xslice, yslice)) + return torch.stack(logdetjacs, dim=self.dim) + + @property + def bijective(self) -> bool: # type: ignore[override] + return all(t.bijective for t in self.transforms) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.stack([t.domain for t in self.transforms], self.dim) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.stack([t.codomain for t in self.transforms], self.dim) + + +class CumulativeDistributionTransform(Transform): + """ + Transform via the cumulative distribution function of a probability distribution. + + Args: + distribution (Distribution): Distribution whose cumulative distribution function to use for + the transformation. + + Example:: + + # Construct a Gaussian copula from a multivariate normal. + base_dist = MultivariateNormal( + loc=torch.zeros(2), + scale_tril=LKJCholesky(2).sample(), + ) + transform = CumulativeDistributionTransform(Normal(0, 1)) + copula = TransformedDistribution(base_dist, [transform]) + """ + + bijective = True + codomain = constraints.unit_interval + sign = +1 + + def __init__(self, distribution: Distribution, cache_size: int = 0) -> None: + super().__init__(cache_size=cache_size) + self.distribution = distribution + + @property + def domain(self) -> Optional[constraints.Constraint]: # type: ignore[override] + return self.distribution.support + + def _call(self, x): + return self.distribution.cdf(x) + + def _inverse(self, y): + return self.distribution.icdf(y) + + def log_abs_det_jacobian(self, x, y): + return self.distribution.log_prob(x) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return CumulativeDistributionTransform(self.distribution, cache_size=cache_size) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/uniform.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/uniform.py new file mode 100644 index 0000000000000000000000000000000000000000..230bc9b31d786f307a9619f6eaabaada53226d90 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/uniform.py @@ -0,0 +1,109 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Uniform"] + + +class Uniform(Distribution): + r""" + Generates uniformly distributed random samples from the half-open interval + ``[low, high)``. + + Example:: + + >>> m = Uniform(torch.tensor([0.0]), torch.tensor([5.0])) + >>> m.sample() # uniformly distributed in the range [0.0, 5.0) + >>> # xdoctest: +SKIP + tensor([ 2.3418]) + + Args: + low (float or Tensor): lower range (inclusive). + high (float or Tensor): upper range (exclusive). + """ + + has_rsample = True + + @property + def arg_constraints(self): + # TODO allow (loc,scale) parameterization to allow independent constraints. + return { + "low": constraints.less_than(self.high), + "high": constraints.greater_than(self.low), + } + + @property + def mean(self) -> Tensor: + return (self.high + self.low) / 2 + + @property + def mode(self) -> Tensor: + return nan * self.high + + @property + def stddev(self) -> Tensor: + return (self.high - self.low) / 12**0.5 + + @property + def variance(self) -> Tensor: + return (self.high - self.low).pow(2) / 12 + + def __init__( + self, + low: Union[Tensor, float], + high: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.low, self.high = broadcast_all(low, high) + + if isinstance(low, _Number) and isinstance(high, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.low.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Uniform, _instance) + batch_shape = torch.Size(batch_shape) + new.low = self.low.expand(batch_shape) + new.high = self.high.expand(batch_shape) + super(Uniform, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property(is_discrete=False, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.interval(self.low, self.high) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + rand = torch.rand(shape, dtype=self.low.dtype, device=self.low.device) + return self.low + rand * (self.high - self.low) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + lb = self.low.le(value).type_as(self.low) + ub = self.high.gt(value).type_as(self.low) + return torch.log(lb.mul(ub)) - torch.log(self.high - self.low) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + result = (value - self.low) / (self.high - self.low) + return result.clamp(min=0, max=1) + + def icdf(self, value): + result = value * (self.high - self.low) + self.low + return result + + def entropy(self): + return torch.log(self.high - self.low) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..01307b88f211577520186e4cb4bbbf2dbdfdefd2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/utils.py @@ -0,0 +1,221 @@ +from collections.abc import Callable, Sequence +from functools import update_wrapper +from typing import Any, Final, Generic, Optional, overload, TypeVar, Union + +import torch +import torch.nn.functional as F +from torch import SymInt, Tensor +from torch.overrides import is_tensor_like +from torch.types import _dtype, _Number, Device, Number + + +euler_constant: Final[float] = 0.57721566490153286060 # Euler Mascheroni Constant + +__all__ = [ + "broadcast_all", + "logits_to_probs", + "clamp_probs", + "probs_to_logits", + "lazy_property", + "tril_matrix_to_vec", + "vec_to_tril_matrix", +] + + +# FIXME: Use (*values: *Ts) -> tuple[Tensor for T in Ts] if Mapping-Type is ever added. +# See https://github.com/python/typing/issues/1216#issuecomment-2126153831 +def broadcast_all(*values: Union[Tensor, Number]) -> tuple[Tensor, ...]: + r""" + Given a list of values (possibly containing numbers), returns a list where each + value is broadcasted based on the following rules: + - `torch.*Tensor` instances are broadcasted as per :ref:`_broadcasting-semantics`. + - Number instances (scalars) are upcast to tensors having + the same size and type as the first tensor passed to `values`. If all the + values are scalars, then they are upcasted to scalar Tensors. + + Args: + values (list of `Number`, `torch.*Tensor` or objects implementing __torch_function__) + + Raises: + ValueError: if any of the values is not a `Number` instance, + a `torch.*Tensor` instance, or an instance implementing __torch_function__ + """ + if not all(is_tensor_like(v) or isinstance(v, _Number) for v in values): + raise ValueError( + "Input arguments must all be instances of Number, " + "torch.Tensor or objects implementing __torch_function__." + ) + if not all(is_tensor_like(v) for v in values): + options: dict[str, Any] = dict(dtype=torch.get_default_dtype()) + for value in values: + if isinstance(value, torch.Tensor): + options = dict(dtype=value.dtype, device=value.device) + break + new_values = [ + v if is_tensor_like(v) else torch.tensor(v, **options) for v in values + ] + return torch.broadcast_tensors(*new_values) + return torch.broadcast_tensors(*values) + + +def _standard_normal( + shape: Sequence[Union[int, SymInt]], + dtype: Optional[_dtype], + device: Optional[Device], +) -> Tensor: + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .normal_() + return torch.normal( + torch.zeros(shape, dtype=dtype, device=device), + torch.ones(shape, dtype=dtype, device=device), + ) + return torch.empty(shape, dtype=dtype, device=device).normal_() + + +def _sum_rightmost(value: Tensor, dim: int) -> Tensor: + r""" + Sum out ``dim`` many rightmost dimensions of a given tensor. + + Args: + value (Tensor): A tensor of ``.dim()`` at least ``dim``. + dim (int): The number of rightmost dims to sum out. + """ + if dim == 0: + return value + required_shape = value.shape[:-dim] + (-1,) + return value.reshape(required_shape).sum(-1) + + +def logits_to_probs(logits: Tensor, is_binary: bool = False) -> Tensor: + r""" + Converts a tensor of logits into probabilities. Note that for the + binary case, each value denotes log odds, whereas for the + multi-dimensional case, the values along the last dimension denote + the log probabilities (possibly unnormalized) of the events. + """ + if is_binary: + return torch.sigmoid(logits) + return F.softmax(logits, dim=-1) + + +def clamp_probs(probs: Tensor) -> Tensor: + """Clamps the probabilities to be in the open interval `(0, 1)`. + + The probabilities would be clamped between `eps` and `1 - eps`, + and `eps` would be the smallest representable positive number for the input data type. + + Args: + probs (Tensor): A tensor of probabilities. + + Returns: + Tensor: The clamped probabilities. + + Examples: + >>> probs = torch.tensor([0.0, 0.5, 1.0]) + >>> clamp_probs(probs) + tensor([1.1921e-07, 5.0000e-01, 1.0000e+00]) + + >>> probs = torch.tensor([0.0, 0.5, 1.0], dtype=torch.float64) + >>> clamp_probs(probs) + tensor([2.2204e-16, 5.0000e-01, 1.0000e+00], dtype=torch.float64) + + """ + eps = torch.finfo(probs.dtype).eps + return probs.clamp(min=eps, max=1 - eps) + + +def probs_to_logits(probs: Tensor, is_binary: bool = False) -> Tensor: + r""" + Converts a tensor of probabilities into logits. For the binary case, + this denotes the probability of occurrence of the event indexed by `1`. + For the multi-dimensional case, the values along the last dimension + denote the probabilities of occurrence of each of the events. + """ + ps_clamped = clamp_probs(probs) + if is_binary: + return torch.log(ps_clamped) - torch.log1p(-ps_clamped) + return torch.log(ps_clamped) + + +T = TypeVar("T", contravariant=True) +R = TypeVar("R", covariant=True) + + +class lazy_property(Generic[T, R]): + r""" + Used as a decorator for lazy loading of class attributes. This uses a + non-data descriptor that calls the wrapped method to compute the property on + first call; thereafter replacing the wrapped method into an instance + attribute. + """ + + def __init__(self, wrapped: Callable[[T], R]) -> None: + self.wrapped: Callable[[T], R] = wrapped + update_wrapper(self, wrapped) # type:ignore[arg-type] + + @overload + def __get__( + self, instance: None, obj_type: Any = None + ) -> "_lazy_property_and_property[T, R]": ... + + @overload + def __get__(self, instance: T, obj_type: Any = None) -> R: ... + + def __get__( + self, instance: Union[T, None], obj_type: Any = None + ) -> "R | _lazy_property_and_property[T, R]": + if instance is None: + return _lazy_property_and_property(self.wrapped) + with torch.enable_grad(): + value = self.wrapped(instance) + setattr(instance, self.wrapped.__name__, value) + return value + + +class _lazy_property_and_property(lazy_property[T, R], property): + """We want lazy properties to look like multiple things. + + * property when Sphinx autodoc looks + * lazy_property when Distribution validate_args looks + """ + + def __init__(self, wrapped: Callable[[T], R]) -> None: + property.__init__(self, wrapped) + + +def tril_matrix_to_vec(mat: Tensor, diag: int = 0) -> Tensor: + r""" + Convert a `D x D` matrix or a batch of matrices into a (batched) vector + which comprises of lower triangular elements from the matrix in row order. + """ + n = mat.shape[-1] + if not torch._C._get_tracing_state() and (diag < -n or diag >= n): + raise ValueError(f"diag ({diag}) provided is outside [{-n}, {n - 1}].") + arange = torch.arange(n, device=mat.device) + tril_mask = arange < arange.view(-1, 1) + (diag + 1) + vec = mat[..., tril_mask] + return vec + + +def vec_to_tril_matrix(vec: Tensor, diag: int = 0) -> Tensor: + r""" + Convert a vector or a batch of vectors into a batched `D x D` + lower triangular matrix containing elements from the vector in row order. + """ + # +ve root of D**2 + (1+2*diag)*D - |diag| * (diag+1) - 2*vec.shape[-1] = 0 + n = ( + -(1 + 2 * diag) + + ((1 + 2 * diag) ** 2 + 8 * vec.shape[-1] + 4 * abs(diag) * (diag + 1)) ** 0.5 + ) / 2 + eps = torch.finfo(vec.dtype).eps + if not torch._C._get_tracing_state() and (round(n) - n > eps): + raise ValueError( + f"The size of last dimension is {vec.shape[-1]} which cannot be expressed as " + + "the lower triangular part of a square D x D matrix." + ) + n = round(n.item()) if isinstance(n, torch.Tensor) else round(n) + mat = vec.new_zeros(vec.shape[:-1] + torch.Size((n, n))) + arange = torch.arange(n, device=vec.device) + tril_mask = arange < arange.view(-1, 1) + (diag + 1) + mat[..., tril_mask] = vec + return mat diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/von_mises.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/von_mises.py new file mode 100644 index 0000000000000000000000000000000000000000..f946e4a6548122056a760e719db0f9b0ded1b1eb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/von_mises.py @@ -0,0 +1,223 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +import torch.jit +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all, lazy_property + + +__all__ = ["VonMises"] + + +def _eval_poly(y, coef): + coef = list(coef) + result = coef.pop() + while coef: + result = coef.pop() + y * result + return result + + +_I0_COEF_SMALL = [ + 1.0, + 3.5156229, + 3.0899424, + 1.2067492, + 0.2659732, + 0.360768e-1, + 0.45813e-2, +] +_I0_COEF_LARGE = [ + 0.39894228, + 0.1328592e-1, + 0.225319e-2, + -0.157565e-2, + 0.916281e-2, + -0.2057706e-1, + 0.2635537e-1, + -0.1647633e-1, + 0.392377e-2, +] +_I1_COEF_SMALL = [ + 0.5, + 0.87890594, + 0.51498869, + 0.15084934, + 0.2658733e-1, + 0.301532e-2, + 0.32411e-3, +] +_I1_COEF_LARGE = [ + 0.39894228, + -0.3988024e-1, + -0.362018e-2, + 0.163801e-2, + -0.1031555e-1, + 0.2282967e-1, + -0.2895312e-1, + 0.1787654e-1, + -0.420059e-2, +] + +_COEF_SMALL = [_I0_COEF_SMALL, _I1_COEF_SMALL] +_COEF_LARGE = [_I0_COEF_LARGE, _I1_COEF_LARGE] + + +def _log_modified_bessel_fn(x, order=0): + """ + Returns ``log(I_order(x))`` for ``x > 0``, + where `order` is either 0 or 1. + """ + assert order == 0 or order == 1 + + # compute small solution + y = x / 3.75 + y = y * y + small = _eval_poly(y, _COEF_SMALL[order]) + if order == 1: + small = x.abs() * small + small = small.log() + + # compute large solution + y = 3.75 / x + large = x - 0.5 * x.log() + _eval_poly(y, _COEF_LARGE[order]).log() + + result = torch.where(x < 3.75, small, large) + return result + + +@torch.jit.script_if_tracing +def _rejection_sample(loc, concentration, proposal_r, x): + done = torch.zeros(x.shape, dtype=torch.bool, device=loc.device) + # pyrefly: ignore [bad-assignment] + while not done.all(): + u = torch.rand((3,) + x.shape, dtype=loc.dtype, device=loc.device) + u1, u2, u3 = u.unbind() + z = torch.cos(math.pi * u1) + f = (1 + proposal_r * z) / (proposal_r + z) + c = concentration * (proposal_r - f) + accept = ((c * (2 - c) - u2) > 0) | ((c / u2).log() + 1 - c >= 0) + if accept.any(): + # pyrefly: ignore [no-matching-overload] + x = torch.where(accept, (u3 - 0.5).sign() * f.acos(), x) + done = done | accept + return (x + math.pi + loc) % (2 * math.pi) - math.pi + + +class VonMises(Distribution): + """ + A circular von Mises distribution. + + This implementation uses polar coordinates. The ``loc`` and ``value`` args + can be any real number (to facilitate unconstrained optimization), but are + interpreted as angles modulo 2 pi. + + Example:: + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = VonMises(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # von Mises distributed with loc=1 and concentration=1 + tensor([1.9777]) + + :param torch.Tensor loc: an angle in radians. + :param torch.Tensor concentration: concentration parameter + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "concentration": constraints.positive} + support = constraints.real + has_rsample = False + + def __init__( + self, + loc: Tensor, + concentration: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.concentration = broadcast_all(loc, concentration) + batch_shape = self.loc.shape + event_shape = torch.Size() + super().__init__(batch_shape, event_shape, validate_args) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_prob = self.concentration * torch.cos(value - self.loc) + log_prob = ( + log_prob + - math.log(2 * math.pi) + - _log_modified_bessel_fn(self.concentration, order=0) + ) + return log_prob + + @lazy_property + def _loc(self) -> Tensor: + return self.loc.to(torch.double) + + @lazy_property + def _concentration(self) -> Tensor: + return self.concentration.to(torch.double) + + @lazy_property + def _proposal_r(self) -> Tensor: + kappa = self._concentration + # pyrefly: ignore [unsupported-operation] + tau = 1 + (1 + 4 * kappa**2).sqrt() + rho = (tau - (2 * tau).sqrt()) / (2 * kappa) + # pyrefly: ignore [unsupported-operation] + _proposal_r = (1 + rho**2) / (2 * rho) + # second order Taylor expansion around 0 for small kappa + _proposal_r_taylor = 1 / kappa + kappa + return torch.where(kappa < 1e-5, _proposal_r_taylor, _proposal_r) + + @torch.no_grad() + def sample(self, sample_shape=torch.Size()): + """ + The sampling algorithm for the von Mises distribution is based on the + following paper: D.J. Best and N.I. Fisher, "Efficient simulation of the + von Mises distribution." Applied Statistics (1979): 152-157. + + Sampling is always done in double precision internally to avoid a hang + in _rejection_sample() for small values of the concentration, which + starts to happen for single precision around 1e-4 (see issue #88443). + """ + shape = self._extended_shape(sample_shape) + x = torch.empty(shape, dtype=self._loc.dtype, device=self.loc.device) + return _rejection_sample( + self._loc, self._concentration, self._proposal_r, x + ).to(self.loc.dtype) + + def expand(self, batch_shape, _instance=None): + try: + return super().expand(batch_shape) + except NotImplementedError: + validate_args = self.__dict__.get("_validate_args") + loc = self.loc.expand(batch_shape) + concentration = self.concentration.expand(batch_shape) + return type(self)(loc, concentration, validate_args=validate_args) + + @property + def mean(self) -> Tensor: + """ + The provided mean is the circular one. + """ + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @lazy_property + def variance(self) -> Tensor: # type: ignore[override] + """ + The provided variance is the circular one. + """ + return ( + 1 + - ( + _log_modified_bessel_fn(self.concentration, order=1) + - _log_modified_bessel_fn(self.concentration, order=0) + ).exp() + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/weibull.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/weibull.py new file mode 100644 index 0000000000000000000000000000000000000000..588e9f37ac4dcaafc2eb651ef4d9b591f928bc5f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/weibull.py @@ -0,0 +1,97 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exponential import Exponential +from torch.distributions.gumbel import euler_constant +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, PowerTransform +from torch.distributions.utils import broadcast_all + + +__all__ = ["Weibull"] + + +class Weibull(TransformedDistribution): + r""" + Samples from a two-parameter Weibull distribution. + + Example: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Weibull(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Weibull distribution with scale=1, concentration=1 + tensor([ 0.4784]) + + Args: + scale (float or Tensor): Scale parameter of distribution (lambda). + concentration (float or Tensor): Concentration parameter of distribution (k/shape). + validate_args (bool, optional): Whether to validate arguments. Default: None. + """ + + arg_constraints = { + "scale": constraints.positive, + "concentration": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.positive + + def __init__( + self, + scale: Union[Tensor, float], + concentration: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.scale, self.concentration = broadcast_all(scale, concentration) + self.concentration_reciprocal = self.concentration.reciprocal() + base_dist = Exponential( + torch.ones_like(self.scale), validate_args=validate_args + ) + transforms = [ + PowerTransform(exponent=self.concentration_reciprocal), + AffineTransform(loc=0, scale=self.scale), + ] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Weibull, _instance) + new.scale = self.scale.expand(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + new.concentration_reciprocal = new.concentration.reciprocal() + base_dist = self.base_dist.expand(batch_shape) + transforms = [ + PowerTransform(exponent=new.concentration_reciprocal), + AffineTransform(loc=0, scale=new.scale), + ] + super(Weibull, new).__init__(base_dist, transforms, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.scale * torch.exp(torch.lgamma(1 + self.concentration_reciprocal)) + + @property + def mode(self) -> Tensor: + return ( + self.scale + * ((self.concentration - 1) / self.concentration) + ** self.concentration.reciprocal() + ) + + @property + def variance(self) -> Tensor: + return self.scale.pow(2) * ( + torch.exp(torch.lgamma(1 + 2 * self.concentration_reciprocal)) + - torch.exp(2 * torch.lgamma(1 + self.concentration_reciprocal)) + ) + + def entropy(self): + return ( + euler_constant * (1 - self.concentration_reciprocal) + + torch.log(self.scale * self.concentration_reciprocal) + + 1 + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/wishart.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/wishart.py new file mode 100644 index 0000000000000000000000000000000000000000..51f89b1f91588a9109c764e71f5d178f66a6a827 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/distributions/wishart.py @@ -0,0 +1,347 @@ +# mypy: allow-untyped-defs +import math +import warnings +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.multivariate_normal import _precision_to_scale_tril +from torch.distributions.utils import lazy_property +from torch.types import _Number, _size, Number + + +__all__ = ["Wishart"] + +_log_2 = math.log(2) + + +def _mvdigamma(x: Tensor, p: int) -> Tensor: + assert x.gt((p - 1) / 2).all(), "Wrong domain for multivariate digamma function." + return torch.digamma( + x.unsqueeze(-1) + - torch.arange(p, dtype=x.dtype, device=x.device).div(2).expand(x.shape + (-1,)) + ).sum(-1) + + +def _clamp_above_eps(x: Tensor) -> Tensor: + # We assume positive input for this function + return x.clamp(min=torch.finfo(x.dtype).eps) + + +class Wishart(ExponentialFamily): + r""" + Creates a Wishart distribution parameterized by a symmetric positive definite matrix :math:`\Sigma`, + or its Cholesky decomposition :math:`\mathbf{\Sigma} = \mathbf{L}\mathbf{L}^\top` + + Example: + >>> # xdoctest: +SKIP("FIXME: scale_tril must be at least two-dimensional") + >>> m = Wishart(torch.Tensor([2]), covariance_matrix=torch.eye(2)) + >>> m.sample() # Wishart distributed with mean=`df * I` and + >>> # variance(x_ij)=`df` for i != j and variance(x_ij)=`2 * df` for i == j + + Args: + df (float or Tensor): real-valued parameter larger than the (dimension of Square matrix) - 1 + covariance_matrix (Tensor): positive-definite covariance matrix + precision_matrix (Tensor): positive-definite precision matrix + scale_tril (Tensor): lower-triangular factor of covariance, with positive-valued diagonal + Note: + Only one of :attr:`covariance_matrix` or :attr:`precision_matrix` or + :attr:`scale_tril` can be specified. + Using :attr:`scale_tril` will be more efficient: all computations internally + are based on :attr:`scale_tril`. If :attr:`covariance_matrix` or + :attr:`precision_matrix` is passed instead, it is only used to compute + the corresponding lower triangular matrices using a Cholesky decomposition. + 'torch.distributions.LKJCholesky' is a restricted Wishart distribution.[1] + + **References** + + [1] Wang, Z., Wu, Y. and Chu, H., 2018. `On equivalence of the LKJ distribution and the restricted Wishart distribution`. + [2] Sawyer, S., 2007. `Wishart Distributions and Inverse-Wishart Sampling`. + [3] Anderson, T. W., 2003. `An Introduction to Multivariate Statistical Analysis (3rd ed.)`. + [4] Odell, P. L. & Feiveson, A. H., 1966. `A Numerical Procedure to Generate a SampleCovariance Matrix`. JASA, 61(313):199-203. + [5] Ku, Y.-C. & Bloomfield, P., 2010. `Generating Random Wishart Matrices with Fractional Degrees of Freedom in OX`. + """ + + support = constraints.positive_definite + has_rsample = True + _mean_carrier_measure = 0 + + @property + def arg_constraints(self): + return { + "covariance_matrix": constraints.positive_definite, + "precision_matrix": constraints.positive_definite, + "scale_tril": constraints.lower_cholesky, + "df": constraints.greater_than(self.event_shape[-1] - 1), + } + + def __init__( + self, + df: Union[Tensor, Number], + covariance_matrix: Optional[Tensor] = None, + precision_matrix: Optional[Tensor] = None, + scale_tril: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + assert (covariance_matrix is not None) + (scale_tril is not None) + ( + precision_matrix is not None + ) == 1, ( + "Exactly one of covariance_matrix or precision_matrix or scale_tril may be specified." + ) + + param = next( + p + for p in (covariance_matrix, precision_matrix, scale_tril) + if p is not None + ) + + if param.dim() < 2: + raise ValueError( + "scale_tril must be at least two-dimensional, with optional leading batch dimensions" + ) + + if isinstance(df, _Number): + batch_shape = torch.Size(param.shape[:-2]) + self.df = torch.tensor(df, dtype=param.dtype, device=param.device) + else: + batch_shape = torch.broadcast_shapes(param.shape[:-2], df.shape) + self.df = df.expand(batch_shape) + event_shape = param.shape[-2:] + + if self.df.le(event_shape[-1] - 1).any(): + raise ValueError( + f"Value of df={df} expected to be greater than ndim - 1 = {event_shape[-1] - 1}." + ) + + if scale_tril is not None: + # pyrefly: ignore [read-only] + self.scale_tril = param.expand(batch_shape + (-1, -1)) + elif covariance_matrix is not None: + # pyrefly: ignore [read-only] + self.covariance_matrix = param.expand(batch_shape + (-1, -1)) + elif precision_matrix is not None: + # pyrefly: ignore [read-only] + self.precision_matrix = param.expand(batch_shape + (-1, -1)) + + if self.df.lt(event_shape[-1]).any(): + warnings.warn( + "Low df values detected. Singular samples are highly likely to occur for ndim - 1 < df < ndim.", + stacklevel=2, + ) + + super().__init__(batch_shape, event_shape, validate_args=validate_args) + self._batch_dims = [-(x + 1) for x in range(len(self._batch_shape))] + + if scale_tril is not None: + self._unbroadcasted_scale_tril = scale_tril + elif covariance_matrix is not None: + self._unbroadcasted_scale_tril = torch.linalg.cholesky(covariance_matrix) + else: # precision_matrix is not None + self._unbroadcasted_scale_tril = _precision_to_scale_tril(precision_matrix) + + # Chi2 distribution is needed for Bartlett decomposition sampling + self._dist_chi2 = torch.distributions.chi2.Chi2( + df=( + self.df.unsqueeze(-1) + - torch.arange( + self._event_shape[-1], + dtype=self._unbroadcasted_scale_tril.dtype, + device=self._unbroadcasted_scale_tril.device, + ).expand(batch_shape + (-1,)) + ) + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Wishart, _instance) + batch_shape = torch.Size(batch_shape) + cov_shape = batch_shape + self.event_shape + new._unbroadcasted_scale_tril = self._unbroadcasted_scale_tril.expand(cov_shape) + new.df = self.df.expand(batch_shape) + + new._batch_dims = [-(x + 1) for x in range(len(batch_shape))] + + if "covariance_matrix" in self.__dict__: + new.covariance_matrix = self.covariance_matrix.expand(cov_shape) + if "scale_tril" in self.__dict__: + new.scale_tril = self.scale_tril.expand(cov_shape) + if "precision_matrix" in self.__dict__: + new.precision_matrix = self.precision_matrix.expand(cov_shape) + + # Chi2 distribution is needed for Bartlett decomposition sampling + new._dist_chi2 = torch.distributions.chi2.Chi2( + df=( + new.df.unsqueeze(-1) + - torch.arange( + self.event_shape[-1], + dtype=new._unbroadcasted_scale_tril.dtype, + device=new._unbroadcasted_scale_tril.device, + ).expand(batch_shape + (-1,)) + ) + ) + + super(Wishart, new).__init__(batch_shape, self.event_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @lazy_property + def scale_tril(self) -> Tensor: + return self._unbroadcasted_scale_tril.expand( + self._batch_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + return ( + self._unbroadcasted_scale_tril + @ self._unbroadcasted_scale_tril.transpose(-2, -1) + ).expand(self._batch_shape + self._event_shape) + + @lazy_property + def precision_matrix(self) -> Tensor: + identity = torch.eye( + self._event_shape[-1], + device=self._unbroadcasted_scale_tril.device, + dtype=self._unbroadcasted_scale_tril.dtype, + ) + return torch.cholesky_solve(identity, self._unbroadcasted_scale_tril).expand( + self._batch_shape + self._event_shape + ) + + @property + def mean(self) -> Tensor: + return self.df.view(self._batch_shape + (1, 1)) * self.covariance_matrix + + @property + def mode(self) -> Tensor: + factor = self.df - self.covariance_matrix.shape[-1] - 1 + factor[factor <= 0] = nan + return factor.view(self._batch_shape + (1, 1)) * self.covariance_matrix + + @property + def variance(self) -> Tensor: + V = self.covariance_matrix # has shape (batch_shape x event_shape) + diag_V = V.diagonal(dim1=-2, dim2=-1) + return self.df.view(self._batch_shape + (1, 1)) * ( + V.pow(2) + torch.einsum("...i,...j->...ij", diag_V, diag_V) + ) + + def _bartlett_sampling(self, sample_shape=torch.Size()): + p = self._event_shape[-1] # has singleton shape + + # Implemented Sampling using Bartlett decomposition + noise = _clamp_above_eps( + self._dist_chi2.rsample(sample_shape).sqrt() + ).diag_embed(dim1=-2, dim2=-1) + + i, j = torch.tril_indices(p, p, offset=-1) + noise[..., i, j] = torch.randn( + torch.Size(sample_shape) + self._batch_shape + (int(p * (p - 1) / 2),), + dtype=noise.dtype, + device=noise.device, + ) + chol = self._unbroadcasted_scale_tril @ noise + return chol @ chol.transpose(-2, -1) + + def rsample( + self, sample_shape: _size = torch.Size(), max_try_correction=None + ) -> Tensor: + r""" + .. warning:: + In some cases, sampling algorithm based on Bartlett decomposition may return singular matrix samples. + Several tries to correct singular samples are performed by default, but it may end up returning + singular matrix samples. Singular samples may return `-inf` values in `.log_prob()`. + In those cases, the user should validate the samples and either fix the value of `df` + or adjust `max_try_correction` value for argument in `.rsample` accordingly. + """ + + if max_try_correction is None: + max_try_correction = 3 if torch._C._get_tracing_state() else 10 + + sample_shape = torch.Size(sample_shape) + sample = self._bartlett_sampling(sample_shape) + + # Below part is to improve numerical stability temporally and should be removed in the future + is_singular = self.support.check(sample) + if self._batch_shape: + is_singular = is_singular.amax(self._batch_dims) + + if torch._C._get_tracing_state(): + # Less optimized version for JIT + for _ in range(max_try_correction): + sample_new = self._bartlett_sampling(sample_shape) + sample = torch.where(is_singular, sample_new, sample) + + is_singular = ~self.support.check(sample) + if self._batch_shape: + is_singular = is_singular.amax(self._batch_dims) + + else: + # More optimized version with data-dependent control flow. + if is_singular.any(): + warnings.warn("Singular sample detected.", stacklevel=2) + + for _ in range(max_try_correction): + sample_new = self._bartlett_sampling(is_singular[is_singular].shape) + sample[is_singular] = sample_new + + is_singular_new = ~self.support.check(sample_new) + if self._batch_shape: + is_singular_new = is_singular_new.amax(self._batch_dims) + is_singular[is_singular.clone()] = is_singular_new + + if not is_singular.any(): + break + + return sample + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return ( + -nu + * ( + p * _log_2 / 2 + + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1) + .log() + .sum(-1) + ) + - torch.mvlgamma(nu / 2, p=p) + + (nu - p - 1) / 2 * torch.linalg.slogdet(value).logabsdet + - torch.cholesky_solve(value, self._unbroadcasted_scale_tril) + .diagonal(dim1=-2, dim2=-1) + .sum(dim=-1) + / 2 + ) + + def entropy(self): + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return ( + (p + 1) + * ( + p * _log_2 / 2 + + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1) + .log() + .sum(-1) + ) + + torch.mvlgamma(nu / 2, p=p) + - (nu - p - 1) / 2 * _mvdigamma(nu / 2, p=p) + + nu * p / 2 + ) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return -self.precision_matrix / 2, (nu - p - 1) / 2 + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + p = self._event_shape[-1] + return (y + (p + 1) / 2) * ( + -torch.linalg.slogdet(-2 * x).logabsdet + _log_2 * p + ) + torch.mvlgamma(y + (p + 1) / 2, p=p) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..63f6d927a1f912ab7d65c2dbbedb797410ea913d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/__init__.py @@ -0,0 +1,611 @@ +import logging +import os +import warnings +import zipfile +from collections.abc import Callable, Mapping +from typing import Any +from typing_extensions import deprecated + +import torch +import torch.utils._pytree as pytree +from torch.fx.passes.infra.pass_base import PassResult +from torch.types import FileLike + + +__all__ = [ + "AdditionalInputs", + "Constraint", + "CustomDecompTable", + "default_decompositions", + "Dim", + "dims", + "draft_export", + "export_for_training", + "export", + "ExportBackwardSignature", + "ExportedProgram", + "ExportGraphSignature", + "FlatArgsAdapter", + "load", + "ModuleCallEntry", + "ModuleCallSignature", + "register_dataclass", + "save", + "ShapesCollection", + "unflatten", + "UnflattenedModule", +] + +# To make sure export specific custom ops are loaded +import torch.export.custom_ops + +from .decomp_utils import CustomDecompTable +from .dynamic_shapes import AdditionalInputs, Constraint, Dim, dims, ShapesCollection +from .exported_program import ( + default_decompositions, + ExportedProgram, + ModuleCallEntry, + ModuleCallSignature, +) +from .graph_signature import ExportBackwardSignature, ExportGraphSignature +from .unflatten import FlatArgsAdapter, unflatten, UnflattenedModule + + +PassType = Callable[[torch.fx.GraphModule], PassResult | None] + +log: logging.Logger = logging.getLogger(__name__) + + +@deprecated( + "`torch.export.export_for_training` is deprecated and will be removed in PyTorch 2.10. " + "Please use `torch.export.export` instead, which is functionally equivalent.", + category=FutureWarning, +) +def export_for_training( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: Mapping[str, Any] | None = None, + *, + dynamic_shapes: dict[str, Any] | tuple[Any, ...] | list[Any] | None = None, + strict: bool = False, + preserve_module_call_signature: tuple[str, ...] = (), + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + """ + :func:`export_for_training` takes any nn.Module along with example inputs, and produces a traced graph representing + only the Tensor computation of the function in an Ahead-of-Time (AOT) fashion, + which can subsequently be executed with different inputs or serialized. The + traced graph (1) produces normalized operators in the all ATen operator set + (as well as any user-specified custom operators), (2) has eliminated all Python control + flow and data structures (with certain exceptions), and (3) records the set of + shape constraints needed to show that this normalization and control-flow elimination + is sound for future inputs. This API is intended for PT2 quantization training use cases + and will soon be the default IR of torch.export.export in the near future. To read further about + the motivation behind this change, please refer to + https://dev-discuss.pytorch.org/t/why-pytorch-does-not-need-a-new-standardized-operator-set/2206 + With this API, and :func:`run_decompositions()`, you should be able to get inference IR with + your custom decomposition behaviour. + + **Soundness Guarantee** + + See :func:`export()` docstring for more details. + + Args: + mod: We will trace the forward method of this module. + + args: Example positional inputs. + + kwargs: Optional example keyword inputs. + + dynamic_shapes: + An optional argument where the type should either be: + 1) a dict from argument names of ``f`` to their dynamic shape specifications, + 2) a tuple that specifies dynamic shape specifications for each input in original order. + If you are specifying dynamism on keyword args, you will need to pass them in the order that + is defined in the original function signature. + + The dynamic shape of a tensor argument can be specified as either + (1) a dict from dynamic dimension indices to :func:`Dim` types, where it is + not required to include static dimension indices in this dict, but when they are, + they should be mapped to None; or (2) a tuple / list of :func:`Dim` types or None, + where the :func:`Dim` types correspond to dynamic dimensions, and static dimensions + are denoted by None. Arguments that are dicts or tuples / lists of tensors are + recursively specified by using mappings or sequences of contained specifications. + + strict: When enabled (default), the export function will trace the program through + TorchDynamo which will ensure the soundness of the resulting graph. Otherwise, the + exported program will not validate the implicit assumptions baked into the graph and + may cause behavior divergence between the original model and the exported one. This is + useful when users need to workaround bugs in the tracer, or simply want incrementally + enable safety in their models. Note that this does not affect the resulting IR spec + to be different and the model will be serialized in the same way regardless of what value + is passed here. + WARNING: This option is experimental and use this at your own risk. + + preserve_module_call_signature: A list of submodule paths for which the original + calling conventions are preserved as metadata. The metadata will be used when calling + torch.export.unflatten to preserve the original calling conventions of modules. + + Returns: + An :class:`ExportedProgram` containing the traced callable. + + **Acceptable input/output types** + + Acceptable types of inputs (for ``args`` and ``kwargs``) and outputs include: + + - Primitive types, i.e. ``torch.Tensor``, ``int``, ``float``, ``bool`` and ``str``. + - Dataclasses, but they must be registered by calling :func:`register_dataclass` first. + - (Nested) Data structures comprising of ``dict``, ``list``, ``tuple``, ``namedtuple`` and + ``OrderedDict`` containing all above types. + + """ + from ._trace import _export_for_training + + if not isinstance(mod, torch.nn.Module): + raise ValueError( + f"Expected `mod` to be an instance of `torch.nn.Module`, got {type(mod)}." + ) + if isinstance(mod, torch.jit.ScriptModule): + raise ValueError( + "Exporting a ScriptModule is not supported. " + "Maybe try converting your ScriptModule to an ExportedProgram " + "using `TS2EPConverter(mod, args, kwargs).convert()` instead." + ) + return _export_for_training( + mod, + args, + kwargs, + dynamic_shapes, + strict=strict, + preserve_module_call_signature=preserve_module_call_signature, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + + +def export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: Mapping[str, Any] | None = None, + *, + dynamic_shapes: dict[str, Any] | tuple[Any, ...] | list[Any] | None = None, + strict: bool = False, + preserve_module_call_signature: tuple[str, ...] = (), + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + """ + :func:`export` takes any nn.Module along with example inputs, and produces a traced graph representing + only the Tensor computation of the function in an Ahead-of-Time (AOT) fashion, + which can subsequently be executed with different inputs or serialized. The + traced graph (1) produces normalized operators in the functional ATen operator set + (as well as any user-specified custom operators), (2) has eliminated all Python control + flow and data structures (with certain exceptions), and (3) records the set of + shape constraints needed to show that this normalization and control-flow elimination + is sound for future inputs. + + **Soundness Guarantee** + + While tracing, :func:`export()` takes note of shape-related assumptions + made by the user program and the underlying PyTorch operator kernels. + The output :class:`ExportedProgram` is considered valid only when these + assumptions hold true. + + Tracing makes assumptions on the shapes (not values) of input tensors. + Such assumptions must be validated at graph capture time for :func:`export` + to succeed. Specifically: + + - Assumptions on static shapes of input tensors are automatically validated without additional effort. + - Assumptions on dynamic shape of input tensors require explicit specification + by using the :func:`Dim` API to construct dynamic dimensions and by associating + them with example inputs through the ``dynamic_shapes`` argument. + + If any assumption can not be validated, a fatal error will be raised. When that happens, + the error message will include suggested fixes to the specification that are needed + to validate the assumptions. For example :func:`export` might suggest the + following fix to the definition of a dynamic dimension ``dim0_x``, say appearing in the + shape associated with input ``x``, that was previously defined as ``Dim("dim0_x")``:: + + dim = Dim("dim0_x", max=5) + + This example means the generated code requires dimension 0 of input ``x`` to be less + than or equal to 5 to be valid. You can inspect the suggested fixes to dynamic dimension + definitions and then copy them verbatim into your code without needing to change the + ``dynamic_shapes`` argument to your :func:`export` call. + + Args: + mod: We will trace the forward method of this module. + + args: Example positional inputs. + + kwargs: Optional example keyword inputs. + + dynamic_shapes: + An optional argument where the type should either be: + 1) a dict from argument names of ``f`` to their dynamic shape specifications, + 2) a tuple that specifies dynamic shape specifications for each input in original order. + If you are specifying dynamism on keyword args, you will need to pass them in the order that + is defined in the original function signature. + + The dynamic shape of a tensor argument can be specified as either + (1) a dict from dynamic dimension indices to :func:`Dim` types, where it is + not required to include static dimension indices in this dict, but when they are, + they should be mapped to None; or (2) a tuple / list of :func:`Dim` types or None, + where the :func:`Dim` types correspond to dynamic dimensions, and static dimensions + are denoted by None. Arguments that are dicts or tuples / lists of tensors are + recursively specified by using mappings or sequences of contained specifications. + + strict: When disabled (default), the export function will trace the program through + Python runtime, which by itself will not validate some of the implicit assumptions + baked into the graph. It will still validate most critical assumptions like shape + safety. When enabled (by setting ``strict=True``), the export function will trace + the program through TorchDynamo which will ensure the soundness of the resulting + graph. TorchDynamo has limited Python feature coverage, thus you may experience more + errors. Note that toggling this argument does not affect the resulting IR spec to be + different and the model will be serialized in the same way regardless of what value + is passed here. + + preserve_module_call_signature: A list of submodule paths for which the original + calling conventions are preserved as metadata. The metadata will be used when calling + torch.export.unflatten to preserve the original calling conventions of modules. + + Returns: + An :class:`ExportedProgram` containing the traced callable. + + **Acceptable input/output types** + + Acceptable types of inputs (for ``args`` and ``kwargs``) and outputs include: + + - Primitive types, i.e. ``torch.Tensor``, ``int``, ``float``, ``bool`` and ``str``. + - Dataclasses, but they must be registered by calling :func:`register_dataclass` first. + - (Nested) Data structures comprising of ``dict``, ``list``, ``tuple``, ``namedtuple`` and + ``OrderedDict`` containing all above types. + + """ + from ._trace import _export + + if not isinstance(mod, torch.nn.Module): + raise ValueError( + f"Expected `mod` to be an instance of `torch.nn.Module`, got {type(mod)}." + ) + if isinstance(mod, torch.jit.ScriptModule): + raise ValueError( + "Exporting a ScriptModule is not supported. " + "Maybe try converting your ScriptModule to an ExportedProgram " + "using `TS2EPConverter(mod, args, kwargs).convert()` instead." + ) + + try: + return _export( + mod, + args, + kwargs, + dynamic_shapes, + strict=strict, + preserve_module_call_signature=preserve_module_call_signature, + pre_dispatch=True, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + except Exception as e: + draft_export_msg = ( + "The error above occurred when calling torch.export.export. If you would " + "like to view some more information about this error, and get a list " + "of all other errors that may occur in your export call, you can " + "replace your `export()` call with `draft_export()`." + ) + + # For errors that we know can be caught by draft-export, add the message + # to ask users to try out draft-export + if isinstance( + e, + ( + torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode, + torch._subclasses.fake_tensor.UnsupportedOperatorException, + torch._dynamo.exc.UserError, + torch.fx.experimental.symbolic_shapes.ConstraintViolationError, + ), + ): + new_msg = str(e) + "\n\n" + draft_export_msg + e.args = (new_msg,) + elif isinstance(e, RuntimeError) and "no fake impl registered" in str(e): + new_msg = str(e) + "\n\n" + draft_export_msg + e.args = (new_msg,) + raise e + + +DEFAULT_PICKLE_PROTOCOL = 2 + + +def save( + ep: ExportedProgram, + f: FileLike, + *, + extra_files: dict[str, Any] | None = None, + opset_version: dict[str, int] | None = None, + pickle_protocol: int = DEFAULT_PICKLE_PROTOCOL, +) -> None: + """ + + .. warning:: + Under active development, saved files may not be usable in newer versions + of PyTorch. + + Saves an :class:`ExportedProgram` to a file-like object. It can then be + loaded using the Python API :func:`torch.export.load `. + + Args: + ep (ExportedProgram): The exported program to save. + + f (str | os.PathLike[str] | IO[bytes]) A file-like object (has to + implement write and flush) or a string containing a file name. + + extra_files (Optional[Dict[str, Any]]): Map from filename to contents + which will be stored as part of f. + + opset_version (Optional[Dict[str, int]]): A map of opset names + to the version of this opset + + pickle_protocol: can be specified to override the default protocol + + Example:: + + import torch + import io + + + class MyModule(torch.nn.Module): + def forward(self, x): + return x + 10 + + + ep = torch.export.export(MyModule(), (torch.randn(5),)) + + # Save to file + torch.export.save(ep, "exported_program.pt2") + + # Save to io.BytesIO buffer + buffer = io.BytesIO() + torch.export.save(ep, buffer) + + # Save with extra files + extra_files = {"foo.txt": b"bar".decode("utf-8")} + torch.export.save(ep, "exported_program.pt2", extra_files=extra_files) + + """ + if not isinstance(ep, ExportedProgram): + raise TypeError( + f"The 'ep' parameter must be an instance of 'ExportedProgram', got '{type(ep).__name__}' instead." + ) + + from torch.export.pt2_archive._package import package_pt2 + + package_pt2( + f, + exported_programs={"model": ep}, + extra_files=extra_files, + pickle_protocol=pickle_protocol, + opset_version=opset_version, + ) + + +def load( + f: FileLike, + *, + extra_files: dict[str, Any] | None = None, + expected_opset_version: dict[str, int] | None = None, +) -> ExportedProgram: + """ + + .. warning:: + Under active development, saved files may not be usable in newer versions + of PyTorch. + + .. warning:: + :func:`torch.export.load()` uses pickle under the hood to load models. **Never load data from an untrusted source.** + + Loads an :class:`ExportedProgram` previously saved with + :func:`torch.export.save `. + + Args: + f (str | os.PathLike[str] | IO[bytes]): A file-like object (has to + implement write and flush) or a string containing a file name. + + extra_files (Optional[Dict[str, Any]]): The extra filenames given in + this map would be loaded and their content would be stored in the + provided map. + + expected_opset_version (Optional[Dict[str, int]]): A map of opset names + to expected opset versions + + Returns: + An :class:`ExportedProgram` object + + Example:: + + import torch + import io + + # Load ExportedProgram from file + ep = torch.export.load("exported_program.pt2") + + # Load ExportedProgram from io.BytesIO object + with open("exported_program.pt2", "rb") as f: + buffer = io.BytesIO(f.read()) + buffer.seek(0) + ep = torch.export.load(buffer) + + # Load with extra files. + extra_files = {"foo.txt": ""} # values will be replaced with data + ep = torch.export.load("exported_program.pt2", extra_files=extra_files) + print(extra_files["foo.txt"]) + print(ep(torch.randn(5))) + """ + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + + extra_files = extra_files or {} + + from torch.export.pt2_archive._package import load_pt2, PT2ArchiveContents + + try: + pt2_contents = load_pt2( + f, + expected_opset_version=expected_opset_version, + ) + except RuntimeError: + log.warning("Ran into the following error when deserializing", exc_info=True) + pt2_contents = PT2ArchiveContents({}, {}, {}) + + if len(pt2_contents.exported_programs) > 0 or len(pt2_contents.extra_files) > 0: + for k, v in pt2_contents.extra_files.items(): + extra_files[k] = v + + return pt2_contents.exported_programs["model"] + + # TODO: For backward compatibility, we support loading a zip file from 2.7. Delete this path in 2.9(?) + with zipfile.ZipFile(f, "r") as zipf: + if "version" not in zipf.namelist(): + raise RuntimeError( + "We ran into an error when deserializing the saved file. " + "Please check the warnings above for possible errors. " + ) + + log.warning( + "Trying to deserialize for the older format. This version of file is " + "deprecated. Please generate a new pt2 saved file." + ) + + # Check the version + version = zipf.read("version").decode().split(".") + from torch._export.serde.schema import ( + SCHEMA_VERSION, # todo change archive version to schema version + ) + + assert len(version) == len(SCHEMA_VERSION), ( + "Version in the saved file has incorrect length, double check if the file is generated by torch.export.save()" + ) + if version[0] != str(SCHEMA_VERSION[0]): + raise RuntimeError( + f"Serialized version {version} does not match our current " + f"schema version {SCHEMA_VERSION}." + ) + + from torch._export.serde.serialize import deserialize, SerializedArtifact + + # Load serialized_ep and serialized_state_dict from the zip file + + serialized_exported_program: bytes | None = None + serialized_state_dict: bytes | None = None + serialized_constants: bytes | None = None + serialized_example_inputs: bytes | None = None + + for file_info in zipf.infolist(): + file_content = zipf.read(file_info.filename) + + if file_info.filename == "serialized_exported_program.json": + serialized_exported_program = file_content + elif file_info.filename == "serialized_state_dict.json": + warnings.warn("This version of file is deprecated", stacklevel=2) + serialized_state_dict = file_content + elif file_info.filename == "serialized_constants.json": + warnings.warn("This version of file is deprecated", stacklevel=2) + serialized_constants = file_content + elif file_info.filename == "serialized_state_dict.pt": + serialized_state_dict = file_content + elif file_info.filename == "serialized_constants.pt": + serialized_constants = file_content + elif file_info.filename == "serialized_example_inputs.pt": + serialized_example_inputs = file_content + elif file_info.filename.startswith("extra_files"): + filename = file_info.filename.split("/", 1)[1] + extra_files[filename] = file_content.decode("utf-8") + + assert serialized_exported_program is not None + assert serialized_state_dict is not None + assert serialized_constants is not None + assert serialized_example_inputs is not None + artifact: SerializedArtifact = SerializedArtifact( + serialized_exported_program, + serialized_state_dict, + serialized_constants, + serialized_example_inputs, + ) + + # Deserialize ExportedProgram + ep = deserialize(artifact, expected_opset_version) + + return ep + + +def draft_export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: Mapping[str, Any] | None = None, + *, + dynamic_shapes: dict[str, Any] | tuple[Any, ...] | list[Any] | None = None, + preserve_module_call_signature: tuple[str, ...] = (), + strict: bool = False, + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + """ + A version of torch.export.export which is designed to consistently produce + an ExportedProgram, even if there are potential soundness issues, and to + generate a report listing the issues found. + """ + from ._draft_export import draft_export + + return draft_export( + mod=mod, + args=args, + kwargs=kwargs, + dynamic_shapes=dynamic_shapes, + preserve_module_call_signature=preserve_module_call_signature, + strict=strict, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + + +def register_dataclass( + cls: type[Any], + *, + serialized_type_name: str | None = None, +) -> None: + """ + Registers a dataclass as a valid input/output type for :func:`torch.export.export`. + + Args: + cls: the dataclass type to register + serialized_type_name: The serialized name for the dataclass. This is + required if you want to serialize the pytree TreeSpec containing this + dataclass. + + Example:: + + import torch + from dataclasses import dataclass + + + @dataclass + class InputDataClass: + feature: torch.Tensor + bias: int + + + @dataclass + class OutputDataClass: + res: torch.Tensor + + + torch.export.register_dataclass(InputDataClass) + torch.export.register_dataclass(OutputDataClass) + + + class Mod(torch.nn.Module): + def forward(self, x: InputDataClass) -> OutputDataClass: + res = x.feature + x.bias + return OutputDataClass(res=res) + + + ep = torch.export.export(Mod(), (InputDataClass(torch.ones(2, 2), 1),)) + print(ep) + + """ + pytree.register_dataclass(cls, serialized_type_name=serialized_type_name) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_draft_export.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_draft_export.py new file mode 100644 index 0000000000000000000000000000000000000000..81be29fde5a606041d833d72f214b062844548a2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_draft_export.py @@ -0,0 +1,544 @@ +import getpass +import json +import logging +import os +import re +import tempfile +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from enum import IntEnum +from typing import Any + +import torch +import torch._logging._internal +import torch.utils._pytree as pytree +from torch._dynamo.exc import UserError, UserErrorType +from torch._export.passes.insert_custom_op_guards import ( + get_op_profiles, + insert_custom_op_guards, + OpProfile, +) +from torch._utils_internal import log_draft_export_usage + +from ._trace import _export, get_ep_stats +from .dynamic_shapes import _DimHint, _DimHintType, Dim +from .exported_program import ExportedProgram + + +log = logging.getLogger(__name__) + + +class FailureType(IntEnum): + MISSING_FAKE_KERNEL = 1 + DATA_DEPENDENT_ERROR = 2 + GUARD_ADDED = 3 + MISMATCHED_FAKE_KERNEL = 4 + + def __str__(self) -> str: + return self.name + + +def prettify_stack(stack: list[dict[str, str]], str_to_filename: dict[int, str]) -> str: + res = "" + for frame in stack: + if frame["filename"] not in str_to_filename: + continue + + res += f""" + File {str_to_filename[frame["filename"]]}, lineno {frame["line"]}, in {frame["name"]}""" # type: ignore[index] + + res += f"\n {stack[-1]['loc']}" + return res + + +def prettify_frame_locals( + loc: str, locals: dict[str, Any], symbols: dict[str, Any] +) -> str: + local_str = "\n".join(f" {k}: {v}" for k, v in locals.items()) + res = f""" + Locals: +{local_str} +""" + if any(v is not None for v in symbols.values()): + symbol_str = "\n".join( + f" {k}: {v}" for k, v in symbols.items() if v is not None + ) + res += f""" + Symbols: +{symbol_str} +""" + return res + + +def get_loc(filename: str, lineno: int) -> str | None: + try: + with open(filename) as f: + for i, line in enumerate(f): + if i == lineno - 1: + return line.strip() + except FileNotFoundError: + pass + return None + + +class FailureReport: + def __init__( + self, failure_type: FailureType, data: dict[str, Any], xfail: bool = False + ) -> None: + self.failure_type: FailureType = failure_type + self.data: dict[str, Any] = data + self.xfail: bool = xfail + + def __repr__(self) -> str: + return f"FailureReport(failure_type={self.failure_type}, xfail={self.xfail}, data={self.data})" + + def print(self, str_to_filename: dict[int, str]) -> str: + if self.failure_type == FailureType.MISSING_FAKE_KERNEL: + op = self.data["op"] + + return f"""Missing fake kernel. + torch.ops.{op} is missing a fake kernel implementation. + + Please refer to https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ahugy69p2jmz for more detailed instructions on how to write a meta implementation. +""" # noqa: B950 + + elif self.failure_type == FailureType.GUARD_ADDED: + locals_info = ( + prettify_frame_locals(**self.data["frame_locals"]) + if self.data["frame_locals"] + else "" + ) + return f"""Guard Added. + A guard was added during tracing, which might've resulted in some incorrect + tracing or constraint violation error. + Specifically, this guard was added: {self.data["expr"]}, where {self.data["symbol_to_sources"]}. + This occurred at the following stacktrace: {prettify_stack(self.data["user_stack"], str_to_filename)}: + {locals_info} + And the following framework stacktrace: {prettify_stack(self.data["stack"], str_to_filename)}\n + Because of this, we have modified the dynamic shapes structure to be the + following. You can also use torch.export.Dim.AUTO instead to specify your + dynamic shapes, and we will automatically infer the dynamism for you. + ``` + dynamic_shapes = {self.data["new_dynamic_shapes"]} + ``` +""" + + elif self.failure_type == FailureType.DATA_DEPENDENT_ERROR: + locals_info = ( + prettify_frame_locals(**self.data["frame_locals"]) + if self.data["frame_locals"] + else "" + ) + return f"""Data dependent error. + When exporting, we were unable to evaluate the value of `{self.data["expr"]}`. + This was encountered {self.data["occurrences"]} times. + This occurred at the following user stacktrace: {prettify_stack(self.data["user_stack"], str_to_filename)} + {locals_info} + And the following framework stacktrace: {prettify_stack(self.data["stack"], str_to_filename)}\n + As a result, it was specialized to a constant (e.g. `{self.data["result"]}` in the 1st occurrence), and asserts were inserted into the graph. + + Please add `torch._check(...)` to the original code to assert this data-dependent assumption. + Please refer to https://docs.google.com/document/d/1kZ_BbB3JnoLbUZleDT6635dHs88ZVYId8jT-yTFgf3A/edit#heading=h.boi2xurpqa0o for more details. +""" # noqa: B950 + + elif self.failure_type == FailureType.MISMATCHED_FAKE_KERNEL: + op = self.data["op"] + reason = self.data["reason"] + return f"""Mismatched fake kernel. + torch.ops.{op} has a fake kernel implementation, but it has incorrect behavior, based on the real kernel. + The reason for the mismatch is: {reason}. + + Please refer to https://docs.google.com/document/d/1_W62p8WJOQQUzPsJYa7s701JXt0qf2OfLub2sbkHOaU/edit#heading=h.ahugy69p2jmz for more detailed instructions on how to write a fake implementation. +""" # noqa: B950 + + else: + raise ValueError(f"Unknown failure type: {self.failure_type}") + + +class DraftExportReport: + def __init__( + self, + failures: list[FailureReport], + str_to_filename: dict[int, str], + expressions_created: dict[int, dict[str, Any]], + op_profiles: dict[str, set[OpProfile]], + ): + self.failures: list[FailureReport] = failures + self.str_to_filename = str_to_filename + self.expressions_created: dict[int, dict[str, Any]] = expressions_created + self.op_profiles = op_profiles + + def successful(self) -> bool: + return len(self.failures) == 0 or all( + failure.xfail for failure in self.failures + ) + + def __repr__(self) -> str: + return f"DraftExportReport({self.failures})" + + def __str__(self) -> str: + WARNING_COLOR = "\033[93m" + GREEN_COLOR = "\033[92m" + END_COLOR = "\033[0m" + + if self.successful(): + return f"""{GREEN_COLOR} +############################################################################################## +Congratuations: No issues are found during export, and it was able to soundly produce a graph. +You can now change back to torch.export.export() +############################################################################################## +{END_COLOR}""" + + error = f"""{WARNING_COLOR} +################################################################################################### +WARNING: {len(self.failures)} issue(s) found during export, and it was not able to soundly produce a graph. +Please follow the instructions to fix the errors. +################################################################################################### + +""" + + for i, failure in enumerate(self.failures): + error += f"{i + 1}. {failure.print(self.str_to_filename)}\n" + error += END_COLOR + return error + + def apply_suggested_fixes(self) -> None: + raise NotImplementedError("Not implemented yet") + + +@dataclass +class ExpressionCreatedNode: + result_id: int + argument_ids: list[int] + record: dict[str, object] + visited: bool = False + + +class LogRecord: + def __init__(self) -> None: + self.log_count: dict[int, int] = {} + self.logs: list[tuple[str, dict[str, Any]]] = [] + + def _hash(self, element: tuple[str, dict[str, Any]]) -> int: + key, data = element + + if key == "missing_fake_kernel": + return hash((key, data["op"])) + elif key == "mismatched_fake_kernel": + return hash((key, data["op"], data["reason"])) + elif key == "propagate_real_tensors_provenance": + return hash((key, json.dumps(data["user_stack"]))) + elif key == "guard_added": + return hash((key, json.dumps(data["user_stack"]))) + elif key == "create_unbacked_symbol": + return hash((key, json.dumps(data["user_stack"]))) + + return hash((key, json.dumps(data))) + + def try_add(self, element: tuple[str, dict[str, str]]) -> bool: + hash_value = self._hash(element) + if hash_value in self.log_count: + self.log_count[hash_value] += 1 + return False + + self.log_count[hash_value] = 1 + self.logs.append(element) + return True + + def get_log_count(self, element: tuple[str, dict[str, Any]]) -> int: + return self.log_count[self._hash(element)] + + +class CaptureStructuredTrace(torch._logging._internal.LazyTraceHandler): + def __init__(self) -> None: + self.specific_log_keys = [ + "str", + "exported_program", + "propagate_real_tensors_provenance", + "guard_added", + "missing_fake_kernel", + "mismatched_fake_kernel", + "expression_created", + "create_unbacked_symbol", + ] + self.log_record: LogRecord = LogRecord() + self.expression_created_logs: dict[int, ExpressionCreatedNode] = {} + self.symbol_to_expressions: dict[str, list[dict[str, Any]]] = {} + self.logger = logging.getLogger("torch.__trace") + self.prev_get_dtrace = False + + if root_dir := os.environ.get(torch._logging._internal.DTRACE_ENV_VAR): + super().__init__(root_dir) + else: + sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser()) + root_dir = os.path.join( + tempfile.gettempdir(), + "export_" + sanitized_username, + ) + super().__init__(root_dir) + + self.setFormatter(torch._logging._internal.TorchLogsFormatter(trace=True)) + + def __enter__(self) -> "CaptureStructuredTrace": + self.log_record = LogRecord() + self.expression_created_logs = {} + + # Remove the lazy trace handler if it exists + possible_lazy_trace_handlers = [ + handler + for handler in self.logger.handlers + if isinstance(handler, torch._logging._internal.LazyTraceHandler) + ] + for handler in possible_lazy_trace_handlers: + self.logger.removeHandler(handler) + + self.logger.addHandler(self) + self.prev_get_dtrace = torch._logging._internal.GET_DTRACE_STRUCTURED + # pyrefly: ignore [bad-assignment] + torch._logging._internal.GET_DTRACE_STRUCTURED = True + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore[no-untyped-def] + self.log_record = LogRecord() + self.expression_created_logs = {} + self.logger.removeHandler(self) + # pyrefly: ignore [bad-assignment] + torch._logging._internal.GET_DTRACE_STRUCTURED = self.prev_get_dtrace + self.prev_get_dtrace = False + + def emit(self, record: Any) -> None: + def _log_expression_created( + emit_func: Callable[[Any], None], sym_node_id: int + ) -> None: + # Log all the relevant expression_created logs + if sym_node_id is None: + return + if res := self.expression_created_logs.get(sym_node_id, None): + # Don't log the expression if we have already + # printed it beforehand + if not res.visited: + res.visited = True + for arg in res.argument_ids: + _log_expression_created(emit_func, arg) + + emit_func(res.record) + + metadata = record.metadata + for key in self.specific_log_keys: + if key in metadata: + if self.log_record.try_add((key, metadata[key])): + if key == "expression_created": + # We don't want to log all expression_created logs, only + # the ones that are relevant to the + # guards/propagate_real_tensor + self.expression_created_logs[metadata[key]["result_id"]] = ( + ExpressionCreatedNode( + metadata[key]["result_id"], + metadata[key].get("argument_ids", []), + record, + ) + ) + return + + elif key == "propagate_real_tensors_provenance": + _log_expression_created( + super().emit, metadata[key].get("expr_node_id") + ) + + elif key == "guard_added": + if len(metadata[key]["symbol_to_sources"]) == 0: + # We only want to include guards added that are relevant to + # the symbolic shapes corresponding to the inputs which were + # specified in the dynamic_shapes arg. These have a source. + return + elif metadata[key]["prefix"] == "runtime_assert": + # This should've been captured by a + # propagate_real_tensors log + return + + _log_expression_created( + super().emit, metadata[key].get("expr_node_id") + ) + + super().emit(record) + + +def draft_export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: Mapping[str, Any] | None = None, + *, + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None = None, + preserve_module_call_signature: tuple[str, ...] = (), + strict: bool = False, + pre_dispatch: bool = True, + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + start_time = time.time() + kwargs = kwargs or {} + dynamic_shapes = dynamic_shapes or {} + + constraint_violation_msg = None + capture_structured_log = CaptureStructuredTrace() + + with ( + torch._functorch.config.patch( + fake_tensor_propagate_real_tensors=True, + generate_fake_kernels_from_real_mismatches=True, + ), + capture_structured_log, + ): + try: + new_shapes = None + ep = _export( + mod, + args, + kwargs, + dynamic_shapes=dynamic_shapes, + strict=strict, + pre_dispatch=pre_dispatch, + preserve_module_call_signature=preserve_module_call_signature, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + except Exception as exc: + if ( + isinstance(exc, UserError) + and exc.error_type == UserErrorType.CONSTRAINT_VIOLATION + ): + constraint_violation_msg = exc.msg + + def convert_dim_to_auto(dim: Any) -> Any: + if isinstance(dim, Dim): + return Dim.AUTO(min=dim.min, max=dim.max) + elif isinstance(dim, _DimHint) and dim.type == _DimHintType.DYNAMIC: + return Dim.AUTO(min=dim.min, max=dim.max) + return dim + + new_shapes = pytree.tree_map(convert_dim_to_auto, dynamic_shapes) + ep = _export( + mod, + args, + kwargs, + dynamic_shapes=new_shapes, + strict=strict, + pre_dispatch=pre_dispatch, + preserve_module_call_signature=preserve_module_call_signature, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + else: + log_draft_export_usage( + error=True, + export_time=time.time() - start_time, + strict=strict, + message=str(exc), + type=f"{type(exc).__name__}.{type(exc).__qualname__}", + ) + raise exc + + torch._logging.dtrace_structured("exported_program", payload_fn=lambda: str(ep)) + + str_to_filename: dict[int, str] = {} + failures: list[FailureReport] = [] + incorrect_custom_ops: set[str] = set() + expressions_created: dict[int, dict[str, Any]] = {} + + for log_name, log_contents in capture_structured_log.log_record.logs: + failure_type = None + + if log_name == "str": + str_to_filename[log_contents[1]] = log_contents[0] # type: ignore[index] + continue + + elif log_name == "propagate_real_tensors_provenance": + log_contents["occurrences"] = ( + capture_structured_log.log_record.get_log_count( + (log_name, log_contents) + ) + ) + + failure_type = FailureType.DATA_DEPENDENT_ERROR + + elif log_name == "guard_added": + if new_shapes is None: + continue + + failure_type = FailureType.GUARD_ADDED + log_contents["new_dynamic_shapes"] = new_shapes + elif log_name == "missing_fake_kernel": + failure_type = FailureType.MISSING_FAKE_KERNEL + incorrect_custom_ops.add(log_contents["op"]) + + elif log_name == "mismatched_fake_kernel": + failure_type = FailureType.MISMATCHED_FAKE_KERNEL + incorrect_custom_ops.add(log_contents["op"]) + + else: + continue + + assert failure_type is not None + failures.append( + FailureReport( + failure_type, + log_contents, + ) + ) + + for k, v in capture_structured_log.expression_created_logs.items(): + if v.visited: + expressions_created[k] = v.record + + op_profiles = get_op_profiles(ep.graph_module, incorrect_custom_ops) + report = DraftExportReport( + failures, str_to_filename, expressions_created, op_profiles + ) + + # Add asserts around custom ops + insert_custom_op_guards(ep.graph_module, incorrect_custom_ops) + + ep._report = report + if not report.successful(): + log_filename = capture_structured_log.stream.name + + warning_msg = f""" +################################################################################################### +WARNING: {len(report.failures)} issue(s) found during export, and it was not able to soundly produce a graph. +To view the report of failures in an html page, please run the command: + `tlparse {log_filename} --export` +Or, you can view the errors in python by inspecting `print(ep._report)`. +""" + + if len(report.op_profiles) > 0: + warning_msg += f""" +While tracing we found {len(report.op_profiles)} operator(s) which do not have a fake kernel registered. +If you intend to retrace the exported graph or run it with fake tensors, please run it under the +following context manager, which will register a fake kernel for those operators. +``` +with torch._library.fake_profile.unsafe_generate_fake_kernels(ep._report.op_profiles): + # run with fake tensors +``` +""" + + warning_msg += """#################################################################################################""" + + log.warning(warning_msg) + + else: + log.info( + """ +############################################################################################## +Congratuations: No issues are found during export, and it was able to soundly produce a graph. +You can now change back to torch.export.export() +############################################################################################## + """ + ) + + log_draft_export_usage( + error=False, + export_time=time.time() - start_time, + strict=strict, + constraint_violations=constraint_violation_msg, + report=ep._report, + **get_ep_stats(ep), + ) + return ep diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_leakage_detection_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_leakage_detection_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d077bb0c1bc7383d3f3949b48b807eec2a4fde78 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_leakage_detection_utils.py @@ -0,0 +1,112 @@ +import gc +import types +import typing +import weakref + +import torch + + +""" +These functions are used to detect potential fake tensor leakage when using PT2 export. +See NOTE [export non-strict fake tensor leak detection] + +There are some complications that made this logic overly complicated: +1) Python 3.10 and Python 3.12 have different ways of implementing referrer so + we need to account for whether it is ref.__dict__ or the real ref object + +2) There are some internal PT2 references to fake tensors like `TrackedFake` +3) closures, generators, and bound methods can hold fake tensors. +4) global object can hold onto a fake tensor + +In general, these utils are our last resort to detect fake tensors. if the leak happens +within the model attributes, we have a separate mechanism to detect. This tool relies a bit +on garbage collector internal details, so I think it is unsafe to turn on by default, hence +this tool should be used as debugging tool. +""" + + +# Things we never want to flag as leaks +_SKIP_TYPES = ( + types.FrameType, + types.ModuleType, +) + + +def _is_globals_or_locals(obj: typing.Any) -> bool: + # These comparisons only make sense within this frame; still cheap to check. + return obj is globals() or obj is locals() + + +def _is_tracked_fake(obj: typing.Any) -> bool: + return isinstance(obj, torch.fx.experimental.symbolic_shapes.TrackedFake) + + +def _is_gm_meta_like_dict(d: dict, o: typing.Any) -> bool: + # Hope gm.meta was a custom dict we can assert on + return d.get("val") is o + + +def _dict_is_attr_of_tracked_fake(d: dict) -> bool: + """ + Python 3.10 quirk: sometimes the referrer is obj.__dict__ instead of obj. + Check if this dict is exactly the __dict__ of a TrackedFake. + """ + for parent in gc.get_referrers(d): + if ( + hasattr(parent, "__dict__") + and parent.__dict__ is d + and _is_tracked_fake(parent) + ): + return True + return False + + +def find_legit_leaks_from_referrers(active_fakes: weakref.WeakSet) -> weakref.WeakSet: + legit_leak: weakref.WeakSet = weakref.WeakSet() + + # This is so that we don't falsely flag generator to be holding fake tensor + fake_list = list(active_fakes) + fake_list_id = id(fake_list) + + for act in fake_list: + # Track by id to avoid processing duplicate referrers + seen = set() + # Assume it's a leak unless we find only ignorable referrers + flagged = False + + for r in gc.get_referrers(act): + rid = id(r) + if rid in seen: + continue + seen.add(rid) + + # Skip our own fake_list + if rid == fake_list_id: + continue + + # Fast-path: skip obvious non-owners + if _is_globals_or_locals(r): + continue + if isinstance(r, _SKIP_TYPES): + continue + if _is_tracked_fake(r): + # TrackedFake should be ignored + continue + + # Handle dicts carefully (Python 3.10 sometimes shows __dict__) + if isinstance(r, dict): + if _is_gm_meta_like_dict(r, act): + continue + if _dict_is_attr_of_tracked_fake(r): + continue + flagged = True + break + + # Any other referrer we don't explicitly whitelist counts as a leak + flagged = True + break + + if flagged: + legit_leak.add(act) + + return legit_leak diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_auto_functionalized_pass.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_auto_functionalized_pass.py new file mode 100644 index 0000000000000000000000000000000000000000..9c45d136c6f35b373e51be317e77b77506998e27 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_auto_functionalized_pass.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +import torch +from torch._higher_order_ops.auto_functionalize import ( + auto_functionalized, + auto_functionalized_v2, +) +from torch._inductor.fx_passes.post_grad import decompose_auto_functionalized +from torch.export import ExportedProgram +from torch.fx import Graph + + +def remove_self_clone(graph: Graph) -> None: + for node in graph.nodes: + if node.target is torch.ops.aten.copy_.default and node.args[0] == node.args[1]: + node.replace_all_uses_with(node.args[0]) + graph.erase_node(node) + + +def unsafe_remove_auto_functionalized_pass( + ep: ExportedProgram, +) -> ExportedProgram: + """ + This pass removes an instances of the higher order op 'auto_functionalized', + and modifies the calling EP inplace to have the original mutator op. + This pass doesn't perform safety checks to make sure that this inplace mutation is safe. + """ + + with ep.graph_module._set_replace_hook(ep.graph_signature.get_replace_hook()): + for module in ep.graph_module.modules(): + if not isinstance(module, torch.fx.GraphModule): + continue + for node in ep.graph.nodes: + if ( + node.op == "call_function" and node.target is auto_functionalized + ) or ( + node.op == "call_function" and node.target is auto_functionalized_v2 + ): + func = node.args[0] + assert isinstance(func, torch._ops.OpOverload) + # re-inplace everything + node.meta["only_clone_these_tensors"] = [] + decompose_auto_functionalized(ep.graph) + remove_self_clone(ep.graph) + ep.graph.eliminate_dead_code() + + return ep diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_effect_tokens_pass.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_effect_tokens_pass.py new file mode 100644 index 0000000000000000000000000000000000000000..ab25fb62d22bbfb58c1a6cb872bf14cd05c82964 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_remove_effect_tokens_pass.py @@ -0,0 +1,212 @@ +# mypy: allow-untyped-defs +import operator + +import torch +from torch._higher_order_ops.effects import _get_schema, with_effects + +from .exported_program import ExportedProgram +from .graph_signature import ( + CustomObjArgument, + InputKind, + InputSpec, + OutputKind, + OutputSpec, + TokenArgument, +) + + +def _get_custom_obj_for_node(node, inputs_to_lifted_custom_objs, constants): + """Extract the custom object from a node's arguments.""" + custom_obj_node = node + custom_obj_meta = custom_obj_node.meta["val"] # type: ignore[union-attr] + assert isinstance(custom_obj_meta, CustomObjArgument) + + if custom_obj_meta.fake_val: + return custom_obj_meta.fake_val + elif custom_obj_node.name in inputs_to_lifted_custom_objs: # type: ignore[union-attr] + return constants[inputs_to_lifted_custom_objs[custom_obj_node.name]] # type: ignore[union-attr] + else: + raise RuntimeError(f"Unable to find custom obj for node {node}") + + +def _replace_with_effects_node( + node, ep, inputs_to_lifted_custom_objs, output_tokens, input_tokens, module +): + """Replace a with_effects node with the underlying function call.""" + # Get the input nodes + token_node, func, *node_args = node.args + if token_node.op == "placeholder": + input_tokens.append(token_node) + + assert isinstance(func, (torch._ops.OpOverload, torch._ops.HigherOrderOperator)) + + # Get the schema for the function + if func is torch.ops.higher_order.call_torchbind: + custom_obj = _get_custom_obj_for_node( + node_args[0], inputs_to_lifted_custom_objs, ep.constants + ) + schema = _get_schema(func, [custom_obj] + node_args[1:]) + else: + schema = _get_schema(func, node_args) + + # Create the replacement node + with module.graph.inserting_before(node): + new_node = module.graph.call_function(func, tuple(node_args), node.kwargs) + + # Update getitem nodes that extract outputs from with_effects + for user in list(node.users.keys()): + assert user.target is operator.getitem + # getitem(with_effects, 0) is the token node + if user.args[1] == 0: + for user_user in list(user.users.keys()): + if user_user.op == "output": + output_tokens.append(user) + + # Copy metadata from old node to new node + for k, v in node.meta.items(): + new_node.meta[k] = v + if k == "unbacked_bindings": + # Remove the extra layer for effect token + old_bindings = new_node.meta[k] + new_bindings = { + k: path[1:] if path else path for k, path in old_bindings.items() + } + new_node.meta[k] = new_bindings + + # Fix up the getitem nodes based on return count + if len(schema.returns) == 1: + # Single return: replace getitem(with_effects, 1) with the node itself + for user in list(node.users.keys()): + if user.args[1] == 1: + user.replace_all_uses_with(new_node) + new_node.meta["val"] = node.meta["val"][1] + elif len(schema.returns) > 1: + # Multiple returns: shift getitem indices down by 1 + for user in list(node.users.keys()): + if user.args[1] >= 1: + user.args = (new_node, user.args[1] - 1) + new_node.meta["val"] = node.meta["val"][1:] + else: + # No returns + assert len(schema.returns) == 0 + assert len(new_node.users) == 0 + new_node.meta["val"] = None + + +def _replace_invoke_subgraph_node(node, module, output_tokens, input_tokens): + """Replace an invoke_subgraph node to remove the token argument.""" + assert node.args[0].op == "get_attr" + submod = getattr(module, node.args[0].target) + if not submod.meta.get("has_with_effects", False): + return + + # Remove token from inputs + subgraph, identifier, token, *operands = node.args + node.args = (subgraph, identifier, *operands) + if token.op == "placeholder": + input_tokens.append(token) + + # Update getitem nodes to account for removed token output + for user in list(node.users.keys()): + if user.args[1] >= 1: + user.args = (node, user.args[1] - 1) + elif user.args[1] == 0: + for user_user in list(user.users.keys()): + if user_user.op == "output": + output_tokens.append(user) + + +def _remove_effect_tokens(ep: ExportedProgram) -> ExportedProgram: + """ + Removes the existence of tokens from the exported program, including: + - Removes the input and output tokens + - Replaces with_effects(token, func, args) with just func(args) + + This function does an inplace modification on the given ExportedProgram. + """ + inputs_to_lifted_custom_objs = ep.graph_signature.inputs_to_lifted_custom_objs + + # mark submodules with effects as having effects. This will be used in the following pass to remove effects from subgraphs + for _, module in ep.graph_module.named_modules(): + if not isinstance(module, torch.fx.GraphModule): + continue + + with_effect_nodes = [ + node for node in module.graph.nodes if node.target is with_effects + ] + if len(with_effect_nodes) > 0: + module.meta["has_with_effects"] = True + + # Process each module with the replace hook to ensure graph signature is updated + with ep.graph_module._set_replace_hook(ep.graph_signature.get_replace_hook()): + for _, module in ep.graph_module.named_modules(): + if not isinstance(module, torch.fx.GraphModule): + continue + + input_tokens = [] + output_tokens = [] + + # Process with_effects and invoke_subgraph nodes + for node in module.graph.nodes: + if node.target is with_effects: + _replace_with_effects_node( + node, + ep, + inputs_to_lifted_custom_objs, + output_tokens, + input_tokens, + module, + ) + elif node.target is torch.ops.higher_order.invoke_subgraph: + _replace_invoke_subgraph_node( + node, module, output_tokens, input_tokens + ) + + # Remove tokens from the output node + if len(output_tokens) > 0: + output_node = next(reversed(module.graph.find_nodes(op="output"))) + output_args = output_node.args[0] + assert len(output_args) >= len(output_tokens), ( + f"{output_args} output arguments found\n" + f"{output_tokens} output tokens found\n" + f"{module.graph}" + ) + output_node.args = (tuple(output_args[len(output_tokens) :]),) + + module.graph.eliminate_dead_code() + + # Remove tokens from the input placeholders + for node in module.graph.nodes: + if node.op == "placeholder" and node in input_tokens: + module.graph.erase_node(node) + + module.recompile() + + num_tokens: int = 0 + input_token_names: list[str] = [] + new_input_specs: list[InputSpec] = [] + for inp in ep.graph_signature.input_specs: + if inp.kind == InputKind.TOKEN: + num_tokens += 1 + assert isinstance(inp.arg, TokenArgument) + input_token_names.append(inp.arg.name) + else: + new_input_specs.append(inp) + + num_out_tokens: int = 0 + new_output_specs: list[OutputSpec] = [] + output_token_names: list[OutputSpec] = [] + for out in ep.graph_signature.output_specs: + if out.kind == OutputKind.TOKEN: + num_out_tokens += 1 + output_token_names.append(out.arg.name) + else: + new_output_specs.append(out) + + # Update graph signature + ep.graph_signature.input_specs = new_input_specs + ep.graph_signature.output_specs = new_output_specs + + assert num_tokens == num_out_tokens + + return ep diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_safeguard.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_safeguard.py new file mode 100644 index 0000000000000000000000000000000000000000..24ad485ebe7e2a51425fbfca53da267ea2c45551 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_safeguard.py @@ -0,0 +1,44 @@ +# mypy: allow-untyped-defs +import torch +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode +from torch.overrides import TorchFunctionMode + + +class AutogradStateOpsFailSafeguard(TorchFunctionMode): + """ + Detect grad state ops during exporting the graph and fail the process by + raising an error, to avoid unexpected behavior. Those grad mode ops could be: + `torch.no_grad` + `torch.enable_grad` + `torch.set_grad_enabled` + + Export with predispatch mode is exempted. + """ + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + unsupported_grad_mode_ops = [ + torch._C._set_grad_enabled, + ] + # It's only enabled while tracing, by confirming the torch dispatch mode is + # any active PROXY. This is to allow the autograd ops out of tracing. + current_state = torch._C.is_grad_enabled() + if func in unsupported_grad_mode_ops: + assert len(args) == 1 + changed_state = args[0] + mode = torch._C._get_dispatch_mode(torch._C._TorchDispatchModeKey.PROXY) + # Intend to check if it's not the pre_dispatch mode. It's allowed to use + # autograd ops in pre_dispatch mode, e.g. `torch.no_grad` + if ( + mode + and isinstance(mode, ProxyTorchDispatchMode) + and not mode.pre_dispatch + and changed_state != current_state + ): + raise RuntimeError( + f"Encountered autograd state manager op {func} trying to change global autograd state " + "while exporting. This is unsafe because we don't capture this op in torch.export " + "today, hence we can't reflect the user intention soundly. You can fix this by " + "adding a torch.no_grad() context around the export call." + ) + return func(*args, **kwargs) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_swap.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_swap.py new file mode 100644 index 0000000000000000000000000000000000000000..f794ecda02a1dd9b197f51deff299df12ede0791 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_swap.py @@ -0,0 +1,439 @@ +import logging +import operator +import types +from collections import defaultdict + +import torch +import torch.fx._pytree as fx_pytree +import torch.utils._pytree as pytree +from torch.export.exported_program import ( + ConstantArgument, + ExportedProgram, + ModuleCallSignature, +) +from torch.fx.passes.tools_common import legalize_graph, NodeList +from torch.fx.passes.utils.fuser_utils import erase_nodes, fuse_as_graphmodule + + +log = logging.getLogger(__name__) + + +def _get_getitem_users(node: torch.fx.Node) -> set[torch.fx.Node]: + node_users = list(node.users.keys()) + getitem_users = set() + for user in node_users: + if user.op == "output": + continue + + assert user.op == "call_function" and user.target is operator.getitem, ( + f"Expected getitem node as user for {node}, instead got {user}" + ) + getitem_users.update(list(user.users.keys())) + return getitem_users + + +def _try_remove_connecting_pytrees(curr_module_node: torch.fx.Node) -> None: + """ + We want to try to remove extraneous pytree flatten/unflatten calls between modules + calls. Instead of having the following: + graph(): + ... + %foo : [num_users=1] = call_module[target=foo](args = (%getitem_1, %getitem_2), kwargs = {}) + %tree_flatten_spec : [num_users=1] = call_function[target=torch.fx._pytree.tree_flatten_spec](args = (%foo, %_spec_1), kwargs = {}) + %getitem_4 : [num_users=1] = call_function[target=operator.getitem](args = (%tree_flatten_spec, 0), kwargs = {}) + %tree_unflatten_1 : [num_users=2] = call_function[target=torch.utils._pytree.tree_unflatten](args = ([%getitem_4], %_spec_2), kwargs = {}) + %getitem_5 : [num_users=1] = call_function[target=operator.getitem](args = (%tree_unflatten_1, 0), kwargs = {}) + %getitem_7 : [num_users=0] = call_function[target=operator.getitem](args = (%tree_unflatten_1, 1), kwargs = {}) + %getitem_6 : [num_users=1] = call_function[target=operator.getitem](args = (%getitem_5, 0), kwargs = {}) + %bar : [num_users=1] = call_module[target=bar](args = (%getitem_6,), kwargs = {}) + ... + + We could do the following, if we know that all the outputs of `foo` feed into `bar`: + graph(): + ... + %foo : [num_users=1] = call_module[target=foo](args = (%getitem_1, %getitem_2), kwargs = {}) + %bar : [num_users=1] = call_module[target=bar](args = (%getitem_6,), kwargs = {}) + ... + + Currently this optimization only works for the case where all of the outputs + of `foo` go directly into `bar`, and `bar` has no other inputs. + """ # noqa: B950 + + log.debug("Trying to remove pytrees for module call %s", curr_module_node) + + curr_module_users = list(curr_module_node.users.keys()) + assert len(curr_module_users) == 1, ( + f"Expected only one user for module node, instead got {list(curr_module_users)}" + ) + flatten_node = curr_module_users[0] + assert ( + flatten_node.op == "call_function" + and flatten_node.target is fx_pytree.tree_flatten_spec + ) + + flatten_getitem_users = _get_getitem_users(flatten_node) + if len(flatten_getitem_users) != 1: + log.debug( + "More than one user found for flatten node, %s: %s. " + "Unable to fuse it with another unflatten call.", + flatten_node, + flatten_getitem_users, + ) + return + + unflatten_node = next(iter(flatten_getitem_users)) + if not ( + unflatten_node.op == "call_function" + and unflatten_node.target is pytree.tree_unflatten + ): + log.debug( + "Flatten node %s's user is not a pytree.tree_unflatten. " + "Instead it is: %s. Passing...", + flatten_node, + unflatten_node, + ) + return + + for i, arg in enumerate(unflatten_node.args[0]): # type: ignore[union-attr,arg-type] + if arg not in flatten_node.users: + log.debug( + "Module %s's outputs are not all directly used as inputs to " + "the subsequent module. Unable to fuse the connecting " + "flatten/unflatten. The inputs to the subsequent module are: %s. ", + curr_module_node, + unflatten_node.args[0], + ) + return + + if not ( + # pyrefly: ignore [missing-attribute] + arg.op == "call_function" + # pyrefly: ignore [missing-attribute] + and arg.target is operator.getitem + # pyrefly: ignore [missing-attribute] + and arg.args[1] == i + ): + log.debug( + "Module %s's outputs are not all directly used in the same " + "order as outputted. Unable to fuse the connecting " + "flatten/unflatten. The inputs to the " + "subsequent module are: %s. ", + curr_module_node, + unflatten_node.args[0], + ) + return + + # Unflatten has two levels of getitem, because it gets the args and kwargs + unflatten_getitem_getitem_users = set() + unflatten_getitem_users = _get_getitem_users(unflatten_node) + for unflatten_getitem_user in unflatten_getitem_users: + unflatten_getitem_getitem_users.update( + list(unflatten_getitem_user.users.keys()) + ) + + if len(unflatten_getitem_getitem_users) != 1: + log.debug( + "More than one user found for unflatten node, %s: %s. " + "Unable to fuse it with another flatten call.", + unflatten_node, + unflatten_getitem_getitem_users, + ) + return + + next_module_node = next(iter(unflatten_getitem_getitem_users)) + if next_module_node.op != "call_module": + log.debug( + "Unflatten node %s's user is not a call_module. " + "Instead it is: %s. Passing...", + unflatten_node, + next_module_node, + ) + return + + # Directly put the outputs of the current module into the next module + next_module_node.args = (curr_module_node,) + + +def _remove_extraneous_pytrees(gm: torch.fx.GraphModule) -> None: + """ + Remove extraneous pytree flatten/unflatten calls. + + We try a couple of optimizations here: + 1. Remove pytree flatten/unflatten calls between modules + 2. TODO: Remove module's in_spec + initial unflatten call + 3. TODO: Remove module's out_spec + final flatten call + """ + + for node in gm.graph.nodes: + if node.op == "call_module" and node.target != "_guards_fn": + _try_remove_connecting_pytrees(node) + + gm.graph.eliminate_dead_code() + + +def _construct_inputs( + gm: torch.fx.GraphModule, + signature: ModuleCallSignature, + node_name_map: dict[str, torch.fx.Node], +) -> tuple[list[torch.fx.Node], dict[str, torch.fx.Node]]: + tree_unflatten_args: list[torch.fx.Node | None] = [] + for input_ in signature.inputs: + if isinstance(input_, ConstantArgument) and input_.value is None: + # Constants should be directly embedded into the graph and not used + # as inputs + tree_unflatten_args.append(None) + elif input_.name not in node_name_map: + # For unused inputs + tree_unflatten_args.append(None) + else: + tree_unflatten_args.append(node_name_map[input_.name]) + + # Insert unflatten call + from .unflatten import _generate_unflatten + + unflatten_node = _generate_unflatten(gm, tree_unflatten_args, signature.in_spec) + + assert signature.in_spec.num_children == 2 + assert signature.in_spec.type is tuple + args_spec, kwargs_spec = signature.in_spec.children() + assert args_spec.type is tuple + assert kwargs_spec.type is dict + + args_node = gm.graph.call_function(operator.getitem, (unflatten_node, 0)) + args_nodes = [ + gm.graph.call_function(operator.getitem, (args_node, i)) + for i in range(args_spec.num_children) + ] + kwargs_node = gm.graph.call_function(operator.getitem, (unflatten_node, 1)) + kwargs_nodes = { + k: gm.graph.call_function(operator.getitem, (kwargs_node, k)) + for k in kwargs_spec.context + } + return args_nodes, kwargs_nodes + + +def _insert_call_module( + gm: torch.fx.GraphModule, + args_nodes: list[torch.fx.Node], + kwargs_nodes: dict[str, torch.fx.Node], + module_to_swap: torch.nn.Module, + name: str, +) -> torch.fx.Node: + from .unflatten import _assign_attr, _AttrKind + + _assign_attr(module_to_swap, gm, name, _AttrKind.MODULE) + module_node = gm.graph.call_module(name, tuple(args_nodes), kwargs_nodes) # type: ignore[arg-type] + return module_node + + +def _deconstruct_outputs( + gm: torch.fx.GraphModule, + signature: ModuleCallSignature, + module_node: torch.fx.Node, + node_name_map: dict[str, torch.fx.Node], + orig_outputs: tuple[torch.fx.Node, ...], +) -> None: + from .unflatten import _generate_flatten_spec + + flatten_node = _generate_flatten_spec(gm, module_node, signature.out_spec) + + for i, orig_output in enumerate(orig_outputs): + # Use Proxy to record getitem access. + proxy_out = torch.fx.Proxy(flatten_node)[i].node # type: ignore[index] + orig_output.replace_all_uses_with(proxy_out, propagate_meta=True) + + node_name_map[orig_output.name] = proxy_out + + +def _swap_module_helper( + gm: torch.fx.GraphModule, + modules_to_swap: dict[str, torch.nn.Module], + module_call_graph: dict[str, ModuleCallSignature], +) -> torch.fx.GraphModule: + log.debug("Starting graph:") + log.debug(gm.graph) + + legalize_graph(gm) + + partitions: dict[str, NodeList] = defaultdict(list) + + node_name_map: dict[str, torch.fx.Node] = { + node.name: node for node in gm.graph.nodes + } + + # TODO: Handle the duplicate module case + for node in gm.graph.nodes: + if nn_module_stack := node.meta.get("nn_module_stack"): + for path, _ in nn_module_stack.values(): + if path in modules_to_swap: + partitions[path].append(node) + break + + for name, nodes in partitions.items(): + """ + Given a graph like the following, and we want to swap out the submodule "foo": + graph(): + %x : [num_users=1] = placeholder[target=x] + %y : [num_users=2] = placeholder[target=y] + %add : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%y, %x), kwargs = {}), nn_module_stack = {"foo": ("foo", torch.nn.Module)} + %sub : [num_users=1] = call_function[target=torch.ops.aten.sub.Tensor](args = (%y, %add), kwargs = {}), nn_module_stack = {"bar": ("bar", torch.nn.Module)} + return (sub,) + + We will first partition out foo's subgraph: + graph(): + %x : [num_users=1] = placeholder[target=x] + %y : [num_users=2] = placeholder[target=y] + %add : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%y, %x), kwargs = {}) + return add + + And then insert an unflatten + call_module + flatten to replace the subgraph: + graph(): + %x : [num_users=1] = placeholder[target=x] + %y : [num_users=1] = placeholder[target=y] + + %_spec_0 : [num_users=1] = get_attr[target=_spec_0] + %tree_unflatten : [num_users=2] = call_function[target=torch.utils._pytree.tree_unflatten](args = ([%x, %y], %_spec_0), kwargs = {}) + %getitem : [num_users=2] = call_function[target=operator.getitem](args = (%tree_unflatten, 0), kwargs = {}) + %getitem_1 : [num_users=1] = call_function[target=operator.getitem](args = (%getitem, 0), kwargs = {}) + %getitem_2 : [num_users=1] = call_function[target=operator.getitem](args = (%getitem, 1), kwargs = {}) + %getitem_3 : [num_users=0] = call_function[target=operator.getitem](args = (%tree_unflatten, 1), kwargs = {}) + %foo : [num_users=0] = call_module[target=foo](args = (%getitem_1, %getitem_2), kwargs = {}) + %_spec_1 : [num_users=1] = get_attr[target=_spec_1] + %tree_flatten_spec : [num_users=1] = call_function[target=torch.fx._pytree.tree_flatten_spec](args = (None, %_spec_1), kwargs = {}) + %getitem_4 : [num_users=1] = call_function[target=operator.getitem](args = (%tree_flatten_spec, 0), kwargs = {}) + + %sub : [num_users=1] = call_function[target=torch.ops.aten.sub.Tensor](args = (%y, %getitem_4), kwargs = {}) + return (%sub,) + + The `tree_unflatten` call will construct tensor inputs into the input + format needed by the swapped eager module. + The `call_module` node should now reference the swapped torch.nn.Module. + The `tree_flatten_spec` call will deconstruct the eager outputs of the + swapped module into tensors. + """ # noqa: B950 + + submod_name = name.replace(".", "_") + sub_gm, orig_inputs, orig_outputs = fuse_as_graphmodule( + gm, nodes, f"fused_{submod_name}" + ) + + log.debug("Fused subgraph nodes:") + log.debug(sub_gm.graph) + + signature: ModuleCallSignature = module_call_graph[name] + + args_nodes, kwargs_nodes = _construct_inputs(gm, signature, node_name_map) + module_node = _insert_call_module( + gm, args_nodes, kwargs_nodes, modules_to_swap[name], name + ) + _deconstruct_outputs(gm, signature, module_node, node_name_map, orig_outputs) + + erase_nodes(gm, nodes) + + log.debug("Swapped graph:") + log.debug(gm.graph) + + legalize_graph(gm) + + log.debug("Before removing extraneous pytrees:") + log.debug(gm.graph) + + _remove_extraneous_pytrees(gm) + log.debug("After removing extraneous pytrees:") + log.debug(gm.graph) + + gm.recompile() + + return gm + + +def _fix_input_output_signature( + gm: torch.fx.GraphModule, signature: ModuleCallSignature +) -> None: + """ + Given the unlifted module from calling ep.module(), we want to remove the + pytree processing from the graph module's PyTreeCodeGen and instead make it + nodes inside of the graph. This allows us to do some optimizations, like + remove these pytree calls if it is unnecessary, and makes the PyTree part + more obvious to graph passes. + """ + from torch.export.unflatten import _generate_flatten, _generate_unflatten + + # Remove the registered pytree codegen because we will take care of it + # through inserting pytree nodes into the graph + gm.graph._codegen = torch.fx.graph.CodeGen() + + old_placeholders = [node for node in gm.graph.nodes if node.op == "placeholder"] + + new_placeholders = [] + forward_arg_names = signature.forward_arg_names + if forward_arg_names is None: + forward_arg_names = [] + assert signature.in_spec.num_children == 2 + arg_spec = signature.in_spec.child(0) + kwarg_spec = signature.in_spec.child(1) + assert arg_spec.type is tuple + assert kwarg_spec.type is dict + for i in range(arg_spec.num_children): + forward_arg_names.append(f"arg_{i}") + forward_arg_names.extend(kwarg_spec.context) + + for arg in forward_arg_names: + with gm.graph.inserting_before(old_placeholders[0]): + new_placeholders.append(gm.graph.placeholder(arg)) + + # Insert flatten call for the inputs + with gm.graph.inserting_before(old_placeholders[0]): + flat_node = _generate_flatten(gm, tuple(new_placeholders)) + for i, old_placeholder in enumerate(old_placeholders): + old_placeholder.op = "call_function" + old_placeholder.target = operator.getitem + old_placeholder.args = (flat_node, i) + + # Insert unflatten call for the outputs + output_node = next(node for node in gm.graph.nodes if node.op == "output") + with gm.graph.inserting_before(output_node): + unflat = _generate_unflatten(gm, output_node.args[0], signature.out_spec) + output_node.args = (unflat,) + + gm.recompile() + + +def _swap_modules( + ep: ExportedProgram, modules_to_swap: dict[str, torch.nn.Module] +) -> torch.fx.GraphModule: + """ + Unlifts the given ExportedProgram into a fx.GraphModule, and then swaps + previously traced modules with new eager modules specified. Returns a + fx.GraphModule with a custom forward function. + + Args: + ep (ExportedProgram): Exported program to modify + modules_to_swap (Dict[str, torch.nn.Module]): Mapping from module fqn to + eager module to swap with. The specified module fqn should have also + been specified in the `preserve_module_call_signature` argument to + torch.export so that we know how to restore the calling convention + to this argument. + run_with_interpreter: Whether or not to run the graph using + fx.Interpreter. Setting to true will help result in better error + messages and easier debugging, but it has found to result in a QPS + drop. + """ + module_call_graph = { + entry.fqn: entry.signature for entry in ep.module_call_graph if entry.signature + } + + gm = ep.module() + gm.validate_inputs = False # type: ignore[assignment] + gm.graph.eliminate_dead_code() # type: ignore[operator, union-attr] + assert isinstance(gm, torch.fx.GraphModule) + _fix_input_output_signature(gm, ep.module_call_graph[0].signature) + + gm.module_call_graph = ep.module_call_graph + gm.train = types.MethodType(type(gm).train, gm) # type: ignore[assignment] + gm.eval = types.MethodType(type(gm).eval, gm) # type: ignore[assignment] + + assert isinstance(gm, torch.fx.GraphModule) + gm = _swap_module_helper(gm, modules_to_swap, module_call_graph) + + return gm diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_trace.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..0a6a9036506f4d83719cfb311388adcdf2e9fb70 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_trace.py @@ -0,0 +1,2463 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +import dataclasses +import functools +import inspect +import logging +import re +import sys +import time +import warnings +from collections.abc import Callable +from contextlib import contextmanager, ExitStack, nullcontext +from itertools import chain +from typing import Any, TYPE_CHECKING, TypeAlias +from unittest import mock + + +if TYPE_CHECKING: + import weakref + +import torch +import torch._dynamo +import torch.fx +import torch.utils._pytree as pytree +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo.exc import UserError, UserErrorType +from torch._export.db.logging import ( + exportdb_error_message, + get_class_if_classified_error, +) +from torch._export.non_strict_utils import ( + _fakify_module_inputs, + _fakify_script_objects, + _gather_constant_attrs, + _NonStrictTorchFunctionHandler, + _override_builtin_ops, + make_constraints, + make_fake_inputs, + produce_guards_and_solve_constraints, +) +from torch._export.passes.collect_tracepoints_pass import CollectTracepointsPass +from torch._export.passes.lift_constants_pass import ( + _materialize_and_lift_constants, + ConstantAttrMap, +) +from torch._export.utils import ( + _collect_param_buffer_metadata, + _compiling_state_context, + _fakify_params_buffers, + _populate_param_buffer_metadata_to_new_gm, + _update_gm_meta_if_possible, + apply_runtime_assertion_pass, + placeholder_naming_pass, + placeholder_prefixes, +) +from torch._export.verifier import SpecViolationError +from torch._export.wrappers import _wrap_submodules +from torch._functorch._aot_autograd.graph_capture_wrappers import create_functional_call +from torch._functorch._aot_autograd.input_output_analysis import ( + _graph_input_names, + _graph_output_names, +) +from torch._functorch._aot_autograd.schemas import GraphSignature +from torch._functorch._aot_autograd.subclass_utils import get_subclass_typing_container +from torch._functorch._aot_autograd.utils import ( + create_tree_flattened_fn, + register_buffer_assignment_hook, +) +from torch._functorch.aot_autograd import ( + _detect_attribute_assignment, + aot_export_joint_with_descriptors, +) +from torch._guards import detect_fake_mode, tracing, TracingContext +from torch._library.fake_class_registry import FakeScriptObject +from torch._logging import dtrace_structured +from torch._subclasses.fake_tensor import FakeTensorMode +from torch._utils_internal import log_export_usage +from torch.export._leakage_detection_utils import find_legit_leaks_from_referrers +from torch.export._unlift import _check_input_constraints_pre_hook +from torch.export.dynamic_shapes import ( + _check_dynamic_shapes, + _combine_args, + _DimHintType, + _IntWrapper, + _process_dynamic_shapes, +) +from torch.export.exported_program import OutputKind +from torch.fx._symbolic_trace import _ConstantAttributeType +from torch.fx.experimental.proxy_tensor import ( + get_proxy_slot, + make_fx, + PreDispatchTorchFunctionMode, + track_tensor_tree, +) +from torch.fx.experimental.symbolic_shapes import ( + ConstraintViolationError, + free_unbacked_symbols, + GuardOnDataDependentSymNode, + ShapeEnv, +) +from torch.fx.graph import _PyTreeInfo +from torch.utils._pytree import TreeSpec +from torch.utils._sympy.value_ranges import ValueRangeError + +from .exported_program import ( + _disable_prexisiting_fake_mode, + ExportedProgram, + InputKind, + ModuleCallEntry, + ModuleCallSignature, +) +from .graph_signature import _convert_to_export_graph_signature, ExportGraphSignature + + +log = logging.getLogger(__name__) + +# Type alias for dynamic shapes specification +_DynamicShapesSpec: TypeAlias = dict[str, Any] | tuple[Any, ...] | list[Any] + + +@dataclasses.dataclass +class ExportDynamoConfig: + """ + Manage Export-specific configurations of Dynamo. + """ + + allow_rnn: bool = True + reorderable_logging_functions: set[Callable] = dataclasses.field( + default_factory=set + ) + # Emit runtime asserts after AOTAutograd instead. + # This isn't really necessary, and isn't much more efficient since the runtime asserts pass does CSE, + # but if we want to reason more about what guards/runtime asserts to emit, + # this makes it a bit cleaner to do from the export side. Also no real point in running this twice. + do_not_emit_runtime_asserts: bool = True + specialize_int: bool = True + specialize_float: bool = True + assume_static_by_default: bool = False + automatic_dynamic_shapes: bool = False + capture_dynamic_output_shape_ops: bool = True + capture_scalar_outputs: bool = True + prefer_deferred_runtime_asserts_over_guards: bool = False + replay_side_effects: bool = False + side_effect_replay_policy: str = "warn" + + +@dataclasses.dataclass +class ATenExportArtifact: + gm: torch.fx.GraphModule + sig: ExportGraphSignature + constants: dict[str, _ConstantAttributeType] + + +@dataclasses.dataclass(frozen=True) +class ExportArtifact: + aten: ATenExportArtifact + in_spec: TreeSpec + out_spec: TreeSpec + fake_mode: FakeTensorMode + module_call_specs: dict[str, dict[str, pytree.TreeSpec]] + + +DEFAULT_EXPORT_DYNAMO_CONFIG = ExportDynamoConfig() +DEFAULT_EXPORT_DYNAMO_CONFIG.reorderable_logging_functions = { + logging.critical, + logging.debug, + logging.error, + logging.exception, + logging.info, + logging.log, + logging.warning, + print, + warnings.warn, +} + + +@contextmanager +def _ignore_backend_decomps(): + orig_mkldnn_flag = torch.backends.mkldnn.set_flags(False) + orig_nnpack_flag = torch.backends.nnpack.set_flags(False) + try: + yield + finally: + torch.backends.mkldnn.set_flags(*orig_mkldnn_flag) + torch.backends.nnpack.set_flags(*orig_nnpack_flag) + + +@contextmanager +def _disable_custom_triton_op_functional_decomposition(): + old = torch._functorch.config.decompose_custom_triton_ops + try: + # pyrefly: ignore [bad-assignment] + torch._functorch.config.decompose_custom_triton_ops = False + yield torch._functorch.config.decompose_custom_triton_ops + finally: + torch._functorch.config.decompose_custom_triton_ops = old + + +def custom_triton_ops_decomposition_disabled(): + return not torch._functorch.config.decompose_custom_triton_ops + + +def _fixup_key(x): + return "L__self__" + _strip_root(x) + + +def _strip_root(x): + if isinstance(x, str) and x.startswith("_export_root"): + stripped = x[len("_export_root") :] + return stripped.removeprefix(".") + return x + + +def _is_bogus_const_name(name: str): + splitted_names = name.split(".") + if len(splitted_names) < 1: + return True + + return splitted_names[-1].startswith("lifted_tensor") + + +def _rewrite_tracepoint_node(gm: torch.fx.GraphModule): + """ + In-place modify input graph module by replacing the export tracepoint with a new node + that has the same target and args, but with the _export_root stripped from path. + """ + for node in gm.graph.nodes: + if node.target is torch.ops.higher_order._export_tracepoint: + if "path" in node.kwargs: + path = _strip_root(node.kwargs["path"]) + with gm.graph.inserting_before(node): + new_node = gm.graph.create_node( + "call_function", + torch.ops.higher_order._export_tracepoint, + args=node.args, + kwargs={ + "path": path, + "kind": node.kwargs["kind"], + }, + ) + new_node.meta = node.meta + node.replace_all_uses_with(new_node) + gm.graph.erase_node(node) + + +def detect_shape_env(inputs: Any = None): + shape_envs = [] + + for i, flat_input in enumerate(inputs): + if isinstance(flat_input, torch.SymInt): + shape_envs.append((flat_input.node.shape_env, "symint input", i)) + + if shape_envs: + shape_env, desc1, i1 = shape_envs[0] + for m, desc2, i2 in shape_envs[1:]: + assert shape_env is m, ( + f"shape env ({shape_env}) from {desc1} {i1} doesn't match mode ({m}) from {desc2} {i2}\n\n" + f"shape env from {desc1} {i1} allocated at:\n{shape_env.stack}\n" + f"shape env from {desc2} {i2} allocated at:\n{m.stack}" + ) + return shape_env + else: + return None + + +def _extract_fake_inputs(gm, args, kwargs): + """ + Given a graph module, extract fakified input tensors from the metadata of + its placeholders, and map them to the structure of given args and kwargs. + Also return the fake mode used to fakify those inputs. + """ + fake_inps: list[Any] = [] + fake_vals: list[Any] = [] + for node in gm.graph.nodes: + if node.op == "placeholder": + fake_inps.append(node.meta.get("val")) + else: + fake_vals.append(node.meta.get("example_value")) + + if in_shuffle_graph := getattr(gm, "_in_shuffle_graph", None): + flat_args = pytree.tree_leaves((args, kwargs)) + node_map = { + node: i + for i, node in enumerate( + next(iter(reversed(in_shuffle_graph.graph.nodes))).args[0] + ) + if node.op == "placeholder" + } + new_fake_inps: list[Any] = [] + for i, node in enumerate( + in_shuffle_graph.graph.find_nodes(op="placeholder")[1:] + ): + if node in node_map: + new_fake_inps.append(fake_inps[node_map[node]]) + else: + new_fake_inps.append(flat_args[i]) + fake_inps = new_fake_inps + # We get both because now we might have a combination of symint and tensor + # inputs, and we want to check that the shape env is consistent between + # both. Unfortunately we can't see what fake mode is attached to the shape + # env, then we can just compare fake modes. + detected_fake_mode = detect_fake_mode(fake_inps + fake_vals) + detected_shape_env = detect_shape_env(fake_inps + fake_vals) + + if detected_fake_mode: + if detected_shape_env: + assert detected_shape_env is detected_fake_mode.shape_env, ( + "Detected shape env does not match fake mode's shape env" + ) + fake_mode = detected_fake_mode + elif detected_shape_env: + fake_mode = FakeTensorMode(shape_env=detected_shape_env, export=True) + else: + fake_mode = FakeTensorMode(shape_env=ShapeEnv(), export=True) + + count = 0 + + def lookup_fake(x): + nonlocal count + val = fake_inps[count] if isinstance(x, (int, torch.Tensor)) else x + count += 1 + return val + + fake_args = pytree.tree_map(lookup_fake, args) + fake_kwargs = pytree.tree_map(lookup_fake, kwargs) + + return fake_args, fake_kwargs, fake_mode + + +def _replace_param_buffer_names(param_buffer_table, sig): + for spec in sig.input_specs: + if spec.kind in ( + InputKind.PARAMETER, + InputKind.BUFFER, + ): + spec.target = param_buffer_table[spec.target] + for spec in sig.output_specs: + if spec.kind in ( + OutputKind.BUFFER_MUTATION, + OutputKind.GRADIENT_TO_PARAMETER, + ): + spec.target = param_buffer_table[spec.target] + + +def _convert_to_positional_args(orig_arg_names, args, kwargs): + assert len(orig_arg_names) == len(args) + len(kwargs), ( + f"Total number of arg names is expected to be {len(orig_arg_names)} " + f"but got {len(args)} positional args, {len(kwargs)} kwargs." + ) + reordered_kwargs = [kwargs[kw_name] for kw_name in orig_arg_names[len(args) :]] + return ( + *args, + *reordered_kwargs, + ) + + +def _normalize_nn_module_stack(gm_torch_level, root_cls): + # Append a root module to every nn_module_stack. + root = "L['self']" + root_key = re.sub(r"[^a-zA-Z0-9]", "_", root) + for gm in gm_torch_level.modules(): + if not isinstance(gm, torch.fx.GraphModule): + continue + for node in gm.graph.nodes: + if node.op in ["placeholder", "output"]: + continue + add_root = True + if nn_module_stack := node.meta.get("nn_module_stack", {}): + path, ty = next(iter(nn_module_stack.values())) + # After deserializing the class `ty` might not exist anymore so + # it could be a string + if inspect.isclass(ty) and issubclass(ty, torch.nn.Module): + # TODO Figure out why sometimes we have root sometimes we don't. + if path == root and ty is root_cls: + add_root = False + else: + assert isinstance(ty, str) + if add_root: + + def normalize_path(path): + if path == "L['self']": + return "" + if path.startswith("L['self']."): + return path[len("L['self'].") :] + return path + + nn_module_stack = { + root_key: (root, root_cls.__module__ + "." + root_cls.__qualname__), + # pyrefly: ignore [unbound-name] + **nn_module_stack, + } + node.meta["nn_module_stack"] = { + key: (normalize_path(path), ty) + for key, (path, ty) in nn_module_stack.items() + } + + +def _get_param_buffer_mapping( + original_module: torch.nn.Module, + traced_module: torch.nn.Module, +) -> dict[str, str]: + """ + Returns a mapping of parameter/buffer names from the new module to the + original model. This is to help with restoring the FQN for parameter/buffers + of a traced module to what the original module contains. + """ + + param_lookup: dict[int, str] = {} + buffer_lookup: dict[int, str] = {} + for name, param in original_module.named_parameters(remove_duplicate=False): + if param_lookup.get(id(param)) is None: + # we only want to keep the first occurrence of a parameter to guarantee parity of original and traced module. + param_lookup[id(param)] = name + for name, buffer in original_module.named_buffers(remove_duplicate=False): + buffer_lookup[id(buffer)] = name + + param_buffer_table: dict[str, str] = {} + for dynamo_name, dynamo_param in traced_module.named_parameters( + remove_duplicate=False + ): + assert dynamo_name not in param_buffer_table + if id(dynamo_param) in param_lookup: + param_buffer_table[dynamo_name] = param_lookup[id(dynamo_param)] + + for dynamo_name, dynamo_buffer in traced_module.named_buffers( + remove_duplicate=False + ): + assert dynamo_name not in param_buffer_table + if id(dynamo_buffer) in buffer_lookup: + param_buffer_table[dynamo_name] = buffer_lookup[id(dynamo_buffer)] + + return param_buffer_table + + +def _preserve_requires_grad_pass( + gm: torch.fx.GraphModule, + sig: ExportGraphSignature, + fake_params_buffers: dict[str, torch.Tensor], + constants: dict[str, _ConstantAttributeType], + flat_fake_args: list[Any], +): + placeholders = [node for node in gm.graph.nodes if node.op == "placeholder"] + assert len(sig.input_specs) == len(placeholders) + i = 0 + for node, spec in zip(placeholders, sig.input_specs): + if spec.kind in ( + InputKind.PARAMETER, + InputKind.BUFFER, + ): + assert spec.target is not None + node.meta["val"].requires_grad = fake_params_buffers[ + spec.target + ].requires_grad + elif spec.kind == InputKind.USER_INPUT: + fake_arg = flat_fake_args[i] + if isinstance(fake_arg, torch.Tensor): + node.meta["val"].requires_grad = fake_arg.requires_grad + i += 1 + elif spec.kind == InputKind.CONSTANT_TENSOR: + assert spec.target is not None + constant = constants[spec.target] + if isinstance(constant, torch.Tensor): + # If the tensor is not leaf, it should already have a correct requires grad field + if node.meta["val"].is_leaf: + node.meta["val"].requires_grad = constant.requires_grad + else: + assert node.meta["val"].requires_grad == constant.requires_grad + elif spec.kind in (InputKind.CUSTOM_OBJ, InputKind.TOKEN): + continue + else: + raise AssertionError(spec.kind) + + +def _remap_constants( + orig_constant_attrs: ConstantAttrMap, + graph_signature: ExportGraphSignature, + constants: dict[str, _ConstantAttributeType], +) -> None: + """Rewrite the graph signature and constants table to use the FQN from the original module.""" + remap_table: dict[str, list[str]] = {} + for name, value in constants.items(): + if value in orig_constant_attrs: + remap_table[name] = orig_constant_attrs[value] + + for spec in graph_signature.input_specs: + if spec.kind in ( + InputKind.CONSTANT_TENSOR, + InputKind.CUSTOM_OBJ, + ): + orig_target = spec.target + assert orig_target is not None + targets = remap_table.get(orig_target, [orig_target]) + spec.target = targets[0] + + constant = constants[orig_target] + del constants[orig_target] + for target in targets: + constants[target] = constant + + +def _replace_unbacked_bindings(gm: torch.fx.GraphModule) -> None: + """ + When we run an interpreter-based pass over a GraphModule, execution of data-dependent operators + will produce example values with new unbacked symbols. To track that the new/old symbols are equivalent, + we used to rely on the unbacked_renamings mapping. This led to problematic metadata where the unbacked_bindings + keys mapped new symbols (u2) to paths containing old symbols (u0) in the example values, or worse, backed symbols + or constants (e.g. if the original unbacked was replaced/specialized). Additionally this created problems with + de/serialized programs, since we didn't comprehensively serialize ShapeEnv/unbacked renamings/node bindings. + + This pass attempts a simpler way of handling these for export, by throwing away the previously computed bindings, and re-running + the pattern match used in compute_unbacked_bindings. This ensures we keep the original symbols contained in the example values, + or delete bindings if they've been replaced/specialized. + """ + from torch._export.utils import _get_shape_env_from_gm + from torch.fx.experimental.symbolic_shapes import _free_unbacked_symbols_with_path + from torch.utils._sympy.symbol import symbol_is_type, SymT + + if (shape_env := _get_shape_env_from_gm(gm)) is None: + return + + base_unbacked_symbols = { + symbol + for symbol in shape_env.var_to_range + if symbol_is_type(symbol, (SymT.UNBACKED_INT, SymT.UNBACKED_FLOAT)) + and symbol not in shape_env.unbacked_renamings + } + for node in gm.graph.nodes: + node.meta.pop("unbacked_bindings", None) + if (val := node.meta.get("val")) is not None and ( + unbacked_bindings := _free_unbacked_symbols_with_path( + val, + (), + shape_env=shape_env, + pending=base_unbacked_symbols, + simplify=True, + ) + ): + node.meta["unbacked_bindings"] = unbacked_bindings + + +def _produce_aten_artifact( + *, + gm: torch.fx.GraphModule, + mod, + constant_attrs, + graph_signature, + pre_dispatch, + fake_args, + fake_kwargs, + fake_params_buffers, + _prettify_placeholder_names=True, +) -> ATenExportArtifact: + """ + This is a helper function that is shared between export_to_aten_ir and export_to_aten_ir_make_fx + to produce the aten artifact. (export compatible graph module + signature) + + It does: + 1. Applies runtime assertion pass + 2. Recompute unbacked_bindings pass + 3. Populate meta val when missing + 4. Lift constants as placeholders + 5. Replace raw autograd and autocast ops with HOPs + 6. Prettify names for placeholders + 7. Preserve requires_grad value on node meta val + """ + # Run runtime asserts pass before creating input/output specs, since size-related CSE/DCE might affect output signature. + # Overwrite output specs afterwards. + flat_fake_args = pytree.tree_leaves((fake_args, fake_kwargs)) + gm, graph_signature = apply_runtime_assertion_pass(gm, graph_signature) + + # Simplify unbacked_bindings by recomputing them. + # Useful for any pass that's interpreter-based and might call rebind_unbacked(), + # e.g. AOTAutograd in this case. + _replace_unbacked_bindings(gm) + + total_non_user_inputs = ( + len(graph_signature.parameters) + + len(graph_signature.buffers) + + len(graph_signature.input_tokens) + ) + set_missing_meta_vals(gm, flat_fake_args, total_non_user_inputs) + + export_graph_signature: ExportGraphSignature | None + export_graph_signature = _convert_to_export_graph_signature( + graph_signature, gm, _get_non_persistent_buffers(mod) + ) + + # script objects are always stored in constants no matter whether they're initial inputs or + # they're lifted in aot" before rewrite_script_object_meta + constants = _materialize_and_lift_constants( + gm, export_graph_signature, constant_attrs + ) + + if pre_dispatch: + from torch._export.passes.replace_autocast_with_hop_pass import ( + replace_autocast_with_hop_pass, + ) + from torch._export.passes.replace_set_grad_with_hop_pass import ( + replace_set_grad_with_hop_pass, + ) + + # Note: replace_set_grad_with_hop_pass need to be after lift_constant_pass because + # a getattr of a constant tensor doesn't have meta["val"] until after lift_constant_pass. + # If replace_set_grad_with_hop_pass is before lift_constant_pass, + # and the constant_tensor is passed as input of the set grad hop, the placeholder's + # meta["val"] will be None and fails our verifier for placeholder. + gm, export_graph_signature = replace_set_grad_with_hop_pass( + gm, export_graph_signature + ) + + gm, export_graph_signature = replace_autocast_with_hop_pass( + gm, export_graph_signature + ) + + # Remove nn_module_stack, stack_trace metadata from all placeholders/inputs nodes. + for _mod in gm.modules(): + if not isinstance(_mod, torch.fx.GraphModule): + continue + for node in _mod.graph.nodes: + if node.op in ["placeholder", "output"]: + node.meta.pop("nn_module_stack", None) + node.meta.pop("stack_trace", None) + + # Prettify names for placeholder nodes. + assert export_graph_signature is not None + if _prettify_placeholder_names: + placeholder_naming_pass( + gm, + export_graph_signature, + mod, + fake_args, + fake_kwargs, + fake_params_buffers, + constants, + ) + + _preserve_requires_grad_pass( + gm, export_graph_signature, fake_params_buffers, constants, flat_fake_args + ) + + return ATenExportArtifact( + gm, + export_graph_signature, + constants, + ) + + +def _rename_constants_nodes( + gm: torch.fx.GraphModule, + graph_signature: ExportGraphSignature, +) -> None: + """ + For strict mode, rename constants nodes that were previously annotated as buffers. + """ + # handle name collisions with existing constants + node_names = {node.name for node in gm.graph.nodes} + + def rename_constant(name): + if name in node_names: + n = 1 + while (dup_name := f"{name}_{n}") in node_names: + n += 1 + name = dup_name + node_names.add(name) + return name + + # use input specs to map names from buffers to constants + buffer_prefix = placeholder_prefixes[InputKind.BUFFER] + const_prefix = placeholder_prefixes[InputKind.CONSTANT_TENSOR] + buffer_to_constant = {} + for spec in graph_signature.input_specs: + if spec.kind == InputKind.CONSTANT_TENSOR and not spec.arg.name.startswith( + const_prefix + ): + if spec.arg.name.startswith(buffer_prefix): # map from buffer to constants + c_name = rename_constant( + const_prefix + spec.arg.name[len(buffer_prefix) :] + ) + else: # lifted constant + c_name = rename_constant(const_prefix + spec.arg.name) + buffer_to_constant[spec.arg.name] = c_name + spec.arg.name = c_name + for spec in graph_signature.output_specs: + if spec.arg.name in buffer_to_constant: + spec.arg.name = buffer_to_constant[spec.arg.name] + + # Rename constants nodes for all modules + for mod in gm.modules(): + if not isinstance(mod, torch.fx.GraphModule): + continue + for node in mod.graph.nodes: + if node.name in buffer_to_constant: + node.name = node.target = buffer_to_constant[node.name] + mod.recompile() + + +def _restore_state_dict( + original_module: torch.nn.Module, traced_module: torch.fx.GraphModule +) -> None: + """ + Restores the state dict of the traced module to that of the original module. + """ + param_buffer_table = _get_param_buffer_mapping(original_module, traced_module) + # Don't want to change the convention of previous call. + param_buffer_table_reverse = {v: k for k, v in param_buffer_table.items()} + + # Replace state dict attr names with the fqn + for name, _ in list( + chain( + original_module.named_parameters(remove_duplicate=False), + # pyrefly: ignore [bad-argument-type] + original_module.named_buffers(remove_duplicate=False), + ) + ): + if name in param_buffer_table_reverse: + dynamo_name = param_buffer_table_reverse[name] + param = torch.fx.graph_module._get_attr(traced_module, dynamo_name) + torch.fx.graph_module._assign_attr(param, traced_module, name) + torch.fx.graph_module._del_attr(traced_module, dynamo_name) + + # Replace graph getattr nodes with the correct name + for node in traced_module.graph.nodes: + if node.op == "get_attr": + attr_name = node.target + if attr_name in param_buffer_table: + node.target = param_buffer_table[attr_name] + + traced_module.recompile() + + +def _get_module_hierarchy(mod: torch.nn.Module) -> dict[str, str]: + return { + name: type(m).__name__ for name, m in mod.named_modules(remove_duplicate=False) + } + + +def _make_module_call_graph( + in_spec: TreeSpec, + out_spec: TreeSpec, + module_call_signatures: dict[str, ModuleCallSignature], + forward_arg_names: list[str] | None = None, +) -> list[ModuleCallEntry]: + original = [ + ModuleCallEntry(fqn=fqn, signature=module_call_signatures.get(fqn)) + for fqn in _EXPORT_MODULE_HIERARCHY # type: ignore[union-attr] + ] + assert original[0].fqn == "" + original[0].signature = ModuleCallSignature( + inputs=[], + outputs=[], + in_spec=in_spec, + out_spec=out_spec, + forward_arg_names=forward_arg_names, + ) + additional = [ + ModuleCallEntry(fqn=fqn, signature=signature) + for fqn, signature in module_call_signatures.items() + if fqn not in _EXPORT_MODULE_HIERARCHY # type: ignore[operator] + ] + return [*original, *additional] + + +class _ExportModuleSpecTrackerDict(dict): + pass + + +def _export_to_torch_ir( + f: Callable, + args: tuple[Any, ...], + kwargs: dict[str, Any] | None = None, + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None = None, + *, + preserve_module_call_signature: tuple[str, ...] = (), + disable_constraint_solver: bool = False, + prefer_deferred_runtime_asserts_over_guards: bool = False, + restore_fqn: bool = True, + _log_export_usage: bool = True, + same_signature: bool = True, +) -> torch.fx.GraphModule: + """ + Traces either an nn.Module's forward function or just a callable with PyTorch + operations inside and produce a torch.fx.GraphModule in torch IR. + """ + + if _log_export_usage: + log_export_usage(event="export.private_api", flags={"_export_to_torch_ir"}) + + if not isinstance(args, tuple): + raise UserError( + UserErrorType.INVALID_INPUT, + f"Expecting `args` to be a tuple of example positional inputs, got {type(args)}", + ) + + kwargs = kwargs or {} + + # Map ints to a wrapper structure to help us mark it as dynamic, if it is + # dynamic. We will unwrap ints in fakify later. + args, kwargs = pytree.tree_map_only(int, _IntWrapper, (args, kwargs)) + + combined_args = _combine_args(f, args, kwargs) + _check_dynamic_shapes(combined_args, dynamic_shapes) + constraints = _process_dynamic_shapes(combined_args, dynamic_shapes) + + # Unwrap static ints -- in the case where we have an empty graph + # containing just integer computation, dynamo will run its generated + # bytecode with these args/kwargs, which will error because we cannot + # directly apply int operations on IntWrapper. So we will just unwrap + # them here. + args, kwargs = pytree.tree_map_only( + _IntWrapper, + lambda a: a.val + if a.dynamism is None or a.dynamism.type == _DimHintType.STATIC + else a, + (args, kwargs), + ) + + dynamo_cfg = dataclasses.replace( + DEFAULT_EXPORT_DYNAMO_CONFIG, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + + def use_legacy_dynamo_graph_capture() -> bool: + return bool( + constraints # dynamic shape + or dynamic_shapes # dynamic shape + or isinstance(f, torch.fx.GraphModule) # retracing + or preserve_module_call_signature # unflatten + or torch._functorch.config.fake_tensor_propagate_real_tensors # draft + or torch._export.config.use_legacy_dynamo_graph_capture + ) + + with torch._dynamo.config.patch(dataclasses.asdict(dynamo_cfg)): + try: + module_call_specs: dict[str, dict[str, pytree.TreeSpec]] = ( + _ExportModuleSpecTrackerDict() + ) + ctx = nullcontext() + if not isinstance(f, torch.fx.GraphModule): + ctx = _wrap_submodules( # type: ignore[assignment] + f, preserve_module_call_signature, module_call_specs + ) + with ctx, _ignore_backend_decomps(): + if torch._export.config.use_new_tracer_experimental: + from torch._dynamo.functional_export import ( + _dynamo_graph_capture_for_export, + dynamo_graph_capture_for_export, + ) + + if use_legacy_dynamo_graph_capture(): + dynamo_graph_capture = _dynamo_graph_capture_for_export( + f, constraints=constraints, dynamic_shapes=dynamic_shapes + ) + else: + dynamo_graph_capture = torch._dynamo.config.patch( + replay_side_effects=False + )(dynamo_graph_capture_for_export(f)) + # We can't serialize entire fake mode yet, so this is to make sure + # things like copy.deepcopy(ep.graph_module) not crash. + # see test_export.py::test_custom_tag_metadata_re_export + # Once we delete the old strict export, we can use + gm_torch_level = dynamo_graph_capture(*args, **kwargs) + # We can't serialize entire fake mode yet, so this is to make sure + # things like copy.deepcopy(ep.graph_module) not crash. + # see test_export.py::test_custom_tag_metadata_re_export + # Once we delete the old strict export, we can use this fake mode in the + # subsequent logic when lowering to aten IR. + del gm_torch_level.meta["fake_mode"] + + else: + gm_torch_level, _ = torch._dynamo.export( + f, + dynamic_shapes=dynamic_shapes, # type: ignore[arg-type] + constraints=constraints, # type: ignore[arg-type] + assume_static_by_default=True, + tracing_mode="symbolic", + disable_constraint_solver=disable_constraint_solver, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + _log_export_usage=_log_export_usage, + same_signature=same_signature, + )( + *args, + **kwargs, + ) + gm_torch_level.meta["module_call_specs"] = module_call_specs + except (ConstraintViolationError, ValueRangeError) as e: + raise UserError(UserErrorType.CONSTRAINT_VIOLATION, str(e)) # noqa: B904 + except GuardOnDataDependentSymNode as e: + raise UserError( # noqa: B904 + UserErrorType.ANTI_PATTERN, + f"Consider annotating your code using torch._check*(). {str(e)}", + case_name="constrain_as_size_example", + ) + + if isinstance(f, torch.nn.Module) and restore_fqn: + _restore_state_dict(f, gm_torch_level) + + return gm_torch_level + + +def _aot_export_joint_with_descriptors( + stack, + mod, + args, + *, + kwargs, + decompositions, + fake_params_buffers, + _record_nn_module_stack=True, +): + from torch._functorch._aot_autograd.graph_compile import aot_stage2_export + from torch._functorch._aot_autograd.input_output_analysis import ( + create_graph_signature, + ) + + joint_with_descriptors = aot_export_joint_with_descriptors( + stack, + mod, + args, + kwargs=kwargs, + decompositions=decompositions, + _record_nn_module_stack=_record_nn_module_stack, + ) + # Convert JointWithDescriptors to graph module and ViewAndMutationMeta + gm, fw_metadata = aot_stage2_export( + joint_with_descriptors._aot_state, + joint_with_descriptors._aot_graph_capture, + ) + + assert isinstance(gm, torch.fx.GraphModule) + + # Create GraphSignature from the metadata + graph_signature = create_graph_signature( + gm, + fw_metadata, + joint_with_descriptors.in_spec, + joint_with_descriptors.out_spec, + user_args_flat=pytree.tree_leaves((args, kwargs)), + params_and_buffers_flat=list(fake_params_buffers.values()), + param_names=joint_with_descriptors.params_spec, + buffer_names=joint_with_descriptors.buffers_spec, + trace_joint=False, + num_user_fw_outs=None, + loss_index=None, + ) + return gm, graph_signature + + +def _export_to_aten_ir( + mod: torch.nn.Module, + fake_args, + fake_kwargs, + fake_params_buffers, + constant_attrs: ConstantAttrMap, + produce_guards_callback=None, + *, + transform=lambda x: x, # TODO(zhxchen17) Revisit if this is needed later. + pre_dispatch=False, + decomp_table=None, + _prettify_placeholder_names: bool = True, + decompose_custom_triton_ops: bool = False, +) -> ATenExportArtifact: + custom_triton_ops_decomposition_ctx = ( + nullcontext + if decompose_custom_triton_ops + else _disable_custom_triton_op_functional_decomposition + ) + # This _reparameterize_module makes sure inputs and module.params/buffers have the same fake_mode, + # otherwise aot_export_module will error out because it sees a mix of fake_modes. + # And we want aot_export_module to use the fake_tensor mode in dynamo to keep the pipeline easy to reason about. + with ExitStack() as stack: + stack.enter_context( + torch.nn.utils.stateless._reparametrize_module( + mod, + fake_params_buffers, + tie_weights=True, + strict=True, + stack_weights=True, + ) + ) + stack.enter_context(_ignore_backend_decomps()) + stack.enter_context(_compiling_state_context()) + stack.enter_context(custom_triton_ops_decomposition_ctx()) + stack.enter_context(torch.no_grad()) + + gm, graph_signature = transform(_aot_export_joint_with_descriptors)( + stack, + mod, + fake_args, + kwargs=fake_kwargs, + decompositions=decomp_table, + fake_params_buffers=fake_params_buffers, + _record_nn_module_stack=True, + ) + + def _maybe_fixup_gm_and_output_node_meta(old_gm, new_gm): + if isinstance(old_gm, torch.fx.GraphModule): + if hasattr(old_gm, "meta"): + new_gm.meta.update(old_gm.meta) + old_output_node = list(old_gm.graph.nodes)[-1] + new_output_node = list(new_gm.graph.nodes)[-1] + assert old_output_node.op == "output" and new_output_node.op == "output" + # make sure we don't override any meta + if "desc" in new_output_node.meta: + del new_output_node.meta["desc"] + new_output_node.meta.update(old_output_node.meta) + + # TODO unfortunately preserving graph-level metadata and output node's meta + # is not working well with aot_export. So we manually copy it. + # (The node-level meta is addressed above.) + _maybe_fixup_gm_and_output_node_meta(mod, gm) + + # Run produce guards before we handle runtime asserts. + # This means we run the export solver before the runtime asserts pass. + # Right now this doesn't mean much - the export solver is only there for suggested fixes, + # and we won't even get to constraint solving if that's needed. + # But if in future we want to control what runtime asserts are emitted for export, + # or rely on produce_guards + solver for some simplification on runtime asserts, this probably makes sense. + if produce_guards_callback: + try: + produce_guards_callback(gm) + except (ConstraintViolationError, ValueRangeError) as e: + raise UserError(UserErrorType.CONSTRAINT_VIOLATION, str(e)) # noqa: B904 + + return _produce_aten_artifact( + gm=gm, + mod=mod, + constant_attrs=constant_attrs, + graph_signature=graph_signature, + pre_dispatch=pre_dispatch, + fake_args=fake_args, + fake_kwargs=fake_kwargs, + fake_params_buffers=fake_params_buffers, + _prettify_placeholder_names=_prettify_placeholder_names, + ) + + +def _get_forward_arg_names( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any] | None = None, +) -> list[str]: + """ + Gets the argument names to forward that are used, for restoring the + original signature when unlifting the exported program module. + - Positional args: retain the original argument names, and enumerate + *args as args_0, args_1, ... + - Keyword args: retain the original kwarg names in the order specified + by the user. This order seems to matter for the current state of + export lifted modules. + """ + sig = inspect.signature(mod.forward) + _args = sig.bind_partial(*args).arguments + + names: list[str] = [] + for name, value in _args.items(): + # handle variable number of positional args + if sig.parameters[name].kind == inspect._ParameterKind.VAR_POSITIONAL: + names.extend([f"{name}_{i}" for i, _ in enumerate(value)]) + else: + names.append(name) + # order of kwargs matters for input spec + if kwargs: + names.extend([kwarg for kwarg, _ in kwargs.items()]) + + return names + + +def _get_non_persistent_buffers(mod: torch.nn.Module) -> set[str]: + """ + Returns set of non-persistent buffers in a module and its submodules. + """ + result: set[str] = set() + for name, m in mod.named_modules(remove_duplicate=False): + if name: + result.update(f"{name}.{b}" for b in m._non_persistent_buffers_set) + else: + result.update(m._non_persistent_buffers_set) + return result + + +def _rewrite_dynamo_tensor_constants( + orig_mod_buffers: set[torch.Tensor], + traced_mod_buffers: dict[str, torch.Tensor], + graph_signature: ExportGraphSignature, + constants: dict[str, _ConstantAttributeType], +) -> None: + """ + Dynamo erroneously marks tensor attributes on modules as buffers. + Rewrite them to be tensor constants. + """ + for spec in graph_signature.input_specs: + if spec.kind == InputKind.BUFFER: + assert spec.target is not None + value = traced_mod_buffers[spec.target] + if value not in orig_mod_buffers: + # This was a tensor constant erroneously marked as a buffer. + # Convert it into a constant in the graph signature, and add its + # value to the constants table. + spec.kind = InputKind.CONSTANT_TENSOR + constants[spec.target] = value # type: ignore[arg-type] + + +def _move_non_persistent_buffers_to_tensor_constants( + orig_mod: torch.nn.Module, + graph_signature: ExportGraphSignature, + constants: dict[str, _ConstantAttributeType], +) -> None: + """ + Moves non-persistent buffers to tensor constants. + """ + for spec in graph_signature.input_specs: + if spec.kind == InputKind.BUFFER and not spec.persistent: + assert spec.target is not None + assert spec.target not in constants + constants[spec.target] = orig_mod.get_buffer(spec.target) # type: ignore[arg-type] + + +def _verify_nn_module_stack(graph_module: torch.fx.GraphModule) -> None: + """ + Perform nn_module_stack checks on the graph. + Current constraints: + For the top level graph: + - populated for 'call_function', 'get_attr' + - None for 'placeholder', 'output' + For submodule graphs: + - None for 'placeholder', output' + + TODO(pianpwk): make this a consistent node-level check once nn_module_stack is populated for cond submodules. + """ + # Check top-level graph for all nodes, all graphs for placeholder & output nodes + for i, mod in enumerate([graph_module] + list(graph_module.modules())): + if not isinstance(mod, torch.fx.GraphModule): + continue + for node in mod.graph.nodes: + if node.op in ["call_function", "get_attr"]: + if i == 0: + if ( + nn_module_stack := node.meta.get("nn_module_stack", None) + ) is None: + raise SpecViolationError( + f"Node {node} of type {node.op} is missing nn_module_stack metadata" + ) + if not all( + isinstance(k, str) + and isinstance(v, tuple) + and len(v) == 2 + and all(isinstance(x, str) for x in v) + for k, v in nn_module_stack.items() + ): + raise SpecViolationError( + f"Node {node} of type {node.op} has incorrect nn_module_stack metadata format" + f"expected Dict[str, Tuple[str, str]], but got {nn_module_stack}" + ) + elif node.op in ["placeholder", "output"]: + if node.meta.get("nn_module_stack", None): + raise SpecViolationError( + f"Node {node} of type {node.op} contains nn_module_stack metadata, this should be None" + ) + + +def _verify_stack_trace(graph_module: torch.fx.GraphModule) -> None: + """ + Perform stack trace checks on the graph. + Constraints: + - None or non-empty str for 'call_function', 'get_attr' + - None for 'placeholder', 'output' + """ + for mod in [graph_module, *graph_module.modules()]: + if not isinstance(mod, torch.fx.GraphModule): + continue + for node in graph_module.graph.nodes: + stack_trace = node.meta.get("stack_trace", None) + if node.op in ["call_function", "get_attr"]: + if not (stack_trace is None or isinstance(stack_trace, str)): + raise SpecViolationError( + f"Node {node} of type {node.op} has invalid stack_trace metadata, " + f"expected a string or None but instead found: {stack_trace}" + ) + elif node.op in ["placeholder", "output"]: + if stack_trace: + raise SpecViolationError( + f"Node {node} of type {node.op} contains stack_trace metadata, " + f"expected None but instead found: {stack_trace}" + ) + + +def _verify_placeholder_names( + gm: torch.fx.GraphModule, sig: ExportGraphSignature +) -> None: + """ + Performs a sanity check on the placeholder node names. + - User input nodes: no restrictions, should match the original forward() signature + - Params/buffers/constants/custom_obj/token nodes: should start with prefixes defined in + """ + name_to_kind = {spec.arg.name: spec.kind for spec in sig.input_specs} + for mod in gm.modules(): + if not isinstance(mod, torch.fx.GraphModule): + continue + for node in mod.graph.nodes: + if node.op == "placeholder": + if node.name not in name_to_kind: + continue + node_kind = name_to_kind[node.name] + prefix = placeholder_prefixes[node_kind] + if not node.name.startswith(prefix): + raise SpecViolationError( + f"Placeholder node name {node.name} does not follow spec for {node_kind}, name should have prefix: {prefix}" + ) + + +def get_ep_stats(ep: ExportedProgram) -> dict[str, Any]: + op_count = 0 + op_set = set() + for m in ep.graph_module.modules(): + if not isinstance(m, torch.fx.GraphModule): + continue + for node in m.graph.nodes: + if node.op != "call_function": + continue + op_count += 1 + assert hasattr(node.target, "__module__") + assert hasattr(node.target, "__name__") + op_set.add(f"{node.target.__module__}.{node.target.__name__}") + return {"op_count": op_count, "op_set": op_set} + + +_EXPORT_FLAGS: set[str] | None = None +_EXPORT_MODULE_HIERARCHY: dict[str, str] | None = None + + +def _log_export_wrapper(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + global _EXPORT_FLAGS, _EXPORT_MODULE_HIERARCHY + try: + start = time.time() + ep = fn(*args, **kwargs) + end = time.time() + log_export_usage( + event="export.time", + metrics=end - start, + flags=_EXPORT_FLAGS, + **get_ep_stats(ep), + ) + except Exception as e: + t = type(e) + error_type = t.__module__ + "." + t.__qualname__ + case_name = get_class_if_classified_error(e) + if case_name is not None: + log.error(exportdb_error_message(case_name)) + log_export_usage( + event="export.error.classified", + type=error_type, + message=str(e), + flags=_EXPORT_FLAGS, + ) + else: + log_export_usage( + event="export.error.unclassified", + type=error_type, + message=str(e), + flags=_EXPORT_FLAGS, + ) + + if hasattr(e, "partial_fx_graph"): + print( + e.partial_fx_graph, + file=sys.stderr, + ) + + raise e + finally: + _EXPORT_FLAGS = None + _EXPORT_MODULE_HIERARCHY = None + + return ep + + return wrapper + + +def _process_jit_trace_inputs_for_export(example_inputs, example_kwarg_inputs): + if not isinstance(example_inputs, (tuple, list, dict)): + example_inputs = (example_inputs,) + + elif isinstance(example_inputs, list): + example_inputs = tuple(example_inputs) + + elif ( + isinstance(example_inputs, (torch.Tensor, dict)) + and example_kwarg_inputs is None + ): + example_inputs = (example_inputs,) + + if example_kwarg_inputs is None: + example_kwarg_inputs = {} + return example_inputs, example_kwarg_inputs + + +def _get_original_state_dict(mod: torch.nn.Module) -> dict[str, Any]: + # Explicitly not calling mode.state_dict() as we do not want the module state for serialization + # but the running module state so we can always match by id() the entries here with the graph inputs + named_parameters = dict(mod.named_parameters(remove_duplicate=False)) + named_buffers = dict(mod.named_buffers(remove_duplicate=False)) + original_state_dict = named_parameters | named_buffers + + non_persistent_buffers = _get_non_persistent_buffers(mod) + for k in non_persistent_buffers: + original_state_dict.pop(k, None) + + return original_state_dict + + +def _process_export_inputs( + mod: torch.nn.Module, + args: tuple[object, ...], + kwargs: dict[str, object] | None, + dynamic_shapes: _DynamicShapesSpec + | torch.export.AdditionalInputs + | torch.export.ShapesCollection + | None, +) -> tuple[ + tuple[object, ...], + dict[str, object], + TreeSpec, + _DynamicShapesSpec | None, + Callable[[ExportedProgram], None], +]: + """ + Process and validate export inputs for the torch.export API. + + This function validates the input arguments, normalizes kwargs, computes input tree specs, + and handles special dynamic shapes cases like AdditionalInputs and ShapesCollection. + + Args: + mod: The PyTorch module to be exported. + args: Tuple of example positional inputs for the module. + kwargs: Optional dictionary of example keyword inputs. + dynamic_shapes: Optional specification for dynamic shapes. Can be: + - dict mapping argument names to dynamic shape specifications + - tuple/list specifying dynamic shapes for each input in order + - torch.export.AdditionalInputs object with verification callback + - torch.export.ShapesCollection object + + Returns: + A tuple containing: + - args: Validated tuple of positional inputs + - kwargs: Normalized dictionary of keyword inputs (empty dict if None was passed) + - original_in_spec: TreeSpec representing the flattened input structure + - dynamic_shapes: Processed dynamic shapes specification + - verify_additional_inputs: Callback function for additional input verification + + Raises: + UserError: If args is not a tuple. + """ + if not isinstance(args, tuple): + raise UserError( + UserErrorType.INVALID_INPUT, + f"Expecting `args` to be a tuple of example positional inputs, got {type(args)}", + ) + kwargs = kwargs if kwargs is not None else {} + if pytree.is_namedtuple_instance(args): + args = tuple(args) + + _, original_in_spec = pytree.tree_flatten((args, kwargs)) + + verify_additional_inputs: Callable[[ExportedProgram], None] + out_dynamic_shapes: _DynamicShapesSpec | None + if isinstance(dynamic_shapes, torch.export.AdditionalInputs): + verify_additional_inputs = dynamic_shapes.verify # type: ignore[assignment] + out_dynamic_shapes = dynamic_shapes.dynamic_shapes(mod, args, kwargs) # type: ignore[assignment] + else: + verify_additional_inputs = lambda ep: None # noqa: E731 + if isinstance(dynamic_shapes, torch.export.ShapesCollection): + out_dynamic_shapes = dynamic_shapes.dynamic_shapes(mod, args, kwargs) # type: ignore[assignment] + else: + out_dynamic_shapes = dynamic_shapes + + return args, kwargs, original_in_spec, out_dynamic_shapes, verify_additional_inputs + + +def _get_module_call_graph( + export_artifact: ExportArtifact, + preserve_module_call_signature: tuple[str, ...], + strict_mode_export: bool, + forward_arg_names: list[str] | None = None, +) -> tuple[torch.fx.GraphModule, list[ModuleCallEntry]]: + """ + In-place modify the graph module in export_artifact, remove _export_tracepoint nodes and + return module_call_graph. + """ + gm: torch.fx.GraphModule = export_artifact.aten.gm + export_graph_signature: ExportGraphSignature = export_artifact.aten.sig + module_call_specs: dict[str, dict[str, TreeSpec]] = ( + export_artifact.module_call_specs + ) + in_spec: TreeSpec = export_artifact.in_spec + out_spec: TreeSpec = export_artifact.out_spec + + # Make module signatures. + module_call_signatures: dict[str, ModuleCallSignature] = {} + for fqn, specs in module_call_specs.items(): + mod_fqn = _strip_root(fqn) if not strict_mode_export else fqn + module_call_signatures[mod_fqn] = ModuleCallSignature( + inputs=[], + outputs=[], + in_spec=specs["in_spec"], + out_spec=specs["out_spec"], + forward_arg_names=None, # we only propagate forward_arg_names for the top level module + ) + + if len(preserve_module_call_signature) > 0: + if not strict_mode_export: + _rewrite_tracepoint_node(gm) + res = CollectTracepointsPass(module_call_signatures, export_graph_signature)(gm) + assert res is not None + gm = res.graph_module + + assert _EXPORT_MODULE_HIERARCHY is not None + module_call_graph = _make_module_call_graph( + in_spec, + out_spec, + module_call_signatures, + forward_arg_names, + ) + return gm, module_call_graph + + +def _get_range_constraints( + mod: torch.nn.Module, + export_artifact: ExportArtifact, + args, + kwargs, + dynamic_shapes, +): + gm: torch.fx.GraphModule = export_artifact.aten.gm + export_graph_signature: ExportGraphSignature = export_artifact.aten.sig + fake_mode: FakeTensorMode = export_artifact.fake_mode + num_lifted = next( + ( + i + for i, s in enumerate(export_graph_signature.input_specs) + if s.kind == InputKind.USER_INPUT + ), + len(export_graph_signature.input_specs), + ) + combined_args = _combine_args(mod, args, kwargs) + + # This is because we trace based on the kwargs passed in from user + # not based on the signature. I feel it would be better to just enforce + # one ordering at the start of tracing to avoid confusions, but that is + # bigger refactor, so do this to unblock for now. + combined_args_traced_order = {} + for arg in combined_args: + if arg not in kwargs: + combined_args_traced_order[arg] = combined_args[arg] + + for key in kwargs: + combined_args_traced_order[key] = kwargs[key] + + combined_args = combined_args_traced_order + + range_constraints = make_constraints( + fake_mode, + gm, + combined_args, + dynamic_shapes, + num_lifted, + ) + return range_constraints + + +def _get_inline_constraints(fake_mode: FakeTensorMode): + assert fake_mode.shape_env is not None + return { + k: v + for k, v in fake_mode.shape_env.var_to_range.items() + if free_unbacked_symbols(k) + } + + +@contextmanager +def patch_forward(obj: torch.nn.Module, new_method): + """Helper method to make it easier to cleanly torch.export() a method on a + module that is not `forward`. + """ + # Save the original method + original_method = obj.forward + + # Patch the method + obj.forward = new_method.__get__(obj, obj.__class__) + + try: + yield + finally: + # Restore the original method + obj.forward = original_method + + +@contextmanager +def _temp_disable_texpr_fuser(): + original_state = torch._C._jit_texpr_fuser_enabled() + torch._C._jit_set_texpr_fuser_enabled(False) + try: + yield + finally: + torch._C._jit_set_texpr_fuser_enabled(original_state) + + +def _strict_export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any], + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, + preserve_module_call_signature: tuple[str, ...], + orig_in_spec: TreeSpec, + prefer_deferred_runtime_asserts_over_guards: bool, + _to_aten_func: Callable, +) -> ExportArtifact: + """ + _to_aten_func can either be `_export_to_aten_ir_make_fx` or `_export_to_aten_ir` + """ + + gm_torch_level = _export_to_torch_ir( + mod, + args, + kwargs, + dynamic_shapes, + preserve_module_call_signature=preserve_module_call_signature, + restore_fqn=False, # don't need to restore because we will do it later + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + _log_export_usage=False, + ) + + # We detect the fake_mode by looking at gm_torch_level's placeholders, this is the fake_mode created in dynamo. + ( + fake_args, + fake_kwargs, + dynamo_fake_mode, + ) = _extract_fake_inputs(gm_torch_level, args, kwargs) + + fake_params_buffers = _fakify_params_buffers(dynamo_fake_mode, gm_torch_level) + + # First, we want to pass through the graph to try populating + # val field for getattr if there is anything missing. + # This can happen when quantization adds extra params and forgets + # to update "val" + for node in gm_torch_level.graph.nodes: + if node.op == "get_attr" and "val" not in node.meta: + attr = getattr(gm_torch_level, node.target) + # Checks if it is not a HigherOrderOp branch or a module + if not isinstance(attr, torch.nn.Module): + assert dynamo_fake_mode is not None, ( + "Cannot find dynamo_fake_mode. This could be due to the exported graph module have no placeholders." + ) + node.meta["val"] = dynamo_fake_mode.from_tensor( + attr, static_shapes=True + ) + + # Fix the graph output signature to be tuple if scalar + + # gm_torch_level.graph._codegen is made a _PyTreeCodeGen in rewrite_signature in eval_frame.py + assert isinstance(gm_torch_level.graph._codegen, torch.fx.graph._PyTreeCodeGen) + + # Calling gm_torch_level._out_spec is not safe because gm_torch_level might be + # a _LazyGraphModule, which does not populate _out_spec when calling recompile(). + # TODO: Fix recompile() in _LazyGraphModule. T207713214 + out_spec = orig_out_spec = gm_torch_level.graph._codegen.pytree_info.out_spec + + # Used to get rid of lint type error. + assert out_spec is not None + assert orig_out_spec is not None + + # aot_export expect the return type to always be a tuple. + if out_spec.type not in (list, tuple): + out_spec = pytree.treespec_tuple([out_spec]) + + orig_arg_names = gm_torch_level.graph._codegen.pytree_info.orig_args # type: ignore[attr-defined] + + gm_torch_level.graph._codegen.pytree_info = _PyTreeInfo( + orig_arg_names, + gm_torch_level._in_spec, + out_spec, + ) + gm_torch_level.recompile() + + _normalize_nn_module_stack(gm_torch_level, type(mod)) + + params_buffers_to_node_meta = _collect_param_buffer_metadata(gm_torch_level) + + # When aot_export lifts the params, we lose metadata (e.g. source_fn_stack, stack_trace) + # from the param nodes as they are treated as fresh inputs + # Therefore, we manually extract them before calling into aot_export + # params_buffers_to_node_meta = _collect_param_buffer_metadata(gm_torch_level) + + constant_attrs = _gather_constant_attrs(mod) + param_buffer_table: dict[str, str] = _get_param_buffer_mapping(mod, gm_torch_level) + + # Dynamo does not track which buffers were registered as non-persistent. This info + # is available in the original module, so we transfer it to the traced module. Also, + # since we didn't restore original param/buffer names yet, we must use traced names. + non_persistent_buffers = _get_non_persistent_buffers(mod) + reverse_name_lookup = {orig: traced for traced, orig in param_buffer_table.items()} + gm_torch_level._non_persistent_buffers_set = { + reverse_name_lookup[name] + for name in non_persistent_buffers + if name in reverse_name_lookup + } + + tx = TracingContext(dynamo_fake_mode) + with ( + dynamo_fake_mode, + tracing(tx), + mock.patch.object(dynamo_fake_mode, "allow_non_fake_inputs", True), + ): + aten_export_artifact = _to_aten_func( + gm_torch_level, + # NOTE: graph module expects only positional args + _convert_to_positional_args(orig_arg_names, fake_args, fake_kwargs), + {}, + fake_params_buffers, + constant_attrs, + ) + + # Decompose for readability. + gm = aten_export_artifact.gm + export_graph_signature = aten_export_artifact.sig + constants = aten_export_artifact.constants + + _populate_param_buffer_metadata_to_new_gm( + params_buffers_to_node_meta, gm, export_graph_signature + ) + + # Do some cleanups on the graph module to restore the state dict to the + # expected form. Each of these steps should probably get fixed upstream. + # 1. Remove tensor constants that were added as buffers. + _rewrite_dynamo_tensor_constants( + orig_mod_buffers=set(mod.buffers()), + traced_mod_buffers=dict(gm_torch_level.named_buffers()), + graph_signature=export_graph_signature, + constants=constants, + ) + # 2. Restore FQN of param/buffers + _replace_param_buffer_names(param_buffer_table, export_graph_signature) + + # 3. Move non-persistent buffers to tensor constants + _move_non_persistent_buffers_to_tensor_constants( + mod, export_graph_signature, constants + ) + + # 4. Rewrite constants to have the same FQN as the original module. + _remap_constants(constant_attrs, export_graph_signature, constants) + + # 5. Rename constants nodes in graph module from buffers to constants + _rename_constants_nodes(gm, export_graph_signature) + + return ExportArtifact( + aten=aten_export_artifact, + in_spec=orig_in_spec, + out_spec=orig_out_spec, + fake_mode=dynamo_fake_mode, + module_call_specs=gm_torch_level.meta["module_call_specs"], + ) + + +def _export_to_aten_ir_make_fx( + mod: torch.nn.Module, + fake_args, + fake_kwargs, + fake_params_buffers, + constant_attrs: ConstantAttrMap, + produce_guards_callback=None, + transform=lambda x: x, +) -> ATenExportArtifact: + def _make_fx_helper(stack, mod, args, kwargs, **flags): + kwargs = kwargs or {} + + named_parameters = dict(mod.named_parameters(remove_duplicate=False)) + named_buffers = dict(mod.named_buffers(remove_duplicate=False)) + + params_and_buffers = {**named_parameters, **named_buffers} + params_and_buffers_flat, params_spec = pytree.tree_flatten(params_and_buffers) + params_and_buffers_flat = tuple(params_and_buffers_flat) + + param_len = len(named_parameters) + buffer_len = len(named_buffers) + params_len = len(params_and_buffers) + + functional_call = create_functional_call( + mod, params_spec, params_len, store_orig_mod=True + ) + + params_buffers_args: list[Any] = [] + params_buffers_args.extend(params_and_buffers_flat) + params_buffers_args.extend(args) + + flat_fn, out_spec = create_tree_flattened_fn( + functional_call, params_buffers_args, kwargs + ) + flat_args, in_spec = pytree.tree_flatten((params_buffers_args, kwargs)) + + @functools.wraps(flat_fn) + def wrapped_fn(*args): + return tuple(flat_fn(*args)) + + with enable_python_dispatcher(): + ctx = nullcontext() + non_strict_root = getattr(mod, "_export_root", None) + if non_strict_root is not None: + ctx = _detect_attribute_assignment(non_strict_root) # type: ignore[assignment] + + # For any buffer that is assigned, we want to associate it to the final proxy node + # that it is assigned to. This node can then be copied into the buffer. + assigned_buffers: dict[str, str] = {} + hook = register_buffer_assignment_hook( + non_strict_root, assigned_buffers + ) + + def custom_getattribute(self, attr, *, original_getattr, attrs_to_proxy): + """ + The idea here is that we override subclass getattr methods to proxy + inner tensors and metadata. Because of infinite loop shenanigans, we have + to manually construct the getattr proxy nodes without relying on torch function + system. + """ + out = original_getattr(self, attr) + if attr in attrs_to_proxy: + if torch._C._is_torch_function_mode_enabled(): + if isinstance(out, torch.Tensor): + # When we get here there is no guarantee that we will hit the + # PreDispatchTorchFunctionMode, so we manually peak into the torch + # function mode list and tweak the PreDispatchTorchFunctionMode. + # This has side effect of proxying stuff like + # proxy.node.meta["val"] = extract_val(val) because at that time, torch function + # mode is still active. It seems bad to turn it off inside proxy_tensor.py, so + # I guess we will just rely on DCE for now to remove extra stuff like detach + torch_function_mode_stack = ( + torch.overrides._get_current_function_mode_stack() + ) + for mode in torch_function_mode_stack: + if isinstance(mode, PreDispatchTorchFunctionMode): + tracer = mode.tracer + proxy = get_proxy_slot(self, tracer).proxy + inner_proxy = tracer.create_proxy( + "call_function", + torch.ops.export.access_subclass_inner_tensor.default, + (proxy, attr), + {}, + ) + track_tensor_tree( + out, inner_proxy, constant=None, tracer=tracer + ) + return out + + @contextmanager + def override_getattribute_for_subclasses(args): + """ + Context manager that temporarily monkey patches + tensor.__getattribute__ so that we can intercept it at + torch_function layer. + """ + + # Dictionary that tracks subclass type to original getattr function + # and the attributes we can proxy. + tensor_type_to_old_getattribute: dict[ + type[torch.Tensor], tuple[Callable, set[str]] + ] = {} + for arg in args: + subclass_types_to_instances: dict[ + type[torch.Tensor], list[type[torch.Tensor]] + ] = get_subclass_typing_container(arg) + for subclass_type in subclass_types_to_instances: + if subclass_type not in tensor_type_to_old_getattribute: + assert len(subclass_types_to_instances[subclass_type]) > 0 + instance = subclass_types_to_instances[subclass_type][0] + # Query subclass specific attrs + attrs_to_proxy = set(dir(instance)) - set(dir(torch.Tensor)) + tensor_type_to_old_getattribute[subclass_type] = ( + subclass_type.__getattribute__, # type: ignore[attr-defined] + attrs_to_proxy, + ) + + try: + for k, ( + old_getattr, + attrs_to_proxy, + ) in tensor_type_to_old_getattribute.items(): + custom = functools.partialmethod( + custom_getattribute, + original_getattr=old_getattr, + attrs_to_proxy=attrs_to_proxy, + ) + k.__getattribute__ = custom # type: ignore[assignment, attr-defined] + yield + finally: + for k, (old_getattr, _) in tensor_type_to_old_getattribute.items(): + k.__getattribute__ = old_getattr # type: ignore[method-assign, attr-defined] + + @contextmanager + def _maybe_restore_grad_state(): + """ + When pre-dispatch export accidentally change grad state, we restore it back. + This can happen when we are calling torch._C._set_grad_enabled directly in the + forward. + """ + old_state = torch.is_grad_enabled() + try: + yield + finally: + torch._C._set_grad_enabled(old_state) + + with ( + ctx, + override_getattribute_for_subclasses(flat_args), + _maybe_restore_grad_state(), + ): + gm = make_fx( + wrapped_fn, + record_module_stack=True, + pre_dispatch=True, + )(*flat_args) + + if non_strict_root is not None: + input_names = _graph_input_names(gm) + buffer_input_names = { + name: input_names[param_len + i] + for i, (name, buf) in enumerate(non_strict_root._buffers.items()) + if buf is not None + } + output_node = list(gm.graph.nodes)[-1] + # We copy nodes corresponding to buffer assignments to buffers in the graph. + for buf, name in assigned_buffers.items(): # type: ignore[possibly-undefined] + buf_node = _find_node(gm, buffer_input_names[buf]) + name_node = _find_node(gm, name) + with gm.graph.inserting_before(output_node): + new_node = gm.graph.create_node( + "call_function", + torch.ops.aten.copy_.default, + args=(buf_node, name_node), + ) + new_node.meta = name_node.meta + + hook.remove() # type: ignore[possibly-undefined] + + def _is_impure(node): + if node.op == "call_function" and node.target in ( + # In export, we ignore any op that is related to + # eager mode profiling call. The expectation is + # that either runtimes provide their own profiling + # OR user wrap the compiled region on a profiling in + # later stage. + torch.ops.profiler._record_function_enter.default, + torch.ops.profiler._record_function_enter_new.default, + torch.ops.profiler._record_function_exit._RecordFunction, + # In theory, we could fix this dead detach and getattr nodes + # from subclass tensors if we carefully rewrite track_tensor_tree + # in a way that it doesn't do any tensor methods. + torch.ops.aten.detach.default, + torch.ops.export.access_subclass_inner_tensor.default, + ): + return False + return True + + gm.graph.eliminate_dead_code(_is_impure) + + # create graph signature + assert out_spec.spec is not None, "out_spec.spec is None!" + input_names = _graph_input_names(gm) + output_names = _graph_output_names(gm) + sig = GraphSignature( + parameters=list(named_parameters), + buffers=list(named_buffers), + user_inputs=input_names[params_len:], + user_outputs=output_names, + inputs_to_parameters=dict(zip(input_names[0:param_len], named_parameters)), + inputs_to_buffers=dict( + zip(input_names[param_len : param_len + buffer_len], named_buffers) + ), + buffers_to_mutate={}, + parameters_to_mutate={}, + user_inputs_to_mutate={}, + in_spec=in_spec, + out_spec=out_spec.spec, + backward_signature=None, + input_tokens=[], + output_tokens=[], + ) + return gm, sig + + # This _reparameterize_module makes sure inputs and module.params/buffers have the same fake_mode, + # otherwise aot_export_module will error out because it sees a mix of fake_modes. + # And we want aot_export_module to use the fake_tensor mode in dynamo to keep the pipeline easy to reason about. + with ExitStack() as stack: + stack.enter_context( + torch.nn.utils.stateless._reparametrize_module( + mod, + fake_params_buffers, + tie_weights=True, + strict=True, + stack_weights=True, + ) + ) + stack.enter_context(_ignore_backend_decomps()) + stack.enter_context(_compiling_state_context()) + gm, graph_signature = transform(_make_fx_helper)( + stack, + mod, + fake_args, + trace_joint=False, + kwargs=fake_kwargs, + ) + + # [NOTE] In training IR, we don't run + # any DCE as a result we preserve constant + # nodes in the graph. make_fx invariant is that + # they don't guarantee every node gets a meta['val'] + # field. Since the actual value is already hardcoded in + # graph, the node.meta here actually doesn't matter. But + # we do this to make spec verifier happy. + for node in gm.graph.nodes: + if ( + node.op == "call_function" + and len(node.users) == 0 + and "val" not in node.meta + ): + node.meta["val"] = None + + if isinstance(mod, torch.fx.GraphModule) and hasattr(mod, "meta"): + gm.meta.update(mod.meta) + + # See comment in _export_to_aten_ir() + if produce_guards_callback: + try: + produce_guards_callback(gm) + except (ConstraintViolationError, ValueRangeError) as e: + raise UserError(UserErrorType.CONSTRAINT_VIOLATION, str(e)) # noqa: B904 + + return _produce_aten_artifact( + gm=gm, + mod=mod, + constant_attrs=constant_attrs, + graph_signature=graph_signature, + pre_dispatch=True, + fake_args=fake_args, + fake_kwargs=fake_kwargs, + fake_params_buffers=fake_params_buffers, + ) + + +def set_missing_meta_vals(gm, flat_args, num_params_buffers): + # Sets missing metadata to address two problems: + # 1. aot_export adds symint metadata for placeholders with int values; since + # these become specialized, we replace such metadata with the original values. + # 2. any tensor attributes that are not params / buffers, i.e., are constants + # need to have their metadata set before lifting them because it is needed + # for computing the exported program's signature. + index = 0 + for node in gm.graph.nodes: + if node.op == "placeholder": + if index >= num_params_buffers: + user_arg = flat_args[index - num_params_buffers] + if not isinstance(user_arg, torch.Tensor): + node.meta["val"] = user_arg + index += 1 + + +def _find_node(gm: torch.fx.GraphModule, name: str) -> torch.fx.Node: + return next(iter(node for node in gm.graph.nodes if node.name == name)) + + +def _non_strict_export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any], + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, + preserve_module_call_signature: tuple[str, ...], + orig_in_spec: TreeSpec, + prefer_deferred_runtime_asserts_over_guards: bool, + _to_aten_func: Callable, +) -> ExportArtifact: + """ + _to_aten_func can either be `_export_to_aten_ir_make_fx` or `_export_to_aten_ir` + """ + + out_spec: TreeSpec | None = None + in_spec: TreeSpec | None = None + + module_call_specs: dict[str, dict[str, pytree.TreeSpec]] = {} + + def _tuplify_outputs(aot_export): + def _aot_export_non_strict(stack, mod, args, *, kwargs=None, **flags): + kwargs = kwargs or {} + + class Wrapper(torch.nn.Module): + def __init__(self, mod): + super().__init__() + self._export_root = mod + + def forward(self, *args, **kwargs): + nonlocal out_spec + nonlocal in_spec + mod = self._export_root + _, in_spec = pytree.tree_flatten((args, kwargs)) + if isinstance(mod, torch.fx.GraphModule): + # NOTE: We're going to run this graph module with an fx interpreter, + # which will not run any forward hooks. Thus, ideally, we should run + # all forward hooks here. But the general logic for running them is + # complicated (see nn/module.py), and probably not worth duplicating. + # Instead we only look for, and run, an export-specific forward hook. + if ( + _check_input_constraints_pre_hook + in mod._forward_pre_hooks.values() + ): + _check_input_constraints_pre_hook(mod, args, kwargs) + with torch.fx.traceback.preserve_node_meta(): + args = (*args, *kwargs.values()) + tree_out = torch.fx.Interpreter(mod).run(*args) + else: + tree_out = mod(*args, **kwargs) + flat_outs, out_spec = pytree.tree_flatten(tree_out) + return tuple(flat_outs) + + wrapped_mod = Wrapper(mod) + # Patch export_root to the signatures so that wrapper module correctly populates the + # in/out spec + new_preserved_call_signatures = [ + "_export_root." + i for i in preserve_module_call_signature + ] + ctx = nullcontext() + if not isinstance(mod, torch.fx.GraphModule): + ctx = _wrap_submodules( # type: ignore[assignment] + wrapped_mod, new_preserved_call_signatures, module_call_specs + ) + with ctx: + gm, sig = aot_export(stack, wrapped_mod, args, kwargs=kwargs, **flags) + log.debug("Exported program from AOTAutograd:\n%s", gm) + + sig.parameters = pytree.tree_map(_strip_root, sig.parameters) + sig.buffers = pytree.tree_map(_strip_root, sig.buffers) + sig.inputs_to_buffers = pytree.tree_map(_strip_root, sig.inputs_to_buffers) + sig.inputs_to_parameters = pytree.tree_map( + _strip_root, sig.inputs_to_parameters + ) + sig.buffers_to_mutate = pytree.tree_map(_strip_root, sig.buffers_to_mutate) + sig.parameters_to_mutate = pytree.tree_map( + _strip_root, sig.parameters_to_mutate + ) + + for node in gm.graph.nodes: + if "nn_module_stack" in node.meta: + nn_module_stack = node.meta["nn_module_stack"] + node.meta["nn_module_stack"] = { + _fixup_key(key): val + for key, val in pytree.tree_map( + _strip_root, nn_module_stack + ).items() + } + + return gm, sig + + return _aot_export_non_strict + + ( + fake_mode, + fake_args, + fake_kwargs, + equalities_inputs, + original_signature, + dynamic_shapes, + ) = make_fake_inputs( + mod, + args, + kwargs, + dynamic_shapes, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, # for shape env initialization + ) + + fake_params_buffers = _fakify_params_buffers(fake_mode, mod) + + def _produce_guards_callback(gm): + return produce_guards_and_solve_constraints( + fake_mode=fake_mode, + gm=gm, + dynamic_shapes=dynamic_shapes, + equalities_inputs=equalities_inputs, + original_signature=original_signature, + ) + + tx = TracingContext(fake_mode) + + # We also need to attach dynamo configs as these will be used in HOOs that + # use torch.compile, like cond + dynamo_config = dataclasses.asdict(DEFAULT_EXPORT_DYNAMO_CONFIG) + dynamo_config["do_not_emit_runtime_asserts"] = ( + False # We want to emit runtime asserts + ) + + with ( + fake_mode, + _NonStrictTorchFunctionHandler(), + tracing(tx), + torch._dynamo.config.patch(dynamo_config), + ): + with ( + _fakify_script_objects(mod, fake_args, fake_kwargs, fake_mode) as ( + patched_mod, + new_fake_args, + new_fake_kwargs, + new_fake_constant_attrs, + map_fake_to_real, + ), + _fakify_module_inputs(fake_args, fake_kwargs, fake_mode), + _override_builtin_ops(), + ): + # _to_aten_func is _export_to_aten_ir when using the default non-strict export + # We need to pass positional args correctly + aten_export_artifact = _to_aten_func( + patched_mod, + new_fake_args, + new_fake_kwargs, + fake_params_buffers, + new_fake_constant_attrs, + produce_guards_callback=_produce_guards_callback, + transform=_tuplify_outputs, + ) + # aten_export_artifact.constants contains only fake script objects, we need to map them back + aten_export_artifact.constants = { + fqn: map_fake_to_real[obj] if isinstance(obj, FakeScriptObject) else obj + for fqn, obj in aten_export_artifact.constants.items() + } + + _move_non_persistent_buffers_to_tensor_constants( + mod, aten_export_artifact.sig, aten_export_artifact.constants + ) + + assert out_spec is not None + assert in_spec is not None + + return ExportArtifact( + aten=aten_export_artifact, + in_spec=in_spec, + out_spec=out_spec, + fake_mode=fake_mode, + module_call_specs=module_call_specs, + ) + + +@_log_export_wrapper +@_disable_prexisiting_fake_mode +def _export_for_training( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any] | None = None, + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None = None, + *, + strict: bool = True, + preserve_module_call_signature: tuple[str, ...] = (), + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + global _EXPORT_MODULE_HIERARCHY + _EXPORT_MODULE_HIERARCHY = _get_module_hierarchy(mod) + + ( + args, + kwargs, + orig_in_spec, + dynamic_shapes, + verify_additional_inputs, + ) = _process_export_inputs(mod, args, kwargs, dynamic_shapes) + + original_state_dict = _get_original_state_dict(mod) + + has_ambient_mode = False + if not strict: + flat_args, _ = pytree.tree_flatten((args, kwargs)) + has_ambient_mode = torch._guards.detect_fake_mode(flat_args) is not None + + # Call the appropriate export function based on the strictness of tracing. + export_func = _strict_export if strict else _non_strict_export + + if not strict and torch._export.config.detect_non_strict_fake_tensor_leaks: + from torch._subclasses.fake_tensor import fake_tensor_tls + + fake_tensor_tls.non_strict_export_fake_tensor_tracker.clear() + + export_artifact = export_func( + mod=mod, + args=args, + kwargs=kwargs, + dynamic_shapes=dynamic_shapes, + preserve_module_call_signature=preserve_module_call_signature, + orig_in_spec=orig_in_spec, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + _to_aten_func=_export_to_aten_ir_make_fx, + ) + + # If we are tracing with fake inputs, it is expected to + # see fake tensor constants. + if not strict and not has_ambient_mode: + for const, val in export_artifact.aten.constants.items(): + if isinstance( + val, torch._subclasses.fake_tensor.FakeTensor + ) and _is_bogus_const_name(const): + error_msg = ( + f"We found a fake tensor in the exported program constant's list. " + f"This typically means our tracing system encountered an op that " + f"we can't trace through. For the potential source, you can refer to " + f"following model attribute: {const}. " + f"Please file an issue on github. " + ) + if torch._export.config.error_on_lifted_constant_tensors: + raise RuntimeError(error_msg) + else: + warnings.warn(error_msg, stacklevel=2) + + export_graph_signature = export_artifact.aten.sig + + forward_arg_names = _get_forward_arg_names(mod, args, kwargs) + inline_constraints = _get_inline_constraints(export_artifact.fake_mode) + # The unbacked symint symbols are updated in aot_export + # so we serialize them here instead of inside dynamo. + # Note: _get_range_constraints depends on "inline_constraints" to be set. + export_artifact.aten.gm.meta["inline_constraints"] = inline_constraints + range_constraints = _get_range_constraints( + mod, + export_artifact, + args, + kwargs, + dynamic_shapes, + ) + # The returned the gm is in-place modified + gm, module_call_graph = _get_module_call_graph( + export_artifact, + preserve_module_call_signature, + strict, + forward_arg_names, + ) + + _verify_nn_module_stack(gm) + _verify_stack_trace(gm) + _verify_placeholder_names(gm, export_graph_signature) + + _update_gm_meta_if_possible(gm, mod) + + from torch._export.verifier import TrainingIRVerifier + + exported_program = ExportedProgram( + root=gm, + graph=gm.graph, + graph_signature=export_graph_signature, + state_dict=original_state_dict, + range_constraints=range_constraints, + module_call_graph=module_call_graph, + example_inputs=(args, kwargs), + constants=export_artifact.aten.constants, + verifiers=[TrainingIRVerifier], + ) + + verify_additional_inputs(exported_program) + + if not strict and torch._export.config.detect_non_strict_fake_tensor_leaks: + # See NOTE [export non-strict fake tensor leak detection] + from torch._subclasses.fake_tensor import fake_tensor_tls + from torch.fx.experimental.proxy_tensor import ( + _FAKE_TENSOR_ID_TO_PROXY_MAP_FOR_EXPORT, + ) + + active_fakes = fake_tensor_tls.non_strict_export_fake_tensor_tracker + legit_leak: weakref.WeakSet = find_legit_leaks_from_referrers(active_fakes) + leak_sources: list[str] = [] + if len(legit_leak) > 0: + for fake_val in legit_leak: + if id(fake_val) in _FAKE_TENSOR_ID_TO_PROXY_MAP_FOR_EXPORT: + stack_trace = _FAKE_TENSOR_ID_TO_PROXY_MAP_FOR_EXPORT[ + id(fake_val) + ].meta.get("stack_trace", "") + + # Get shape and dtype info + shape_info = f"shape={fake_val.shape}, dtype={fake_val.dtype}" + leak_info = f"FakeTensor({shape_info}): {stack_trace}" + leak_sources.append(leak_info) + + # Format the warning message more nicely + leak_details = "\n ".join(leak_sources) + warnings.warn( + f"Detected {len(legit_leak)} fake tensors that are still alive after export.\n" + f"This is likely result of torch.export.export not being able to track side effects " + f"that is happening outside of model scope.\n\n" + f"Leaked tensors:\n {leak_details}\n\n" + f"Alternatively, please file a bug report to PyTorch team for further debugging help.", + stacklevel=2, + ) + + del legit_leak + + return exported_program + + +@_log_export_wrapper +@_disable_prexisiting_fake_mode +def _export( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: dict[str, Any] | None = None, + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None = None, + *, + strict: bool = True, + preserve_module_call_signature: tuple[str, ...] = (), + pre_dispatch: bool = False, + prefer_deferred_runtime_asserts_over_guards: bool = False, +) -> ExportedProgram: + """ + Traces either an nn.Module's forward function or just a callable with PyTorch + operations inside and produce a ExportedProgram. + + Args: + mod: the `nn.Module` to trace. + + args: example positional inputs. + + kwargs: optional example keyword inputs. + + dynamic_shapes: + An optional argument where the type should either be: + 1) a dict from argument names of ``f`` to their dynamic shape specifications, + 2) a tuple that specifies dynamic shape specifications for each input in original order. + If you are specifying dynamism on keyword args, you will need to pass them in the order that + is defined in the original function signature. + + The dynamic shape of a tensor argument can be specified as either + (1) a dict from dynamic dimension indices to :func:`Dim` types, where it is + not required to include static dimension indices in this dict, but when they are, + they should be mapped to None; or (2) a tuple / list of :func:`Dim` types or None, + where the :func:`Dim` types correspond to dynamic dimensions, and static dimensions + are denoted by None. Arguments that are dicts or tuples / lists of tensors are + recursively specified by using mappings or sequences of contained specifications. + + preserve_module_call_signature: A list of submodule paths for which the original + calling conventions are preserved as metadata. + + prefer_deferred_runtime_asserts_over_guards: + With the current dynamic shapes language for dims and derived dims, we can run into constraints + that are not expressible with the language. For example, flattening a matrix and adding to a vector, + both fully dynamic (i.e. x.reshape([-1]) + y) emits a guard s0 * s1 = s2, which is not expressible. + By default, we either raise a constraint violation error or specialize to static values. + If this flag is set to True, we avoid erroring out and instead allow complex constraints to exist as runtime + assertions in the graph. The sympy interpreter (torch/utils/_sympy/interp.py) will produce the math ops + required to compute and assert the value of the guard (e.g. sym_size_int, eq, _assert_scalar). + Additionally, if TORCH_DYNAMO_DO_NOT_EMIT_RUNTIME_ASSERTS=1 is specified, we will allow complex constraints + while not emitting runtime asserts, returning a cleaner graph with lesser guarantees around dynamic shapes. + + Returns: + An ExportedProgram containing the traced module. + """ + + from torch._utils_internal import export_training_ir_rollout_check + + global _EXPORT_FLAGS, _EXPORT_MODULE_HIERARCHY + _EXPORT_MODULE_HIERARCHY = _get_module_hierarchy(mod) + + flags = set() + flags.add("strict" if strict else "non_strict") + flags.add("pre_dispatch" if pre_dispatch else "aot_dispatch") + _EXPORT_FLAGS = flags + + log_export_usage(event="export.enter", flags=_EXPORT_FLAGS) + + dtrace_structured("export", payload_fn=lambda: "start!") + + # NOTE Export training IR rollout + # Old export calls export._trace(pre_dispatch=True) + # and there are still lot of internal/OSS callsites that + # use export._trace(pre_dispatch=True) directly. Therefore, + # it makes more sense to do the switch here. + # export_training_ir_rollout_check returns True in OSS + # while internally it returns False UNLESS otherwise specified. + if pre_dispatch and export_training_ir_rollout_check(): + ep = _export_for_training( + mod, + args, + kwargs, + dynamic_shapes, + strict=strict, + preserve_module_call_signature=preserve_module_call_signature, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + ) + dtrace_structured("exported_program", payload_fn=lambda: str(ep)) + return ep + + ( + args, + kwargs, + original_in_spec, + dynamic_shapes, + verify_additional_inputs, + ) = _process_export_inputs(mod, args, kwargs, dynamic_shapes) + + original_state_dict = _get_original_state_dict(mod) + + # Call the appropriate export function based on the strictness of tracing. + export_func = _strict_export if strict else _non_strict_export + + export_artifact = export_func( # type: ignore[operator] + mod=mod, + args=args, + kwargs=kwargs, + dynamic_shapes=dynamic_shapes, + preserve_module_call_signature=preserve_module_call_signature, + orig_in_spec=original_in_spec, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + _to_aten_func=functools.partial( + _export_to_aten_ir, + pre_dispatch=pre_dispatch, + ), + ) + export_graph_signature: ExportGraphSignature = export_artifact.aten.sig + + forward_arg_names = _get_forward_arg_names(mod, args, kwargs) + inline_constraints = _get_inline_constraints(export_artifact.fake_mode) + # The unbacked symint symbols are updated in aot_export + # so we serialize them here instead of inside dynamo. + # Note: this step must be before _get_range_constraints. + export_artifact.aten.gm.meta["inline_constraints"] = inline_constraints + range_constraints = _get_range_constraints( + mod, + export_artifact, + args, + kwargs, + dynamic_shapes, + ) + gm, module_call_graph = _get_module_call_graph( + export_artifact, + preserve_module_call_signature, + strict, + forward_arg_names, + ) + + _verify_nn_module_stack(gm) + _verify_stack_trace(gm) + _verify_placeholder_names(gm, export_graph_signature) + + # Remove Proxy because they cannot be deepcopied or pickled. + torch._export.utils.remove_proxy_from_state_dict(original_state_dict, in_place=True) + + from torch._export.verifier import Verifier + + _update_gm_meta_if_possible(gm, mod) + + exported_program = ExportedProgram( + root=gm, + graph=gm.graph, + graph_signature=export_graph_signature, + state_dict=original_state_dict, + range_constraints=range_constraints, + module_call_graph=module_call_graph, + example_inputs=(args, kwargs), + constants=export_artifact.aten.constants, + verifiers=[Verifier], + ) + + dtrace_structured("exported_program", payload_fn=lambda: str(exported_program)) + + verify_additional_inputs(exported_program) + return exported_program diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_tree_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_tree_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a14551ee3400cb88077ea918d65fd03002494c1a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_tree_utils.py @@ -0,0 +1,65 @@ +from collections.abc import Callable +from typing import Any + +from torch.utils._pytree import Context, TreeSpec + + +def reorder_kwargs(user_kwargs: dict[str, Any], spec: TreeSpec) -> dict[str, Any]: + """Reorder user-provided kwargs to match the order in `spec`. `spec` is + expected to be the in_spec of an exported program, i.e. the spec that + results from flattening `(args, kwargs)`. + + We need this to provide consistent input ordering, such so that users can + pass in foo(a=a, b=b) OR foo(b=b, a=a) and receive the same result. + """ + # Make sure that the spec is actually shaped like (args, kwargs) + assert spec.type is tuple + assert spec.num_children == 2 + kwargs_spec = spec.child(1) + assert kwargs_spec.type is dict + + if set(user_kwargs) != set(kwargs_spec.context): + raise ValueError( + f"Ran into a kwarg keyword mismatch: " + f"Got the following keywords {list(user_kwargs)} but expected {kwargs_spec.context}" + ) + + reordered_kwargs = {} + for kw in kwargs_spec.context: + reordered_kwargs[kw] = user_kwargs[kw] + + return reordered_kwargs + + +def is_equivalent( + spec1: TreeSpec, + spec2: TreeSpec, + equivalence_fn: Callable[[type | None, Context, type | None, Context], bool], +) -> bool: + """Customizable equivalence check for two TreeSpecs. + + Arguments: + spec1: The first TreeSpec to compare + spec2: The second TreeSpec to compare + equivalence_fn: A function to determine the equivalence of two + TreeSpecs by examining their types and contexts. It will be called like: + + equivalence_fn(spec1.type, spec1.context, spec2.type, spec2.context) + + This function will be applied recursively to all children. + + Returns: + True if the two TreeSpecs are equivalent, False otherwise. + """ + if not equivalence_fn(spec1.type, spec1.context, spec2.type, spec2.context): + return False + + # Recurse on children + if spec1.num_children != spec2.num_children: + return False + + for child_spec1, child_spec2 in zip(spec1.children(), spec2.children()): + if not is_equivalent(child_spec1, child_spec2, equivalence_fn): + return False + + return True diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_unlift.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_unlift.py new file mode 100644 index 0000000000000000000000000000000000000000..bf7e1aed91bae87400cda79a1be492f71254ac4c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_unlift.py @@ -0,0 +1,878 @@ +# mypy: allow-untyped-defs +import copy +import inspect +import math +import warnings +from collections.abc import Sequence +from itertools import chain +from typing import Any + +import sympy + +import torch +import torch.utils._pytree as pytree +from torch._export.non_strict_utils import ( + _enter_enable_graph_inputs_of_type_nn_module, + _exit_enable_graph_inputs_of_type_nn_module, + _get_graph_inputs_of_type_nn_module, +) +from torch._export.passes.add_runtime_assertions_for_constraints_pass import ( + _convert_range_to_int, +) +from torch._export.utils import _check_input_constraints_for_graph +from torch.export.unflatten import _assign_attr, _AttrKind +from torch.fx.experimental.proxy_tensor import _pytree_subclasses_that_lose_info +from torch.fx.graph import _PyTreeCodeGen, _PyTreeInfo +from torch.fx.traceback import NodeSource, NodeSourceAction +from torch.utils._sympy.solve import try_solve +from torch.utils._sympy.value_ranges import ValueRanges + +from ._remove_effect_tokens_pass import _remove_effect_tokens +from ._tree_utils import reorder_kwargs +from .exported_program import ( + ExportedProgram, + ExportGraphSignature, + InputKind, + OutputKind, +) + + +def eq_spec(self: pytree.TreeSpec, other: pytree.TreeSpec) -> bool: + """ + Refinement of TreeSpec.__eq__ where, e.g., torch.Size(...) matches tuple(...). + See _pytree_subclasses_that_lose_info in proxy_tensor.py for more details. + """ + + def _normalize_type(t): + return str(_pytree_subclasses_that_lose_info.get(t, t)) + + def _match_normalized_structure(a, b): + if a is b: + return True + if _normalize_type(a.type) != _normalize_type(b.type): + return False + if a.type is dict and b.type is dict: + # in the case of dict, the context is list of keys and we allow the keys to be in any order + if set(a.context) != set(b.context): + return False + elif a.context != b.context: + return False + if a.num_children != b.num_children: + return False + return all( + _match_normalized_structure(a, b) + for a, b in zip(a.children(), b.children()) + ) + + return _match_normalized_structure(self, other) + + +def _check_inputs_match(args, kwargs, in_spec: pytree.TreeSpec) -> list: + reordered_kwargs = reorder_kwargs(kwargs, in_spec) + flat_args_with_path, received_spec = pytree.tree_flatten_with_path( + (args, reordered_kwargs) + ) + + if not eq_spec(received_spec, in_spec): + raise ValueError( # noqa: B904 + "Trying to flatten user inputs with exported input tree spec: \n" + f"{in_spec}\n" + "but actually got inputs with tree spec of: \n" + f"{received_spec}.\n" + "Please check that the inputs have the same number and type of " + "args and kwargs as the ones you used when tracing." + ) + + return flat_args_with_path + + +def _force_ep_signature_match(ep_guards_code: list[str], input_paths): + # TODO (tmanlaibaatar) + # This is band-aid solution to export new tracer replacing + # shape env sources to flat_args. The real fix should be replacing + # shape env sources to original user sources but this is quite + # involved because you need to carefully construct new sources using + # dynamo and replace all instances of it inside shape env. But it is + # lot easier to manipulate after we turn them into strings and only + # time we use these guards is during retracing or running exported program, + # so it is probably ok to have "not useful" guards on ep for now. + name_mapping = {} + for idx, path in enumerate(input_paths): + name_mapping[f"L['flat_args'][{idx}]"] = f"L{pytree.keystr(path)}" + + new_guards_code = [] + for guard in ep_guards_code: + for old_name, new_name in name_mapping.items(): + guard = guard.replace(old_name, new_name) + new_guards_code.append(guard) + + return new_guards_code + + +def _force_gm_signature_match(ep_guards_code: list[str], signature): + """ + The signature of the originally exported module may not match + the signature of the unlifted graph module extracted from the + exported program. The guards code extracted from the exported + program is based on the former, but the generated guards fn is + based on the latter; thus we need to reconcile any such diff. + """ + + import re + + # Handle case where signatures may differ in var args. + orig_arg_names = set() + for g in ep_guards_code: + # match substrings of the form L[''][] + orig_arg_names.update(re.findall(r"L\[\'([^\']+)\'\]\[([0-9]+)\]", g)) + + sig_arg_names = set() + for n in signature.parameters: + # match substrings of the form _ + sig_arg_names.update(re.findall(r"(.+)_([0-9]+)", n)) + + # replace L[''][] with L['_'] + new_guards_code = ep_guards_code + for match in orig_arg_names: + if match in sig_arg_names: + base, idx = match + new_guards_code = [ + g.replace(f"L['{base}'][{idx}]", f"L['{base}_{idx}']") + for g in new_guards_code + ] + + return new_guards_code + + +def _convert_guards_code_to_fn( + guards_code: list[str], + paths_of_placeholders: list[pytree.KeyPath], +): + """ + Generates Python code given guards code and paths of placeholders. + We assume that, based on source information, + - the tracer generates the guards code + - the input spec generates the paths of placeholders. + + Example: + + Suppose we are given the guards code "L['z']['k'].size()[1] == 3" + and we are given that ['z']['k'] is the path of placeholder #2. + Then we will generate: + ``` + torch._assert( + args[2].size()[0] == 3, + "Guard failed: z['k'].size()[0] == 3", + ) + ``` + + FAQ: Why do we generate code based on (flattened) args instead of + the original (unflattened) inputs? Because this would require + inserting an additional pytree.unflatten call in our graph. + + FAQ: Why do we not emit RuntimeError on guard failure as we used to? + Because it is inconvenient :/, get used to AssertionError instead. + """ + + import ast + + from torch.fx.experimental.symbolic_shapes import SYMPY_INTERP + + actual_guards_code = [] + shadow_guards_code = [] + for c in guards_code: + a, s = c, c + for idx, path in enumerate(paths_of_placeholders): + # e.g., replace L['z']['k'] with args[2] for Python code (actual) + a = a.replace("L" + pytree.keystr(path), f"args[{idx}]") + # e.g., replace L['z']['k'] with z['k'] for error message (shadow) + s = s.replace( + "L" + pytree.keystr(path), + path[0].key + pytree.keystr(path[1:]), # type: ignore[attr-defined] + ) + actual_guards_code.append(a) + shadow_guards_code.append(s.replace("\n", "")) + + # generate function code as str + code_str = "\ndef _(*args):\n" + for actual, shadow in zip(actual_guards_code, shadow_guards_code): + # printing guards code may potentially introduce redundant parens; + # we can normalize them out for readability by parsing/unparsing + # NOTE: this is not necessary for correctness, just deemed desirable + _shadow = ast.unparse(ast.parse(shadow, mode="eval")) + # actual code and shadow error message + code_str += f' torch._assert({actual}, "Guard failed: {_shadow}")\n' + code_str += " return\n" + + # populate namespace with sympy globals, materialize function (named `_`) + namespace = {**SYMPY_INTERP} + exec(code_str, namespace) + + # create and return a module whose forward is the materialized function + # NOTE: we want Dynamo to trace through this module, to repopulate guards: + # otherwise we would lose them when retracing + # NOTE: calling this module will be a side effect (no users): so it must + # be marked impure to avoid being not cleaned up by DCE + guards_fn = GuardsFn() + guards_fn.forward = torch._dynamo.dont_skip_tracing(namespace["_"]) # type: ignore[call-overload, method-assign] + guards_fn._is_impure = True # type: ignore[assignment] + return guards_fn + + +@torch._dynamo.disable +def _check_input_constraints_for_module(self, args, kwargs): + flat_args_with_path = _check_inputs_match(args, kwargs, self._in_spec) + _check_input_constraints_for_graph( + self.graph.find_nodes(op="placeholder"), + flat_args_with_path, + self.range_constraints, + ) + + +def _check_input_constraints_pre_hook(self, args, kwargs): + # preserve current behavior for clients that do not want any validation + if not self.validate_inputs: + return + + # when a guards function exists, assume that the graph does calls it! + # so we do not need to check input constraints...but we still want + # to check inputs match, otherwise we'd get obscure pytree errors + if hasattr(self, "_guards_fn"): + _check_inputs_match(args, kwargs, self._in_spec) + return + + # NOTE: for some reason, Dynamo is tracing into this, we should see why and + # put compile at the right place. Until then, we can skip the input + # constraint checks. + if not torch.compiler.is_dynamo_compiling(): + _check_input_constraints_for_module(self, args, kwargs) + + +def _unlift_inputs_as_getattr( + gm: torch.fx.GraphModule, + lifted_inputs: Sequence[str | None], +) -> tuple[dict[str, torch.fx.Node], dict[str, torch.fx.Node]]: + """ + Unlift inputs referring to params/buffers/constants as getattr nodes in the + graph + """ + unlifted_name_to_node = {} + input_name_to_node = {} + + placeholder_nodes = [node for node in gm.graph.nodes if node.op == "placeholder"] + assert len(lifted_inputs) == len(placeholder_nodes) + for input_node, lifted_node in zip(placeholder_nodes, lifted_inputs): + if lifted_node is None: + input_name_to_node[input_node.name] = input_node + + else: + with gm.graph.inserting_after(input_node): + # It is fine to ignore this warning because + # it is guaranteed that we will populate this + # attr later. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + getattr_node = gm.graph.get_attr(lifted_node) + input_node.replace_all_uses_with(getattr_node) + metadata = input_node.meta + gm.graph.erase_node(input_node) + getattr_node.meta = metadata + getattr_node.meta["from_node"] = [ + NodeSource( + input_node, + "ExportedProgram.module().unlift()", + [NodeSourceAction.CREATE, NodeSourceAction.REPLACE], + ) + ] + unlifted_name_to_node[lifted_node] = getattr_node + + return unlifted_name_to_node, input_name_to_node + + +def _insert_copy_for_mutations( + gm: torch.fx.GraphModule, + mutated_outputs: Sequence[str | None], + unlifted_name_to_node: dict[str, torch.fx.Node], + input_name_to_node: dict[str, torch.fx.Node], +) -> None: + """ + Find the all the buffers and inputs that were mutated and insert copy_ + operators to reflect mutations. + """ + output_node = gm.graph.output_node() + outputs = pytree.tree_flatten(output_node.args)[0] + assert len(outputs) == len(mutated_outputs) + + user_output_nodes = [] + return_nodes_to_copy = {} + for return_node, mutated_node_name in zip(outputs, mutated_outputs): + if mutated_node_name is None: + user_output_nodes.append(return_node) + continue + + if mutated_node_name in unlifted_name_to_node: + mutated_node = unlifted_name_to_node[mutated_node_name] + elif mutated_node_name in input_name_to_node: + mutated_node = input_name_to_node[mutated_node_name] + else: + raise RuntimeError( + f"Could not find {mutated_node_name} in either buffer or input nodes" + ) + + with gm.graph.inserting_before(output_node): + copy_node = gm.graph.call_function( + torch.ops.aten.copy_.default, (mutated_node, return_node) + ) + return_nodes_to_copy[return_node] = copy_node + + output_args = tuple( + return_nodes_to_copy.get(node, node) for node in user_output_nodes + ) + with gm.graph.inserting_before(output_node): + # Only return user outputs + new_output = gm.graph.output(output_args) + output_node.replace_all_uses_with(new_output) + gm.graph.erase_node(output_node) + new_output.name = output_node.name + new_output.meta.update(output_node.meta) + new_output.meta["from_node"] = [ + NodeSource( + output_node, + "ExportedProgram.module().unlift()", + [NodeSourceAction.CREATE, NodeSourceAction.REPLACE], + ) + ] + + +def _get_codegen( + in_spec: pytree.TreeSpec, + out_spec: pytree.TreeSpec | None, + forward_arg_names: list[str] | None = None, +) -> _PyTreeCodeGen: + """ + Create the codegen for the graph module based on the in/out specs + """ + if forward_arg_names: + names = forward_arg_names + elif ( + in_spec.type is tuple + and in_spec.num_children == 2 + and in_spec.child(0).type is tuple + and in_spec.child(1).type is dict + ): + # if in_spec contains the args (tuple) and kwargs (dict) + names = [f"arg_{i}" for i in range(in_spec.child(0).num_children)] + # add kwarg names + names.extend(in_spec.child(1).context) + else: + names = [f"arg_{i}" for i in range(in_spec.num_children)] + + return _PyTreeCodeGen( + _PyTreeInfo( + names, + in_spec, + out_spec, + ) + ) + + +def _unlift( + gm: torch.fx.GraphModule, + lifted_inputs: Sequence[str | None], + mutated_outputs: Sequence[str | None], + in_spec: pytree.TreeSpec, + out_spec: pytree.TreeSpec | None, + forward_arg_names: list[str] | None = None, +): + """ + Args: + lifted_inputs: A list matching the graph module's input nodes. For + an input node that is referring to a lifted parameter/buffer, this + list will contain the fqn the corresponding attribute. Otherwise, this + list will contain None. This is used to unlift the lifted parameters as + get_attr nodes. + + mutated_outputs: A list matching the graph module's output nodes. For + an output node that is referring to a mutated buffer or user input, this + list will contain the name of the corresponding buffer or user input + that needs to be mutated. Otherwise, this list will contain None. This + is used to re-insert an inplace copy_ operator to copy the mutated + values back to the original node. + """ + unlifted_name_to_node, input_name_to_node = _unlift_inputs_as_getattr( + gm, lifted_inputs + ) + _insert_copy_for_mutations( + gm, mutated_outputs, unlifted_name_to_node, input_name_to_node + ) + gm.graph._codegen = _get_codegen(in_spec, out_spec, forward_arg_names) + gm.graph.lint() + gm.recompile() + return gm + + +def _register_attrs_to_new_gm( + new_gm: torch.fx.GraphModule, + graph_signature: ExportGraphSignature, + state_dict: dict[str, Any], + constants: dict[str, Any], +) -> None: + non_persistent_buffers = set(graph_signature.non_persistent_buffers) + for name in graph_signature.buffers: + if name in non_persistent_buffers: + persistent = False + value = constants[name] + else: + persistent = True + value = state_dict[name] + _assign_attr( + value, new_gm, name, attr_kind=_AttrKind.BUFFER, persistent=persistent + ) + for name in graph_signature.parameters: + value = state_dict[name] + _assign_attr( + value, + new_gm, + name, + attr_kind=_AttrKind.PARAMETER, + ) + + # Technically this doesn't account for the aliased multiple constants but + # it is ok because we have a separate pass later in the stack that populates + # the final gm. + for name in chain( + graph_signature.lifted_custom_objs, graph_signature.lifted_tensor_constants + ): + value = constants[name] + _assign_attr( + value, + new_gm, + name, + attr_kind=_AttrKind.CONSTANT, + ) + + +class _StatefulGraphModuleFactory(type): + """ + Metaclass that ensures a private constructor for _StatefulGraphModule + """ + + def __call__(cls, *args, **kwargs): + raise TypeError( + f"{cls.__module__}.{cls.__qualname__} has no public constructor. " + ) + + def _create(cls, root, graph, range_constraints=None): + return super().__call__( + root, + graph, + range_constraints=range_constraints, + ) + + +class _StatefulGraphModule(torch.fx.GraphModule, metaclass=_StatefulGraphModuleFactory): + def __init__(self, root, graph, range_constraints=None): + super().__init__(root, graph) + # Need to fix up non-persistent buffers. + self.range_constraints = range_constraints or [] + self.validate_inputs = True + + +def _create_stateful_graph_module( + plain_graph_module: torch.fx.GraphModule, + range_constraints, + ep: ExportedProgram, +) -> _StatefulGraphModule: + stateful_gm = _StatefulGraphModule._create( + plain_graph_module, + plain_graph_module.graph, + range_constraints=range_constraints, + ) + + module_types = _get_graph_inputs_of_type_nn_module(ep.example_inputs) + stateful_gm.register_forward_pre_hook( + lambda *args, **kwargs: _enter_enable_graph_inputs_of_type_nn_module( + module_types + ) + ) + stateful_gm.register_forward_pre_hook( + _check_input_constraints_pre_hook, with_kwargs=True + ) + + stateful_gm.register_forward_hook( + lambda *args, **kwargs: _exit_enable_graph_inputs_of_type_nn_module( + module_types + ), + always_call=True, + ) + + # When we have a constant that has requires_grad=True, we need to detach it + # when we unlift as the tensors that require gradients should be registered + # via parameters. But this is problematic when we have aliasing two constants + # because when we call detach, they will become different tensors. This dict + # keeps track of this logic. + original_tensor_to_detached_tensor = {} + + # Fix up lifted tensor constants. + # fx.GraphModule() constructor silently turns a constant attribute of plain_graph_module + # into a buffer in stateful_gm and creates an inconsistency with graph_signature. + # We fix this by de-registering these buffers in lifted_tensor_constants + # and call _assign_attr(attr_kind=CONSTANT) to register them as constants. + for constant_fqn in ep.graph_signature.lifted_tensor_constants: + # Sometimes, the constant can require gradient, this is probably a bug in user code, + # e.g. `self.const = torch.randn(2, 2, requires_grad=True)`. + # We call detach on the constant_val since they're tensor constants and we don't need to + # compute their gradients anyway. + # Users should properly register it as parameter if they want it to require gradient. + buffer = stateful_gm.get_buffer(constant_fqn) + if buffer.requires_grad: + warnings.warn( + f"A model attribute `{constant_fqn}` requires gradient. " + f"but it's not properly registered as a parameter. " + f"torch.export will detach it and treat it as a constant tensor " + f"but please register it as parameter instead.", + stacklevel=2, + ) + detached_buffer = buffer.detach() + original_tensor_to_detached_tensor[buffer] = detached_buffer + buffer = detached_buffer + *prefix, field = constant_fqn.rsplit(".") + submod = torch.fx.graph_module._get_attr_via_attr_list(stateful_gm, prefix) + delattr(submod, field) + _assign_attr(buffer, stateful_gm, constant_fqn, attr_kind=_AttrKind.CONSTANT) + + # Constants are not preserved well when we create a new GraphModule unlike param/buffers + for const_name, value in ep.constants.items(): + if not torch.fx.graph_module._has_attr(stateful_gm, const_name): + if isinstance(value, torch.Tensor): + if value.requires_grad: + warnings.warn( + f"A model attribute `{const_name}` requires gradient " + f"but it's not properly registered as a parameter. " + f"torch.export will detach it and treat it as a constant tensor " + f"but please register it as parameter instead.", + stacklevel=2, + ) + if value in original_tensor_to_detached_tensor: + value = original_tensor_to_detached_tensor[value] + else: + detached_value = value.detach() + original_tensor_to_detached_tensor[value] = detached_value + value = detached_value + _assign_attr( + value, + stateful_gm, + const_name, + attr_kind=_AttrKind.CONSTANT, + ) + + # Fix up non-persistent buffers. torch.fx does not distinguish between + # persistent and non-persistent buffers, so we must restore that distinction + # here. + for buffer in ep.graph_signature.non_persistent_buffers: + _assign_attr( + plain_graph_module.get_buffer(buffer), + stateful_gm, + buffer, + attr_kind=_AttrKind.BUFFER, + persistent=False, + ) + + return stateful_gm + + +def _get_input_paths(example_inputs, signature): + """ + Generate paths of placeholders, needed for generating the guards function. + + NOTE: Here we make use of the example inputs used for export as well as + the signature of the unlifted graph module (not preserved by export). + """ + + args, kwargs = example_inputs + binded = signature.bind(*args, **kwargs) + binded.apply_defaults() + ctx = binded.arguments + flat_example_inputs_with_paths = pytree.tree_leaves_with_path(ctx) + return [path for path, _ in flat_example_inputs_with_paths] + + +def _replace_sources(result_str: str, flat_input_paths: list[Any]): + """ + Given user specified input paths, maybe fix up the guard string + to reflect user path instead of tracer path. + """ + name_mapping = {} + for idx, path in enumerate(flat_input_paths): + name_mapping[f"L['flat_args'][{idx}]"] = f"L{pytree.keystr(path)}" + + replace = result_str + for key, val in name_mapping.items(): + replace = replace.replace(key, val) + return replace + + +def _get_input_guards_for_graph( + placeholders: list[torch.fx.Node], + range_constraints: dict[sympy.Symbol, ValueRanges], + paths_for_placeholders: list[pytree.KeyPath], +): + """ + Guards generated by the tracer include conditions observed in code, but + but do not include some additional checks we typically do in export. + For example, when dynamic shapes get specialized, are specified to be + within a range, or are specified to be in some equational relation, + corresponding input invalidation is done within a pre_hook, specifically, + `_check_input_constraints_for_graph`. + + Here we generate guards corresponding to the checks that happen in + `_check_input_constraints_for_graph`, and add them to the guards already + generated by the tracer. In the future, it may be worthwhile to separate + them so that we can allow clients to turn off one but not the other. + (Looking at you, AOTI.) + + NOTE: We should eventually reconcile this logic with `build_guards` that + is used by AOT Precompile. + """ + + deferred_expressions = [] + new_guards_code = [] + sources: dict[sympy.Expr, str] = {} + + def handle_symint(expr, src): + if len(expr.free_symbols) == 1: + # complex equations (e.g., involving derived dims) need to + # handled later, since we may not have enough information + # just as we are passing through the placeholders in order + deferred_expressions.append((src, expr)) + if expr in sources: + # expressions that appear in multiple sources should force + # inputs corresponding to those sources to be equal + # e.g., x.shape[0] == y.shape[1] + orig_src = sources[expr] + new_guards_code.append(f"{src} == {orig_src}") + else: + sources[expr] = src + # process value ranges as elsewhere in export + min_val, max_val = _convert_range_to_int(range_constraints[expr]) + if min_val > 2: + new_guards_code.append(f"{src} >= {min_val}") + if max_val < math.inf: + new_guards_code.append(f"{src} <= {max_val}") + + for placeholder, path in zip(placeholders, paths_for_placeholders): + src = "L" + pytree.keystr(path) + meta = placeholder.meta["val"] + # specializations + if isinstance(meta, int): + new_guards_code.append(f"{src} == {meta}") + if isinstance(meta, float): + if meta == math.inf: + new_guards_code.append(f"{src} == math.inf") + elif meta == -math.inf: + new_guards_code.append(f"{src} == -math.inf") + else: + new_guards_code.append(f"{src} == {meta}") + elif isinstance(meta, str): + new_guards_code.append(f"{src} == '{meta}'") + # range constraints and equalities + elif isinstance(meta, torch.SymInt) and meta.node.expr in range_constraints: + handle_symint(meta.node.expr, src) + elif isinstance(meta, torch.Tensor): + for i, dim in enumerate(meta.shape): + src = "L" + pytree.keystr(path) + f".size()[{i}]" + if isinstance(dim, int): + # specializations + new_guards_code.append(f"{src} == {dim}") + elif ( + isinstance(dim, torch.SymInt) and dim.node.expr in range_constraints + ): + # range constraints and equalities + handle_symint(dim.node.expr, src) + + unification_map: dict[sympy.Symbol, sympy.Expr] = {} + py_printer = torch.utils._sympy.printers.PythonPrinter() + + # process complex equations (e.g., involving derived dims) + for src, expr in deferred_expressions: + # we know this is the only symbol in expr (see check above) + symbol = next(iter(expr.free_symbols)) + if symbol in sources: + # if s0 is already known to be directly sourced from inputs, + # e.g., z.shape[2], we do not need to do anything further + # (assume we have already processed constraints on s0 above) + continue + + # otherwise s0 has some "hidden" source like 'dim' + # example: src = y.shape[1], expr = s0 + 1 + if symbol in unification_map: + # suppose that we already know that s0 = x.shape[0] * 2 + # so we can emit the guard: x.shape[0] * 2 + 1 = y.shape[1] + substitution = expr.subs(unification_map) + new_guards_code.append( + py_printer.doprint(sympy.Eq(substitution, sympy.Symbol(src))) + ) + else: + # we do not yet know what s0 is, but given s0 + 1 = y.shape[1], + # we can solve for s0...now knowing that s0 = y.shape[1] - 1 + solution = try_solve(sympy.Eq(expr, sympy.Symbol(src)), symbol) + if solution is not None: + definition = solution[1] + unification_map[symbol] = definition + + return new_guards_code + + +def _ok_to_generate_guards_fn(): + patterns = [ + "executorch", + "modai", + "on_device_ai", + "torchao", + ] + # force check_guards=False for files matching `patterns` + # because they have too many calls to .module() and + # do not like any call modules in the graph + # TODO: fix these files to handle guard fns + frame = inspect.currentframe() + while frame is not None: + if any(path in frame.f_code.co_filename for path in patterns): + return False + frame = frame.f_back + + return True + + +def _unlift_exported_program_lifted_states( + ep: ExportedProgram, check_guards=True +) -> torch.fx.GraphModule: + check_guards = check_guards and _ok_to_generate_guards_fn() + + source_node_dict = { + node.name: node for node in ep.graph.nodes if node.op != "placeholder" + } + # placeholder node name might change after deepcopy + placeholder_source_node_dict = { + node.target: node for node in ep.graph.nodes if node.op == "placeholder" + } + + new_gm = torch.fx.GraphModule(ep.graph_module, copy.deepcopy(ep.graph)) + new_gm.meta.update(ep.graph_module.meta) + ep = copy.copy(ep) + ep._graph_module = new_gm + + # TODO T206340015 + if ep.verifiers[0].dialect != "TRAINING": + ep = _remove_effect_tokens(ep) + + _register_attrs_to_new_gm(new_gm, ep.graph_signature, ep.state_dict, ep.constants) + forward_arg_names = ( + sig.forward_arg_names if (sig := ep.module_call_graph[0].signature) else None + ) + lifted_inputs: list[str | None] = [ + ( + in_spec.target + if in_spec.kind + in ( + InputKind.BUFFER, + InputKind.CONSTANT_TENSOR, + InputKind.PARAMETER, + InputKind.CUSTOM_OBJ, + ) + else None + ) + for in_spec in ep.graph_signature.input_specs + ] + + mutated_outputs: list[str | None] = [ + ( + out_spec.target + if out_spec.kind + in ( + OutputKind.BUFFER_MUTATION, + OutputKind.USER_INPUT_MUTATION, + OutputKind.PARAMETER_MUTATION, + ) + else None + ) + for out_spec in ep.graph_signature.output_specs + ] + + for node in new_gm.graph.nodes: + source_node = None + if node.op == "placeholder": + source_node = placeholder_source_node_dict.get(node.target) + else: + if node.name in source_node_dict: + source_node = source_node_dict.get(node.name) + node.meta["from_node"] = [ + NodeSource( + source_node, + "ExportedProgram.module()", + NodeSourceAction.CREATE, + ) + ] + + assert ep.call_spec.in_spec is not None + new_gm = _unlift( + new_gm, + lifted_inputs, + mutated_outputs, + ep.call_spec.in_spec, + ep.call_spec.out_spec, + forward_arg_names=forward_arg_names, + ) + unlift_gm = _create_stateful_graph_module(new_gm, ep.range_constraints, ep) + unlift_gm.meta.update(ep.graph_module.meta) + + # create a _guards_fn submodule and insert a call to it after placeholders + graph = unlift_gm.graph + placeholders = graph.find_nodes(op="placeholder") + if check_guards and placeholders and ep.example_inputs: + sig = inspect.signature(unlift_gm.forward) + input_paths = _get_input_paths( + ep.example_inputs, + sig, + ) + + # TODO (tmanlaibaatar) + # This is band-aid solution to export new tracer replacing + # shape env sources to flat_args. The real fix should be replacing + # shape env sources to original user sources but this is quite + # involved because you need to carefully construct new sources using + # dynamo and replace all instances of it inside shape env. But it is + # lot easier to manipulate after we turn them into strings and only + # time we use these guards is during retracing or running exported program, + # so it is probably ok to have "not useful" guards on ep for now. + ep_guards = [] + for guard in ep._guards_code: + ep_guards.append(_replace_sources(guard, input_paths)) + + guards_code = _get_input_guards_for_graph( + placeholders, ep.range_constraints, input_paths + ) + + ep_guards_code = _force_ep_signature_match(ep._guards_code, input_paths) + ep_guards_code = _force_gm_signature_match(ep_guards_code, sig) + guards_code.extend(ep_guards_code) + unlift_gm._guards_fn = _convert_guards_code_to_fn(guards_code, input_paths) + + root_nn_module_stack = torch.fx._utils.first_call_function_nn_module_stack( + graph + ) + with graph.inserting_after(placeholders[-1]): + node = graph.call_module("_guards_fn", tuple(placeholders)) + node.meta["nn_module_stack"] = root_nn_module_stack + + unlift_gm.recompile() + + return unlift_gm + + +class GuardsFn(torch.nn.Module): + """ + Module class for guard functions. + """ + + def forward(self, *args): + pass diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_wrapper_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_wrapper_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ae7c49d8b860936956a076b41a6df063c353994b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/_wrapper_utils.py @@ -0,0 +1,10 @@ +import torch + + +class _WrapperModule(torch.nn.Module): + def __init__(self, f): # type: ignore[no-untyped-def] + super().__init__() + self.f = f + + def forward(self, *args, **kwargs): # type: ignore[no-untyped-def] + return self.f(*args, **kwargs) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_obj.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_obj.py new file mode 100644 index 0000000000000000000000000000000000000000..12b04215c31fb79af34511606600a856bc5ba6a8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_obj.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass + + +__all__ = ["ScriptObjectMeta"] + + +@dataclass +class ScriptObjectMeta: + """ + Metadata which is stored on nodes representing ScriptObjects. + """ + + # Key into constants table to retrieve the real ScriptObject. + constant_name: str + + class_fqn: str diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..629324c0f503aee3bfdf00a68286b01e35a89483 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/custom_ops.py @@ -0,0 +1,49 @@ +# mypy: allow-untyped-defs +import importlib + +import torch + + +lib = torch.library.Library("export", "FRAGMENT") # noqa: TOR901 + +lib.define( + "access_subclass_inner_tensor(Tensor src_subclass_tensor, str attr) -> Tensor" +) + + +@torch.library.impl(lib, "access_subclass_inner_tensor", "Autograd") +# When running under torch.inference_mode(), we seem to skip AUtograd key +# so we should desugar this op as soon as we start tracing to post-dispatch. +@torch.library.impl(lib, "access_subclass_inner_tensor", "Python") +def _access_subclass_inner_tensor( + src_subclass_tensor: torch.Tensor, attr: str +) -> torch.Tensor: + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + assert is_traceable_wrapper_subclass(src_subclass_tensor) + val = getattr(src_subclass_tensor, attr, None) + if val is None or not isinstance(val, torch.Tensor): + raise RuntimeError( + f"Attribute {attr} is not a tensor or doesn't exist in {src_subclass_tensor}" + ) + return val + + +def _call_custom_autograd_function_in_pre_dispatch(function_cls_name, *args, **kwargs): + """ + Import a custom autograd function by string name and call it. This is pretty bad + because: + 1) There is no schema + + Ideally we should automatically wrap custom autograd functions with a custom op, but + that is too much work because we need to schematize custom autograd functions. For now, + we just hackily put it in the IR. + """ + # Parse module and class name + module_name, class_name = function_cls_name.rsplit(".", 1) + + # Import the module and get the class + module = importlib.import_module(module_name) + function_cls = getattr(module, class_name) + assert hasattr(function_cls, "apply") + return function_cls.apply(*args, **kwargs) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/decomp_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/decomp_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e61d9767ac8ade04521ebf560abd43d53ddd4c9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/decomp_utils.py @@ -0,0 +1,160 @@ +# mypy: allow-untyped-defs +from collections.abc import Callable + +import torch +from torch._export.utils import ( + _collect_all_valid_cia_ops, + _collect_all_valid_cia_ops_for_aten_namespace, + _get_decomp_for_cia, + _is_aten_op, +) + + +__all__ = ["CustomDecompTable"] + + +""" +Core ATen ops with Composite Implicit Autograd dispatch that should be excluded from decomposition +by default. The decomposition logic should eventually exclude all core-tagged CIA ops, but until all +backends are ready, this list allows opt-in one at a time. +""" +PRESERVED_ATEN_CIA_OPS = { + torch.ops.aten.upsample_bilinear2d.vec, + torch.ops.aten.upsample_nearest2d.vec, + # NB: don't use the C++ decomp, because it is not functional! + torch.ops.aten.silu_backward.default, + torch.ops.aten.mish_backward.default, + torch.ops.aten._fused_rms_norm.default, +} + + +class CustomDecompTable(dict[torch._ops.OperatorBase, Callable]): + """ + This is a custom dictionary that is specifically used for handling decomp_table in export. + The reason we need this is because in the new world, you can only *delete* an op from decomp + table to preserve it. This is problematic for custom ops because we don't know when the custom + op will actually be loaded to the dispatcher. As a result, we need to record the custom ops operations + until we really need to materialize it (which is when we run decomposition pass.) + + Invariants we hold are: + 1. All aten decomp is loaded at the init time + 2. We materialize ALL ops when user ever reads from the table to make it more likely + that dispatcher picks up the custom op. + 3. If it is write operation, we don't necessarily materialize + 4. We load the final time during export, right before calling run_decompositions() + + """ + + def __init__(self): + super().__init__() + from torch._decomp import _core_aten_decompositions_post_autograd + + # For aten ops, we load them up in the beginning + self.decomp_table = _core_aten_decompositions_post_autograd() + + for op in _collect_all_valid_cia_ops_for_aten_namespace(): + if op not in PRESERVED_ATEN_CIA_OPS and op not in self.decomp_table: + self.decomp_table[op] = _get_decomp_for_cia(op) + + # This is to track the *pending* deleted custom ops that haven't been materialized yet + self.deleted_custom_ops = set() + # When this is true, there shouldn't be any pending operations in the table. + self.has_materialized = False + + def __getitem__(self, key): + self._materialize_if_needed() + return self.decomp_table.__getitem__(key) + + def __setitem__(self, key, value) -> None: + self.decomp_table.__setitem__(key, value) + + if key in self.deleted_custom_ops: + self.deleted_custom_ops.remove(key) + + def keys(self): + self._materialize_if_needed() + return self.decomp_table.keys() + + def __delitem__(self, key) -> None: + self.pop(key) + + def update(self, other_dict): # type: ignore[override] + for k, v in other_dict.items(): + self.decomp_table.__setitem__(k, v) + + def __missing__(self, key) -> bool: + return not self.__contains__(key) + + def __contains__(self, key) -> bool: + self._materialize_if_needed() + return self.decomp_table.__contains__(key) + + def __len__(self) -> int: + self._materialize_if_needed() + return self.decomp_table.__len__() + + def __iter__(self): + self._materialize_if_needed() + return self.decomp_table.__iter__() + + def __reversed__(self): + self._materialize_if_needed() + return self.decomp_table.__reversed__() + + def copy(self) -> "CustomDecompTable": + new_dict = CustomDecompTable() + new_dict.decomp_table = self.decomp_table.copy() + new_dict.deleted_custom_ops = self.deleted_custom_ops.copy() + new_dict.has_materialized = self.has_materialized + return new_dict + + def pop(self, *args): + def _pop_if_can(key): + if _is_aten_op(key): + return self.decomp_table.pop(key) + + if key in self.decomp_table: + # Even if we materialized it, we should add it to the deleted + # custom ops list so that when we materialize next time, + # we should respect user's intention. + self.deleted_custom_ops.add(key) + return self.decomp_table.pop(key) + + if key in self.deleted_custom_ops: + raise KeyError(f"{key} doesn't exist in the table") + + self.deleted_custom_ops.add(key) + # We would come here when user pops off something that is + # not in the table. In this case, we just pretend that it + # was in the table. + return _get_decomp_for_cia(key) + + if len(args) == 1: + return _pop_if_can(args[0]) + + if len(args) == 2: + try: + return _pop_if_can(args[0]) + except KeyError: + return args[1] + + def items(self): + self._materialize_if_needed() + return self.decomp_table.items() + + def materialize(self) -> dict[torch._ops.OperatorBase, Callable]: + for op in _collect_all_valid_cia_ops(): + if _is_aten_op(op): + continue + elif op in self.decomp_table: + continue + elif op not in self.deleted_custom_ops: + self.decomp_table[op] = _get_decomp_for_cia(op) + + self.has_materialized = True + self.deleted_custom_ops = set() + return {**self.decomp_table} + + def _materialize_if_needed(self) -> None: + if not self.has_materialized: + self.materialize() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/dynamic_shapes.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/dynamic_shapes.py new file mode 100644 index 0000000000000000000000000000000000000000..f97c02da08554ceabdc346579c530f742496d8ef --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/dynamic_shapes.py @@ -0,0 +1,1374 @@ +# mypy: allow-untyped-defs +import dataclasses +import inspect +import logging +import sys +from collections import defaultdict +from collections.abc import Callable +from enum import auto, Enum +from typing import Any, TYPE_CHECKING, Union + +import torch +from torch.utils._pytree import ( + _get_node_type, + BUILTIN_TYPES, + KeyPath, + keystr, + MappingKey, + SequenceKey, + SUPPORTED_NODES, + tree_iter, + tree_map, + tree_map_with_path, + tree_structure, + TreeSpec, +) + +from .exported_program import ExportedProgram + + +if TYPE_CHECKING: + from sympy import Symbol + + from torch._guards import Source + from torch.fx.experimental.symbolic_shapes import ShapeEnv, StrictMinMaxConstraint + +__all__ = [ + "Constraint", + "Dim", + "dims", + "refine_dynamic_shapes_from_suggested_fixes", + "AdditionalInputs", +] + + +log = logging.getLogger(__name__) + + +class _DimHintType(Enum): + """ + Enum for dynamic shape hints. + - AUTO means automatic inference of shape (static or dynamic). + - STATIC means static shape (always specialized). + - DYNAMIC means dynamic, will error out if specialized. + """ + + AUTO = auto() + STATIC = auto() + DYNAMIC = auto() + + +@dataclasses.dataclass +class _DimHint: + """ + Internal class for dynamic shape hints. + - min and max are optional. + - _factory is for UX only, below example: + auto_hint = _DimHint.AUTO() # _factory=True + bounded_hint = auto_hint(min=10, max=100) # Returns new instance with _factory=False + bounded_hint(min=5, max=50) # Will fail, non-factory instance cannot be called + """ + + type: _DimHintType + min: int | None = None + max: int | None = None + _factory: bool | None = True + + @staticmethod + def AUTO(): + return _DimHint(_DimHintType.AUTO) + + @staticmethod + def DYNAMIC(): + return _DimHint(_DimHintType.DYNAMIC) + + @staticmethod + def STATIC(): + return _DimHint(_DimHintType.STATIC) + + def __call__(self, min=None, max=None) -> "_DimHint": + if not self._factory: + raise TypeError(f"'{type(self)}' object is not callable") + assert min is None or min >= 0, "min must be non-negative" + assert max is None or max >= 0, "max must be non-negative" + assert min is None or max is None or min <= max, "min must be <= max" + return _DimHint(self.type, min=min, max=max, _factory=False) + + def __repr__(self): + parts = [self.type.name] + if self.min is not None: + parts.append(f"min={self.min}") + if self.max is not None: + parts.append(f"max={self.max}") + return f"DimHint({', '.join(parts)})" + + +class Dim: + """ + The ``Dim`` class allows users to specify dynamism in their exported + programs. By marking a dimension with a ``Dim``, the compiler associates the + dimension with a symbolic integer containing a dynamic range. + + The API can be used in 2 ways: Dim hints (i.e. automatic dynamic shapes: + ``Dim.AUTO``, ``Dim.DYNAMIC``, ``Dim.STATIC``), or named Dims (i.e. + ``Dim("name", min=1, max=2)``). + + Dim hints provide the lowest barrier to exportability, with the user only + needing to specify if a dimension if dynamic, static, or left for the + compiler to decide (``Dim.AUTO``). The export process will automatically + infer the remaining constraints on min/max ranges and relationships between + dimensions. + + Example:: + + class Foo(nn.Module): + def forward(self, x, y): + assert x.shape[0] == 4 + assert y.shape[0] >= 16 + return x @ y + + + x = torch.randn(4, 8) + y = torch.randn(8, 16) + dynamic_shapes = { + "x": {0: Dim.AUTO, 1: Dim.AUTO}, + "y": {0: Dim.AUTO, 1: Dim.AUTO}, + } + ep = torch.export(Foo(), (x, y), dynamic_shapes=dynamic_shapes) + + Here, export would raise an exception if we replaced all uses of ``Dim.AUTO`` with ``Dim.DYNAMIC``, + as ``x.shape[0]`` is constrained to be static by the model. + + More complex relations between dimensions may also be codegened as runtime assertion nodes by the compiler, + e.g. ``(x.shape[0] + y.shape[1]) % 4 == 0``, to be raised if runtime inputs do not satisfy such constraints. + + You may also specify min-max bounds for Dim hints, e.g. ``Dim.AUTO(min=16, max=32)``, ``Dim.DYNAMIC(max=64)``, + with the compiler inferring the remaining constraints within the ranges. An exception will be raised if + the valid range is entirely outside the user-specified range. + + Named Dims provide a stricter way of specifying dynamism, where exceptions are raised if the compiler + infers constraints that do not match the user specification. For example, exporting the previous + model, the user would need the following ``dynamic_shapes`` argument:: + + s0 = Dim("s0") + s1 = Dim("s1", min=16) + dynamic_shapes = { + "x": {0: 4, 1: s0}, + "y": {0: s0, 1: s1}, + } + ep = torch.export(Foo(), (x, y), dynamic_shapes=dynamic_shapes) + + Named Dims also allow specification of relationships between dimensions, up + to univariate linear relations. For example, the following indicates one + dimension is a multiple of another plus 4:: + + s0 = Dim("s0") + s1 = 3 * s0 + 4 + + """ + + AUTO = _DimHint.AUTO() + DYNAMIC = _DimHint.DYNAMIC() + STATIC = _DimHint.STATIC() + + def __init__(self, name: str, *, min: int | None = None, max: int | None = None): + from torch.utils._sympy.numbers import int_oo + + _min = 0 if min is None else min + _max = int_oo if max is None else max + assert _max > _min, f"Cannot create Dim with inconsistent min={min}, max={max}" + assert name.isidentifier(), f"Dim name must be a valid identifier, got {name}" + self.__name__ = name + self.min = _min + self.max = _max + + def __add__(self, other) -> "Dim": + # e.g., dim + 1 + if type(other) is not int: + raise NotImplementedError( + f"Attempted to add {other} to {self.__name__}, where an integer was expected. " + "(Only increasing linear operations with integer coefficients are supported.)" + ) + return self._derive(lambda x: x + other) + + def __radd__(self, other) -> "Dim": + return self + other + + def __sub__(self, other) -> "Dim": + # e.g., dim - 1 + if type(other) is not int: + raise NotImplementedError( + f"Attempted to subtract {other} from {self.__name__}, where an integer was expected. " + "(Only increasing linear operations with integer coefficients are supported.)" + ) + return self._derive(lambda x: x - other) + + def __rsub__(self, other) -> "Dim": + raise NotImplementedError( + f"Attempted to negate {self.__name__}. " + "(Only increasing linear operations with integer coefficients are supported.)" + ) + + def __mul__(self, other) -> "Dim": + # e.g., dim * 2 + if type(other) is not int or other <= 0: + raise NotImplementedError( + f"Attempted to multiply {other} with {self.__name__}, where a positive integer was expected. " + "(Only increasing linear operations with integer coefficients are supported.)" + ) + return self._derive(lambda x: x * other) + + def __rmul__(self, other) -> "Dim": + return self * other + + def _derived_name(self, fn) -> str: + from sympy import sympify + + return str(fn(sympify(self.__name__))) + + def _derive(self, fn) -> "Dim": + return _DerivedDim(self._derived_name(fn), self, fn) + + @staticmethod + def _readable(name: str, min_: int, max_: int) -> str: + from torch.utils._sympy.numbers import int_oo + + if min_ == 2: + min_ = None # type: ignore[assignment] + if max_ == int_oo: + max_ = None # type: ignore[assignment] + if min_ is None and max_ is None: + return f"Dim('{name}')" + if min_ is None: + return f"Dim('{name}', max={max_})" + if max_ is None: + return f"Dim('{name}', min={min_})" + return f"Dim('{name}', min={min_}, max={max_})" + + def __repr__(self): + return Dim._readable(self.__name__, self.min, self.max) + + +_Dim = Dim # TODO(pianpwk): remove after it's no longer internally breaking + + +class _StaticDim(Dim): + """ + Class for static :func:`Dim` types. + + This class is only for setting and checking static dim constraints, + and the user should never interact with it. + """ + + def __init__(self, value: int): + self.__name__ = str(value) + self.value = value + + @property + def min(self): # type: ignore[override] + return self.value # type: ignore[attr-defined] + + @property + def max(self): # type: ignore[override] + return self.value # type: ignore[attr-defined] + + +class _DerivedDim(Dim): + """ + Class for derived :func:`Dim` types. + + Currently we only support increasing linear expressions with integer coefficients. + In other words, a derived Dim can always be written in the form Ax + B, where + x is a regular Dim (i.e., non-derived Dim), A and B are integers, and A is positive. + (In particular, the latter ensures that x < y => Ax + B < Ay + B.) + These restrictions on the form of derived Dims makes the metatheory simpler: e.g., + it simplifies computing ranges for derived Dims, solving for underlying regular Dims, + deciding equalities between derived Dims, and so on. + + The function lambda x: Ax + B is expressed by `fn`, where x is a normal Dim, `root`. + The range of a derived Dim is computed by mapping `fn` over the range of its `root`. + """ + + def __init__(self, name: str, root: Dim, fn: Callable): + self.__name__ = name + self.root = root + self.fn = fn + + @property + def min(self): # type: ignore[override] + # assume that self.fn is an increasing function + # TODO(avik): use sympy value range analysis instead? + from sympy import Integer + + from torch.utils._sympy.numbers import int_oo + + if self.root.min is -int_oo: # type: ignore[attr-defined] + return -int_oo # fn not needed cuz increasing + + _min_symint = self.fn(Integer(self.root.min)) # type: ignore[attr-defined] + root = self.root # type: ignore[attr-defined] + assert _min_symint >= 0, ( + f"Expected derived min value of {self.__name__} to be >= 0. " + f"Please specify an appropriate min value for {root.__name__} " + f"(currently {root.min})." + ) + return int(_min_symint) + + @property + def max(self): # type: ignore[override] + # assume that self.fn is an increasing function + # TODO(avik): use sympy value range analysis instead? + from sympy import Integer + + from torch.utils._sympy.numbers import int_oo + + if self.root.max is int_oo: # type: ignore[attr-defined] + return int_oo # fn not needed cuz increasing + + _max_symint = self.fn(Integer(self.root.max)) # type: ignore[attr-defined] + root = self.root # type: ignore[attr-defined] + assert _max_symint <= sys.maxsize - 1, ( + f"Expected derived max value of {self.__name__} to be <= {sys.maxsize - 1}. " + f"Please specify an appropriate max value for {root.__name__} " + f"(currently {root.max})." + ) + return int(_max_symint) + + def _derive(self, fn): + # We support nesting, e.g., 2*dim + 1. + # This is implemented by composing operations on the same root. + # As a consequence, roots are always regular Dims (i.e., not derived Dims). + return _DerivedDim( + self._derived_name(fn), + self.root, + lambda x: fn(self.fn(x)), + ) + + def __repr__(self): + return self.__name__ + + +def dims( + *names: str, min: int | None = None, max: int | None = None +) -> tuple[Dim, ...]: + """ + Util to create multiple :func:`Dim` types. + + Returns: + A tuple of :func:`Dim` types. + """ + return tuple(Dim(name, min=min, max=max) for name in names) # type: ignore[misc] + + +@dataclasses.dataclass +class _ConstraintTarget: + """ + This represents input tensor dimensions. + """ + + t_id: int + dim: int + + +@dataclasses.dataclass +class _Constraint(_ConstraintTarget): + """ + This represents a Dim describing a constraint target. + + `name` is the name of the Dim. + `constraint_range` contains the min/max bounds of the Dim. + """ + + name: str + constraint_range: "StrictMinMaxConstraint" + + def _clone_with_range(self, lower=0, upper=None): + # Import sympy locally + from torch.fx.experimental.symbolic_shapes import StrictMinMaxConstraint + from torch.utils._sympy.numbers import int_oo + from torch.utils._sympy.value_ranges import ValueRanges + + if upper is None: + upper = int_oo + + constraint_range = StrictMinMaxConstraint( + vr=self.constraint_range.vr & ValueRanges(lower=lower, upper=upper), + warn_only=False, + ) + return _Constraint( + self.t_id, + self.dim, + self.name, + constraint_range, + ) + + def __ge__(self, lower): + return self._clone_with_range(lower=lower) + + def __gt__(self, lower): + return self._clone_with_range(lower=lower + 1) + + def __le__(self, upper): + return self._clone_with_range(upper=upper) + + def __lt__(self, upper): + return self._clone_with_range(upper=upper - 1) + + def __bool__(self): + # NOTE(avik): We do not support compound expressions like a <= x <= b. + # This is because Python implicitly desugars them into bool(a <= x) and bool(x <= b), + # and moreover, enforces that any overload of __bool__ must return True or False. + # FWIW, sympy also raises TypeError in this case. + raise TypeError( + "Cannot determine truth value of _Constraint. " + "If you are trying to combine _Constraint's with logical connectives, " + "you can specify them separately instead." + ) + + @property + def serializable_spec(self): + # We need a serialization compatible format of the constraint so that it + # can be savedin the graph module w/o breaking the module serialization. + # The saved constraints will be used directly for the post-exporting pass + # that converts constraints to runtime assertion. The saved constraints + # will not be saved in the serialized module. + # TODO: A better way is needed. Currently we use 't_id' to map the constraint, + # which is not reliable + return { + "t_id": self.t_id, + "dim": self.dim, + "min": self.constraint_range.vr.lower, + "max": self.constraint_range.vr.upper, + } + + +@dataclasses.dataclass +class _PhantomRoot: + """ + This represents the root of a derived Dim where the root does not directly + specify the shape of any input dimension, but the derived Dim does. + + e.g., the input shapes 2*dim and dim + 1 are related via a "phantom" dim. + + The fields `name`, `constraint_range`, and `val` carried by a phantom root + help create a symbol for it. Any derived dims with this phantom root are + backed by expressions over this symbol. + """ + + name: str + constraint_range: "StrictMinMaxConstraint" + val: int + + +@dataclasses.dataclass +class _DerivedConstraint(_ConstraintTarget): + """ + This represents a derived Dim, whose root is either a regular constraint target + (which directly specifies the shape of some input dimension) or a phantom root + (which does so indirectly). + + It can be thought of as a subclass of `_Constraint`, except that it does not + support <, <=, >, >= operations. + """ + + name: str + constraint_range: "StrictMinMaxConstraint" + root: _ConstraintTarget | _PhantomRoot + fn: Callable + + @property + def serializable_spec(self): + # same as _Constraint.serializable_spec + return { + "t_id": self.t_id, + "dim": self.dim, + "min": self.constraint_range.vr.lower, + "max": self.constraint_range.vr.upper, + } + + +@dataclasses.dataclass +class _RelaxedConstraint(_ConstraintTarget): + """ + This represents a dim marked with Dim.AUTO/DYNAMIC (i.e. mark_dynamic() or maybe_mark_dynamic()), + which leaves relations & min/max ranges for inference, instead of requiring explicit specification. + The intention is for constraint violations to not be raised if produce_guards() finds equalities or + relations between a _RelaxedConstraint and another type of _Constraint. + """ + + @property + def serializable_spec(self): + return { + "t_id": self.t_id, + "dim": self.dim, + } + + +Constraint = _Constraint | _DerivedConstraint | _RelaxedConstraint + + +@dataclasses.dataclass +class _IntWrapper: + """ + Dummy wrapper class to wrap around integer inputs so that when we parse the + dynamic_shapes structure, we can mark if any of the integers were marked as + dynamic. + """ + + val: int + # Disallow specifying dynamism + dynamism: _DimHint | int | None = dataclasses.field(init=False, default=None) + + +def _process_equalities( + constraint: Constraint, + get_sources: Callable[[int, int], list["Source"]], + shape_env: "ShapeEnv", + names: dict[str, tuple[int, int]], + source_pairs: list[tuple["Source", "Source"]], + derived_equalities: list[tuple["Source", Union["Source", "Symbol"], Callable]], + phantom_symbols: dict[str, "Symbol"], + relaxed_sources: set["Source"], +): + """ + Updates `source_pairs`, `derived_equalities`, and `phantom_symbols` (which become + fields of `EqualityConstraint`) based on a given input `constraint`. + """ + + sources = get_sources(constraint.t_id, constraint.dim) + if not sources: # empty sources due to unused shapes + return + + source, *other_sources = sources + # When t.size()[dim] maps to src0, src1, ..., srcN, we add + # constraints that make src0 "equal" to src1, ..., srcN. + source_pairs.extend((source, other_source) for other_source in other_sources) + if isinstance(constraint, _Constraint): + if constraint.name in names: + shared_t_id, shared_dim = names[constraint.name] + other_sources = get_sources(shared_t_id, shared_dim) + source_pairs.extend( + (source, other_source) for other_source in other_sources + ) + else: + names[constraint.name] = (constraint.t_id, constraint.dim) + elif isinstance(constraint, _DerivedConstraint): + # branch based on the root of the _DerivedConstraint + if not isinstance(constraint.root, _PhantomRoot): + # either root points to an input source + root = get_sources(constraint.root.t_id, constraint.root.dim)[0] + else: + # or root points to a phantom symbol + if constraint.root.name in phantom_symbols: + root = phantom_symbols[constraint.root.name] + else: + # create a phantom symbol in the shape env based on the _PhantomRoot + root = shape_env.create_symbol( + val=constraint.root.val, + source=torch._dynamo.source.ConstantSource(constraint.root.name), + dynamic_dim=torch.fx.experimental.symbolic_shapes.DimDynamic.DYNAMIC, + constraint_dim=constraint.root.constraint_range, + ) + phantom_symbols[constraint.root.name] = root + + fn = constraint.fn + # A derived equality (source, root, fn) informally corresponds to source = fn(root). + # Here source describes an input and root might describe another input or a phantom symbol. + derived_equalities.append((source, root, fn)) + elif isinstance(constraint, _RelaxedConstraint): + relaxed_sources.add(source) + + +def _tree_map_with_path( + func: Callable[..., Any], + tree: Any, + *dynamic_shapes: Any, + tree_name: str | None = None, +) -> Any: + """ + Customized tree_map for mapping pytrees to dynamic_shapes. + + For built-in types (e.g., standard collections) this behaves exactly like tree_map. + + OTOH for a user-defined class C registered with pytree, we cannot assume that a C + containing tensors can be mapped to a C containing dynamic shapes (i.e., C may not + be a polymorphic container). In that case we use the flattened form of C instead. + Thus a C(**tensors) that flattens to (**tensors) will map to (**dynamic_shapes). + + Args: + func: function to apply to each (int, float, str, bool, None, torch.Tensor) + tree: input pytree + dynamic_shapes: zero or more (typically one) dynamic_shapes to match + + Returns: + output pytree mapping func to each (int, float, str, bool, None, torch.Tensor) + """ + + def is_leaf(t): + # BUILTIN_TYPES is a subset of SUPPORTED_NODES, the latter being all types + # registered with pytree. Types *not* in BUILTIN_TYPES include primitive types + # (int, float, str, bool, None, torch.Tensor), which are not in SUPPORTED_NODES, + # as well as user-defined classes registered with pytree, which are. + return _get_node_type(t) not in BUILTIN_TYPES + + def f(path, t, *dynamic_shapes): + typ = _get_node_type(t) + # typ is not in BUILTIN_TYPES + if typ in SUPPORTED_NODES: + # thus typ is a user-defined class registered with pytree, + # in which case flatten and recurse + return tree_map_with_path( + f, + SUPPORTED_NODES[typ].flatten_fn(t)[0], + *dynamic_shapes, + is_leaf=is_leaf, + ) + else: + return func(path, t, *dynamic_shapes) + + try: + return tree_map_with_path(f, tree, *dynamic_shapes, is_leaf=is_leaf) + except ValueError as e: + if "mismatch" in e.args[0]: + # When PyTree finds a structural mismatch between tree and dynamic_shapes, + # the error message is unfortunately quite horrible. Let's fix that. + assert dynamic_shapes, "Cannot be a mismatch if there is no dynamic_shapes" + assert tree_name, "Must provide a tree_name when there might be a mismatch" + + def _key(type_, context, i): + # derive a PyTree key given the type, context, and child # of a TreeSpec + if type_ is dict: + return MappingKey(context[i]) + if type_ in (list, tuple): + assert context is None + return SequenceKey(i) + raise AssertionError(f"Did not expect type {type_}") + + def raise_mismatch_error(msg): + from torch._dynamo.exc import UserError, UserErrorType + + raise UserError( + UserErrorType.INVALID_INPUT, + f"Detected mismatch between the structure of `{tree_name}` and `dynamic_shapes`: {msg}", + case_name="dynamic_shapes_validation", + ) + + def _compare( + treespec: TreeSpec, other_treespec: TreeSpec, path: KeyPath + ) -> None: + # raise an error at the point where tree and dynamic_shapes differ, + # including the path to that point and the reason for the difference + rendered_path = keystr(path) + if treespec.is_leaf(): + return + if other_treespec.is_leaf(): + raise_mismatch_error( + f"`{tree_name}{rendered_path}` is a {treespec.type}, " + f"but `dynamic_shapes{rendered_path}` is not" + ) + if treespec.type != other_treespec.type: + raise_mismatch_error( + f"`{tree_name}{rendered_path}` is a {treespec.type}, " + f"but `dynamic_shapes{rendered_path}` is a {other_treespec.type}" + ) + if treespec.num_children != other_treespec.num_children: + raise_mismatch_error( + f"`{tree_name}{rendered_path}` has {treespec.num_children} elements, " + f"but `dynamic_shapes{rendered_path}` has {other_treespec.num_children} elements" + ) + if treespec.type is dict: + # context, children could be out of order + if set(treespec.context) != set(other_treespec.context): + raise_mismatch_error( + f"`{tree_name}{rendered_path}` has keys {treespec.context}, " + f"but `dynamic_shapes{rendered_path}` has keys {other_treespec.context}" + ) + _remap = dict( + zip(other_treespec.context, other_treespec.children()) + ) + other_children = [_remap[k] for k in treespec.context] + else: + other_children = other_treespec.children() + for i, (child, other_child) in enumerate( + zip(treespec.children(), other_children) + ): + _compare( + child, + other_child, + path + (_key(treespec.type, treespec.context, i),), + ) + + treespec = tree_structure(tree, is_leaf=is_leaf) + for other_tree in dynamic_shapes: + other_treespec = tree_structure(other_tree, is_leaf) + _compare(treespec, other_treespec, ()) + raise + + +def _combine_args(f, args, kwargs) -> dict[str, Any]: + # combine args and kwargs following the signature of f, as it happens + # in the body of f when called with *args, **kwargs + if isinstance(f, ExportedProgram): + f = f.module() + + signature = ( + inspect.signature(f.forward) + if isinstance(f, torch.nn.Module) + else inspect.signature(f) + ) + kwargs = kwargs if kwargs is not None else {} + return signature.bind(*args, **kwargs).arguments + + +class ShapesCollection: + """ + Builder for dynamic_shapes. + Used to assign dynamic shape specifications to tensors that appear in inputs. + + This is useful particularly when :func:`args` is a nested input structure, and it's + easier to index the input tensors, than to replicate the structure of :func:`args` in + the :func:`dynamic_shapes` specification. + + Example:: + + args = {"x": tensor_x, "others": [tensor_y, tensor_z]} + + dim = torch.export.Dim(...) + dynamic_shapes = torch.export.ShapesCollection() + dynamic_shapes[tensor_x] = (dim, dim + 1, 8) + dynamic_shapes[tensor_y] = {0: dim * 2} + # This is equivalent to the following (now auto-generated): + # dynamic_shapes = {"x": (dim, dim + 1, 8), "others": [{0: dim * 2}, None]} + + torch.export(..., args, dynamic_shapes=dynamic_shapes) + + To specify dynamism for integers, we need to first wrap the integers using + _IntWrapper so that we have a "unique identification tag" for each integer. + + Example:: + + args = {"x": tensor_x, "others": [int_x, int_y]} + # Wrap all ints with _IntWrapper + mapped_args = pytree.tree_map_only(int, lambda a: _IntWrapper(a), args) + + dynamic_shapes = torch.export.ShapesCollection() + dynamic_shapes[tensor_x] = (dim, dim + 1, 8) + dynamic_shapes[mapped_args["others"][0]] = Dim.DYNAMIC + + # This is equivalent to the following (now auto-generated): + # dynamic_shapes = {"x": (dim, dim + 1, 8), "others": [Dim.DYNAMIC, None]} + + torch.export(..., args, dynamic_shapes=dynamic_shapes) + """ + + def __init__(self): + self._shapes = {} + + def __setitem__(self, t, shape): + assert isinstance(t, (torch.Tensor, _IntWrapper)), ( + f"Cannot assign shape to non-tensor or non-_IntWrapper type {type(t)}" + ) + + # TODO(avik): check that shape is indeed a Shape + + t_id = id(t) + if t_id in self._shapes: + _shape = self._shapes[t_id] + assert shape == _shape, ( + f"Shapes assigned to input do not match: expected {_shape}, got {shape}" + ) + else: + self._shapes[id(t)] = shape + + def __getitem__(self, t): + t_id = id(t) + if t_id not in self._shapes: + self._shapes[t_id] = {} + return self._shapes[t_id] + + def __len__(self): + return len(self._shapes) + + def dynamic_shapes(self, m, args, kwargs=None): + """ + Generates the :func:`dynamic_shapes` pytree structure according to :func:`args` and :func:`kwargs`. + """ + + t_ids = set() + + def find_shape(path, t): + t_id = id(t) + if t_id in self._shapes: + t_ids.add(t_id) + return self._shapes[t_id] + else: + return None + + combined_args = _combine_args(m, args, kwargs) + dynamic_shapes = _tree_map_with_path(find_shape, combined_args) + if any(t_id not in t_ids for t_id in self._shapes): + raise ValueError( + "Some tensors that were assigned shapes were not found in args. " + "Maybe such tensors were copied when passing them as args? " + "Maybe such tensors are contained in classes that were not registered with pytree?" + ) + return dynamic_shapes + + +class AdditionalInputs: + """ + Infers dynamic_shapes based on additional inputs. + + This is useful particularly for deployment engineers who, on the one hand, may + have access to ample testing or profiling data that can provide a fair sense of + representative inputs for a model, but on the other hand, may not know enough + about the model to guess which input shapes should be dynamic. + + Input shapes that are different than the original are considered dynamic; conversely, + those that are the same as the original are considered static. Moreover, we verify + that the additional inputs are valid for the exported program. This guarantees that + tracing with them instead of the original would have generated the same graph. + + Example:: + + args0, kwargs0 = ... # example inputs for export + + # other representative inputs that the exported program will run on + dynamic_shapes = torch.export.AdditionalInputs() + dynamic_shapes.add(args1, kwargs1) + ... + dynamic_shapes.add(argsN, kwargsN) + + torch.export(..., args0, kwargs0, dynamic_shapes=dynamic_shapes) + """ + + def __init__(self): + self._examples = [] + + def add(self, args, kwargs=None): + """ + Additional input :func:`args` and :func:`kwargs`. + """ + + assert type(args) is tuple, f"Representative args {args} must be a tuple" + assert kwargs is None or type(kwargs) is dict, ( + f"Representative kwargs {kwargs} must be None or a dict" + ) + self._examples.append((args, kwargs)) + + def dynamic_shapes(self, m, args, kwargs=None): + """ + Infers a :func:`dynamic_shapes` pytree structure by merging shapes of the + original input :func:`args` and :func:`kwargs` and of each additional input + args and kwargs. + """ + + dynamic_shapes, *other_dynamic_shapes = [ + _tree_map_with_path( + lambda path, t: tuple(t.shape) if isinstance(t, torch.Tensor) else t, + _combine_args(m, args, kwargs), + ) + for args, kwargs in [(args, kwargs), *self._examples] + ] + + def _mark_dynamism(v, *other_vs): + if not all(type(v) is type(other) for other in other_vs): + raise ValueError( + "The following inputs were found to have differing types, " + f"so they cannot be marked as dynamic: {(v,) + other_vs}." + ) + + if isinstance(v, int) and not isinstance(v, bool): + if all(other_v == v for other_v in other_vs): + return None + else: + return Dim.DYNAMIC + else: + if not all(other_v == v for other_v in other_vs): + raise ValueError( + "The following inputs were found to have differing values, " + f"but they cannot be marked as dynamic: {(v,) + other_vs}." + ) + return None + + return tree_map( + _mark_dynamism, + dynamic_shapes, + *other_dynamic_shapes, + is_leaf=lambda i: type(i) is int, + ) + + def verify(self, ep): + """ + Verifies that an exported program is valid for each additional input. + """ + + epm = ep.module() + for args, kwargs in self._examples: + torch.export._unlift._check_input_constraints_for_module( + epm, args, kwargs or {} + ) + + +def _warn_on_None_dynamic_shape_dimension(): + msg = ( + "Using None as a dynamic shape dimension is deprecated. " + "Please use Dim.STATIC instead" + ) + # TODO(avik): raise an error in the future + log.warning(msg) + + +def _check_dynamic_shapes( + combined_args: dict[str, Any], + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, +): + """ + Checks the dynamic_shapes specification for correctness, + using combined args + kwargs as reference for inputs structure. + """ + from torch._dynamo.exc import UserError, UserErrorType + + if dynamic_shapes is None or len(dynamic_shapes) == 0: + return + if isinstance(dynamic_shapes, (tuple, list)): + combined_args = type(dynamic_shapes)(combined_args.values()) # type: ignore[assignment, misc] + + bounds: dict[str, tuple[int, int]] = {} + + def check_same_bounds(dim): + if dim.__name__ in bounds: + min_, max_ = bounds[dim.__name__] + if dim.min != min_ or dim.max != max_: + this_ = Dim._readable(dim.__name__, min_, max_) + that_ = Dim._readable(dim.__name__, dim.min, dim.max) + raise UserError( + UserErrorType.INVALID_INPUT, + f"Found different definitions {this_} and {that_} " + f"for the same symbolic dimension {dim}!", + ) + else: + bounds[dim.__name__] = (dim.min, dim.max) + + def check_symbols(path, tensor, shape): + if isinstance(shape, dict): + for i, dim in shape.items(): + if isinstance(dim, Dim): + check_same_bounds(dim) + elif dim is None: + _warn_on_None_dynamic_shape_dimension() + elif not (isinstance(dim, (int, _DimHint))): + raise UserError( + UserErrorType.INVALID_INPUT, + f"Unexpected dimension mapped to index {i} in input tensor shape {shape} " + f"specified at `dynamic_shapes{keystr(path)}` " + f"(expected None, an int, a Dim, Dim.AUTO, Dim.STATIC, or Dim.DYNAMIC, " + f" but got {dim!r} instead)", + case_name="dynamic_shapes_validation", + ) + elif isinstance(shape, (tuple, list)): + if len(shape) != len(tensor.shape): + raise UserError( + UserErrorType.INVALID_INPUT, + f"Expected dynamic shape spec {shape} specified at `dynamic_shapes{keystr(path)}` " + f"to have the same length as the actual tensor shape {tensor.shape} " + f"(expected {len(tensor.shape)}, but got {len(shape)} instead)", + case_name="dynamic_shapes_validation", + ) + for i, dim in enumerate(shape): + if isinstance(dim, Dim): + check_same_bounds(dim) + elif dim is None: + _warn_on_None_dynamic_shape_dimension() + elif not (isinstance(dim, (int, _DimHint))): + raise UserError( + UserErrorType.INVALID_INPUT, + f"Unexpected dimension #{i} in input tensor shape {shape} " + f"specified at `dynamic_shapes{keystr(path)}` " + f"(expected None, an int, a Dim, Dim.AUTO, Dim.STATIC, or Dim.DYNAMIC, " + f"but got {dim!r} instead)", + case_name="dynamic_shapes_validation", + ) + elif shape is not None: + raise UserError( + UserErrorType.INVALID_INPUT, + f"Unexpected input tensor shape {shape} specified at `dynamic_shapes{keystr(path)}` " + f"(expected either a list/tuple of dimensions, or a dict mapping indices to dimensions," + f" where each dimension is an int, a Dim, Dim.AUTO, Dim.STATIC, or Dim.DYNAMIC)", + case_name="dynamic_shapes_validation", + ) + + assert isinstance(dynamic_shapes, (dict, tuple, list)) + if isinstance(dynamic_shapes, dict): + got_keys = list(dynamic_shapes.keys()) + expected_arg_names = list(combined_args.keys()) + if sorted(got_keys) != sorted(expected_arg_names): + msg = ( + f"When `dynamic_shapes` is specified as a dict, its top-level keys " + f"must be the arg names {expected_arg_names} of `inputs`, but " + f"here they are {got_keys}. " + ) + if ( + len(combined_args) == 1 + and expected_arg_names[0] not in got_keys + and isinstance(combined_args[expected_arg_names[0]], dict) + ): + msg += ( + "Since here `inputs` is a list/tuple enclosing a single dict, " + "maybe you just forgot to enclose `dynamic_shapes` in a list/tuple?" + ) + else: + msg += ( + "Alternatively, you could also ignore arg names entirely " + "and specify `dynamic_shapes` as a list/tuple matching `inputs`." + ) + raise UserError( + UserErrorType.INVALID_INPUT, msg, case_name="dynamic_shapes_validation" + ) + + def check_shape(path, t, dynamic_shape): + if isinstance(t, torch.Tensor): + check_symbols(path, t, dynamic_shape) + elif isinstance(t, _IntWrapper): + if isinstance(dynamic_shape, _Dim): + raise ValueError( + "Unable to specify input integers as dynamic through named " + "Dims. Please use Dim.AUTO/DYNAMIC instead." + ) + assert dynamic_shape is None or isinstance(dynamic_shape, (int, _DimHint)) + else: + if dynamic_shape is not None: + rendered_path = keystr(path) + raise UserError( + UserErrorType.INVALID_INPUT, + f"Cannot associate shape {dynamic_shape} specified at `dynamic_shapes{rendered_path}` " + f"to non-tensor type {type(t)} at `inputs{rendered_path}` (expected None)", + case_name="dynamic_shapes_validation", + ) + + _tree_map_with_path(check_shape, combined_args, dynamic_shapes, tree_name="inputs") + + +def _process_dynamic_shapes( + combined_args: dict[str, Any], + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, +) -> list[Constraint]: + """ + Reads the dynamic_shapes specification and produces a list of constraints. + """ + from torch._dynamo.exc import UserError, UserErrorType + + if dynamic_shapes is None or len(dynamic_shapes) == 0: + # we run with dynamic by default, so no need to produce constraints + return [] + if isinstance(dynamic_shapes, (tuple, list)): + combined_args = type(dynamic_shapes)(combined_args.values()) # type: ignore[assignment, misc] + + # map of Dim names representing input shape dimensions to constraints on them + symbols: dict[str, list[Constraint]] = defaultdict(list) + # track roots that do not directly represent input shape dimensions + phantom_roots: dict[str, _PhantomRoot] = {} + derived_constraints_with_phantom_root: list[_DerivedConstraint] = [] + # list of constraints to return + constraints: list[Constraint] = [] + + def to_constraint(dim, tensor, i): + import sympy + + from torch.fx.experimental.symbolic_shapes import StrictMinMaxConstraint + from torch.utils._sympy.solve import try_solve + from torch.utils._sympy.value_ranges import ValueRanges + + def root_value(): + # given tensor.shape[i] is the value of dim = fn(root), + # find the value of root + symbol = sympy.Symbol(dim.root.__name__, integer=True) + expr = dim.fn(symbol) + solution = try_solve(sympy.Eq(expr, tensor.shape[i]), symbol) + if solution is not None: + return int(solution[1]) + else: + raise UserError( # noqa: B904 + UserErrorType.CONSTRAINT_VIOLATION, + f"Expected shape[{i}] = {tensor.shape[i]} of input Tensor to be " + f"of the form {expr}, where {symbol} is an integer", + ) + + if isinstance(dim, _DerivedDim): + # generate a _DerivedConstraint where the root is: + # - either a _ConstraintTarget (if dim.root directly describes an input shape) + # - or a _PhantomRoot (otherwise) + dim_root = dim.root # type: ignore[attr-defined] + if dim_root.__name__ in symbols: + # root represents an input shape dimension + root_constraint = symbols[dim_root.__name__][0] + root = _ConstraintTarget( + root_constraint.t_id, + root_constraint.dim, + ) + elif dim_root.__name__ not in phantom_roots: + # create a phantom root + root = _PhantomRoot( # type: ignore[assignment] + name=dim_root.__name__, + constraint_range=StrictMinMaxConstraint( + vr=ValueRanges(lower=dim_root.min, upper=dim_root.max), + warn_only=False, + ), + val=root_value(), + ) + phantom_roots[dim_root.__name__] = root # type: ignore[assignment] + else: + root = phantom_roots[dim_root.__name__] # type: ignore[assignment] + constraint = _DerivedConstraint( + id(tensor), + i, + dim.__name__, + StrictMinMaxConstraint( + vr=ValueRanges(lower=dim.min, upper=dim.max), + warn_only=False, + ), + root, + dim.fn, # type: ignore[attr-defined] + ) + if isinstance(root, _PhantomRoot): + # NOTE(avik): since we have not processed all inputs yet, we may replace this + # with a root that does represent an input shape dimension later (see below) + derived_constraints_with_phantom_root.append(constraint) + elif isinstance(dim, _StaticDim): + constraint = _Constraint( # type: ignore[assignment] + id(tensor), + i, + dim.__name__, + StrictMinMaxConstraint( + vr=ValueRanges(lower=dim.value, upper=dim.value), # type: ignore[attr-defined] + warn_only=False, + ), + ) + else: + assert isinstance(dim, Dim) + constraint = _Constraint( # type: ignore[assignment] + id(tensor), + i, + dim.__name__, + StrictMinMaxConstraint( + vr=ValueRanges(lower=dim.min, upper=dim.max), # type: ignore[attr-defined] + warn_only=False, + ), + ) + return constraint + + def _parse_tensor_dim(tensor, idx, dim) -> None: + def _create_static_dim(tensor, i, value): + return _StaticDim(value) + + if isinstance(dim, (int, Dim)): + if isinstance(dim, int): + dim = _create_static_dim(tensor, idx, dim) + constraint = to_constraint(dim, tensor, idx) + symbols[dim.__name__].append(constraint) + elif isinstance(dim, _DimHint): + if dim.type == _DimHintType.AUTO: + torch._dynamo.maybe_mark_dynamic(tensor, idx) + elif dim.type == _DimHintType.STATIC: + torch._dynamo.mark_static(tensor, idx) + elif dim.type == _DimHintType.DYNAMIC: + torch._dynamo.mark_dynamic(tensor, idx) + constraints.append(_RelaxedConstraint(id(tensor), idx)) + elif dim is None: + torch._dynamo.mark_static(tensor, idx) + + def update_symbols(path, tensor, shape): + # clean out decorators from user side, or previous export call + # we also delete these attributes in non_strict_utils.py/make_constraints() + tensor._dynamo_weak_dynamic_indices = set() + tensor._dynamo_dynamic_indices = set() + tensor._dynamo_dynamic_range = set() + tensor._dynamo_static_indices = set() + tensor._dynamo_unbacked_indices = set() + + if isinstance(shape, dict): + for i, dim in shape.items(): + _parse_tensor_dim(tensor, i, dim) + elif isinstance(shape, (tuple, list)): + for i, dim in enumerate(shape): + _parse_tensor_dim(tensor, i, dim) + elif shape is None: + for i in range(tensor.dim()): + _parse_tensor_dim(tensor, i, None) + + def assoc_shape(path, t, dynamic_shape): + if isinstance(t, torch.Tensor): + update_symbols(path, t, dynamic_shape) + elif isinstance(t, _IntWrapper): + # If tensor dimensions are marked as dynamic, the tensors themselves + # get marked using mark_dynamic. However since we can't mark + # integers as dynamic, we first wrap integers in this class, and + # then set the `dim` field of the class with the dynamic shapes dim + # to mark the integer as dynamic. + t.dynamism = dynamic_shape + + _tree_map_with_path(assoc_shape, combined_args, dynamic_shapes, tree_name="inputs") + + for derived_constraint_with_phantom_root in derived_constraints_with_phantom_root: + phantom_root_name = derived_constraint_with_phantom_root.root.name # type: ignore[union-attr] + if phantom_root_name in symbols: + # We found an input shape dimension corresponding to this name, so we + # do not need a phantom symbol for it after all. + # NOTE(avik): Overall we want to maintain the invariant that roots that + # are phantom symbols are really "phantom," i.e., they cannot be represented + # by any input source. This is important when we are deciding derived equalities, + # since we can focus our attention exclusively on input sources: deciding + # derived equalities involving phantom symbols are, in comparison, trivial. + derived_constraint_with_phantom_root.root = symbols[phantom_root_name][0] + + for dynamic_dims in symbols.values(): + constraints.extend(dynamic_dims) + + return constraints + + +def _get_dim_name_mapping( + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any] | None, +): + name_to_dim = {} + for dim in tree_iter(dynamic_shapes, is_leaf=lambda x: isinstance(x, Dim)): + if dim is None: + # NOTE: this must denote a non-Tensor or automatic at this point. + continue + if isinstance(dim, int): + continue + elif isinstance(dim, Dim): + name_to_dim[dim.__name__] = dim + if isinstance(dim, _DerivedDim): + name_to_dim[dim.root.__name__] = dim.root # type: ignore[attr-defined] + else: + assert isinstance(dim, _DimHint) + return name_to_dim + + +def refine_dynamic_shapes_from_suggested_fixes( + msg: str, + dynamic_shapes: dict[str, Any] | tuple[Any] | list[Any], +) -> dict[str, Any] | tuple[Any] | list[Any]: + """ + When exporting with :func:`dynamic_shapes`, export may fail with a ConstraintViolation error if the specification + doesn't match the constraints inferred from tracing the model. The error message may provide suggested fixes - + changes that can be made to :func:`dynamic_shapes` to export successfully. + + Example ConstraintViolation error message:: + + Suggested fixes: + + dim = Dim('dim', min=3, max=6) # this just refines the dim's range + dim = 4 # this specializes to a constant + dy = dx + 1 # dy was specified as an independent dim, but is actually tied to dx with this relation + + This is a helper function that takes the ConstraintViolation error message and the original :func:`dynamic_shapes` spec, + and returns a new :func:`dynamic_shapes` spec that incorporates the suggested fixes. + + Example usage:: + + try: + ep = export(mod, args, dynamic_shapes=dynamic_shapes) + except torch._dynamo.exc.UserError as exc: + new_shapes = refine_dynamic_shapes_from_suggested_fixes( + exc.msg, dynamic_shapes + ) + ep = export(mod, args, dynamic_shapes=new_shapes) + + """ + + import re + + import sympy + + from torch._dynamo.exc import UserError, UserErrorType + from torch.fx.experimental.symbolic_shapes import _is_supported_equivalence + + try: + shape_fixes_msg = msg.split("Suggested fixes:")[1].strip() + except Exception as exc: + raise UserError( + UserErrorType.INVALID_INPUT, + "Suggested fixes not found in error message given to refine_dynamic_shapes_from_suggested_fixes()", + ) from exc + + # build shape_fixes dictionary + shape_fixes = {} + for fix in shape_fixes_msg.split("\n"): + fix = fix.strip() + if match := re.match(r"(.*) = Dim\('(.*)'.*\)", fix): + name = match.group(1) + _min, _max = None, None + if match_min := re.match(r".* = Dim\('.*', min\=([0-9]+).*\)", fix): + _min = int(match_min.group(1)) + if match_max := re.match(r".* = Dim\('.*'.*max\=([0-9]+)\)", fix): + _max = int(match_max.group(1)) + shape_fixes[name] = Dim(name, min=_min, max=_max) + else: + name, expr = fix.split(" = ") + expr = sympy.sympify(expr) + if isinstance(expr, sympy.Number): + # static, integer + shape_fixes[name] = int(expr) # type: ignore[assignment] + else: + # relation or derived dim + shape_fixes[name] = expr + + name_to_dim = _get_dim_name_mapping(dynamic_shapes) + + # track derived dim roots + roots: set[str] = set() + for k, c in shape_fixes.items(): + assert isinstance(c, (int, Dim, _DerivedDim, sympy.Expr)) + if isinstance(c, sympy.Expr): # check dim/derived dim expression + assert _is_supported_equivalence(c) + shape_fixes[k] = c + roots.add(str(next(iter(c.free_symbols)))) + if isinstance(c, _DerivedDim): + roots.add(c.root.__name__) # type: ignore[attr-defined] + + # check keys are existing dims or new roots + for k in shape_fixes: + assert k in name_to_dim or k in roots + + # cache so we don't produce multiple derived dim objects + derived_dim_cache: dict[str, _DerivedDim] = {} + + def apply_fixes(path, dim, dummy): + if dim is None or isinstance(dim, int): # not dynamic + return dim + elif dim.__name__ in shape_fixes: # directly fix + fix = shape_fixes[dim.__name__] + if isinstance(fix, sympy.Expr): # now derived or related + if str(fix) in derived_dim_cache: + return derived_dim_cache[str(fix)] + else: + symbol = next(iter(fix.free_symbols)) + # try to locate symbol + if symbol.name in shape_fixes: + root = shape_fixes[symbol.name] + else: + assert symbol.name in name_to_dim + root = name_to_dim[symbol.name] + # figure out value of fix + modulus, remainder = sympy.polys.polytools.div(fix, symbol) + dim = root + if modulus != 1: + dim = int(modulus) * dim + if remainder != 0: + dim = dim + int(remainder) + derived_dim_cache[str(fix)] = dim + return dim + else: + return fix + elif isinstance(dim, _DerivedDim) and dim.root.__name__ in shape_fixes: # type: ignore[attr-defined] + if dim.__name__ in derived_dim_cache: + return derived_dim_cache[dim.__name__] + else: # evaluate new derived value based on root + _dim = dim.fn(shape_fixes[dim.root.__name__]) # type: ignore[attr-defined] + derived_dim_cache[dim.__name__] = _dim + return _dim + return dim # unchanged dim + + return _tree_map_with_path(apply_fixes, dynamic_shapes, dynamic_shapes) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/exported_program.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/exported_program.py new file mode 100644 index 0000000000000000000000000000000000000000..9193aa788952539b8a1fce5d8d67e65cd508d967 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/exported_program.py @@ -0,0 +1,1719 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +import contextlib +import copy +import dataclasses +import functools +import operator +import types +import warnings +from collections import defaultdict +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any, final, NamedTuple, TYPE_CHECKING + +from torch._guards import tracing, TracingContext +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._library.fake_class_registry import FakeScriptObject +from torch._subclasses.fake_impls import ( + _deregister_op_impl, + _is_op_registered_to_fake_rule, + register_op_impl, +) +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx._symbolic_trace import _ConstantAttributeType +from torch.fx._utils import first_call_function_nn_module_stack +from torch.fx.graph import _PyTreeCodeGen, _PyTreeInfo +from torch.fx.immutable_collections import immutable_dict, immutable_list +from torch.fx.passes.runtime_assert import insert_deferred_runtime_asserts + + +if TYPE_CHECKING: + # Import the following modules during type checking to enable code intelligence features, + # such as auto-completion in tools like pylance, even when these modules are not explicitly + # imported in user code. + + import sympy + + from torch.utils._sympy.value_ranges import ValueRanges + +import torch +import torch.utils._pytree as pytree +from torch._export.utils import ( + _build_cache, + _collect_all_valid_cia_ops, + _collect_and_set_constant_attrs, + _collect_param_buffer_metadata, + _detect_fake_mode_from_gm, + _fakify_params_buffers, + _get_decomp_for_cia, + _is_preservable_cia_op, + _name_hoo_subgraph_placeholders, + _override_graph_signature_for_temp_registered_constants, + _overwrite_signature_for_non_persistent_buffers, + _populate_param_buffer_metadata_to_new_gm, + _register_constants_as_buffers, + _rename_without_collisions, + _special_op_to_preserve_cia, + placeholder_naming_pass, +) +from torch._export.verifier import Verifier +from torch._guards import detect_fake_mode +from torch._subclasses.fake_tensor import unset_fake_temporarily +from torch.export._tree_utils import is_equivalent, reorder_kwargs +from torch.export.decomp_utils import CustomDecompTable +from torch.fx._compatibility import compatibility +from torch.fx.passes.infra.pass_base import PassResult +from torch.fx.passes.infra.pass_manager import PassManager + +from .graph_signature import ( # noqa: F401 + ArgumentSpec, + ConstantArgument, + CustomObjArgument, + ExportGraphSignature, + InputKind, + InputSpec, + OutputKind, + OutputSpec, + SymBoolArgument, + SymFloatArgument, + SymIntArgument, + TensorArgument, + TokenArgument, +) + + +__all__ = [ + "ExportedProgram", + "ModuleCallEntry", + "ModuleCallSignature", + "default_decompositions", +] + + +PassType = Callable[[torch.fx.GraphModule], PassResult | None] + + +@dataclasses.dataclass +class ModuleCallSignature: + inputs: list[ArgumentSpec] + outputs: list[ArgumentSpec] + in_spec: pytree.TreeSpec + out_spec: pytree.TreeSpec + forward_arg_names: list[str] | None = None + + def replace_all_uses_with(self, original_node, new_node): + for i in self.inputs: + if i.name == original_node.name: + i.name = new_node.name + for o in self.outputs: + if o.name == original_node.name: + o.name = new_node.name + + +@dataclasses.dataclass +class ModuleCallEntry: + fqn: str + signature: ModuleCallSignature | None = None + + +def _disable_prexisiting_fake_mode(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + with unset_fake_temporarily(): + return fn(*args, **kwargs) + + return wrapper + + +def _fx_collection_equivalence_fn( + spec1_type: type | None, + spec1_context: pytree.Context, + spec2_type: type | None, + spec2_context: pytree.Context, +) -> bool: + """Treat containers and their immutable variants as the same type. Otherwise + compare as normal. + """ + if spec1_type is None or spec2_type is None: + return spec1_type is spec2_type and spec1_context == spec2_context + + if issubclass(spec1_type, (dict, immutable_dict)) and issubclass( + spec2_type, (dict, immutable_dict) + ): + return spec1_context == spec2_context + + if issubclass(spec1_type, (list, immutable_list)) and issubclass( + spec2_type, (list, immutable_list) + ): + return spec1_context == spec2_context + + return spec1_type is spec2_type and spec1_context == spec2_context + + +# This list is compiled from DispatchKey.cpp. +# The idea is that we use these keys to override +# CIA decomp in export +_AUTOGRAD_ALIAS_BACKEND_KEYS_TO_OVERRIDE = [ + torch._C.DispatchKey.AutogradCPU, + torch._C.DispatchKey.AutogradCUDA, + torch._C.DispatchKey.AutogradMeta, + torch._C.DispatchKey.AutogradXLA, + torch._C.DispatchKey.AutogradLazy, + torch._C.DispatchKey.AutogradIPU, + torch._C.DispatchKey.AutogradXPU, + torch._C.DispatchKey.AutogradMPS, + torch._C.DispatchKey.AutogradHPU, + torch._C.DispatchKey.AutogradPrivateUse1, + torch._C.DispatchKey.AutogradPrivateUse2, + torch._C.DispatchKey.AutogradPrivateUse3, +] + + +# This list is compiled from DispatchKey.cpp. +# The idea is that we use these keys to add +# python kernels that directly uses default +# CIA decomp +# See NOTE Registering old CIA to Backend kernel +_BACKEND_KEYS_TO_OVERRIDE = [ + torch._C.DispatchKey.CPU, + torch._C.DispatchKey.CUDA, + torch._C.DispatchKey.Meta, + torch._C.DispatchKey.XLA, + torch._C.DispatchKey.Lazy, + torch._C.DispatchKey.IPU, + torch._C.DispatchKey.XPU, + torch._C.DispatchKey.MPS, + torch._C.DispatchKey.HPU, +] + + +@contextmanager +def _override_composite_implicit_decomp(cia_ops_to_callable): + # This function overrides CompositeImplicitAutograd decomp for + # functional composite ops that user specified. Ideally we want to not-decompose + # ALL composite ops but today's C++ functinalization relies on + # the fact that it is working with the opset after decomp is run. + # Hence we can only do it for functional ops. One caveat is that + # there are some composite ops that lie about their schema (claimed to be + # functional but not really aka dropout), for these cases, we just decompose. + saved_tables = {} + patched_ops = set() + for op_overload, decomp_callable in cia_ops_to_callable.items(): + saved_tables[op_overload] = op_overload.py_kernels.copy() + patched_ops.add(op_overload) + for override_dispatch_key in _AUTOGRAD_ALIAS_BACKEND_KEYS_TO_OVERRIDE: + if override_dispatch_key not in op_overload.py_kernels: + # TODO (tmanlaibaatar)https://github.com/pytorch/pytorch/issues/129430 + op_overload.py_impl(override_dispatch_key)( + autograd_not_implemented(op_overload, deferred_error=True) + ) + # See NOTE: Registering old CIA to Backend kernel + # It is important that we cache this before we override py_kernels. + orig_cia_callable = _get_decomp_for_cia(op_overload) + if torch._C.DispatchKey.CompositeImplicitAutograd in op_overload.py_kernels: + del op_overload.py_kernels[torch._C.DispatchKey.CompositeImplicitAutograd] + + op_overload.py_impl(torch._C.DispatchKey.CompositeImplicitAutograd)( + decomp_callable + ) + + # [NOTE] Directly registering fake tensor rule to CIA ops + # The problem we are facing here is if your CIA custom rule + # says we want to preserve the op, we will return NotImplemented. + # Unfortunately, this will invoke meta device tracing in fake tensor + # resulting in divergent behaviour for CIA kernels that has device based + # branching (one case is torch.ops.aten.scaled_dot_product.attention) + # To get around this issue, we register direct fake impl so that we + # run the kernel before we actually try to decompose the op in FakeTensorMode. + # Note that is a no-op in most cases, because: + # 1) In post dispatch tracing, CIA would have already decomposed + # 2) Most CIA impl are device agnostic. + def _force_dispatch_to_orig_cia_callable(fake_tensor_mode, op, *args, **kwargs): + orig_cia_callable = kwargs["original_callable"] + del kwargs["original_callable"] + with fake_tensor_mode: + return orig_cia_callable(*args, **kwargs) + + if not _is_op_registered_to_fake_rule(op_overload): + register_op_impl(op_overload)( + functools.partial( + _force_dispatch_to_orig_cia_callable, + original_callable=orig_cia_callable, + ) + ) + + for key in _BACKEND_KEYS_TO_OVERRIDE: + if key not in op_overload.py_kernels: + # [NOTE] Registering old CIA to Backend kernel + # We always register original CIA behavior to the backend keys kernel + # The reason is when we are fake tensor prop-ing or executing real kernel, + # we end up calling an operator on respective backend, which in python dispatcher, + # will resolve into CIA key. (see resolve_key in torch/_ops.py) + # As a result, this CIA now will call into the custom user defined + # CIA which can cause a problem. + # To make it more concrete, the case we are handling is: + # (1) there is a tensor constant we are performing constant propagation + # on during tracing + # (2) we invoke an op underneath autograd (either because we are below autograd, + # or we are tracing in inference mode), so one of the backend keys gets hit + # (3) the op we are invoking has a CIA impl that normally runs in eager mode + # (and the user wants to tweak this CIA impl during tracing, but during + # const-prop we want the original CIA to run + op_overload.py_impl(key)(orig_cia_callable) + + try: + yield + finally: + for op in patched_ops: + op.py_kernels.clear() + op.py_kernels.update(saved_tables[op]) + op._dispatch_cache.clear() + _deregister_op_impl(op) + + +def _split_decomp_table_to_cia_and_python_decomp( + decomp_table: dict[torch._ops.OperatorBase, Callable], +) -> tuple[dict[torch._ops.OperatorBase, Callable], ...]: + all_preservable_cia_ops = set(_collect_all_valid_cia_ops()) + cia_ops_to_callable = {} + + for op in list(decomp_table.keys()): + # TODO we are silently allowing non-safe(non-functional) ops through a crack + # due to core aten decomp table having non-functional entries. Once we have + # a tighter check around core aten decomp, we should warn users about them. + # Tracking issue: (https://github.com/pytorch/pytorch/issues/135759) + + # if it is a valid CIA op we can mess with in export, we check if it is: + # 1. Has been marked as to be decomposed. Example: + # decomp_table = decomp_table_to_core_aten() + # del decomp_table[aten.linear] + # In this case, user says decompose everything except for aten.linear + # 2. Has been marked with custom decomp behaviour. Example: + # decomp_table = {aten.linear: some_op} + # For (1), we want to remove all the CIA ops that weren't handled by user as + # it suggests they are safe to decompose, so we should remove from preservable_list. + # for (2), we just plumb the custom decomp to AOTDIspatcher. + # In both cases, we want to remove this CIA op from the decomp_table as it is special + # handled. + if op in all_preservable_cia_ops: + cia_ops_to_callable[op] = decomp_table[op] + all_preservable_cia_ops.remove(op) + del decomp_table[op] + # If it is a custom op, we want to still preserve or do whatever + # with it if it is a functional CIA. The reason we don't remove + # from CIA list is because we don't query custom ops. + elif _is_preservable_cia_op(op): + op_name = op.name() + assert not op_name.startswith("aten"), "This should be a custom op" + cia_ops_to_callable[op] = decomp_table[op] + + # If we reached here, it means user intentionally deleted these CIA ops from + # decomp table. + for k in all_preservable_cia_ops: + cia_ops_to_callable[k] = _special_op_to_preserve_cia + + return cia_ops_to_callable, decomp_table + + +def default_decompositions() -> "CustomDecompTable": + """ + This is the default decomposition table which contains decomposition of + all ATEN operators to core aten opset. Use this API together with + :func:`run_decompositions()` + """ + return CustomDecompTable() + + +def _decompose_and_get_gm_with_new_signature_constants( + ep: "ExportedProgram", + *, + cia_to_decomp: dict[torch._ops.OperatorBase, Callable], + python_decomp_table: dict[torch._ops.OperatorBase, Callable], + joint_loss_index: int | None, + decompose_custom_triton_ops, +): + from torch._export.passes.lift_constants_pass import _materialize_and_lift_constants + from torch._functorch.aot_autograd import aot_export_module + from torch.export._trace import ( + _disable_custom_triton_op_functional_decomposition, + _export_to_aten_ir, + _ignore_backend_decomps, + _verify_nn_module_stack, + _verify_placeholder_names, + _verify_stack_trace, + ) + from torch.fx.experimental.symbolic_shapes import ShapeEnv + + def _is_joint_ir_decomp(ep, joint_loss_index): + return ( + joint_loss_index is not None + or ep.graph_signature.backward_signature is not None + ) + + if not _is_joint_ir_decomp(ep, joint_loss_index): + mod = ep.module() + + wrapped_params_buffers = { + **dict(mod.named_parameters(remove_duplicate=False)), + **dict(mod.named_buffers(remove_duplicate=False)), + } + + from torch._functorch._aot_autograd.subclass_parametrization import ( + unwrap_tensor_subclass_parameters, + ) + + # [NOTE] Unwrapping subclasses AOT + # In torch.compile, the subclass unwrapping/wrapping happen at runtime + # but at export, this is impossible as it is intended to be run on + # C++ environment. As a result, we unwrap subclass parameters AOT. After this, + # ExportedProgram state_dict won't be same as eager model because eager model + # could have subclass weights while ExportedProgram will have desugared versions. + # This is fine because run_decompositions is supposed to specialize to post-autograd + # graph where the subclass desugaring is supposed to happen. + unwrap_tensor_subclass_parameters(mod) + unwrapped_params_buffers = { + **dict(mod.named_parameters(remove_duplicate=False)), + **dict(mod.named_buffers(remove_duplicate=False)), + } + + # TODO T204030333 + fake_mode = _detect_fake_mode_from_gm(ep.graph_module) + if fake_mode is None: + fake_mode = FakeTensorMode(shape_env=ShapeEnv(), export=True) + + # Fix the graph output signature to be tuple if scalar + out_spec = mod._out_spec + + assert isinstance(mod.graph._codegen, _PyTreeCodeGen) + orig_arg_names = mod.graph._codegen.pytree_info.orig_args + + # aot_export expect the return type to always be a tuple. + assert out_spec is not None + if out_spec.type not in (list, tuple): + out_spec = pytree.treespec_tuple([out_spec]) + + mod.graph._codegen = _PyTreeCodeGen( + _PyTreeInfo( + orig_arg_names, + mod._in_spec, + out_spec, + ) + ) + + mod.recompile() + + # the exported module will store constants & non-persistent buffers such that + # retracing treats them as persistent buffers, so we inform the constants lifting pass + # and overwrite the new graph signature using the previous program. + _collect_and_set_constant_attrs(ep.graph_signature, ep.constants, mod) + + # When we have a module with constant attributes, AotDispatcher doesn't actually + # wrap them as functional tensors, because dynamo would have already made it buffer. + # In non-strict case, however, AotDispatcher can intercept constants, causing it to not + # functionalize the operators that are operating on constant tensors. Since dynamo already + # wraps constants as buffers, we temporarily register the constants as buffers and undo this + # operation after AOTDispatcher is done. + temp_registered_constants = _register_constants_as_buffers( + mod, ep.state_dict, ep.graph_signature.non_persistent_buffers + ) + + # get params & buffers after excluding constants + fake_params_buffers = _fakify_params_buffers(fake_mode, mod) + + params_buffers_to_node_meta = _collect_param_buffer_metadata(mod) + + # TODO (tmanlaibaatar) Ideally run_decomp should just call _non_strict_export + # but due to special handling of constants as non-persistent buffers make it little + # difficult. But we should unify this code path together. T206837815 + from torch._export.non_strict_utils import ( + _enable_graph_inputs_of_type_nn_module, + _fakify_script_objects, + ) + + retracing_args = [] + for node in mod.graph.nodes: + if node.op == "placeholder": + if isinstance(node.meta["val"], CustomObjArgument): + real_script_obj = None + if node.meta["val"].fake_val is None: + real_script_obj = ep.constants[node.meta["val"].name] + else: + real_script_obj = node.meta["val"].fake_val.real_obj + retracing_args.append(real_script_obj) + else: + retracing_args.append(node.meta["val"]) + + tx = TracingContext(fake_mode) + + with ( + fake_mode, + _override_composite_implicit_decomp( + cia_to_decomp, + ), + _enable_graph_inputs_of_type_nn_module(ep.example_inputs), + tracing(tx), + ): + retracing_args_unwrapped = pytree.tree_unflatten( + retracing_args, mod._in_spec + ) + # this requires empty kwargs, but not in pytree.flattened format + with _fakify_script_objects( + mod, + ( + *retracing_args_unwrapped[0], + *retracing_args_unwrapped[1].values(), + ), + {}, + fake_mode, + ) as ( + patched_mod, + new_fake_args, + new_fake_kwargs, + new_fake_constant_attrs, + map_fake_to_real, + ): + aten_export_artifact = _export_to_aten_ir( + patched_mod, + new_fake_args, + new_fake_kwargs, + fake_params_buffers, + new_fake_constant_attrs, + decomp_table=python_decomp_table, + _prettify_placeholder_names=False, + decompose_custom_triton_ops=decompose_custom_triton_ops, + ) + + # aten_export_artifact.constants contains only fake script objects, we need to map them back + aten_export_artifact.constants = { + fqn: ( + map_fake_to_real[obj] + if isinstance(obj, FakeScriptObject) + else obj + ) + for fqn, obj in aten_export_artifact.constants.items() + } + + gm = aten_export_artifact.gm + new_graph_signature = aten_export_artifact.sig + + # In the previous step, we assume constants as buffers for AOTDispatcher to + # functianalize properly, so undo that here + new_graph_signature = ( + _override_graph_signature_for_temp_registered_constants( + new_graph_signature, temp_registered_constants + ) + ) + + _populate_param_buffer_metadata_to_new_gm( + params_buffers_to_node_meta, gm, new_graph_signature + ) + + # overwrite signature for non-persistent buffers + new_graph_signature = _overwrite_signature_for_non_persistent_buffers( + ep.graph_signature, new_graph_signature + ) + + constants = _materialize_and_lift_constants( + gm, new_graph_signature, new_fake_constant_attrs + ) + + placeholder_naming_pass( + gm, + new_graph_signature, + patched_mod, + new_fake_args, + new_fake_kwargs, + fake_params_buffers, + constants, + ) + + _verify_nn_module_stack(gm) + _verify_stack_trace(gm) + _verify_placeholder_names(gm, new_graph_signature) + + gm, new_graph_signature = _remove_unnecessary_copy_op_pass( + gm, new_graph_signature + ) + + # When we apply parameterization rule to unwrap + # subclasses, the state dict will now have different + # desugared parameters. We need to manually filter those + # and update the ep.state_dict. Ideally, we should just return + # the state dict of ep.module but ep.module only stores params + # buffers that participate in forward. If we undo this behavior, + # it would break some downstream users. + new_state_dict = { + **ep.state_dict, + **{ + name: p + for name, p in unwrapped_params_buffers.items() + if name not in wrapped_params_buffers + }, + } + + for name, p in wrapped_params_buffers.items(): + # Buffers can be persistent/non-persistent + if name not in new_state_dict: + assert not isinstance(p, torch.nn.Parameter) + + if name in new_state_dict: + if name not in unwrapped_params_buffers: + new_state_dict.pop(name) + + return gm, new_graph_signature, new_state_dict + + old_placeholders = [ + node for node in ep.graph_module.graph.nodes if node.op == "placeholder" + ] + fake_args = [node.meta["val"] for node in old_placeholders] + + buffers_to_remove = [name for name, _ in ep.graph_module.named_buffers()] + for name in buffers_to_remove: + delattr(ep.graph_module, name) + + # TODO(zhxhchen17) Return the new graph_signature directly. + fake_mode_det = detect_fake_mode(fake_args) + fake_mode_ctx = contextlib.nullcontext() if fake_mode_det is None else fake_mode_det # type: ignore[assignment] + custom_triton_ops_decomposition_ctx = ( + contextlib.nullcontext + if decompose_custom_triton_ops + else _disable_custom_triton_op_functional_decomposition + ) + with ( + _ignore_backend_decomps(), + fake_mode_ctx, + _override_composite_implicit_decomp(cia_to_decomp), + custom_triton_ops_decomposition_ctx(), + ): + gm, graph_signature = aot_export_module( + ep.graph_module, + fake_args, + decompositions=python_decomp_table, + trace_joint=joint_loss_index is not None, + output_loss_index=( + joint_loss_index if joint_loss_index is not None else None + ), + ) + gm.graph.eliminate_dead_code() + + # Update the signatures with the new placeholder names in case they + # changed when calling aot_export + def update_arg(old_arg, new_ph): + if isinstance(old_arg, ConstantArgument): + return old_arg + elif isinstance(old_arg, TensorArgument): + return TensorArgument(name=new_ph.name) + elif isinstance(old_arg, SymIntArgument): + return SymIntArgument(name=new_ph.name) + elif isinstance(old_arg, SymFloatArgument): + return SymFloatArgument(name=new_ph.name) + elif isinstance(old_arg, SymBoolArgument): + return SymBoolArgument(name=new_ph.name) + raise RuntimeError(f"Type of old_arg not supported: {type(old_arg)}") + + new_placeholders = [node for node in gm.graph.nodes if node.op == "placeholder"] + new_outputs: tuple[torch.fx.Node, ...] = tuple(gm.graph.output_node().args[0]) # type: ignore[arg-type] + + # rename the placeholders + assert len(new_placeholders) == len(old_placeholders) + for old_ph, new_ph in zip(old_placeholders, new_placeholders): + new_ph.name = new_ph.target = old_ph.name + + # handle name collisions with newly decomposed graph nodes + name_map = {} + find_available: dict[str, int] = defaultdict(int) + used_names: set[str] = set() + for ph in new_placeholders: + name_map[ph.name] = ph.name + _build_cache(ph.name, find_available, used_names) + for node in gm.graph.nodes: + if node.op == "placeholder": + continue + node.name = _rename_without_collisions( + name_map, find_available, used_names, node.name, node.name + ) + + # propagate names to higher order op subgraphs + _name_hoo_subgraph_placeholders(gm) + + # Run this pass before creating input/output specs, since size-related CSE/DCE might affect output signature. + # Overwrite output specs afterwards. + from torch._export.passes._node_metadata_hook import ( + _node_metadata_hook, + _set_node_metadata_hook, + ) + from torch._functorch._aot_autograd.input_output_analysis import _graph_output_names + + if not torch._dynamo.config.do_not_emit_runtime_asserts: + stack_trace = ( + 'File "torch/fx/passes/runtime_assert.py", line 24, ' + "in insert_deferred_runtime_asserts" + ) + shape_env = _get_shape_env(gm) + if shape_env is not None: + with _set_node_metadata_hook( + gm, + functools.partial( + _node_metadata_hook, metadata={"stack_trace": stack_trace} + ), + ): + insert_deferred_runtime_asserts( + gm, + shape_env, + f"exported program: {first_call_function_nn_module_stack(gm.graph)}", + export=True, + ) + + # update output specs + gm.recompile() + for output, name in zip(new_outputs, _graph_output_names(gm)): + if name is not None: + output.name = name + + # To match the output target with correct input for input mutations + # need to find the old to new placeholder map + old_new_placeholder_map = { + spec.arg.name: new_placeholders[i].name + for i, spec in enumerate(ep.graph_signature.input_specs) + if not isinstance(spec.arg, ConstantArgument) + } + + input_specs = [ + InputSpec( + spec.kind, + update_arg(spec.arg, new_placeholders[i]), + spec.target, + spec.persistent, + ) + for i, spec in enumerate(ep.graph_signature.input_specs) + ] + + output_specs = [] + + # handle buffer & input mutations; these appear before loss output & gradients + # (1) ep.graph_signature.input_specs tells us types of inputs + # (2) graph_signature.user_inputs tells us node input names in order + # (3) graph_signature.user_inputs_to_mutate tells us buffer & input mutations + # map (3) -> (2) for input order, -> (1) for input type + user_inputs_index = {name: i for i, name in enumerate(graph_signature.user_inputs)} + mutation_names = list(graph_signature.user_inputs_to_mutate.keys()) + assert mutation_names == [node.name for node in new_outputs[: len(mutation_names)]] + for output_name, input_name in graph_signature.user_inputs_to_mutate.items(): + i = user_inputs_index[input_name] + input_spec = ep.graph_signature.input_specs[i] + assert input_spec.kind in (InputKind.USER_INPUT, InputKind.BUFFER) + output_kind = ( + OutputKind.BUFFER_MUTATION + if input_spec.kind == InputKind.BUFFER + else OutputKind.USER_INPUT_MUTATION + ) + target = ( + input_spec.target + if input_spec.kind == InputKind.BUFFER + else input_spec.arg.name + ) + output_specs.append( + OutputSpec( + kind=output_kind, + arg=TensorArgument(name=output_name), + target=target, + ) + ) + + # handle actual user outputs + for i, spec in enumerate(ep.graph_signature.output_specs): + output_specs.append( + OutputSpec( + OutputKind.LOSS_OUTPUT if i == joint_loss_index else spec.kind, + update_arg(spec.arg, new_outputs[len(mutation_names) + i]), + old_new_placeholder_map.get(spec.target, spec.target), + ) + ) + + if joint_loss_index is not None: + assert graph_signature.backward_signature is not None + gradients = graph_signature.backward_signature.gradients_to_user_inputs + assert len(graph_signature.user_inputs) == len(ep.graph_signature.input_specs) + specs = { + graph_signature.user_inputs[i]: spec + for i, spec in enumerate(ep.graph_signature.input_specs) + if isinstance(spec.arg, TensorArgument) + } + for node in new_outputs[len(output_specs) :]: + source = gradients[node.name] + spec = specs[source] # type: ignore[index] + if spec.kind == InputKind.PARAMETER: + kind = OutputKind.GRADIENT_TO_PARAMETER + target = spec.target + elif spec.kind == InputKind.USER_INPUT: + kind = OutputKind.GRADIENT_TO_USER_INPUT + target = source + else: + raise AssertionError(f"Unknown input kind: {spec.kind}") + output_specs.append( + OutputSpec( + kind, + TensorArgument(name=node.name), + target, + ) + ) + + assert len(new_placeholders) == len(old_placeholders) + + new_graph_signature = ExportGraphSignature( + input_specs=input_specs, output_specs=output_specs + ) + # NOTE: aot_export adds symint metadata for placeholders with int + # values; since these become specialized, we replace such metadata with + # the original values. + # Also, set the param/buffer metadata back to the placeholders. + for old_node, new_node in zip(old_placeholders, new_placeholders): + if not isinstance(old_node.meta["val"], torch.Tensor): + new_node.meta["val"] = old_node.meta["val"] + + if ( + new_node.target in new_graph_signature.inputs_to_parameters + or new_node.target in new_graph_signature.inputs_to_buffers + ): + for k, v in old_node.meta.items(): + new_node.meta[k] = v + return gm, new_graph_signature, ep.state_dict + + +def _remove_unnecessary_copy_op_pass( + gm: torch.fx.GraphModule, new_graph_signature: ExportGraphSignature +) -> tuple[torch.fx.GraphModule, ExportGraphSignature]: + """ + Removes redundant copy_ node that was introduced due to mutated buffer. + """ + with gm._set_replace_hook(new_graph_signature.get_replace_hook()): + for node in gm.graph.nodes: + if node.op == "output": + args, _ = pytree.tree_flatten(node.args) + for out in args: + if isinstance(out, torch.fx.Node) and ( + out.name in new_graph_signature.buffers_to_mutate + or out.name in new_graph_signature.parameters_to_mutate + ): + if ( + out.op == "call_function" + and out.target is torch.ops.aten.copy.default + ): + out.replace_all_uses_with(out.args[1]) # type: ignore[arg-type] + gm.graph.erase_node(out) + gm.recompile() + return gm, new_graph_signature + + +def _common_getitem_elimination_pass( + gm: torch.fx.GraphModule, graph_signature, module_call_graph +): + with gm._set_replace_hook(graph_signature.get_replace_hook()): + for module in gm.modules(): + if not isinstance(module, torch.fx.GraphModule): + continue + + node_id: dict[torch.fx.Node, str] = {} + getitems: dict[str, torch.fx.Node] = {} + for node in list(module.graph.nodes): + if node.op == "call_function" and node.target is operator.getitem: + source, idx = node.args + new_id = f"{node_id[source]}.{idx}" + if new_id in getitems: + node.replace_all_uses_with(getitems[new_id]) + for entry in module_call_graph: + if entry.signature is not None: + entry.signature.replace_all_uses_with( + node, getitems[new_id] + ) + module.graph.erase_node(node) + else: + getitems[new_id] = node + node_id[node] = new_id + else: + node_id[node] = node.name + + +def _get_updated_module_call_graph( + old_gm: torch.fx.GraphModule, + old_graph_signature: ExportGraphSignature, + gm: torch.fx.GraphModule, + graph_signature: ExportGraphSignature, + old_module_call_graph: list[ModuleCallEntry], +): + new_module_call_graph = copy.deepcopy(old_module_call_graph) + + old_nodes = {node.name: node for node in old_gm.graph.nodes} + + old_graph_params_buffers = { + **old_graph_signature.inputs_to_parameters, + **old_graph_signature.inputs_to_buffers, + } + new_graph_params_buffers = { + **graph_signature.inputs_to_parameters, + **graph_signature.inputs_to_buffers, + } + + # use node-level provenance metadata to create a map + # from old node names to new node names + provenance: dict[str, str] = {} + + user_input_counter = 0 + old_user_input_names = [ + node.target for node in old_gm.graph.nodes if node.op == "placeholder" + ] + old_user_input_names = list( + filter( + lambda x: x not in old_graph_params_buffers + and x not in old_graph_signature.input_tokens, + old_user_input_names, + ) + ) + new_user_input_names = [ + node.target for node in gm.graph.nodes if node.op == "placeholder" + ] + + for node in gm.graph.nodes: + if history := node.meta.get("from_node", []): + provenance[history[-1].name] = node.name + + # For params and buffers, we might have applied parameterizaiton rule + # so that the names might have changed. But for user inputs, we know we + # must preserve the old name. + elif node.op == "placeholder": + if not ( + node.target in new_graph_params_buffers + or node.target in graph_signature.input_tokens + ): + if node.target in new_user_input_names: + assert isinstance(node.name, str) + old_name = old_user_input_names[user_input_counter] + assert isinstance(old_name, str) + provenance[old_name] = node.name + user_input_counter += 1 + + # For all the parameters and buffers, we first see + # if they are result of parametrizations and if they + # are, we log them and error later + old_param_to_desugared = defaultdict(list) + for name, target in new_graph_params_buffers.items(): + # if the parameters are not parametrized, the naming won't change. + if not target.startswith("parametrizations."): + # If we are in strict mode, we can't just reuse the param names + if name in old_graph_params_buffers: + provenance[name] = name + else: + old_target = ".".join(target.split(".")[1:-1]) + old_param_to_desugared[old_target].append(name) + + # map old names to new names in module call signatures + for entry in new_module_call_graph: + signature = entry.signature + if signature is None: + continue + for x in [*signature.inputs, *signature.outputs]: + # We noticed that submodule is taking subclass as input. we can't + # preserve signature here. + if x.name in old_param_to_desugared: + raise ValueError( + f"It looks like {x.name} is a tensor subclass. " + f"Preserving submodule that takes subclass parameter is not supported" + f" in inference IR because we desugar them, resulting in more tensors" + ) + + if x.name in provenance: + x.name = provenance[x.name] + + # This can happen when aten.to is called at graph boundaries. + # Basically aten.to at post-dispatch level can either be copy + # or alias. In the alias case, we will no-op it so it will + # disappear from the graph. If we detect such case, we should + # reuse the input to aten.to as the new input to the submodule. + # Technically this can happen for other maybe aliasing ops, + # but aten.to is probably the most common one. + elif x.name in old_nodes: + old_node = old_nodes[x.name] + if old_node.op == "call_function" and old_node.target in [ + torch.ops.aten.to.dtype_layout, + torch.ops.aten.to.device, + torch.ops.aten.to.dtype, + ]: + old_target = old_node.args[0].name + if old_target not in provenance: + raise ValueError( + f"It looks like {old_target} is a tensor subclass. " + f"Preserving submodule that takes subclass parameter is not supported" + f" in inference IR because we desugar them, resulting in more tensors" + ) + + x.name = provenance[old_target] + + return new_module_call_graph + + +def _decompose_exported_program( + ep, + *, + cia_to_decomp: dict[torch._ops.OperatorBase, Callable], + python_decomp_table: dict[torch._ops.OperatorBase, Callable], + joint_loss_index: int | None, + decompose_custom_triton_ops: bool, +): + ( + gm, + new_graph_signature, + state_dict, + ) = _decompose_and_get_gm_with_new_signature_constants( + ep, + cia_to_decomp=cia_to_decomp, + python_decomp_table=python_decomp_table, + joint_loss_index=joint_loss_index, + decompose_custom_triton_ops=decompose_custom_triton_ops, + ) + + # The signatures of ep.module_call_graph refer to input / output nodes of + # the original graph module. However, the new graph module may have + # new nodes due to decompositions. So we need to update these signatures + # in the decomposed exported program's module_call_graph. + new_module_call_graph = _get_updated_module_call_graph( + ep.graph_module, + ep.graph_signature, + gm, + new_graph_signature, + ep.module_call_graph, + ) + + # TODO unfortunately preserving graph-level metadata is not + # working well with aot_export. So we manually copy it. + # (The node-level meta is addressed above.) + gm.meta.update(ep.graph_module.meta) + + new_range_constraints = _get_updated_range_constraints( + gm, + ep.range_constraints, + ) + + exported_program = ExportedProgram( + root=gm, + graph=gm.graph, + graph_signature=new_graph_signature, + state_dict=state_dict, + range_constraints=new_range_constraints, + module_call_graph=new_module_call_graph, + example_inputs=ep.example_inputs, + constants=ep.constants, + ) + return exported_program + + +class ExportedProgram: + """ + Package of a program from :func:`export`. It contains + an :class:`torch.fx.Graph` that represents Tensor computation, a state_dict containing + tensor values of all lifted parameters and buffers, and various metadata. + + You can call an ExportedProgram like the original callable traced by + :func:`export` with the same calling convention. + + To perform transformations on the graph, use ``.module`` property to access + an :class:`torch.fx.GraphModule`. You can then use + `FX transformation `_ + to rewrite the graph. Afterwards, you can simply use :func:`export` + again to construct a correct ExportedProgram. + """ + + _graph_module: torch.fx.GraphModule + """The underlying GraphModule containing the exported computation graph.""" + + _graph_signature: ExportGraphSignature + """The signature containing input/output specifications for the graph.""" + + _state_dict: dict[str, Any] + """Dictionary containing parameter and buffer values from the original module.""" + + _range_constraints: "dict[sympy.Symbol, ValueRanges]" + """Symbolic shape constraints for dynamic shapes in the graph.""" + + _module_call_graph: list[ModuleCallEntry] + """Call graph information tracking module hierarchy and signatures.""" + + _example_inputs: tuple[tuple[Any, ...], dict[str, Any]] | None + """Example inputs used during export, stored as (args, kwargs) tuple.""" + + _constants: dict[str, _ConstantAttributeType] + """Dictionary of constant values used in the graph.""" + + _verifiers: list[type[Verifier]] + """List of verifier classes used to validate the exported program.""" + + _guards_code: list[str] + + def __init__( + self, + root: torch.nn.Module | dict[str, Any], + graph: torch.fx.Graph, + graph_signature: ExportGraphSignature, + state_dict: dict[str, torch.Tensor | torch.nn.Parameter], + range_constraints: "dict[sympy.Symbol, Any]", + module_call_graph: list[ModuleCallEntry], + example_inputs: tuple[tuple[Any, ...], dict[str, Any]] | None = None, + constants: dict[str, _ConstantAttributeType] | None = None, + *, + verifiers: list[type[Verifier]] | None = None, + ): + # Remove codegen related things from the graph. It should just be a flat graph. + graph._codegen = torch.fx.graph.CodeGen() + self._graph_module = _create_graph_module_for_export(root, graph) + if isinstance(root, torch.fx.GraphModule): + self._graph_module.meta.update(root.meta) + + _common_getitem_elimination_pass( + self._graph_module, graph_signature, module_call_graph + ) + self._graph_signature: ExportGraphSignature = graph_signature + self._state_dict: dict[str, Any] = state_dict + self._range_constraints: dict[sympy.Symbol, ValueRanges] = range_constraints + assert module_call_graph is not None + self._module_call_graph: list[ModuleCallEntry] = module_call_graph + self._example_inputs = example_inputs + + self._constants = constants or {} + + verifiers = verifiers or [Verifier] + assert all(issubclass(v, Verifier) for v in verifiers) + self._verifiers = verifiers + # Validate should be always the last step of the constructor. + self.validate() + + self._guards_code = _convert_guards_to_code(self._graph_module) + + @property + @compatibility(is_backward_compatible=False) + def graph_module(self): + return self._graph_module + + @graph_module.setter + @compatibility(is_backward_compatible=False) + def graph_module(self, value): + raise RuntimeError("Unable to set ExportedProgram's graph_module attribute.") + + @property + @compatibility(is_backward_compatible=False) + def graph(self): + return self.graph_module.graph + + @graph.setter + @compatibility(is_backward_compatible=False) + def graph(self, value): + raise RuntimeError("Unable to set ExportedProgram's graph attribute.") + + @property + @compatibility(is_backward_compatible=False) + def graph_signature(self): + return self._graph_signature + + @graph_signature.setter + @compatibility(is_backward_compatible=False) + def graph_signature(self, value): + raise RuntimeError("Unable to set ExportedProgram's graph_signature attribute.") + + @property + @compatibility(is_backward_compatible=False) + def state_dict(self): + return self._state_dict + + @state_dict.setter + @compatibility(is_backward_compatible=False) + def state_dict(self, value): + raise RuntimeError("Unable to set ExportedProgram's state_dict attribute.") + + @compatibility(is_backward_compatible=False) + def parameters(self) -> Iterator[torch.nn.Parameter]: + """ + Returns an iterator over original module's parameters. + """ + for _, param in self.named_parameters(): + yield param + + @compatibility(is_backward_compatible=False) + def named_parameters(self) -> Iterator[tuple[str, torch.nn.Parameter]]: + """ + Returns an iterator over original module parameters, yielding + both the name of the parameter as well as the parameter itself. + """ + for param_name in self.graph_signature.parameters: + yield param_name, self.state_dict[param_name] + + @compatibility(is_backward_compatible=False) + def buffers(self) -> Iterator[torch.Tensor]: + """ + Returns an iterator over original module buffers. + """ + for _, buf in self.named_buffers(): + yield buf + + @compatibility(is_backward_compatible=False) + def named_buffers(self) -> Iterator[tuple[str, torch.Tensor]]: + """ + Returns an iterator over original module buffers, yielding + both the name of the buffer as well as the buffer itself. + """ + non_persistent_buffers = set(self.graph_signature.non_persistent_buffers) + for buffer_name in self.graph_signature.buffers: + if buffer_name in non_persistent_buffers: + yield buffer_name, self.constants[buffer_name] + else: + yield buffer_name, self.state_dict[buffer_name] + + @property + @compatibility(is_backward_compatible=False) + def range_constraints(self): + return self._range_constraints + + @range_constraints.setter + @compatibility(is_backward_compatible=False) + def range_constraints(self, value): + raise RuntimeError( + "Unable to set ExportedProgram's range_constraints attribute." + ) + + @property + @compatibility(is_backward_compatible=False) + def module_call_graph(self): + return self._module_call_graph + + @module_call_graph.setter + @compatibility(is_backward_compatible=False) + def module_call_graph(self, value): + raise RuntimeError( + "Unable to set ExportedProgram's module_call_graph attribute." + ) + + @property + @compatibility(is_backward_compatible=False) + def example_inputs(self): + return self._example_inputs + + @example_inputs.setter + @compatibility(is_backward_compatible=False) + def example_inputs(self, value): + # This is allowed + + if value is None: + self._example_inputs = value + return + + if not ( + isinstance(value, tuple) + and len(value) == 2 + and isinstance(value[0], tuple) + and isinstance(value[1], dict) + ): + raise ValueError( + "Example inputs should be a tuple containing example arguments (as " + "a tuple), and example kwargs (as a dictionary)." + ) + + args, kwargs = value + from ._unlift import _check_inputs_match + + _check_inputs_match(args, kwargs, self.call_spec.in_spec) + + self._example_inputs = value + + @property + @compatibility(is_backward_compatible=False) + def call_spec(self): + class CallSpec(NamedTuple): + in_spec: pytree.TreeSpec | None + out_spec: pytree.TreeSpec | None + + if len(self.module_call_graph) == 0: + return CallSpec(in_spec=None, out_spec=None) + assert self.module_call_graph[0].fqn == "" + return CallSpec( + in_spec=self.module_call_graph[0].signature.in_spec, + out_spec=self.module_call_graph[0].signature.out_spec, + ) + + @call_spec.setter + @compatibility(is_backward_compatible=False) + def call_spec(self, value): + raise RuntimeError("Unable to set ExportedProgram's call_spec attribute.") + + @property + @compatibility(is_backward_compatible=False) + def verifier(self) -> Any: + return self._verifiers[0] + + @verifier.setter + @compatibility(is_backward_compatible=False) + def verifier(self, value): + raise RuntimeError("Unable to set ExportedProgram's verifier attribute.") + + @property + @compatibility(is_backward_compatible=False) + def dialect(self) -> str: + assert self._verifiers is not None + return self._verifiers[0].dialect + + @dialect.setter + @compatibility(is_backward_compatible=False) + def dialect(self, value): + raise RuntimeError("Unable to set ExportedProgram's dialect attribute.") + + @property + @compatibility(is_backward_compatible=False) + def verifiers(self): + return self._verifiers + + @verifiers.setter + @compatibility(is_backward_compatible=False) + def verifiers(self, value): + raise RuntimeError("Unable to set ExportedProgram's verifiers attribute.") + + @property + @compatibility(is_backward_compatible=False) + def tensor_constants(self): + return self._constants + + @tensor_constants.setter + @compatibility(is_backward_compatible=False) + def tensor_constants(self, value): + raise RuntimeError( + "Unable to set ExportedProgram's tensor_constants attribute." + ) + + @property + @compatibility(is_backward_compatible=False) + def constants(self): + return self._constants + + @constants.setter + @compatibility(is_backward_compatible=False) + def constants(self, value): + raise RuntimeError("Unable to set ExportedProgram's constants attribute.") + + def _get_flat_args_with_check(self, args, kwargs): + """Flatten args, kwargs using pytree, then, check specs. + + Args: + args: List[Any] original args passed to __call__ + kwargs: Dict[str, Any] original kwargs passed to __call + + Returns: + A tuple of (flat_args, received_spec) + flat_args is flattened args / kwargs + received_spec is the pytree spec produced while flattening the + tuple (args, kwargs) + """ + in_spec = self.call_spec.in_spec + if in_spec is not None: + kwargs = reorder_kwargs(kwargs, in_spec) + flat_args_with_path, received_spec = pytree.tree_flatten_with_path( + (args, kwargs) + ) + self._check_input_constraints(flat_args_with_path) + flat_args = tuple(x[1] for x in flat_args_with_path) + return flat_args, received_spec + + def _graph_module_flat_inputs(self, args: Any, kwargs: Any) -> Any: + """Transform args, kwargs of __call__ to args for graph_module. + + self.graph_module takes stuff from state dict as inputs. + The invariant is for ep: ExportedProgram is + ep(args, kwargs) == + ep.postprocess(ep.graph_module(ep.graph_module_flat_inputs(args, kwargs))) + """ + + in_spec = self.call_spec.in_spec + flat_args, received_spec = self._get_flat_args_with_check(args, kwargs) + if in_spec is not None and not is_equivalent( + received_spec, in_spec, _fx_collection_equivalence_fn + ): + raise ValueError( + "Trying to flatten user inputs with exported input tree spec: \n" + f"{in_spec}\n" + "but actually got inputs with tree spec of: \n" + f"{received_spec}" + ) + + additional_inputs = [] + for input_ in self.graph_signature.input_specs: + if input_.kind == InputKind.USER_INPUT: + continue + elif input_.kind in ( + InputKind.PARAMETER, + InputKind.BUFFER, + ): + if input_.persistent is False: + # This is a non-persistent buffer, grab it from our + # constants instead of the state dict. + additional_inputs.append(self.constants[input_.target]) + else: + additional_inputs.append(self.state_dict[input_.target]) + elif input_.kind in ( + InputKind.CONSTANT_TENSOR, + InputKind.CUSTOM_OBJ, + ): + additional_inputs.append(self.constants[input_.target]) + additional_inputs = tuple(additional_inputs) + + # NOTE: calling convention is first params, then buffers, then args as user supplied them. + # See: torch/_functorch/aot_autograd.py#L1034 + return additional_inputs + flat_args + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError( + "Unable to call ExportedProgram directly. " + "You should use `exported_program.module()` instead." + ) + + def __str__(self) -> str: + graph_module = self.graph_module.print_readable( + print_output=False, colored=False + ).replace("\n", "\n ") + graph_signature = str(self.graph_signature).replace("\n", "\n ") + string = ( + "ExportedProgram:\n" + f" {graph_module}\n" + f"Graph signature: {graph_signature}\n" + f"Range constraints: {self.range_constraints}\n" + ) + return string + + def module(self, check_guards=True) -> torch.fx.GraphModule: + """ + Returns a self contained GraphModule with all the parameters/buffers inlined. + + - When `check_guards=True` (default), a `_guards_fn` submodule is generated + and a call to a `_guards_fn` submodule is inserted right after placeholders + in the graph. This module checks guards on inputs. + - When `check_guards=False`, a subset of these checks are performed by a + forward pre-hook on the graph module. No `_guards_fn` submodule is generated. + + """ + from ._unlift import _unlift_exported_program_lifted_states + + module = _unlift_exported_program_lifted_states(self, check_guards=check_guards) + + def _train(self, mode: bool = True): + raise NotImplementedError("Calling train() is not supported yet.") + + def _eval(self, mode: bool = True): + raise NotImplementedError("Calling eval() is not supported yet.") + + module.train = types.MethodType(_train, module) # type: ignore[method-assign] + module.eval = types.MethodType(_eval, module) # type: ignore[method-assign] + return module + + def _num_lifted_params_buffers(self): + return next( + ( + i + for i, s in enumerate(self._graph_signature.input_specs) + if s.kind == InputKind.USER_INPUT + ), + len(self._graph_signature.input_specs), + ) + + @_disable_prexisiting_fake_mode + def run_decompositions( + self, + decomp_table: dict[torch._ops.OperatorBase, Callable] | None = None, + decompose_custom_triton_ops: bool = False, + ) -> "ExportedProgram": + """ + Run a set of decompositions on the exported program and returns a new + exported program. By default we will run the Core ATen decompositions to + get operators in the + `Core ATen Operator Set `_. + + For now, we do not decompose joint graphs. + + Args: + decomp_table: + An optional argument that specifies decomp behaviour for Aten ops + (1) If None, we decompose to core aten decompositions + (2) If empty, we don't decompose any operator + + + Some examples: + + If you don't want to decompose anything + + .. code-block:: python + + ep = torch.export.export(model, ...) + ep = ep.run_decompositions(decomp_table={}) + + If you want to get a core aten operator set except for certain operator, you can do following: + + .. code-block:: python + + ep = torch.export.export(model, ...) + decomp_table = torch.export.default_decompositions() + decomp_table[your_op] = your_custom_decomp + ep = ep.run_decompositions(decomp_table=decomp_table) + """ + _decomp_table = ( + default_decompositions() if decomp_table is None else dict(decomp_table) + ) + + if isinstance(_decomp_table, CustomDecompTable): + _decomp_table = _decomp_table.materialize() + + # Note [Separating decomp_table into CIA decomps and non-CIA decomps] + # At this point, we have a decomp_table that contains decomp behaviour for + # both CIA and post-autograd ops. + # We need to separate the op into two categories: + # 1. CIA op: These are the ops that we want to override + # CompositeImplicitAutograd decomp for. For them, we need to use _override_composite_implicit_decomp + # context manager to plumb it through AOTDispatcher + # 2. Non-CIA op: These ops are only relevant after AOTDIspatcher runs, so just + # checking if they are statically functional is enough. + # For joint IR case tho, we need to use the old path because we can't register + # custom decomps this way because we can't use context manager as it installs + # autograd_error node. + ( + cia_to_decomp, + python_decomp_table, + ) = _split_decomp_table_to_cia_and_python_decomp(_decomp_table) + + return _decompose_exported_program( + self, + cia_to_decomp=cia_to_decomp, + python_decomp_table=python_decomp_table, + joint_loss_index=None, + decompose_custom_triton_ops=decompose_custom_triton_ops, + ) + + def _transform_do_not_use(self, *passes: PassType) -> "ExportedProgram": + pm = PassManager(list(passes)) + # Since we abstractly run the passes, we need to disable backend decomp here + # again. + from torch.export._trace import _ignore_backend_decomps + + with _ignore_backend_decomps(): + res = pm(self.graph_module) + transformed_gm = res.graph_module if res is not None else self.graph_module + assert transformed_gm is not None + + # pyrefly: ignore [missing-attribute] + if transformed_gm is self.graph_module and not res.modified: + return self + + # TODO(zhxchen17) Remove this. + def _get_updated_graph_signature( + old_signature: ExportGraphSignature, + new_gm: torch.fx.GraphModule, + ) -> ExportGraphSignature: + """ + Update the graph signature's user_input/user_outputs. + """ + new_input_specs = [] + for i, node in enumerate(new_gm.graph.nodes): + if node.op != "placeholder": + break + + assert i < len(old_signature.input_specs), ( + "Number of inputs changed after transformation" + ) + old_input_spec = old_signature.input_specs[i] + arg = ( + old_input_spec.arg + if isinstance( + old_input_spec.arg, (ConstantArgument, CustomObjArgument) + ) + else type(old_input_spec.arg)(node.name) + ) + new_input_specs.append( + InputSpec( + old_input_spec.kind, + arg, + old_input_spec.target, + old_input_spec.persistent, + ) + ) + + output_node = list(new_gm.graph.nodes)[-1] + assert output_node.op == "output" + + new_output_specs = [] + for i, node in enumerate(output_node.args[0]): + assert i < len(old_signature.output_specs), ( + "Number of outputs changed after transformation" + ) + old_output_spec = old_signature.output_specs[i] + arg = ( + old_output_spec.arg + if isinstance( + old_output_spec.arg, (ConstantArgument, CustomObjArgument) + ) + else type(old_output_spec.arg)(node.name) + ) + new_output_specs.append( + OutputSpec(old_output_spec.kind, arg, old_output_spec.target) + ) + + new_signature = ExportGraphSignature( + input_specs=new_input_specs, output_specs=new_output_specs + ) + return new_signature + + transformed_ep = ExportedProgram( + root=transformed_gm, + graph=transformed_gm.graph, + graph_signature=_get_updated_graph_signature( + self.graph_signature, transformed_gm + ), + state_dict=self.state_dict, + range_constraints=_get_updated_range_constraints( + transformed_gm, + self.range_constraints, + ), + module_call_graph=copy.deepcopy(self._module_call_graph), + example_inputs=self.example_inputs, + constants=self.constants, + verifiers=self.verifiers, + ) + transformed_ep.graph_module.meta.update(self.graph_module.meta) + # pyrefly: ignore [missing-attribute] + transformed_ep.graph_module.meta.update(res.graph_module.meta) + return transformed_ep + + def _check_input_constraints(self, flat_args_with_path): + from torch._export.utils import _check_input_constraints_for_graph + + placeholders = [p for p in self.graph.nodes if p.op == "placeholder"] + input_placeholders = [ + p + for p, s in zip(placeholders, self.graph_signature.input_specs) + if s.kind == InputKind.USER_INPUT + ] + _check_input_constraints_for_graph( + input_placeholders, flat_args_with_path, self.range_constraints + ) + + @compatibility(is_backward_compatible=False) + def validate(self): + self._validate() + + # TODO: remove this + @final + def _validate(self): + assert len(self.verifiers) > 0, ( + "ExportedProgram must have at least one verifier." + ) + for v in self.verifiers: + v().check(self) + + # TODO(zhxchen17) Formalize this. + def _update( + self, + graph_module, + graph_signature, + *, + state_dict=None, + constants=None, + verifiers=None, + ) -> "ExportedProgram": + return ExportedProgram( + root=graph_module, + graph=graph_module.graph, + graph_signature=graph_signature, + state_dict=state_dict if state_dict is not None else self.state_dict, + range_constraints=copy.deepcopy(self.range_constraints), + module_call_graph=copy.deepcopy(self._module_call_graph), + example_inputs=self.example_inputs, + constants=constants if constants is not None else self.constants, + verifiers=verifiers if verifiers is not None else self.verifiers, + ) + + +def _get_shape_env(gm): + vals = [ + node.meta["val"] + for node in gm.graph.nodes + if node.meta.get("val", None) is not None + ] + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(vals) + if fake_mode is not None: + return fake_mode.shape_env + for v in vals: + if isinstance(v, torch.SymInt): + return v.node.shape_env + + +def _get_updated_range_constraints( + gm: torch.fx.GraphModule, + old_range_constraints: "dict[sympy.Symbol, Any] | None" = None, +) -> "dict[sympy.Symbol, Any]": + assert old_range_constraints is not None + + shape_env = _get_shape_env(gm) + if shape_env is None: + return {} + + range_constraints = copy.copy(old_range_constraints) + range_constraints = { + k: v for k, v in range_constraints.items() if k not in shape_env.replacements + } + # Only when we have an unbacked symint, and it's used as constructor inputs, + # runtime_var_to_range will make a difference compated to var_to_range. + # e.g. [2, oo) -> [0, oo) + for k, v in shape_env.var_to_range.items(): + if k not in shape_env.replacements and k not in range_constraints: + range_constraints[k] = v + return range_constraints + + +def _create_graph_module_for_export(root, graph): + try: + gm = torch.fx.GraphModule(root, graph) + except SyntaxError: + # If custom objects stored in memory are being used in the graph, + # the generated python code will result in a syntax error on the custom + # object, since it is unable to parse the in-memory object. However + # we can still run the graph eagerly through torch.fx.Interpreter, + # so we will bypass this error. + warnings.warn( + "Unable to execute the generated python source code from " + "the graph. The graph module will no longer be directly callable, " + "but you can still run the ExportedProgram, and if needed, you can " + "run the graph module eagerly using torch.fx.Interpreter.", + stacklevel=2, + ) + gm = torch.fx.GraphModule(root, torch.fx.Graph()) + gm._graph = graph + + return gm + + +def _convert_guards_to_code(graph_module): + shape_env = _get_shape_env(graph_module) + if shape_env is None: + return [] + + local_vars = { + var + for var, sources in shape_env.var_to_sources.items() + if all( + not isinstance(source, torch._dynamo.source.ConstantSource) + for source in sources + ) + } + py_printer = torch.fx.experimental.symbolic_shapes.ShapeGuardPythonPrinter( + shape_env.var_to_sources, lambda s: s.name, shape_env.var_to_sources + ) + ret = [ + py_printer.doprint(guard.expr) + for guard in shape_env.guards + if guard.expr.free_symbols.issubset(local_vars) + ] + # TODO Figure out how to resolve guards containing weight sizes. + # This is not a big deal as _guards_code is mostly empty today. + return [guard for guard in ret if "L['self']" not in guard] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/graph_signature.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/graph_signature.py new file mode 100644 index 0000000000000000000000000000000000000000..a85e57024cdd1bd47760ffdb7715c8bc070e8903 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/graph_signature.py @@ -0,0 +1,729 @@ +# mypy: allow-untyped-defs +import dataclasses +from collections.abc import Collection, Mapping +from enum import auto, Enum +from typing import TYPE_CHECKING, Union + +from torch._library.fake_class_registry import FakeScriptObject +from torch._library.opaque_object import get_opaque_type_name, is_opaque_type +from torch._subclasses.fake_tensor import is_fake + + +if TYPE_CHECKING: + import torch + from torch._functorch._aot_autograd.schemas import GraphSignature + +__all__ = [ + "ConstantArgument", + "CustomObjArgument", + "ExportBackwardSignature", + "ExportGraphSignature", + "InputKind", + "InputSpec", + "OutputKind", + "OutputSpec", + "SymIntArgument", + "SymFloatArgument", + "SymBoolArgument", + "TensorArgument", +] + + +@dataclasses.dataclass +class TensorArgument: + name: str + + +@dataclasses.dataclass +class TokenArgument: + name: str + + +@dataclasses.dataclass +class SymIntArgument: + name: str + + +@dataclasses.dataclass +class SymFloatArgument: + name: str + + +@dataclasses.dataclass +class SymBoolArgument: + name: str + + +@dataclasses.dataclass +class CustomObjArgument: + name: str + class_fqn: str + fake_val: FakeScriptObject | None = None + + +@dataclasses.dataclass +class ConstantArgument: + name: str + value: int | float | bool | str | None + + +ArgumentSpec = Union[ + TensorArgument, + SymIntArgument, + SymFloatArgument, + SymBoolArgument, + ConstantArgument, + CustomObjArgument, + TokenArgument, +] + + +class InputKind(Enum): + USER_INPUT = auto() + PARAMETER = auto() + BUFFER = auto() + CONSTANT_TENSOR = auto() + CUSTOM_OBJ = auto() + TOKEN = auto() + + +@dataclasses.dataclass +class InputSpec: + kind: InputKind + arg: ArgumentSpec + target: str | None + persistent: bool | None = None + + def __post_init__(self): + if self.kind == InputKind.BUFFER: + assert self.persistent is not None, ( + "Failed to specify persistent flag on BUFFER." + ) + assert isinstance( + self.arg, + ( + TensorArgument, + SymIntArgument, + SymFloatArgument, + SymBoolArgument, + ConstantArgument, + CustomObjArgument, + TokenArgument, + ), + ), f"got {type(self.arg)}" + + def __str__(self): + target = "" if self.target is None else f" target='{self.target}'" + persistent = "" if self.persistent is None else f" persistent={self.persistent}" + return f"{str(self.arg.name)}: {str(self.kind.name)}{target}{persistent}" + + +class OutputKind(Enum): + USER_OUTPUT = auto() + LOSS_OUTPUT = auto() + BUFFER_MUTATION = auto() + PARAMETER_MUTATION = auto() + GRADIENT_TO_PARAMETER = auto() + GRADIENT_TO_USER_INPUT = auto() + USER_INPUT_MUTATION = auto() + TOKEN = auto() + + +@dataclasses.dataclass +class OutputSpec: + kind: OutputKind + arg: ArgumentSpec + target: str | None + + def __post_init__(self): + assert isinstance( + self.arg, + ( + TensorArgument, + SymIntArgument, + SymFloatArgument, + SymBoolArgument, + ConstantArgument, + TokenArgument, + CustomObjArgument, + ), + ), self.arg + + def __str__(self): + target = "" if self.target is None else f" target='{self.target}'" + return f"{str(self.arg.name)}: {str(self.kind.name)}{target}" + + +@dataclasses.dataclass +class ExportBackwardSignature: + gradients_to_parameters: dict[str, str] + gradients_to_user_inputs: dict[str, str] + loss_output: str + + +@dataclasses.dataclass +class ExportGraphSignature: + """ + :class:`ExportGraphSignature` models the input/output signature of Export Graph, + which is a fx.Graph with stronger invariants guarantees. + + Export Graph is functional and does not access "states" like parameters + or buffers within the graph via ``getattr`` nodes. Instead, :func:`export` + guarantees that parameters, buffers, and constant tensors are lifted out of + the graph as inputs. Similarly, any mutations to buffers are not included + in the graph either, instead the updated values of mutated buffers are + modeled as additional outputs of Export Graph. + + The ordering of all inputs and outputs are:: + + Inputs = [*parameters_buffers_constant_tensors, *flattened_user_inputs] + Outputs = [*mutated_inputs, *flattened_user_outputs] + + e.g. If following module is exported:: + + class CustomModule(nn.Module): + def __init__(self) -> None: + super(CustomModule, self).__init__() + + # Define a parameter + self.my_parameter = nn.Parameter(torch.tensor(2.0)) + + # Define two buffers + self.register_buffer("my_buffer1", torch.tensor(3.0)) + self.register_buffer("my_buffer2", torch.tensor(4.0)) + + def forward(self, x1, x2): + # Use the parameter, buffers, and both inputs in the forward method + output = ( + x1 + self.my_parameter + ) * self.my_buffer1 + x2 * self.my_buffer2 + + # Mutate one of the buffers (e.g., increment it by 1) + self.my_buffer2.add_(1.0) # In-place addition + + return output + + + mod = CustomModule() + ep = torch.export.export(mod, (torch.tensor(1.0), torch.tensor(2.0))) + + Resulting Graph is non-functional:: + + graph(): + %p_my_parameter : [num_users=1] = placeholder[target=p_my_parameter] + %b_my_buffer1 : [num_users=1] = placeholder[target=b_my_buffer1] + %b_my_buffer2 : [num_users=2] = placeholder[target=b_my_buffer2] + %x1 : [num_users=1] = placeholder[target=x1] + %x2 : [num_users=1] = placeholder[target=x2] + %add : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%x1, %p_my_parameter), kwargs = {}) + %mul : [num_users=1] = call_function[target=torch.ops.aten.mul.Tensor](args = (%add, %b_my_buffer1), kwargs = {}) + %mul_1 : [num_users=1] = call_function[target=torch.ops.aten.mul.Tensor](args = (%x2, %b_my_buffer2), kwargs = {}) + %add_1 : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%mul, %mul_1), kwargs = {}) + %add_ : [num_users=0] = call_function[target=torch.ops.aten.add_.Tensor](args = (%b_my_buffer2, 1.0), kwargs = {}) + return (add_1,) + + Resulting ExportGraphSignature of the non-functional Graph would be:: + + # inputs + p_my_parameter: PARAMETER target='my_parameter' + b_my_buffer1: BUFFER target='my_buffer1' persistent=True + b_my_buffer2: BUFFER target='my_buffer2' persistent=True + x1: USER_INPUT + x2: USER_INPUT + + # outputs + add_1: USER_OUTPUT + + To get a functional Graph, you can use :func:`run_decompositions`:: + + mod = CustomModule() + ep = torch.export.export(mod, (torch.tensor(1.0), torch.tensor(2.0))) + ep = ep.run_decompositions() + + Resulting Graph is functional:: + + graph(): + %p_my_parameter : [num_users=1] = placeholder[target=p_my_parameter] + %b_my_buffer1 : [num_users=1] = placeholder[target=b_my_buffer1] + %b_my_buffer2 : [num_users=2] = placeholder[target=b_my_buffer2] + %x1 : [num_users=1] = placeholder[target=x1] + %x2 : [num_users=1] = placeholder[target=x2] + %add : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%x1, %p_my_parameter), kwargs = {}) + %mul : [num_users=1] = call_function[target=torch.ops.aten.mul.Tensor](args = (%add, %b_my_buffer1), kwargs = {}) + %mul_1 : [num_users=1] = call_function[target=torch.ops.aten.mul.Tensor](args = (%x2, %b_my_buffer2), kwargs = {}) + %add_1 : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%mul, %mul_1), kwargs = {}) + %add_2 : [num_users=1] = call_function[target=torch.ops.aten.add.Tensor](args = (%b_my_buffer2, 1.0), kwargs = {}) + return (add_2, add_1) + + Resulting ExportGraphSignature of the functional Graph would be:: + + # inputs + p_my_parameter: PARAMETER target='my_parameter' + b_my_buffer1: BUFFER target='my_buffer1' persistent=True + b_my_buffer2: BUFFER target='my_buffer2' persistent=True + x1: USER_INPUT + x2: USER_INPUT + + # outputs + add_2: BUFFER_MUTATION target='my_buffer2' + add_1: USER_OUTPUT + + """ + + input_specs: list[InputSpec] + output_specs: list[OutputSpec] + + # A list of parameters uniquely identified by mangled fully qualified name + @property + def parameters(self) -> Collection[str]: + return tuple( + s.target + for s in self.input_specs + if s.kind == InputKind.PARAMETER + if isinstance(s.target, str) + ) + + # A list of buffers uniquely identified by mangled fully qualified name + @property + def buffers(self) -> Collection[str]: + return tuple( + s.target + for s in self.input_specs + if s.kind == InputKind.BUFFER + if isinstance(s.target, str) + ) + + @property + def non_persistent_buffers(self) -> Collection[str]: + return tuple( + s.target + for s in self.input_specs + if s.kind == InputKind.BUFFER + if s.persistent is False + if isinstance(s.target, str) + ) + + # A list of lifted constant tensors + @property + def lifted_tensor_constants(self) -> Collection[str]: + return tuple( + s.target + for s in self.input_specs + if s.kind == InputKind.CONSTANT_TENSOR + if isinstance(s.target, str) + ) + + @property + def lifted_custom_objs(self) -> Collection[str]: + return tuple( + s.target + for s in self.input_specs + if s.kind == InputKind.CUSTOM_OBJ + if isinstance(s.target, str) + ) + + # Graph node names of pytree-flattened inputs of original program + @property + def user_inputs(self) -> Collection[int | float | bool | None | str]: + user_inputs: list[int | float | bool | None | str] = [] + for s in self.input_specs: + if s.kind != InputKind.USER_INPUT: + continue + + if isinstance( + s.arg, + ( + TensorArgument, + SymIntArgument, + SymFloatArgument, + SymBoolArgument, + CustomObjArgument, + ), + ): + user_inputs.append(s.arg.name) + elif isinstance(s.arg, ConstantArgument): + user_inputs.append(s.arg.value) + else: + raise RuntimeError(f"{s.arg} is not a valid user inputs") + return tuple(user_inputs) + + # Graph node names of pytree-flattened outputs of original program + # For joint-graph purposes, will include the loss output. + @property + def user_outputs(self) -> Collection[int | float | bool | None | str]: + user_outputs: list[int | float | bool | None | str] = [] + for s in self.output_specs: + if s.kind not in [ + OutputKind.USER_OUTPUT, + OutputKind.LOSS_OUTPUT, + ]: + continue + + if isinstance( + s.arg, + (TensorArgument, SymIntArgument, SymFloatArgument, SymBoolArgument), + ): + user_outputs.append(s.arg.name) + elif isinstance(s.arg, ConstantArgument): + user_outputs.append(s.arg.value) + elif isinstance(s.arg, CustomObjArgument): + user_outputs.append(s.arg.name) + else: + raise RuntimeError(f"{s.arg} is not a valid user output") + return tuple(user_outputs) + + # A dictionary mapping graph input node names to parameters. If a graph input + # name is found in this dictionary, it is guaranteed to be a lifted parameter. + @property + def inputs_to_parameters(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.input_specs + if s.kind == InputKind.PARAMETER + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + # A dictionary mapping graph input node names to buffers. If a graph input + # name is found in this dictionary, it is guaranteed to be a lifted buffer. + @property + def inputs_to_buffers(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) # type: ignore[union-attr, misc] + for s in self.input_specs + if s.kind == InputKind.BUFFER + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + # A dictionary mapping graph output node names to buffers that are mutated in the + # original program. Buffers that are not mutated will not be found in this dictionary. + @property + def buffers_to_mutate(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.output_specs + if s.kind == OutputKind.BUFFER_MUTATION + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + @property + def parameters_to_mutate(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.output_specs + if s.kind == OutputKind.PARAMETER_MUTATION + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + @property + def user_inputs_to_mutate(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.output_specs + if s.kind == OutputKind.USER_INPUT_MUTATION + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + # A dictionary mapping graph input node names to lifted tensor constants. + @property + def inputs_to_lifted_tensor_constants(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.input_specs + if s.kind == InputKind.CONSTANT_TENSOR + and isinstance(s.arg, TensorArgument) + and isinstance(s.target, str) + ) + + @property + def inputs_to_lifted_custom_objs(self) -> Mapping[str, str]: + return _immutable_dict( + (s.arg.name, s.target) + for s in self.input_specs + if s.kind == InputKind.CUSTOM_OBJ + and isinstance(s.arg, CustomObjArgument) + and isinstance(s.target, str) + ) + + @property + def backward_signature(self) -> ExportBackwardSignature | None: + loss_output = None + gradients_to_parameters: dict[str, str] = {} + gradients_to_user_inputs: dict[str, str] = {} + for spec in self.output_specs: + if spec.kind == OutputKind.LOSS_OUTPUT: + assert loss_output is None + assert isinstance(spec.arg, TensorArgument) + loss_output = spec.arg.name + elif spec.kind == OutputKind.GRADIENT_TO_PARAMETER: + assert isinstance(spec.target, str) + assert isinstance(spec.arg, TensorArgument) + gradients_to_parameters[spec.arg.name] = spec.target + elif spec.kind == OutputKind.GRADIENT_TO_USER_INPUT: + assert isinstance(spec.target, str) + assert isinstance(spec.arg, TensorArgument) + gradients_to_user_inputs[spec.arg.name] = spec.target + + if loss_output is None: + return None + + return ExportBackwardSignature( + loss_output=loss_output, + gradients_to_parameters=gradients_to_parameters, + gradients_to_user_inputs=gradients_to_user_inputs, + ) + + # Map from assertion dependency token index to assertion dep token output + # name in output. The shape of output after aot_autograd will be like: + # (updated_inputs, user_outputs, dep_token). + @property + def assertion_dep_token(self) -> Mapping[int, str] | None: + return None + + @property + def input_tokens(self) -> Collection[str]: + input_tokens = [] + for s in self.input_specs: + if s.kind == InputKind.TOKEN: + assert isinstance(s.arg, TokenArgument) + input_tokens.append(s.arg.name) + return tuple(input_tokens) + + @property + def output_tokens(self) -> Collection[str]: + output_tokens = [] + for s in self.output_specs: + if s.kind == OutputKind.TOKEN: + assert isinstance(s.arg, TokenArgument) + output_tokens.append(s.arg.name) + return tuple(output_tokens) + + def __post_init__(self) -> None: + assertion_dep_token = self.assertion_dep_token + if assertion_dep_token is None: + return + assert len(assertion_dep_token) == 1 + assertion_dep_token_index = next(iter(assertion_dep_token.keys())) + assert ( + len(self.user_outputs) + len(self.buffers_to_mutate) + == assertion_dep_token_index + ) + + def replace_all_uses(self, old: str, new: str): + """ + Replace all uses of the old name with new name in the signature. + """ + assert isinstance(old, str) + assert isinstance(new, str) + arg_types = ( + TensorArgument, + SymIntArgument, + SymFloatArgument, + SymBoolArgument, + CustomObjArgument, + TokenArgument, + ) + for o in self.output_specs: + if isinstance(o.arg, arg_types): + if o.arg.name == old: + o.arg.name = new + for i in self.input_specs: + if isinstance(i.arg, arg_types): + if i.arg.name == old: + i.arg.name = new + + def get_replace_hook(self, replace_inputs=False): + def _(old, new, user): + if user.op == "output": + self.replace_all_uses(old.name, new) + if replace_inputs and old.op == "placeholder": + self.replace_all_uses(old.name, new) + + return _ + + def __str__(self): + input_specs = "\n".join(str(s) for s in self.input_specs) + output_specs = "\n".join(str(s) for s in self.output_specs) + return f"\n# inputs\n{input_specs}\n\n# outputs\n{output_specs}\n" + + +def _immutable_dict(items): + """ + Creates a mapping where items cannot be added, deleted, or updated. + NOTE: The immutability is shallow (like tuple is an immutable collection). + """ + from types import MappingProxyType + + return MappingProxyType(dict(items)) + + +def _make_argument_spec(node, token_names) -> ArgumentSpec: + from torch import ScriptObject, SymBool, SymFloat, SymInt + from torch._library.fake_class_registry import FakeScriptObject + + if isinstance(node, (int, bool, float, type(None), str)): + # For const outputs we just directly return this + return ConstantArgument(name="", value=node) + + assert "val" in node.meta, ( + f"{node} is not a constant or a node with a 'val' metadata field" + ) + val = node.meta["val"] + if node.name in token_names: + return TokenArgument(name=node.name) + elif is_fake(val): + return TensorArgument(name=node.name) + elif isinstance(val, SymInt): + return SymIntArgument(name=node.name) + elif isinstance(val, SymFloat): + return SymFloatArgument(name=node.name) + elif isinstance(val, SymBool): + return SymBoolArgument(name=node.name) + elif isinstance(val, ScriptObject): + return CustomObjArgument(name=node.name, class_fqn=val._type().qualified_name()) # type: ignore[attr-defined] + elif isinstance(val, FakeScriptObject): + return CustomObjArgument( + name=node.name, class_fqn=val.script_class_name, fake_val=val + ) + elif is_opaque_type(type(val)): + return CustomObjArgument( + name=node.name, class_fqn=get_opaque_type_name(type(val)), fake_val=val + ) + elif isinstance(val, (int, bool, str, float, type(None))): + return ConstantArgument(name=node.name, value=val) + else: + raise AssertionError( + f"Encountered an unsupported object of type {type(val)} " + f"while writing the metadata for exported program" + ) + + +def _convert_to_export_graph_signature( + graph_signature: "GraphSignature", + gm: "torch.fx.GraphModule", + non_persistent_buffers: set[str], +) -> "ExportGraphSignature": + from torch.utils import _pytree as pytree + + is_joint = graph_signature.backward_signature is not None + + # unpack objects + user_inputs = set(graph_signature.user_inputs) + inputs_to_parameters = graph_signature.inputs_to_parameters + inputs_to_buffers = graph_signature.inputs_to_buffers + user_outputs = set(graph_signature.user_outputs) + buffer_mutations = graph_signature.buffers_to_mutate + parameter_mutations = graph_signature.parameters_to_mutate + user_input_mutations = graph_signature.user_inputs_to_mutate + grad_params = ( + graph_signature.backward_signature.gradients_to_parameter # type: ignore[union-attr] + if is_joint + else {} + ) + grad_user_inputs = ( + graph_signature.backward_signature.gradients_to_user_inputs # type: ignore[union-attr] + if is_joint + else {} + ) + loss_output = ( + graph_signature.backward_signature.loss_output # type: ignore[union-attr] + if is_joint + else None + ) + input_tokens = graph_signature.input_tokens + output_tokens = graph_signature.output_tokens + + inputs = [ + _make_argument_spec(node, input_tokens) + for node in gm.graph.nodes + if node.op == "placeholder" + ] + outputs = [ + _make_argument_spec(node, output_tokens) + for node in pytree.tree_leaves(next(iter(reversed(gm.graph.nodes))).args) + ] + + def to_input_spec(inp: ArgumentSpec) -> InputSpec: + if isinstance(inp, TokenArgument): + return InputSpec(kind=InputKind.TOKEN, arg=inp, target=None) + + if not isinstance(inp, TensorArgument): + return InputSpec(kind=InputKind.USER_INPUT, arg=inp, target=None) + name = inp.name + if name in user_inputs: + return InputSpec(kind=InputKind.USER_INPUT, arg=inp, target=None) + elif name in inputs_to_parameters: + return InputSpec( + kind=InputKind.PARAMETER, + arg=inp, + target=inputs_to_parameters[name], # type: ignore[index] + ) + elif name in inputs_to_buffers: + return InputSpec( + kind=InputKind.BUFFER, + arg=inp, + target=inputs_to_buffers[name], # type: ignore[index] + persistent=(inputs_to_buffers[name] not in non_persistent_buffers), # type: ignore[index] + ) + else: + raise AssertionError(f"Unknown tensor input kind: {name}") + + def to_output_spec(idx: int, o: ArgumentSpec) -> OutputSpec: + if isinstance(o, TokenArgument): + return OutputSpec(kind=OutputKind.TOKEN, arg=o, target=None) + + if not isinstance(o, TensorArgument): + return OutputSpec(kind=OutputKind.USER_OUTPUT, arg=o, target=None) + name = o.name + if idx < len(buffer_mutations) + len(parameter_mutations) + len( + user_input_mutations + ) + len(output_tokens): + if name in buffer_mutations: + return OutputSpec( + kind=OutputKind.BUFFER_MUTATION, + arg=o, + target=buffer_mutations[name], # type: ignore[index] + ) + elif name in parameter_mutations: + return OutputSpec( + kind=OutputKind.PARAMETER_MUTATION, + arg=o, + target=parameter_mutations[name], # type: ignore[index] + ) + elif name in user_input_mutations: + return OutputSpec( + kind=OutputKind.USER_INPUT_MUTATION, + arg=o, + target=user_input_mutations[name], # type: ignore[index] + ) + else: + raise AssertionError(f"Unknown tensor mutation kind: {name}") + else: + if name in user_outputs: + return OutputSpec(kind=OutputKind.USER_OUTPUT, arg=o, target=None) + + elif name in grad_params: + return OutputSpec( + kind=OutputKind.GRADIENT_TO_PARAMETER, + arg=o, + target=grad_params[name], + ) + elif name in grad_user_inputs: + return OutputSpec( + kind=OutputKind.GRADIENT_TO_USER_INPUT, + arg=o, + target=grad_user_inputs[name], + ) + elif name == loss_output: + return OutputSpec(kind=OutputKind.LOSS_OUTPUT, arg=o, target=None) + + else: + raise AssertionError(f"Unknown tensor output kind: {name}") + + input_specs = [to_input_spec(inp) for inp in inputs] + output_specs = [to_output_spec(idx, o) for idx, o in enumerate(outputs)] + return ExportGraphSignature(input_specs=input_specs, output_specs=output_specs) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/export/unflatten.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/unflatten.py new file mode 100644 index 0000000000000000000000000000000000000000..a877e62ce43cf6b5d80fb8599bc48cf21746b3b2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/export/unflatten.py @@ -0,0 +1,1803 @@ +# mypy: allow-untyped-defs +import abc +import copy +import logging +import operator +import re +from collections import defaultdict +from collections.abc import Callable +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import dataclass +from enum import Enum +from typing import Any, cast + +import torch +import torch.fx._pytree as fx_pytree +import torch.utils._pytree as pytree +from torch._library.fake_class_registry import FakeScriptObject +from torch.export import ExportedProgram +from torch.export._tree_utils import reorder_kwargs +from torch.export.exported_program import ( + ConstantArgument, + ExportGraphSignature, + InputKind, + ModuleCallSignature, + SymBoolArgument, + SymFloatArgument, + SymIntArgument, + TensorArgument, +) +from torch.fx._symbolic_trace import is_fx_symbolic_tracing +from torch.fx.graph_module import _get_attr, _get_attr_via_attr_list, _print_readable +from torch.utils._pytree import GetAttrKey, SequenceKey + +from ._remove_effect_tokens_pass import _remove_effect_tokens + + +log = logging.getLogger(__name__) + + +__all__ = [ + "FlatArgsAdapter", + "InterpreterModule", + "InterpreterModuleDispatcher", + "UnflattenedModule", + "unflatten", +] + + +class _AttrKind(Enum): + PARAMETER = "parameter" + BUFFER = "buffer" + CONSTANT = "constant" + MODULE = "module" + + +@dataclass(frozen=True) +class _TensorID: + """Custom tensor identifier containing storage, stride, and size information.""" + + untyped_storage: torch.UntypedStorage + stride: tuple + size: tuple + storage_offset: int + + +RUN_WITH_INTERPRETER = True + + +@contextmanager +def _disable_interpreter(): + global RUN_WITH_INTERPRETER + old_flag = RUN_WITH_INTERPRETER + RUN_WITH_INTERPRETER = False + try: + yield + finally: + RUN_WITH_INTERPRETER = old_flag + + +# Assign attribute 'from_obj' to the qualified name 'target' on 'to_module +# This installs empty Modules where none exist yet if they are subpaths of target +def _assign_attr( + from_obj: torch.Tensor | torch.ScriptObject | torch.nn.Module, + to_module: torch.nn.Module, + target: str, + attr_kind: _AttrKind, + persistent: bool = True, +): + *prefix, field = target.split(".") + # We need to generate all submodules of `to_module` that are at `prefix` and + # variants of `prefix` that differ only by call name. All of these submodules + # will then be assigned `from_obj` at `field` so that they can share this attribute. + # For example, if target is foo.bar.f, foo has another call name foo@1, + # and bar has other call names bar@1, bar@2, then we will assign f to + # foo.bar, foo.bar@1, foo.bar@2, foo@1.bar, foo@1.bar@1, foo@1.bar@2. + to_modules = {to_module} + for item in prefix: + ts: set[torch.nn.Module] = set() + for to_module in to_modules: + if not hasattr(to_module, item): + setattr(to_module, item, torch.nn.Module()) + ts.update( + t_call # type: ignore[misc] + for k, t_call in to_module._modules.items() + if _is_call_name(k, item) + ) + to_modules = ts + + for to_module in to_modules: + if attr_kind == _AttrKind.PARAMETER: + assert isinstance(from_obj, torch.nn.Parameter) + to_module.register_parameter(field, from_obj) + elif attr_kind == _AttrKind.BUFFER: + assert isinstance(from_obj, torch.Tensor) + to_module.register_buffer(field, from_obj, persistent=persistent) + elif attr_kind == _AttrKind.CONSTANT: + assert not isinstance(from_obj, FakeScriptObject), ( + "FakeScriptObject should only exist during tracing." + ) + assert isinstance( + from_obj, + ( + torch.Tensor, + torch.ScriptObject, + ), + ) + setattr(to_module, field, from_obj) + elif attr_kind == _AttrKind.MODULE: + assert isinstance(from_obj, torch.nn.Module) + setattr(to_module, field, from_obj) + + +class _SubmoduleBase: + _ty: str | None + + def type_name(self) -> str | None: + """ + Subclass of this class - InterpreterModule, InterpreterModuleDispatcher, represents + corresponding model in eager model. To get this type information for those modules + in eager model we need to use this method. + """ + return self._ty + + +class InterpreterModule(_SubmoduleBase, torch.nn.Module): + """A module that uses torch.fx.Interpreter to execute instead of the usual + codegen that GraphModule uses. This provides better stack trace information + and makes it easier to debug execution. + """ + + graph_module: torch.fx.GraphModule | None + + def __init__( + self, + graph: torch.fx.Graph, + ty: str | None = None, + ): + super().__init__() + self.graph = graph + self._ty = ty + self.graph.owning_module = self # type: ignore[assignment] + self._run_with_interpreter = RUN_WITH_INTERPRETER + + def forward(self, *args, **kwargs): + assert self.graph_module is not None, "Didn't finalize this InterpreterModule" + if not is_fx_symbolic_tracing() and ( + torch.compiler.is_dynamo_compiling() or not self._run_with_interpreter + ): + # Dynamo cannot trace through torch.fx.Interpreter, so fall back to + # GraphModule codegen in this instance. + # Patch the codegened forward to run with this InterpreterModule, + # so attribute accesses, etc. are on this module instead. + return type(self.graph_module).forward(self, *args, **kwargs) + else: + if kwargs: + # Handle **kwargs. FX only natively supports positional + # arguments (through placeholders). So in order to pass in + # kwargs, we must correspond the names of the placeholders with + # the keys in the kwarg dict. + arg_list = list(args) + kwarg_names = self.arg_names[len(arg_list) :] + arg_list.extend( + kwargs[kwarg_name] + for kwarg_name in kwarg_names + if kwarg_name in kwargs + ) + + # Assert that the kwargs passed in exactly match the positional + # arguments specified by the GraphModule. This should be + # guaranteed by the unflattening process. + assert len(kwarg_names) == len(kwargs) + assert len(arg_list) == len(self.arg_names) + args = tuple(arg_list) + + return torch.fx.Interpreter(self, graph=self.graph).run( + *args, enable_io_processing=False + ) + + def finalize(self): + # We need to "finalize" because GraphModule populates its own state_dict + # based on the get_attrs observed in the graph. So we need to fully + # construct the graph and call _sink_params before generating this + # GraphModule. + + # need to set `graph_module` directly on the dict to avoid it getting + # registered as a submodule. + self.__dict__["graph_module"] = torch.fx.GraphModule(self, self.graph) + self.graph.lint() + + # Cache arg names for kwarg handling (see forward()) + self.arg_names = [] + for node in self.graph.nodes: + if node.op == "placeholder": + self.arg_names.append(node.target) + + def print_readable( + self, + print_output=True, + include_stride=False, + include_device=False, + colored=False, + ): + return _print_readable( + self, + "InterpreterModule", + print_output, + include_stride, + include_device, + colored, + ) + + +class InterpreterModuleDispatcher(_SubmoduleBase, torch.nn.Module): + """ + A module that carries a sequence of InterpreterModules corresponding to + a sequence of calls of that module. Each call to the module dispatches + to the next InterpreterModule, and wraps back around after the last. + """ + + def __init__(self, attrs: set[str], call_modules: list[InterpreterModule]): + super().__init__() + assert call_modules + self._modules = call_modules[0]._modules + for accessor in attrs: + setattr(self, accessor, getattr(call_modules[0], accessor)) + self._ty = call_modules[0]._ty + self._call_modules = call_modules + self._num_calls = 0 + + def forward(self, *args, **kwargs): + call_module = self._call_modules[self._num_calls] + self._num_calls = (self._num_calls + 1) % len(self._call_modules) + try: + return call_module(*args, **kwargs) + except Exception: + self._num_calls = 0 + raise + + def call_modules(self): + return self._call_modules + + def print_readable( + self, + print_output=True, + include_stride=False, + include_device=False, + colored=False, + ): + outputs = [ + mod.print_readable( + print_output, + include_stride, + include_device, + colored, + ) + for mod in self._call_modules + ] + return "\n".join(outputs) + + +class FlatArgsAdapter(abc.ABC): + """ + Adapts input arguments with ``input_spec`` to align ``target_spec``. + """ + + @abc.abstractmethod + def adapt( + self, + target_spec: pytree.TreeSpec, + input_spec: pytree.TreeSpec, + input_args: list[Any], + metadata: dict[str, Any] | None = None, + obj: Any | None = None, + ) -> list[Any]: + """NOTE: This adapter may mutate given ``input_args_with_path``.""" + ... + + def get_flat_arg_paths(self) -> list[str]: + """Returns a list of paths that are used to access the flat args.""" + return [] + + +class UnflattenedModule(_SubmoduleBase, torch.nn.Module): + def __init__( + self, + export_module: ExportedProgram, + flat_args_adapter: FlatArgsAdapter | None = None, + ): + super().__init__() + if export_module.graph_signature.backward_signature is not None: + raise ValueError("Unflattening on JointExportModule NYI") + + def _id(obj): + """Returns _TensorID dataclass for tensors, otherwise id().""" + if isinstance(obj, torch.Tensor): + return _TensorID( + untyped_storage=obj.untyped_storage(), + stride=obj.stride(), + size=obj.size(), + storage_offset=obj.storage_offset(), # type: ignore[arg-type] + ) + return id(obj) + + fqn_list = [entry.fqn for entry in export_module.module_call_graph] + assert fqn_list[0] == "" + export_graph = deepcopy(export_module.graph) + self.graph_signature = deepcopy(export_module.graph_signature) + self.graph = torch.fx.Graph() + self.graph.owning_module = self # type: ignore[assignment] + self.module_call_graph = deepcopy(export_module.module_call_graph) + self.flat_args_adapter = flat_args_adapter + + self.meta = export_module.graph_module.meta + self.meta["unflattened_module"] = self + + # Flag to indicate whether args have been adapted. + self.adapted = False + self._run_with_interpreter = RUN_WITH_INTERPRETER + + _inplace_buffer_and_input_mutations(export_graph, self.graph_signature) + _fix_nn_module_stacks(export_graph) + self._ty = _root_module_type(export_graph) + + self.ivals = _IVals() + # for any intermediate value of a mutation that is read, track the mutation + seen_modules, seen_attrs = _outline_submodules(export_graph, self) + # for each read intermediate value of a mutation, find where it was created, + # and perform the mutation + self.ivals.update(seen_modules.values()) + # move attributes that correspond to graph arguments for HOPs + # from exported program to unflattened submodules + _copy_graph_attrs(export_module._graph_module, self, seen_attrs) + + self.range_constraints = export_module.range_constraints + self.equality_constraints: list = [] + + # aliasing/unused param or buffer issues: + # in strict-mode export, dynamo export will deduplicate aliased tensors, + # and ignore unused tensors. For aliasing, this causes issues when some aliases + # are unused, and we're unable to match the placeholder node to the correct FQN. + # This leads to the graph signature potentially having the wrong target FQN, + # and downstream issues where parameters are assigned to the wrong target attribute, + # mismatching the relevant placeholder node in the unflattened module. + # To resolve this we restore (_assign_attr) all aliased/unused tensors in + # the state_dict as module attributes, but only keep the used tensors in the + # graph's forward pass (_sink_params). + state_dict = export_module.state_dict + assigned_params: set[str] = set() # tracking unused params + id_to_param: dict[ + int | _TensorID, torch.nn.Parameter + ] = {} # handling weight-sharing + for name in self.graph_signature.parameters: # this loop adds used params + param = state_dict[name] + if _id(param) not in id_to_param: + id_to_param[_id(param)] = torch.nn.Parameter( + param.clone(), requires_grad=param.requires_grad + ) + + _assign_attr( + id_to_param[_id(param)], + self, + name, + attr_kind=_AttrKind.PARAMETER, + ) + assigned_params.add(name) + + non_persistent_buffers = set(self.graph_signature.non_persistent_buffers) + assigned_buffers: set[str] = set() # tracking unused buffers + id_to_buffer: dict[int | _TensorID, tuple[torch.nn.Parameter, bool]] = {} + for name in self.graph_signature.buffers: # this loop adds used buffers + if name in non_persistent_buffers: + persistent = False + buffer = export_module.constants[name] + else: + persistent = True + buffer = state_dict[name] + + if _id(buffer) not in id_to_buffer: + id_to_buffer[_id(buffer)] = (buffer.clone(), persistent) + + _assign_attr( + id_to_buffer[_id(buffer)][0], + self, + name, + attr_kind=_AttrKind.BUFFER, + persistent=persistent, + ) + assigned_buffers.add(name) + + # restore aliased/unused params and buffers + # these appear in state dict but not graph signature + for name, tensor in state_dict.items(): + if name in assigned_params or name in assigned_buffers: # already assigned + continue + + is_buffer = False + if _id(tensor) in id_to_buffer or not isinstance( + tensor, torch.nn.Parameter + ): # aliased buffer + is_buffer = True + + if is_buffer: + if ( + _id(tensor) not in id_to_buffer + ): # this is completely unused (not weight-sharing) + id_to_buffer[_id(tensor)] = ( + tensor, + True, + ) # assign to respect original model + _assign_attr( + id_to_buffer[_id(tensor)][0], + self, + name, + attr_kind=_AttrKind.BUFFER, + persistent=True, + ) + else: + if _id(tensor) not in id_to_param: # this is unused + id_to_param[_id(tensor)] = tensor + _assign_attr( + id_to_param[_id(tensor)], + self, + name, + attr_kind=_AttrKind.PARAMETER, + ) + + # use id map so we don't double-clone aliased constants + id_to_const: dict[int | _TensorID, torch.Tensor | torch._C.ScriptObject] = {} + for fqn, constant in export_module.constants.items(): + if _id(constant) not in id_to_const: + if isinstance(constant, torch.Tensor): + constant = constant.clone() + id_to_const[_id(constant)] = constant + _constant = id_to_const[_id(constant)] + _assign_attr( + _constant, + self, + fqn, + attr_kind=_AttrKind.CONSTANT, + ) + + # This is to handle parameters/buffers that point to the same tensor + # object id -> list of (node_name, target_name) + consts_map: dict[int | _TensorID, list[tuple[str, str]]] = defaultdict(list) + consts_targets: set[str] = set() + + def add_to_consts_map(obj_id, node_name, target_name): + name_list = consts_map[obj_id] + name_list.append((node_name, target_name)) + + # track aliased/unused params, buffers + # prefer using untyped_storage() over id() when it's available + added_params_buffers: set[str] = set() + for s in self.graph_signature.input_specs: + if s.kind == InputKind.PARAMETER or ( + s.kind == InputKind.BUFFER and s.persistent + ): + assert hasattr(s.arg, "name") + assert isinstance(s.target, str) + add_to_consts_map( + _id(export_module.state_dict[s.target]), + s.arg.name, + s.target, + ) + consts_targets.add(s.target) + added_params_buffers.add(s.target) + elif ( + s.kind == InputKind.BUFFER + and not s.persistent + or s.kind == InputKind.CONSTANT_TENSOR + or s.kind == InputKind.CUSTOM_OBJ + ): + assert hasattr(s.arg, "name") + assert isinstance(s.target, str) + add_to_consts_map( + _id(export_module.constants[s.target]), + s.arg.name, + s.target, + ) + consts_targets.add(s.target) + + # add constants that are aliased and don't appear in graph signature + for const_name, const in export_module.constants.items(): + if const_name not in consts_targets: + const_id = _id(const) + assert const_id in consts_map + ph_name, _ = consts_map[const_id][0] + add_to_consts_map(const_id, ph_name, const_name) + added_params_buffers.add(s.target) + + # add aliased/unused params and buffers that don't appear in graph signature + for fqn, tensor in export_module.state_dict.items(): + if fqn not in added_params_buffers: + tensor_id = _id(tensor) + if tensor_id not in consts_map: + # completely unused (no weight-sharing), ignore. + # this weight doesn't appear in graph module, + # so won't cause FQN assignment issues + continue + ph_name, _ = consts_map[tensor_id][0] + add_to_consts_map(tensor_id, ph_name, fqn) + + # node name -> list of possible targets + inputs_to_state: dict[str, list[str]] = {} + for node_target in consts_map.values(): + targets = [t[1] for t in node_target] + for n, _ in node_target: + inputs_to_state[n] = targets + + _sink_params(self, inputs_to_state, []) + + redirected_call_indices = _deduplicate_modules(seen_modules.values()) + fqn_list = [fqn for fqn in fqn_list if fqn not in redirected_call_indices] + + self._dispatch_modules(redirected_call_indices, consts_targets) + fqn_list = [fqn for fqn in fqn_list if "@" not in fqn] + + # Cache so we don't have to compute this every time. + # NOTE: this needs to be kept in sync with the placeholders in + # self.graph, but currently we have no way to guarantee that. + self.input_placeholders = [ + node for node in self.graph.nodes if node.op == "placeholder" + ] + self.check_input_constraints = True + # TODO(zhxchen17) We can register modules ahead of time instead of reorder later. + fqn_order = {fqn: i for i, fqn in enumerate(fqn_list)} + # In the case of legacy IR, we might be missing some modules from metadata. + for name, _ in self.named_modules(remove_duplicate=False): + if name not in fqn_order: + fqn_order[name] = len(fqn_order) + _reorder_submodules(self, fqn_order) + self.graph.lint() + self.finalize() + + def _print_graph(self): + for fqn, mod in self.named_modules(): + print(fqn + ":") + if hasattr(mod, "graph") and isinstance(mod.graph, torch.fx.Graph): + print(mod.graph) + + def _adapt_flat_args(self, flat_args, in_spec, input): + signature = self.module_call_graph[0].signature + if in_spec == signature.in_spec: + return flat_args + + if self.flat_args_adapter is None: + raise TypeError( + "There is no flat args adapter specified. " + "Are you sure you are calling this with the right arguments? " + ) + else: + flat_args = self.flat_args_adapter.adapt( + target_spec=signature.in_spec, + input_spec=in_spec, + input_args=flat_args, + metadata=self.meta, + obj=input, + ) + + if len(flat_args) != signature.in_spec.num_leaves: + raise TypeError( + f"Flat args adaption failed, number of args mismatch " + f"Adatped: {len(flat_args)} \n" + f"Exported module: {signature.in_spec.num_leaves}" + ) + return flat_args + + def process_forward_inputs(self, *args, **kwargs): + signature = self.module_call_graph[0].signature + + reordered_kwargs = kwargs + if kwargs: + reordered_kwargs = reorder_kwargs(kwargs, signature.in_spec) + + flat_args_with_path, in_spec = pytree.tree_flatten_with_path( + (args, reordered_kwargs) + ) + flat_args = [x[1] for x in flat_args_with_path] + + if is_fx_symbolic_tracing(): + return flat_args + + if in_spec != signature.in_spec: + if not self.adapted: + print( + "Input treespec does not match with exported module's: \n" + f"Input treespec: {in_spec}. ", + f"Exported module treespec: {signature.in_spec}", + ) + print("Adapting flat arg to match exported module's treespec") + flat_args = self._adapt_flat_args(flat_args, in_spec, args) + self.adapted = True + + if self.check_input_constraints: + # Import here to avoid an unfortunate circular dependency. + # TODO(suo): untangle this. + from torch._export.utils import _check_input_constraints_for_graph + + if self.adapted is True: + flat_arg_paths = ( + self.flat_args_adapter.get_flat_arg_paths() + if self.flat_args_adapter + else [] + ) + assert not flat_arg_paths or len(flat_arg_paths) == len(flat_args) + new_flat_args_with_path = [ # type: ignore[var-annotated] + ( + ( + SequenceKey(idx=idx), + GetAttrKey( + name=flat_arg_paths[idx] + if flat_arg_paths + else "" + ), + ), + arg, + ) + for idx, arg in enumerate(flat_args) + ] + else: + new_flat_args_with_path = flat_args_with_path # type: ignore[assignment] + + _check_input_constraints_for_graph( + self.input_placeholders, new_flat_args_with_path, self.range_constraints + ) + + return flat_args + + def forward(self, *args, **kwargs): + flat_args = self.process_forward_inputs(*args, **kwargs) + signature = self.module_call_graph[0].signature + + if is_fx_symbolic_tracing(): + return_val = torch.fx.Interpreter(self, graph=self.graph).run( + *flat_args, enable_io_processing=False + ) + # For scalar return value, fx.Graph wraps in a tuple + if isinstance(return_val, tuple) and len(return_val) == 1: + return return_val[0] + return return_val + + if torch.compiler.is_dynamo_compiling() or not self._run_with_interpreter: + tree_out = type(self.graph_module).forward(self, *flat_args) # type: ignore[union-attr] + else: + tree_out = torch.fx.Interpreter(self, graph=self.graph).run( + *flat_args, enable_io_processing=False + ) + return pytree.tree_unflatten(tree_out, signature.out_spec) + + def finalize(self): + self.__dict__["graph_module"] = torch.fx.GraphModule(self, self.graph) + self.graph.lint() + + def _dispatch_modules(self, redirected_call_indices, consts_targets): + """For a module whose call signatures are preserved, replace + multiple modules corresponding to multiple calls to that module + with a single dispatcher module that tracks which module to call. + """ + + # for each fqn whose module call signature is preserved, + # map that fqn to a list of called modules + called_modules = defaultdict(list) + for entry in self.module_call_graph: + if entry.fqn and entry.signature: + # some modules were removed and their fqns redirected to other + # fqns during deduplication + fqn = entry.fqn + mod = _get_attr(self, redirected_call_indices.get(fqn, fqn)) + base, idx = fqn.split("@") if "@" in fqn else [fqn, "0"] + called_modules[base].append((int(idx), mod)) + + attrs_map = defaultdict(set) + for target in consts_targets: + if "." in target: + orig_fqn, name = target.rsplit(".", 1) + attrs_map[orig_fqn].add(name) + else: + attrs_map[""].add(target) + + # replace multiple call modules with a single dispatcher module + for orig_fqn, indexed_call_modules in called_modules.items(): + call_modules = [mod for _, mod in sorted(indexed_call_modules)] + if len(call_modules) > 1: + for i in range(len(call_modules)): + fqn = _call_name(orig_fqn, i + 1) + if fqn not in redirected_call_indices: + *prefix, name = fqn.split(".") + _get_attr_via_attr_list(self, prefix)._modules.pop(name) + self.set_submodule( + orig_fqn, + InterpreterModuleDispatcher(attrs_map[orig_fqn], call_modules), + ) + + # elide call indices in call modules because they are + # tracked automatically inside the dispatcher module + def elide_call_indices(prefix, graph): + for node in graph.nodes: + if node.op == "call_module": + fqn = node.target.split("@")[0] + path = f"{prefix}.{fqn}" if prefix else fqn + if path in called_modules: + node.target = fqn + + for fqn, mod in self.named_modules(remove_duplicate=False): + if hasattr(mod, "graph"): + elide_call_indices(fqn, mod.graph) + elif hasattr(mod, "_call_modules"): + for mod_ in mod._call_modules: + assert hasattr(mod_, "graph") + elide_call_indices(fqn, mod_.graph) + + def print_readable( + self, + print_output=True, + include_stride=False, + include_device=False, + colored=False, + ): + return _print_readable( + self, + "UnflattenedModule", + print_output, + include_stride, + include_device, + colored, + ) + + +def unflatten( + module: ExportedProgram, flat_args_adapter: FlatArgsAdapter | None = None +) -> UnflattenedModule: + """Unflatten an ExportedProgram, producing a module with the same module + hierarchy as the original eager module. This can be useful if you are trying + to use :mod:`torch.export` with another system that expects a module + hierarchy instead of the flat graph that :mod:`torch.export` usually produces. + + .. note:: The args/kwargs of unflattened modules will not necessarily match + the eager module, so doing a module swap (e.g. :code:`self.submod = + new_mod`) will not necessarily work. If you need to swap a module out, you + need to set the :code:`preserve_module_call_signature` parameter of + :func:`torch.export.export`. + + Args: + module (ExportedProgram): The ExportedProgram to unflatten. + flat_args_adapter (Optional[FlatArgsAdapter]): Adapt flat args if input TreeSpec does not match with exported module's. + + Returns: + An instance of :class:`UnflattenedModule`, which has the same module + hierarchy as the original eager module pre-export. + """ + module = _remove_effect_tokens(module) + m = UnflattenedModule(module, flat_args_adapter) + + # Disable process_forward_inputs as the adapter has many + # non-dynamo-traceable behavior. + m.process_forward_inputs = torch._dynamo.disable( # type: ignore[method-assign] + m.process_forward_inputs, + reason="do not trace into preprocessing the inputs", + recursive=True, + ) + + return m + + +def _inplace_buffer_and_input_mutations( + graph: torch.fx.Graph, + graph_signature: ExportGraphSignature, +) -> None: + """Transform buffer and input mutations from their functionalized form + into copy_ nodes in the graph. + + Functionalization represents a buffer mutation by passing the buffer as + an input and output. For example, consider the eager code: + def forward(self, x): + self.buffer += x + return x * x + + This corresponds to a graph that looks like: + def forward(self, buffer, x): + mutated_buffer = aten.add(buffer, x) + mul = aten.mul(x, x) + return (mutated_buffer, mul) + + We want to inplace this into something that looks like the original + eager code: + def forward(self, buffer, x): + mutated_buffer = aten.add(buffer, x) + buffer.copy_(mutated_buffer) + mul = aten.mul(x, x) + return (mul,) + + Input mutations are handled similarly. + """ + output_node = next(iter(reversed(graph.nodes))) + assert output_node.op == "output" and len(output_node.args) == 1 + return_args = output_node.args[0] + + input_name_to_node = { + node.name: node for node in graph.nodes if node.op == "placeholder" + } + mutation_name_to_input_name = {} + + # Collect mutated buffers. + buffer_fqn_to_input_name = { + buffer_fqn: k for k, buffer_fqn in graph_signature.inputs_to_buffers.items() + } + mutation_name_to_input_name = { + k: buffer_fqn_to_input_name[buffer_fqn] + for k, buffer_fqn in graph_signature.buffers_to_mutate.items() + } + # Collect mutated user inputs. + mutation_name_to_input_name.update(graph_signature.user_inputs_to_mutate) + + num_mutations = len(mutation_name_to_input_name) + + for mutation in return_args[:num_mutations]: + input_name = mutation_name_to_input_name[mutation.name] + input_node = input_name_to_node[input_name] + + with graph.inserting_after(mutation): + # Create a copy_ node that inplaces the mutation. + new_node = graph.create_node( + "call_function", torch.ops.aten.copy_.default, (input_node, mutation) + ) + for k, v in mutation.meta.items(): + new_node.meta[k] = v + # Replace all uses of the previously functional mutation with + # our copy_ node. + mutation.replace_all_uses_with(new_node, lambda x: x is not new_node) + + # Remove the mutated buffer / input from the graph outputs, since we don't + # need to thread it through anymore. + user_outputs = tuple(return_args[num_mutations:]) + output_node.args = ((user_outputs),) + + +def _root_module_type(graph: torch.fx.Graph) -> str | None: + for node in graph.nodes: + if "nn_module_stack" not in node.meta: + continue + + for path, ty in node.meta["nn_module_stack"].values(): + if not path: + return ty + return None + + +def _fix_nn_module_stacks(graph): + # For each nn module stack in the graph, check if the fqns in it represent a stack: + # 1. Each fqn must be a prefix of the next fqn. + # 2. If not, remove the entries starting from the next fqn, emitting a warning. + for node in graph.nodes: + if "nn_module_stack" not in node.meta: + continue + + nn_module_stack = node.meta["nn_module_stack"] + fqns = [ + fqn.split("@")[0] if "@" in fqn else fqn + for fqn, _t in nn_module_stack.values() + ] + + # Check if each FQN is a prefix of the next one + prev_fqn, *next_fqns = fqns + num_valid_indices = 1 # root FQN + for curr_fqn in next_fqns: + # Check if the previous FQN is a prefix of the current one + if _is_prefix(prev_fqn, curr_fqn): + num_valid_indices += 1 + prev_fqn = curr_fqn + else: + # Found a non-prefix FQN, stop here + break + + # If we need to remove entries, create a new stack with only valid entries + if num_valid_indices < len(nn_module_stack): + log.warning( + "nn_module_stack fqns %s at node %s do not form a stack! dropping last %d entries", + fqns, + node, + len(nn_module_stack) - num_valid_indices, + ) + node.meta["nn_module_stack"] = dict( + list(nn_module_stack.items())[:num_valid_indices] + ) + + +def _is_prefix(candidate, target): + """Check whether `candidate` is a prefix of `target`.""" + return len(candidate) < len(target) and target[: len(candidate)] == candidate + + +def _compute_accessor(parent_fqn: str, child_fqn: str) -> str: + if parent_fqn == "": + # Handle the root module correctly. + return child_fqn + + parent_split = parent_fqn.split(".") + child_split = child_fqn.split(".") + + # TODO: support skip connection by inlining the child module. + if child_split[: len(parent_split)] != parent_split: + raise RuntimeError( + f"Child module '{child_fqn}' is not a descendant of parent module '{parent_fqn}'." + "This is currently unsupported." + "Please try to make child module attach to parent module directly." + ) + return ".".join(child_split[len(parent_split) :]) + + +def _check_graph_equivalence(x: torch.nn.Module, y: torch.nn.Module): + def graph_dump(graph: torch.fx.Graph) -> str: + ret = [] + nodes_idx: dict[int, int] = {} + + def arg_dump(arg) -> str: + if isinstance(arg, torch.fx.Node): + return "%" + str(nodes_idx[id(arg)]) + return str(arg) + + for i, node in enumerate(graph.nodes): + args_dump = [str(arg) for arg in pytree.tree_map(arg_dump, node.args)] + args_dump += [ + f"{key}={value}" + for key, value in pytree.tree_map(arg_dump, node.kwargs).items() + ] + target = node.target if node.op in ("call_function", "get_attr") else "" + # pyrefly: ignore [bad-argument-type] + ret.append(f"{i}: {node.op}[{target}]({', '.join(args_dump)})") + nodes_idx[id(node)] = i + return "\n".join(ret) + + assert isinstance(x.graph, torch.fx.Graph) + assert isinstance(y.graph, torch.fx.Graph) + return graph_dump(x.graph) == graph_dump(y.graph) + + +def _add_spec(gm: torch.nn.Module, spec) -> str: + i = 0 + while hasattr(gm, f"_spec_{i}"): + i += 1 + name = f"_spec_{i}" + setattr(gm, name, spec) + return name + + +def _generate_flatten(gm: torch.fx.GraphModule, node) -> torch.fx.Node: + flatten = gm.graph.call_function(pytree.tree_flatten, (node,)) + getitem_0 = gm.graph.call_function(operator.getitem, (flatten, 0)) + return getitem_0 + + +def _generate_flatten_spec( + gm: torch.fx.GraphModule | InterpreterModule | UnflattenedModule, node, spec +) -> torch.fx.Node: + name = _add_spec(gm, spec) + spec_node = gm.graph.get_attr(name) + return gm.graph.call_function(fx_pytree.tree_flatten_spec, (node, spec_node)) + + +def _generate_unflatten( + gm: torch.fx.GraphModule | InterpreterModule | UnflattenedModule, nodes, spec +) -> torch.fx.Node: + name = _add_spec(gm, spec) + spec_node = gm.graph.get_attr(name) + return gm.graph.call_function(pytree.tree_unflatten, (nodes, spec_node)) + + +def _get_submodule(mod: torch.nn.Module, target: str): + *prefix, field = target.split(".") + + for item in prefix: + submod = getattr(mod, item, None) + + if submod is None: + return None + + if not isinstance(submod, torch.nn.Module): + return None + + mod = submod + + return getattr(mod, field, None) + + +def _add_submodule( + mod: torch.nn.Module, + target: str, + module_to_add: torch.nn.Module, + create_module: Callable[[str], torch.nn.Module] | None = None, +): + *prefix, field = target.split(".") + + for i, item in enumerate(prefix): + submod = getattr(mod, item, None) + + if submod is None: + if create_module is not None: + submod = create_module(".".join(prefix[: i + 1])) + else: + submod = torch.nn.Module() + setattr(mod, item, submod) + + if not isinstance(submod, torch.nn.Module): + return False + + mod = submod + + mod.add_module(field, module_to_add) + + +def _call_name(base: str, n: int) -> str: + # Given n >= 0, generate call names to a submodule `base` of the form + # `base`, `base@1`, `base@2`, etc. + return base if n == 1 else f"{base}@{n - 1}" + + +def _is_call_name(call_name: str, base: str) -> bool: + # Recognize when call_name = _call_name(base, n) for some n >= 0. + return re.match(re.escape(base) + r"(@\d+)?$", call_name) is not None + + +class _ModuleFrame: + def __init__( + self, + flat_graph: torch.fx.Graph, + nodes: tuple[torch.fx.Node, ...], + seen_nodes, + seen_modules, + seen_attrs, + created_modules, + parent, + module_stack: list[tuple[str, str | None, int]], + module_id, + module_call_graph: dict[str, ModuleCallSignature], + module: torch.fx.GraphModule | UnflattenedModule | None = None, + ): + self.flat_graph = flat_graph + self.nodes = nodes + self.seen_nodes = seen_nodes + self.seen_modules = seen_modules + self.seen_attrs = seen_attrs + self.created_modules = created_modules + self.parent = parent + self.module_stack = module_stack + self.module_id = module_id + + self.module_call_graph = module_call_graph + self.verbose = False + + self.fqn, ty, num_calls = self.module_stack[-1] + # generate call name for self.fqn + self.child_fqn = _call_name(self.fqn, num_calls + 1) + + self.module: torch.fx.GraphModule | UnflattenedModule | InterpreterModule + if module is not None: + self.module = module + self.ivals = module.ivals if hasattr(module, "ivals") else {} # type: ignore[var-annotated] + else: + self.module = self.created_modules.get( + self.fqn, + InterpreterModule(torch.fx.Graph(), ty=ty), + ) + self.ivals = parent.ivals + + self.graph = self.module.graph + + # Mapping of nodes in the flat graph to nodes in this graph. + self.node_map: dict[torch.fx.Node, torch.fx.Node] = {} + self.node_to_placeholder = {} + + self.parent_call_module: torch.fx.Node | None = None + if parent is not None: + accessor = _compute_accessor(parent.fqn, self.child_fqn) + + def create_module(fqn): + path = f"{parent.fqn}.{fqn}" if parent.fqn else fqn + if path in self.created_modules: + return self.created_modules[path] + submod = InterpreterModule(torch.fx.Graph(), ty=ty) + self.created_modules[path] = submod + return submod + + _add_submodule(parent.module, accessor, self.module, create_module) + self.parent_call_module = parent.graph.call_module(accessor) + if self.seen_modules[self.module_id]: + base_module_frame = self.seen_modules[self.module_id][0] + self.module._modules = base_module_frame.module._modules + self.seen_modules[self.module_id].append( + _SubmoduleEntry( + parent_fqn=self.parent.fqn, + parent_module=self.parent.module, + parent_call_module=self.parent_call_module, + fqn=self.fqn, + call_idx=num_calls + 1, + module=self.module, + ) + ) + + signature = module_call_graph.get(self.child_fqn) + if signature is not None and self.parent is not None: + assert signature.in_spec.num_children == 2 + assert signature.in_spec.type is tuple + args_spec, kwargs_spec = signature.in_spec.children() + assert args_spec.type is tuple + assert kwargs_spec.type is dict + + with self.graph.inserting_after(None): + arg_nodes = [ + self.graph.placeholder(f"_positional_arg_{idx}") + for idx in range(args_spec.num_children) + ] + kwarg_nodes = {} + for name in kwargs_spec.context: + kwarg_nodes[name] = self.graph.placeholder(name) + flat_args = _generate_flatten_spec( + self.module, + (tuple(arg_nodes), kwarg_nodes), + signature.in_spec, + ) + for idx, arg in enumerate(signature.inputs): + flat_arg_node = self.graph.create_node( + op="call_function", + target=operator.getitem, + args=(flat_args, idx), + name=( + arg.name + if not isinstance(arg, ConstantArgument) + else f"_constant_{idx}" + ), + ) + if isinstance(arg, ConstantArgument): + continue + + if arg.name in self.seen_nodes: + flat_arg_node.meta = copy.copy(self.seen_nodes[arg.name].meta) + self.node_to_placeholder[self.seen_nodes[arg.name]] = ( + flat_arg_node + ) + + with self.parent.graph.inserting_before(self.parent_call_module): + input_nodes: list[torch.fx.Node | None] = [] + for input in signature.inputs: + if isinstance(input, ConstantArgument): + input_nodes.append(input.value) # type: ignore[arg-type] + elif input.name not in self.seen_nodes: + input_nodes.append(None) + else: + assert isinstance( + input, + ( + TensorArgument, + SymIntArgument, + SymBoolArgument, + SymFloatArgument, + ), + ) + input_nodes.append( + self.parent.remap_input(self.seen_nodes[input.name]) + ) + + inputs_node = _generate_unflatten( + self.parent.module, + input_nodes, + signature.in_spec, + ) + + args_node = self.parent.graph.call_function( + operator.getitem, (inputs_node, 0) + ) + kwargs_node = self.parent.graph.call_function( + operator.getitem, (inputs_node, 1) + ) + arg_nodes = [ + self.parent.graph.call_function(operator.getitem, (args_node, i)) + for i in range(args_spec.num_children) + ] + kwarg_nodes = { + k: self.parent.graph.call_function( + operator.getitem, (kwargs_node, k) + ) + for k in kwargs_spec.context + } + assert self.parent_call_module is not None + # pyrefly: ignore [bad-assignment] + self.parent_call_module.args = tuple(arg_nodes) + self.parent_call_module.kwargs = kwarg_nodes # type: ignore[assignment] + + def add_placeholder(self, x): + assert self.fqn != "", f"Cannot add placeholder {x} to root module" + assert x.graph is self.flat_graph + # x is not in subgraph, create a new placeholder for subgraph + with self.graph.inserting_before(None): + placeholder_node = self.graph.placeholder(x.name, type_expr=x.type) + # copy all meta fields, even if some fields might be irrelevant for + # the placeholder node + placeholder_node.meta = copy.copy(x.meta) + self.node_to_placeholder[x] = placeholder_node + + def copy_sym_call_function(self, x): + # This only exists because we deduplicate sym_size nodes in the flat export graph, + # and if preserve_module_call_signature is set, we may not be able to pass sym_size + # nodes, or their downstream users, as inputs to submodule calls. + # To avoid this we copy these call_function nodes with sym_type results. + # This should however only be done for sym_type nodes - call_function nodes on tensors + # should not be deduplicated in the first place. + args = pytree.tree_map_only(torch.fx.Node, self.remap_input, x.args) + kwargs = pytree.tree_map_only(torch.fx.Node, self.remap_input, x.kwargs) + node = self.graph.call_function(x.target, args, kwargs) + node.meta = copy.copy(x.meta) + self.node_map[x] = node + return node + + def remap_input(self, x): + assert x.graph is self.flat_graph + if x in self.node_map: + return self.node_map[x] + self.print(f"remap_input({x})") + if x in self.node_to_placeholder: + return self.node_to_placeholder[x] + elif ( + x.op == "placeholder" or self.module_call_graph.get(self.fqn) is None + # allow placeholder creation if we are not preserving module call signature + ): + self.add_placeholder(x) + if self.parent_call_module is not None: + # Important to *prepend* the output to match how we are + # inserting placeholder nodes. + with self.parent.graph.inserting_before(self.parent_call_module): + self.parent_call_module.insert_arg(0, self.parent.remap_input(x)) + return self.node_to_placeholder[x] + elif x.op == "call_function" and ( + x.target + in ( + torch.ops.aten.sym_size.int, + torch.ops.aten.item.default, + torch.ops.aten.unbind.int, + torch.ops.aten.sum.dim_IntList, + torch.ops.aten.view.default, + torch.ops.aten.diff.default, + ) + or (hasattr(x.target, "__module__") and x.target.__module__ == "_operator") + ): + # export deduplicates sym_size nodes, and may need to re-copy them + # if module call signature needs to be preserved + self.copy_sym_call_function(x) + return self.node_map[x] + elif self.module_call_graph.get(self.fqn) is not None: + # x is reading the intermediate value of a mutation, so record it; + # later we will find where it was created and perform the update + return self.ivals.read(self, x) # type: ignore[operator, union-attr] + else: + raise RuntimeError( + f"Could not run remap_input() on op type: {x.op} for node {x}" + ) + + def uplift_common_custom_metadata(self) -> None: + # Copy custom metadata if all nodes have same custom metadata + custom_meta = None + for node in self.node_map.values(): + curr_meta = node.meta.get("custom", {}) + if custom_meta is None: + # first node + custom_meta = curr_meta + continue + + if curr_meta != custom_meta: + custom_meta = {} + break + + if custom_meta: + # Lift common custom metadata to parent node and clear children node's custom metadata + assert self.parent_call_module is not None + self.parent_call_module.meta["custom"] = custom_meta + for node in self.node_map.values(): + del node.meta["custom"] + + def finalize_outputs(self): + self.created_modules.pop(self.fqn, None) + + orig_outputs = [] + + signature = self.module_call_graph.get(self.child_fqn) + if signature is not None and self.parent is not None: + for output in signature.outputs: + if isinstance( + output, + ( + TensorArgument, + SymIntArgument, + SymBoolArgument, + SymFloatArgument, + ConstantArgument, + ), + ): + if output.name in self.seen_nodes: + orig_outputs.append(self.seen_nodes[output.name]) + else: + orig_outputs.append(None) + else: + raise RuntimeError( + f"Unsupported data type for output node: {output}" + ) + + def get_actual_output_node(output): + if output is None: + return None + + seen_node = self.seen_nodes[output.name] + if seen_node in self.node_map: + return self.node_map[seen_node] + elif seen_node in self.node_to_placeholder: + return self.node_to_placeholder[seen_node] + else: + raise RuntimeError( + f"Could not find output node {output}. Graph: {self.graph}" + ) + + tree_out_node = _generate_unflatten( + self.module, + tuple(get_actual_output_node(output) for output in orig_outputs), + signature.out_spec, + ) + parent_out: torch.fx.Node | None = _generate_flatten_spec( + self.parent.module, self.parent_call_module, signature.out_spec + ) + graph_outputs: torch.fx.Node | list[torch.fx.Node] = tree_out_node + else: + graph_outputs = [] + # Iterate through nodes we have copied into self.graph. + for orig_node in self.node_map: + for user_node in orig_node.users: + if user_node.name not in self.seen_nodes: + # external user node, need to expose as an output + orig_outputs.append(orig_node) + graph_outputs.append(self.node_map[orig_node]) + break + + parent_out = self.parent_call_module + if len(graph_outputs) == 1: + graph_outputs = graph_outputs[0] + + assert isinstance(graph_outputs, (list, torch.fx.Node)) + + self.graph.output(graph_outputs) + + # Rewrite outputs in parent module + if parent_out is None: + return + + parent_out.meta["val"] = ( + graph_outputs.meta.get("val") + if isinstance(graph_outputs, torch.fx.Node) + else [o.meta.get("val") for o in graph_outputs] + ) + self.uplift_common_custom_metadata() + + if len(orig_outputs) == 1 and signature is None: + self.parent.node_map[orig_outputs[0]] = parent_out + else: + for i, orig_output in enumerate(orig_outputs): + if orig_output is None: + continue + # Use Proxy to record getitem access. + proxy_out = torch.fx.Proxy(parent_out)[i].node # type: ignore[index] + proxy_out.meta["val"] = orig_output.meta.get("val") + self.parent.node_map[orig_output] = proxy_out + + def copy_node(self, node): + self.print("copying", node.format_node()) + self.node_map[node] = self.graph.node_copy(node, self.remap_input) + self.seen_nodes[node.name] = node + + def run_outer(self): + for i, node in enumerate(self.flat_graph.nodes): + self.print(i, node.meta.get("nn_module_stack"), node.format_node()) + + # Copy all graph inputs + node_idx: int = 0 + node = self.nodes[node_idx] + while node.op == "placeholder": + self.copy_node(node) + node_idx += 1 + node = self.nodes[node_idx] + + self.run_from(node_idx) + + # Copy graph outputs + for node in self.flat_graph.nodes: + if node.op == "output": + self.copy_node(node) + + def print(self, *args, **kwargs): + if self.verbose: + # pyrefly: ignore [not-iterable] + print(*args, **kwargs) + + def run_from(self, node_idx): + module_idx = 0 + # Walk through the graph, building up a new graph with the right submodules + while node_idx < len(self.nodes): + node = self.nodes[node_idx] + assert node.op != "placeholder" + + self.print() + self.print("STEP", node_idx, node.format_node()) + self.print(self.module_stack) + depth = len(self.module_stack) + if node.op == "output": + if depth == 1: + # We want the output node of the original graph to be handled + # specially by the outermost stack frame (in run_outer). So + # skip finalization here. + return node_idx + + # We've reached the end of the graph. Wrap up all the existing stack frames. + self.finalize_outputs() + return node_idx + + if len(node.meta.get("nn_module_stack", {})) == 0: + raise RuntimeError(f"Unable to find nn_module_stack for node {node}") + + nn_module_stack = node.meta["nn_module_stack"] + from torch._export.passes._node_metadata_hook import ( + _EMPTY_NN_MODULE_STACK_KEY, + ) + + if ( + len(nn_module_stack) == 1 + and _EMPTY_NN_MODULE_STACK_KEY in nn_module_stack + ): + # Empty case from the node_metadata_hook + node_module_stack = self.module_stack + else: + node_module_stack = [ + ( + path, + ty if path else None, + int(k.split("@")[-1]) if "@" in k else 0, + ) + for k, (path, ty) in node.meta["nn_module_stack"].items() + ] + + if node_module_stack[:depth] != self.module_stack: + # This means that the current module is done executing and the + # current node is the beginning of a new module. + # + # In this case, we should finalize this module and return without + # incrementing the node counter. + self.finalize_outputs() + self.print("outlining", self.fqn) + self.print(self.graph) + return node_idx + + assert node_module_stack is not None + + if _is_prefix(self.module_stack, node_module_stack): + # This means that the current node represents the execution of a new + # module. + next_module = node_module_stack[depth] + self.print("Creating new stack frame for", next_module) + # Run a nested version of module outliner from the current node + # counter. Once it is complete, continue from that point. + next_module_key = list(node.meta["nn_module_stack"].keys())[depth] + node_idx = _ModuleFrame( + self.flat_graph, + self.nodes, + self.seen_nodes, + self.seen_modules, + self.seen_attrs, + self.created_modules, + self, + self.module_stack + [next_module], + next_module_key.split("@")[0], + self.module_call_graph, + ).run_from(node_idx) + module_idx += 1 + continue + + # The only remaining possibility is that we are in the right stack + # frame. Copy the node into this frame's graph and increment the node counter. + assert node_module_stack == self.module_stack + + if node.op == "get_attr": + # this must be a graph argument for a HOP + self.seen_attrs[self.child_fqn].add(node.target) + + self.copy_node(node) + # pyrefly: ignore [unsupported-operation] + node_idx += 1 + + +@dataclass +class _SubmoduleEntry: + parent_fqn: str + parent_module: torch.nn.Module + parent_call_module: torch.fx.Node + fqn: str + call_idx: int + module: torch.nn.Module + + +def _outline_submodules(orig_graph: torch.fx.Graph, root_module: UnflattenedModule): + seen_nodes: dict[str, torch.fx.Node] = {} + seen_modules: dict[int, list[_SubmoduleEntry]] = defaultdict(list) + seen_attrs: dict[str, set[str]] = defaultdict(set) + created_modules: dict[str, torch.nn.Module] = {} + _ModuleFrame( + orig_graph, + tuple(orig_graph.nodes), + seen_nodes, + seen_modules, + seen_attrs, + created_modules, + None, + [("", None, 0)], + "", + { + entry.fqn: entry.signature + for entry in root_module.module_call_graph + if entry.signature + }, + module=root_module, + ).run_outer() + return seen_modules, seen_attrs + + +def _reorder_submodules( + parent: torch.nn.Module, fqn_order: dict[str, int], prefix: str = "" +): + # TODO Can be optimized by adding submodules ahead of time. + if prefix == "": + for fqn in list(fqn_order.keys())[1:]: + if _get_submodule(parent, fqn) is None: + _add_submodule(parent, fqn, torch.nn.Module()) + + children = [] + for name, child in list(parent._modules.items()): + if child is None: + continue + fqn = prefix + name + _reorder_submodules(child, fqn_order, prefix=fqn.split("@")[0] + ".") + delattr(parent, name) + children.append((fqn_order[fqn], name, child)) + children.sort(key=operator.itemgetter(0)) + for _, name, child in children: + parent.register_module(name, child) + + +class _IVals: + """ + Collect the intermediate values of mutations in a graph. + + Example: in the following graph, suppose that buf_in and buf_out + are the input and output values of a buffer. + + buf_in = placeholder() + ... + ival1 = f0(buf_in, ...) # inside self.n0(...) + ... + ival2 = f1(ival1, ...) # inside self.n1(...) + ... + buf_out = f2(ival2, ...) # inside self.n2(...) + return buf_out, ... + + Here ival1 and ival2 are intermediate values created inside + calls to n0 and n1 respectively, and used inside calls to + n1 and n2 respectively. + """ + + def __init__(self): + # for each fqn, set of node names corresponding to intermediate values + self.node_names_by_fqn = defaultdict(set) + + def _is_mutable(self, target): + if isinstance(target, torch._ops.OpOverload): + return target._schema.is_mutable + return False + + def read(self, mf, node): + """ + Read state corresponding to a given intermediate value. + """ + # we can assume that the node must be from a mutation + assert node.op == "call_function" + b = self._is_mutable(node.target) + print("Checking mutability", node.target, b) + if not b: + # so the mutation was functionalized; + # we will apply the original mutation later (see below) + fqn, _ = next(reversed(node.meta["nn_module_stack"].values())) + self.node_names_by_fqn[fqn].add(node.name) + return mf.remap_input(node.args[0]) + + def update(self, partitions): + """ + Update states corresponding to intermediate values that were read. + """ + for shared_submodules in partitions: + for entry in shared_submodules: + graph = entry.module.graph + node_names = self.node_names_by_fqn[entry.fqn] + nodes = [n for n in graph.nodes if n.name in node_names] + for node in nodes: + # so node must be from a functionalized mutation; + # we perform the original mutation now + with graph.inserting_after(node): + new_node = graph.create_node( + "call_function", + torch.ops.aten.copy_.default, + (node.args[0], node), + ) + new_node.meta = copy.copy(node.meta) + + +def _copy_graph_attrs( + gm: torch.fx.GraphModule, + root_module: UnflattenedModule, + seen_attrs: dict[str, set[str]], +): + for child_fqn, names in seen_attrs.items(): + module = _get_attr(root_module, child_fqn) if child_fqn else root_module + for name in names: + val = getattr(gm, name) + setattr(module, name, val) + + +def _deduplicate_modules(partitions): + redirected_call_indices = {} + for shared_submodules in partitions: + for i, entry in enumerate(shared_submodules): + child_fqn = _call_name(entry.fqn, entry.call_idx) + target = _compute_accessor(entry.parent_fqn, child_fqn) + deduplicated = False + # Iterate over all previously seen modules, and deduplicate if possible + for seen in shared_submodules[:i]: + if _check_graph_equivalence(seen.module, entry.module): + parent = entry.parent_module + # Since graphs are equivalent, we can deduplicate. + # There are two cases. + if seen.fqn == entry.fqn: + # Case 1: The current module has the same fqn as the seen module. + # In this case we have generated a call name that can be optimized away. + # So we remove the current module from the hierarchy and replace + # the current call name with the seen call name in the parent graph. + *prefix, name = target.split(".") + _get_attr_via_attr_list(parent, prefix)._modules.pop(name) + seen_child_fqn = _call_name(seen.fqn, seen.call_idx) + seen_target = _compute_accessor( + entry.parent_fqn, seen_child_fqn + ) + entry.parent_call_module.target = seen_target + redirected_call_indices[child_fqn] = seen_child_fqn + break + elif not deduplicated: + # Case 2: The current module has a different fqn than the seen module. + # In this case we replace the current module with the seen module. + # There should be nothing pointing to the current module any more, + # so it can be garbage collected. + # NOTE: We *do not* replace the current call name with the seen call name + # in the parent graph, because this will lose information on which fqn + # was actually called. However, it is possible that the current call name + # will be optimized away when we find another seen module with the same fqn, + # so we do not break out of the loop yet. + parent.set_submodule(target, seen.module) + deduplicated = True + + return redirected_call_indices + + +def _sink_params( + module: torch.nn.Module, + inputs_to_state: dict[str, list[str]], + scope: list[str], + module_id_to_inputs_removed: dict[int, set[str]] | None = None, +): + """Sink params, buffers, and constants from graph inputs into get_attr nodes. + + Exported modules are purely functional, so they pass their parameters and + buffers in as inputs to the graph. + + To replicate eager's semantics, we need to get them from the module state + via get_attr instead. + + module: GraphModule, potentially containing nested submodules. + inputs_to_state: mapping graph input names to the corresponding key in the state_dict. + scope: tracks where we are in the module hierarchy, so that we can emit the + right `getattr(self, "foo.bar")` calls, etc. + module_id_to_inputs_removed: records inputs removed by child modules, mapping + the module object id to the list of placeholder node names in the child module + that were removed. + """ + if module_id_to_inputs_removed is None: + module_id_to_inputs_removed = defaultdict(set) + + if id(module) in module_id_to_inputs_removed: + return {id(module): module_id_to_inputs_removed[id(module)]} + + # We need to use _modules here instead of named_children(), because we + # explicitly want duplicate modules to show up in the traversal. + for name, submodule in module._modules.items(): + submod_id_to_inputs_removed = _sink_params( + cast("torch.nn.Module", submodule), + inputs_to_state, + scope + [name], + module_id_to_inputs_removed, + ) + for k, v in submod_id_to_inputs_removed.items(): + module_id_to_inputs_removed[k].update(v) + + graph = getattr(module, "graph", None) + if graph is None or len(graph.nodes) == 0: + # Not all modules have graphs defined, if they are empty modules with no operations (like ParameterList) + return module_id_to_inputs_removed + + assert isinstance(graph, torch.fx.Graph) + + inputs = list(filter(lambda n: n.op == "placeholder", graph.nodes)) + the_last_input = None if len(inputs) == 0 else inputs[-1] + + # Also remove from call_module nodes + call_module_nodes = filter(lambda n: n.op == "call_module", graph.nodes) + for node in call_module_nodes: + submodule = _get_attr(module, node.target) + # remove placeholder from call_module node arguments, only if we've + # erased the placeholder node in the corresponding _sink_params() call + if submodule is not None and id(submodule) in module_id_to_inputs_removed: + node.args = tuple( + filter( + lambda n: n.name not in module_id_to_inputs_removed[id(submodule)], + node.args, + ) + ) + + # Filter out inputs_to_state corresponding to current scope. + inputs_to_state_of_scope: dict[torch.fx.Node, list[str]] = {} + for node in inputs: + if node.name not in inputs_to_state: + continue + + state_name = None + for sn in inputs_to_state[node.name]: + sn_split = sn.split(".") + if sn_split[: len(scope)] == [x.split("@")[0] for x in scope]: + state_name = sn_split + break + + # If there's a mismatch between scope name and state name, then + # there must be multiple scopes pointing to the same state name, + # meaning some modules are shared. In such case, we can simply skip + # updating the current node because another later iteration will + # take care of this input node when the unique match between scope + # and state name occurs. To make sure this always happen, we should + # enforce the invariant that no placeholder node in the unflattened + # graph appears in inputs_to_state dict, which means all the extra + # input nodes have been handled. + if state_name is None: + continue + + inputs_to_state_of_scope[node] = state_name + + # Record name of remove inputs for return purpose. + inputs_removed: set[str] = set() + + for node, state_name in inputs_to_state_of_scope.items(): + if len(node.users) > 0: + attr_path = state_name[len(scope) :] + state_attr = _get_attr_via_attr_list(module, attr_path) + assert isinstance(state_attr, (torch.Tensor, torch.ScriptObject)) + + # Make sure the newly created get_attr node is placed after the last placeholder node + with graph.inserting_after(the_last_input): + new_node = graph.create_node("get_attr", ".".join(attr_path)) + + node.replace_all_uses_with(new_node, propagate_meta=True) + + graph.erase_node(node) + inputs_removed.add(node.name) + + if isinstance(module, InterpreterModule): + module.finalize() + + return {id(module): inputs_removed} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fft/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fft/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c19325cb03fa6d67e51f68f6499239c66ac024a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fft/__init__.py @@ -0,0 +1,1442 @@ +import torch +from torch._C import _add_docstr, _fft # type: ignore[attr-defined] +from torch._torch_docs import common_args, factory_common_args + + +__all__ = [ + "fft", + "ifft", + "fft2", + "ifft2", + "fftn", + "ifftn", + "rfft", + "irfft", + "rfft2", + "irfft2", + "rfftn", + "irfftn", + "hfft", + "ihfft", + "fftfreq", + "rfftfreq", + "fftshift", + "ifftshift", + "Tensor", +] + +Tensor = torch.Tensor + +# Note: This not only adds the doc strings for the spectral ops, but +# connects the torch.fft Python namespace to the torch._C._fft builtins. + +fft = _add_docstr( + _fft.fft_fft, + r""" +fft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional discrete Fourier transform of :attr:`input`. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: `X[i] = conj(X[-i])`. This function always returns both + the positive and negative frequency terms even though, for real inputs, the + negative frequencies are redundant. :func:`~torch.fft.rfft` returns the + more compact one-sided representation where only the positive frequencies + are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the FFT. + dim (int, optional): The dimension along which to take the one dimensional FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.ifft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(4) + >>> t + tensor([0, 1, 2, 3]) + >>> torch.fft.fft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + + >>> t = torch.tensor([0.+1.j, 2.+3.j, 4.+5.j, 6.+7.j]) + >>> torch.fft.fft(t) + tensor([12.+16.j, -8.+0.j, -4.-4.j, 0.-8.j]) +""".format(**common_args), +) + +ifft = _add_docstr( + _fft.fft_ifft, + r""" +ifft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional inverse discrete Fourier transform of :attr:`input`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the IFFT. + dim (int, optional): The dimension along which to take the one dimensional IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.fft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + >>> torch.fft.ifft(t) + tensor([0.+0.j, 1.+0.j, 2.+0.j, 3.+0.j]) +""".format(**common_args), +) + +fft2 = _add_docstr( + _fft.fft_fft2, + r""" +fft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2 dimensional discrete Fourier transform of :attr:`input`. +Equivalent to :func:`~torch.fft.fftn` but FFTs only the last two dimensions by default. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: ``X[i, j] = conj(X[-i, -j])``. This + function always returns all positive and negative frequency terms even + though, for real inputs, half of these values are redundant. + :func:`~torch.fft.rfft2` returns the more compact one-sided representation + where only the positive frequencies of the last dimension are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ifft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` + between the two transforms. This is required to make + :func:`~torch.fft.ifft2` the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> fft2 = torch.fft.fft2(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.fft2` + here is equivalent to two one-dimensional :func:`~torch.fft.fft` calls: + + >>> two_ffts = torch.fft.fft(torch.fft.fft(x, dim=0), dim=1) + >>> torch.testing.assert_close(fft2, two_ffts, check_stride=False) + +""".format(**common_args), +) + +ifft2 = _add_docstr( + _fft.fft_ifft2, + r""" +ifft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2 dimensional inverse discrete Fourier transform of :attr:`input`. +Equivalent to :func:`~torch.fft.ifftn` but IFFTs only the last two dimensions by default. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.fft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> ifft2 = torch.fft.ifft2(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.ifft2` + here is equivalent to two one-dimensional :func:`~torch.fft.ifft` calls: + + >>> two_iffts = torch.fft.ifft(torch.fft.ifft(x, dim=0), dim=1) + >>> torch.testing.assert_close(ifft2, two_iffts, check_stride=False) + +""".format(**common_args), +) + +fftn = _add_docstr( + _fft.fft_fftn, + r""" +fftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N dimensional discrete Fourier transform of :attr:`input`. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: ``X[i_1, ..., i_n] = conj(X[-i_1, ..., -i_n])``. This + function always returns all positive and negative frequency terms even + though, for real inputs, half of these values are redundant. + :func:`~torch.fft.rfftn` returns the more compact one-sided representation + where only the positive frequencies of the last dimension are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ifftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` + between the two transforms. This is required to make + :func:`~torch.fft.ifftn` the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> fftn = torch.fft.fftn(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.fftn` + here is equivalent to two one-dimensional :func:`~torch.fft.fft` calls: + + >>> two_ffts = torch.fft.fft(torch.fft.fft(x, dim=0), dim=1) + >>> torch.testing.assert_close(fftn, two_ffts, check_stride=False) + +""".format(**common_args), +) + +ifftn = _add_docstr( + _fft.fft_ifftn, + r""" +ifftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N dimensional inverse discrete Fourier transform of :attr:`input`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.fftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> ifftn = torch.fft.ifftn(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.ifftn` + here is equivalent to two one-dimensional :func:`~torch.fft.ifft` calls: + + >>> two_iffts = torch.fft.ifft(torch.fft.ifft(x, dim=0), dim=1) + >>> torch.testing.assert_close(ifftn, two_iffts, check_stride=False) + +""".format(**common_args), +) + +rfft = _add_docstr( + _fft.fft_rfft, + r""" +rfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional Fourier transform of real-valued :attr:`input`. + +The FFT of a real signal is Hermitian-symmetric, ``X[i] = conj(X[-i])`` so +the output contains only the positive frequencies below the Nyquist frequency. +To compute the full output, use :func:`~torch.fft.fft` + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the real input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the real FFT. + dim (int, optional): The dimension along which to take the one dimensional real FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.irfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(4) + >>> t + tensor([0, 1, 2, 3]) + >>> torch.fft.rfft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j]) + + Compare against the full output from :func:`~torch.fft.fft`: + + >>> torch.fft.fft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + + Notice that the symmetric element ``T[-1] == T[1].conj()`` is omitted. + At the Nyquist frequency ``T[-2] == T[2]`` is it's own symmetric pair, + and therefore must always be real-valued. +""".format(**common_args), +) + +irfft = _add_docstr( + _fft.fft_irfft, + r""" +irfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfft`. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfft`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`n`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal length :attr:`n`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + With default arguments, size of the transformed dimension should be (2^n + 1) as argument + `n` defaults to even output size = 2 * (transformed_dim_size - 1) + +Args: + input (Tensor): the input tensor representing a half-Hermitian signal + n (int, optional): Output signal length. This determines the length of the + output signal. If given, the input will either be zero-padded or trimmed to this + length before computing the real IFFT. + Defaults to even output: ``n=2*(input.size(dim) - 1)``. + dim (int, optional): The dimension along which to take the one dimensional real IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.rfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.linspace(0, 1, 5) + >>> t + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + >>> T = torch.fft.rfft(t) + >>> T + tensor([ 2.5000+0.0000j, -0.6250+0.8602j, -0.6250+0.2031j]) + + Without specifying the output length to :func:`~torch.fft.irfft`, the output + will not round-trip properly because the input is odd-length: + + >>> torch.fft.irfft(T) + tensor([0.1562, 0.3511, 0.7812, 1.2114]) + + So, it is recommended to always pass the signal length :attr:`n`: + + >>> roundtrip = torch.fft.irfft(T, t.numel()) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +rfft2 = _add_docstr( + _fft.fft_rfft2, + r""" +rfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional discrete Fourier transform of real :attr:`input`. +Equivalent to :func:`~torch.fft.rfftn` but FFTs only the last two dimensions by default. + +The FFT of a real signal is Hermitian-symmetric, ``X[i, j] = conj(X[-i, -j])``, +so the full :func:`~torch.fft.fft2` output contains redundant information. +:func:`~torch.fft.rfft2` instead omits the negative frequencies in the last +dimension. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.irfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft2` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 10) + >>> rfft2 = torch.fft.rfft2(t) + >>> rfft2.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.fft2`, we have all + elements up to the Nyquist frequency. + + >>> fft2 = torch.fft.fft2(t) + >>> torch.testing.assert_close(fft2[..., :6], rfft2, check_stride=False) + + The discrete Fourier transform is separable, so :func:`~torch.fft.rfft2` + here is equivalent to a combination of :func:`~torch.fft.fft` and + :func:`~torch.fft.rfft`: + + >>> two_ffts = torch.fft.fft(torch.fft.rfft(t, dim=1), dim=0) + >>> torch.testing.assert_close(rfft2, two_ffts, check_stride=False) + +""".format(**common_args), +) + +irfft2 = _add_docstr( + _fft.fft_irfft2, + r""" +irfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfft2`. +Equivalent to :func:`~torch.fft.irfftn` but IFFTs only the last two dimensions by default. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfft2`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.rfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 9) + >>> T = torch.fft.rfft2(t) + + Without specifying the output length to :func:`~torch.fft.irfft2`, the output + will not round-trip properly because the input is odd-length in the last + dimension: + + >>> torch.fft.irfft2(T).size() + torch.Size([10, 8]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.irfft2(T, t.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +rfftn = _add_docstr( + _fft.fft_rfftn, + r""" +rfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N-dimensional discrete Fourier transform of real :attr:`input`. + +The FFT of a real signal is Hermitian-symmetric, +``X[i_1, ..., i_n] = conj(X[-i_1, ..., -i_n])`` so the full +:func:`~torch.fft.fftn` output contains redundant information. +:func:`~torch.fft.rfftn` instead omits the negative frequencies in the +last dimension. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.irfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfftn` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 10) + >>> rfftn = torch.fft.rfftn(t) + >>> rfftn.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.fftn`, we have all + elements up to the Nyquist frequency. + + >>> fftn = torch.fft.fftn(t) + >>> torch.testing.assert_close(fftn[..., :6], rfftn, check_stride=False) + + The discrete Fourier transform is separable, so :func:`~torch.fft.rfftn` + here is equivalent to a combination of :func:`~torch.fft.fft` and + :func:`~torch.fft.rfft`: + + >>> two_ffts = torch.fft.fft(torch.fft.rfft(t, dim=1), dim=0) + >>> torch.testing.assert_close(rfftn, two_ffts, check_stride=False) + +""".format(**common_args), +) + +irfftn = _add_docstr( + _fft.fft_irfftn, + r""" +irfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfftn`. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfftn`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.rfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 9) + >>> T = torch.fft.rfftn(t) + + Without specifying the output length to :func:`~torch.fft.irfft`, the output + will not round-trip properly because the input is odd-length in the last + dimension: + + >>> torch.fft.irfftn(T).size() + torch.Size([10, 8]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.irfftn(T, t.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +hfft = _add_docstr( + _fft.fft_hfft, + r""" +hfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional discrete Fourier transform of a Hermitian +symmetric :attr:`input` signal. + +Note: + + :func:`~torch.fft.hfft`/:func:`~torch.fft.ihfft` are analogous to + :func:`~torch.fft.rfft`/:func:`~torch.fft.irfft`. The real FFT expects + a real signal in the time-domain and gives a Hermitian symmetry in the + frequency-domain. The Hermitian FFT is the opposite; Hermitian symmetric in + the time-domain and real-valued in the frequency-domain. For this reason, + special care needs to be taken with the length argument :attr:`n`, in the + same way as with :func:`~torch.fft.irfft`. + +Note: + Because the signal is Hermitian in the time-domain, the result will be + real in the frequency domain. Note that some input frequencies must be + real-valued to satisfy the Hermitian property. In these cases the imaginary + component will be ignored. For example, any imaginary component in + ``input[0]`` would result in one or more complex frequency terms which + cannot be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`n`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal length :attr:`n`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + With default arguments, size of the transformed dimension should be (2^n + 1) as argument + `n` defaults to even output size = 2 * (transformed_dim_size - 1) + +Args: + input (Tensor): the input tensor representing a half-Hermitian signal + n (int, optional): Output signal length. This determines the length of the + real output. If given, the input will either be zero-padded or trimmed to this + length before computing the Hermitian FFT. + Defaults to even output: ``n=2*(input.size(dim) - 1)``. + dim (int, optional): The dimension along which to take the one dimensional Hermitian FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.ihfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Taking a real-valued frequency signal and bringing it into the time domain + gives Hermitian symmetric output: + + >>> t = torch.linspace(0, 1, 5) + >>> t + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + >>> T = torch.fft.ifft(t) + >>> T + tensor([ 0.5000-0.0000j, -0.1250-0.1720j, -0.1250-0.0406j, -0.1250+0.0406j, + -0.1250+0.1720j]) + + Note that ``T[1] == T[-1].conj()`` and ``T[2] == T[-2].conj()`` is + redundant. We can thus compute the forward transform without considering + negative frequencies: + + >>> torch.fft.hfft(T[:3], n=5) + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + + Like with :func:`~torch.fft.irfft`, the output length must be given in order + to recover an even length output: + + >>> torch.fft.hfft(T[:3]) + tensor([0.1250, 0.2809, 0.6250, 0.9691]) +""".format(**common_args), +) + +ihfft = _add_docstr( + _fft.fft_ihfft, + r""" +ihfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.hfft`. + +:attr:`input` must be a real-valued signal, interpreted in the Fourier domain. +The IFFT of a real signal is Hermitian-symmetric, ``X[i] = conj(X[-i])``. +:func:`~torch.fft.ihfft` represents this in the one-sided form where only the +positive frequencies below the Nyquist frequency are included. To compute the +full output, use :func:`~torch.fft.ifft`. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the real input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the Hermitian IFFT. + dim (int, optional): The dimension along which to take the one dimensional Hermitian IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.hfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(5) + >>> t + tensor([0, 1, 2, 3, 4]) + >>> torch.fft.ihfft(t) + tensor([ 2.0000-0.0000j, -0.5000-0.6882j, -0.5000-0.1625j]) + + Compare against the full output from :func:`~torch.fft.ifft`: + + >>> torch.fft.ifft(t) + tensor([ 2.0000-0.0000j, -0.5000-0.6882j, -0.5000-0.1625j, -0.5000+0.1625j, + -0.5000+0.6882j]) +""".format(**common_args), +) + +hfft2 = _add_docstr( + _fft.fft_hfft2, + r""" +hfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional discrete Fourier transform of a Hermitian symmetric +:attr:`input` signal. Equivalent to :func:`~torch.fft.hfftn` but only +transforms the last two dimensions by default. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the time +domain. By the Hermitian property, the Fourier transform will be real-valued. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ihfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft2` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Starting from a real frequency-space signal, we can generate a + Hermitian-symmetric time-domain signal: + >>> T = torch.rand(10, 9) + >>> t = torch.fft.ihfft2(T) + + Without specifying the output length to :func:`~torch.fft.hfftn`, the + output will not round-trip properly because the input is odd-length in the + last dimension: + + >>> torch.fft.hfft2(t).size() + torch.Size([10, 10]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.hfft2(t, T.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.allclose(roundtrip, T) + True + +""".format(**common_args), +) + +ihfft2 = _add_docstr( + _fft.fft_ihfft2, + r""" +ihfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional inverse discrete Fourier transform of real +:attr:`input`. Equivalent to :func:`~torch.fft.ihfftn` but transforms only the +two last dimensions by default. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.hfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> T = torch.rand(10, 10) + >>> t = torch.fft.ihfft2(t) + >>> t.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.ifft2`, the + Hermitian time-space signal takes up only half the space. + + >>> fftn = torch.fft.ifft2(t) + >>> torch.allclose(fftn[..., :6], rfftn) + True + + The discrete Fourier transform is separable, so :func:`~torch.fft.ihfft2` + here is equivalent to a combination of :func:`~torch.fft.ifft` and + :func:`~torch.fft.ihfft`: + + >>> two_ffts = torch.fft.ifft(torch.fft.ihfft(t, dim=1), dim=0) + >>> torch.allclose(t, two_ffts) + True + +""".format(**common_args), +) + +hfftn = _add_docstr( + _fft.fft_hfftn, + r""" +hfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the n-dimensional discrete Fourier transform of a Hermitian symmetric +:attr:`input` signal. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the time +domain. By the Hermitian property, the Fourier transform will be real-valued. + +Note: + :func:`~torch.fft.hfftn`/:func:`~torch.fft.ihfftn` are analogous to + :func:`~torch.fft.rfftn`/:func:`~torch.fft.irfftn`. The real FFT expects + a real signal in the time-domain and gives Hermitian symmetry in the + frequency-domain. The Hermitian FFT is the opposite; Hermitian symmetric in + the time-domain and real-valued in the frequency-domain. For this reason, + special care needs to be taken with the shape argument :attr:`s`, in the + same way as with :func:`~torch.fft.irfftn`. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. It is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ihfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfftn` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Starting from a real frequency-space signal, we can generate a + Hermitian-symmetric time-domain signal: + >>> T = torch.rand(10, 9) + >>> t = torch.fft.ihfftn(T) + + Without specifying the output length to :func:`~torch.fft.hfftn`, the + output will not round-trip properly because the input is odd-length in the + last dimension: + + >>> torch.fft.hfftn(t).size() + torch.Size([10, 10]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.hfftn(t, T.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.allclose(roundtrip, T) + True + +""".format(**common_args), +) + +ihfftn = _add_docstr( + _fft.fft_ihfftn, + r""" +ihfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N-dimensional inverse discrete Fourier transform of real :attr:`input`. + +:attr:`input` must be a real-valued signal, interpreted in the Fourier domain. +The n-dimensional IFFT of a real signal is Hermitian-symmetric, +``X[i, j, ...] = conj(X[-i, -j, ...])``. :func:`~torch.fft.ihfftn` represents +this in the one-sided form where only the positive frequencies below the +Nyquist frequency are included in the last signal dimension. To compute the +full output, use :func:`~torch.fft.ifftn`. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.hfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> T = torch.rand(10, 10) + >>> ihfftn = torch.fft.ihfftn(T) + >>> ihfftn.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.ifftn`, we have all + elements up to the Nyquist frequency. + + >>> ifftn = torch.fft.ifftn(t) + >>> torch.allclose(ifftn[..., :6], ihfftn) + True + + The discrete Fourier transform is separable, so :func:`~torch.fft.ihfftn` + here is equivalent to a combination of :func:`~torch.fft.ihfft` and + :func:`~torch.fft.ifft`: + + >>> two_iffts = torch.fft.ifft(torch.fft.ihfft(t, dim=1), dim=0) + >>> torch.allclose(ihfftn, two_iffts) + True + +""".format(**common_args), +) + +fftfreq = _add_docstr( + _fft.fft_fftfreq, + r""" +fftfreq(n, d=1.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + +Computes the discrete Fourier Transform sample frequencies for a signal of size :attr:`n`. + +Note: + By convention, :func:`~torch.fft.fft` returns positive frequency terms + first, followed by the negative frequencies in reverse order, so that + ``f[-i]`` for all :math:`0 < i \leq n/2`` in Python gives the negative + frequency terms. For an FFT of length :attr:`n` and with inputs spaced in + length unit :attr:`d`, the frequencies are:: + + f = [0, 1, ..., (n - 1) // 2, -(n // 2), ..., -1] / (d * n) + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. :func:`~torch.fft.fftfreq` follows NumPy's + convention of taking it to be negative. + +Args: + n (int): the FFT length + d (float, optional): The sampling length scale. + The spacing between individual samples of the FFT input. + The default assumes unit spacing, dividing that result by the actual + spacing gives the result in physical frequency units. + +Keyword Args: + {out} + {dtype} + {layout} + {device} + {requires_grad} + +Example: + + >>> torch.fft.fftfreq(5) + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + + For even input, we can see the Nyquist frequency at ``f[2]`` is given as + negative: + + >>> torch.fft.fftfreq(4) + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + +""".format(**factory_common_args), +) + +rfftfreq = _add_docstr( + _fft.fft_rfftfreq, + r""" +rfftfreq(n, d=1.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + +Computes the sample frequencies for :func:`~torch.fft.rfft` with a signal of size :attr:`n`. + +Note: + :func:`~torch.fft.rfft` returns Hermitian one-sided output, so only the + positive frequency terms are returned. For a real FFT of length :attr:`n` + and with inputs spaced in length unit :attr:`d`, the frequencies are:: + + f = torch.arange((n + 1) // 2) / (d * n) + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. Unlike :func:`~torch.fft.fftfreq`, + :func:`~torch.fft.rfftfreq` always returns it as positive. + +Args: + n (int): the real FFT length + d (float, optional): The sampling length scale. + The spacing between individual samples of the FFT input. + The default assumes unit spacing, dividing that result by the actual + spacing gives the result in physical frequency units. + +Keyword Args: + {out} + {dtype} + {layout} + {device} + {requires_grad} + +Example: + + >>> torch.fft.rfftfreq(5) + tensor([0.0000, 0.2000, 0.4000]) + + >>> torch.fft.rfftfreq(4) + tensor([0.0000, 0.2500, 0.5000]) + + Compared to the output from :func:`~torch.fft.fftfreq`, we see that the + Nyquist frequency at ``f[2]`` has changed sign: + >>> torch.fft.fftfreq(4) + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + +""".format(**factory_common_args), +) + +fftshift = _add_docstr( + _fft.fft_fftshift, + r""" +fftshift(input, dim=None) -> Tensor + +Reorders n-dimensional FFT data, as provided by :func:`~torch.fft.fftn`, to have +negative frequency terms first. + +This performs a periodic shift of n-dimensional data such that the origin +``(0, ..., 0)`` is moved to the center of the tensor. Specifically, to +``input.shape[dim] // 2`` in each selected dimension. + +Note: + By convention, the FFT returns positive frequency terms first, followed by + the negative frequencies in reverse order, so that ``f[-i]`` for all + :math:`0 < i \leq n/2` in Python gives the negative frequency terms. + :func:`~torch.fft.fftshift` rearranges all frequencies into ascending order + from negative to positive with the zero-frequency term in the center. + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. :func:`~torch.fft.fftshift` always puts the + Nyquist term at the 0-index. This is the same convention used by + :func:`~torch.fft.fftfreq`. + +Args: + input (Tensor): the tensor in FFT order + dim (int, Tuple[int], optional): The dimensions to rearrange. + Only dimensions specified here will be rearranged, any other dimensions + will be left in their original order. + Default: All dimensions of :attr:`input`. + +Example: + + >>> f = torch.fft.fftfreq(4) + >>> f + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + + >>> torch.fft.fftshift(f) + tensor([-0.5000, -0.2500, 0.0000, 0.2500]) + + Also notice that the Nyquist frequency term at ``f[2]`` was moved to the + beginning of the tensor. + + This also works for multi-dimensional transforms: + + >>> x = torch.fft.fftfreq(5, d=1/5) + 0.1 * torch.fft.fftfreq(5, d=1/5).unsqueeze(1) + >>> x + tensor([[ 0.0000, 1.0000, 2.0000, -2.0000, -1.0000], + [ 0.1000, 1.1000, 2.1000, -1.9000, -0.9000], + [ 0.2000, 1.2000, 2.2000, -1.8000, -0.8000], + [-0.2000, 0.8000, 1.8000, -2.2000, -1.2000], + [-0.1000, 0.9000, 1.9000, -2.1000, -1.1000]]) + + >>> torch.fft.fftshift(x) + tensor([[-2.2000, -1.2000, -0.2000, 0.8000, 1.8000], + [-2.1000, -1.1000, -0.1000, 0.9000, 1.9000], + [-2.0000, -1.0000, 0.0000, 1.0000, 2.0000], + [-1.9000, -0.9000, 0.1000, 1.1000, 2.1000], + [-1.8000, -0.8000, 0.2000, 1.2000, 2.2000]]) + + :func:`~torch.fft.fftshift` can also be useful for spatial data. If our + data is defined on a centered grid (``[-(N//2), (N-1)//2]``) then we can + use the standard FFT defined on an uncentered grid (``[0, N)``) by first + applying an :func:`~torch.fft.ifftshift`. + + >>> x_centered = torch.arange(-5, 5) + >>> x_uncentered = torch.fft.ifftshift(x_centered) + >>> fft_uncentered = torch.fft.fft(x_uncentered) + + Similarly, we can convert the frequency domain components to centered + convention by applying :func:`~torch.fft.fftshift`. + + >>> fft_centered = torch.fft.fftshift(fft_uncentered) + + The inverse transform, from centered Fourier space back to centered spatial + data, can be performed by applying the inverse shifts in reverse order: + + >>> x_centered_2 = torch.fft.fftshift(torch.fft.ifft(torch.fft.ifftshift(fft_centered))) + >>> torch.testing.assert_close(x_centered.to(torch.complex64), x_centered_2, check_stride=False) + + +""", +) + +ifftshift = _add_docstr( + _fft.fft_ifftshift, + r""" +ifftshift(input, dim=None) -> Tensor + +Inverse of :func:`~torch.fft.fftshift`. + +Args: + input (Tensor): the tensor in FFT order + dim (int, Tuple[int], optional): The dimensions to rearrange. + Only dimensions specified here will be rearranged, any other dimensions + will be left in their original order. + Default: All dimensions of :attr:`input`. + +Example: + + >>> f = torch.fft.fftfreq(5) + >>> f + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + + A round-trip through :func:`~torch.fft.fftshift` and + :func:`~torch.fft.ifftshift` gives the same result: + + >>> shifted = torch.fft.fftshift(f) + >>> torch.fft.ifftshift(shifted) + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + +""", +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/func/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/func/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6afd8ad0c45a77d545f52f596d2e75c2ddef20e9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/func/__init__.py @@ -0,0 +1,31 @@ +from torch._functorch.apis import grad, grad_and_value, vmap +from torch._functorch.batch_norm_replacement import replace_all_batch_norm_modules_ +from torch._functorch.eager_transforms import ( + debug_unwrap, + functionalize, + hessian, + jacfwd, + jacrev, + jvp, + linearize, + vjp, +) +from torch._functorch.functional_call import functional_call, stack_module_state + + +__all__ = [ + "grad", + "grad_and_value", + "vmap", + "replace_all_batch_norm_modules_", + "functionalize", + "hessian", + "jacfwd", + "jacrev", + "jvp", + "linearize", + "vjp", + "functional_call", + "stack_module_state", + "debug_unwrap", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/futures/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/futures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d698c6fd8bba046cf460f9834ee6d3542a91610 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/futures/__init__.py @@ -0,0 +1,336 @@ +# mypy: allow-untyped-defs +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +from typing import cast, Generic, Optional, TYPE_CHECKING, TypeVar, Union + +import torch + + +if TYPE_CHECKING: + from collections.abc import Callable + + +__all__ = ["Future", "collect_all", "wait_all"] + + +T = TypeVar("T") +S = TypeVar("S") + + +class Future(torch._C.Future, Generic[T]): + r""" + Wrapper around a ``torch._C.Future`` which encapsulates an asynchronous + execution of a callable, e.g. :meth:`~torch.distributed.rpc.rpc_async`. It + also exposes a set of APIs to add callback functions and set results. + + .. warning:: GPU support is a beta feature, subject to changes. + """ + + def __init__( + self, *, devices: Optional[list[Union[int, str, torch.device]]] = None + ): + r""" + Create an empty unset ``Future``. If the future is intended to hold + values containing CUDA tensors, (a superset of) their CUDA devices must + be specified at construction. (This is only supported if + ``torch.cuda.is_available()`` returns ``True``). This is needed to + ensure proper CUDA stream synchronization. The child futures, returned + by the ``then`` method, will inherit these devices. + + Args: + devices(``List[Union[int, str, torch.device]]``, optional): the set + of devices on which tensors contained in this future's value are + allowed to reside and on which callbacks are allowed to operate. + """ + if devices is None: + devices = [] + super().__init__([torch.device(d) for d in devices]) + + def done(self) -> bool: + r""" + Return ``True`` if this ``Future`` is done. A ``Future`` is done if it + has a result or an exception. + + If the value contains tensors that reside on GPUs, ``Future.done()`` + will return ``True`` even if the asynchronous kernels that are + populating those tensors haven't yet completed running on the device, + because at such stage the result is already usable, provided one + performs the appropriate synchronizations (see :meth:`wait`). + """ + return super().done() + + def wait(self) -> T: + r""" + Block until the value of this ``Future`` is ready. + + If the value contains tensors that reside on GPUs, then an additional + synchronization is performed with the kernels (executing on the device) + which may be asynchronously populating those tensors. Such sync is + non-blocking, which means that ``wait()`` will insert the necessary + instructions in the current streams to ensure that further operations + enqueued on those streams will be properly scheduled after the async + kernels but, once that is done, ``wait()`` will return, even if those + kernels are still running. No further synchronization is required when + accessing and using the values, as long as one doesn't change streams. + + Returns: + The value held by this ``Future``. If the function (callback or RPC) + creating the value has thrown an error, this ``wait`` method will + also throw an error. + """ + return super().wait() + + def value(self) -> T: + r""" + Obtain the value of an already-completed future. + + This method should only be called after a call to :meth:`wait` has + completed, or inside a callback function passed to :meth:`then`. In + other cases this ``Future`` may not yet hold a value and calling + ``value()`` could fail. + + If the value contains tensors that reside on GPUs, then this method will + *not* perform any additional synchronization. This should be done + beforehand, separately, through a call to :meth:`wait` (except within + callbacks, for which it's already being taken care of by :meth:`then`). + + Returns: + The value held by this ``Future``. If the function (callback or RPC) + creating the value has thrown an error, this ``value()`` method will + also throw an error. + """ + return super().value() + + def then(self, callback: Callable[[Future[T]], S]) -> Future[S]: + r""" + Append the given callback function to this ``Future``, which will be run + when the ``Future`` is completed. Multiple callbacks can be added to + the same ``Future``, but the order in which they will be executed cannot + be guaranteed (to enforce a certain order consider chaining: + ``fut.then(cb1).then(cb2)``). The callback must take one argument, which + is the reference to this ``Future``. The callback function can use the + :meth:`value` method to get the value. Note that if this ``Future`` is + already completed, the given callback will be run immediately inline. + + If the ``Future``'s value contains tensors that reside on GPUs, the + callback might be invoked while the async kernels that are populating + those tensors haven't yet finished executing on the device. However, the + callback will be invoked with some dedicated streams set as current + (fetched from a global pool) which will be synchronized with those + kernels. Hence any operation performed by the callback on these tensors + will be scheduled on the device after the kernels complete. In other + words, as long as the callback doesn't switch streams, it can safely + manipulate the result without any additional synchronization. This is + similar to the non-blocking behavior of :meth:`wait`. + + Similarly, if the callback returns a value that contains tensors that + reside on a GPU, it can do so even if the kernels that are producing + these tensors are still running on the device, as long as the callback + didn't change streams during its execution. If one wants to change + streams, one must be careful to re-synchronize them with the original + streams, that is, those that were current when the callback was invoked. + + Args: + callback(``Callable``): a ``Callable`` that takes this ``Future`` as + the only argument. + + Returns: + A new ``Future`` object that holds the return value of the + ``callback`` and will be marked as completed when the given + ``callback`` finishes. + + .. note:: Note that if the callback function throws, either + through the original future being completed with an exception and + calling ``fut.wait()``, or through other code in the callback, the + future returned by ``then`` will be marked appropriately with the + encountered error. However, if this callback later completes + additional futures, those futures are not marked as completed with + an error and the user is responsible for handling completion/waiting + on those futures independently. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> def callback(fut): + ... print(f"RPC return value is {fut.wait()}.") + >>> fut = torch.futures.Future() + >>> # The inserted callback will print the return value when + >>> # receiving the response from "worker1" + >>> cb_fut = fut.then(callback) + >>> chain_cb_fut = cb_fut.then( + ... lambda x : print(f"Chained cb done. {x.wait()}") + ... ) + >>> fut.set_result(5) + RPC return value is 5. + Chained cb done. None + """ + return cast(Future[S], super().then(callback)) + + def add_done_callback(self, callback: Callable[[Future[T]], None]) -> None: + r""" + Append the given callback function to this ``Future``, which will be run + when the ``Future`` is completed. Multiple callbacks can be added to + the same ``Future``, but the order in which they will be executed cannot + be guaranteed. The callback must take one argument, which is the + reference to this ``Future``. The callback function can use the + :meth:`value` method to get the value. Note that if this ``Future`` is + already completed, the given callback will be run inline. + + We recommend that you use the :meth:`then` method as it provides a way + to synchronize after your callback has completed. ``add_done_callback`` + can be cheaper if your callback does not return anything. But both + :meth:`then` and ``add_done_callback`` use the same callback + registration API under the hood. + + With respect to GPU tensors, this method behaves in the same way as + :meth:`then`. + + Args: + callback(``Future``): a ``Callable`` that takes in one argument, + which is the reference to this ``Future``. + + .. note:: Note that if the callback function throws, either + through the original future being completed with an exception and + calling ``fut.wait()``, or through other code in the callback, + error handling must be carefully taken care of. For example, if + this callback later completes additional futures, those futures are + not marked as completed with an error and the user is responsible + for handling completion/waiting on those futures independently. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> def callback(fut): + ... print("This will run after the future has finished.") + ... print(fut.wait()) + >>> fut = torch.futures.Future() + >>> fut.add_done_callback(callback) + >>> fut.set_result(5) + This will run after the future has finished. + 5 + """ + super().add_done_callback(callback) + + def set_result(self, result: T) -> None: + r""" + Set the result for this ``Future``, which will mark this ``Future`` as + completed and trigger all attached callbacks. Note that a ``Future`` + cannot be marked completed twice. + + If the result contains tensors that reside on GPUs, this method can be + called even if the asynchronous kernels that are populating those + tensors haven't yet completed running on the device, provided that the + streams on which those kernels were enqueued are set as the current ones + when this method is called. Put simply, it's safe to call this method + immediately after launching those kernels, without any additional + synchronization, as long as one doesn't change streams in between. This + method will record events on all the relevant current streams and will + use them to ensure proper scheduling for all the consumers of this + ``Future``. + + Args: + result (object): the result object of this ``Future``. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> import threading + >>> import time + >>> def slow_set_future(fut, value): + ... time.sleep(0.5) + ... fut.set_result(value) + >>> fut = torch.futures.Future() + >>> t = threading.Thread( + ... target=slow_set_future, + ... args=(fut, torch.ones(2) * 3) + ... ) + >>> t.start() + >>> print(fut.wait()) + tensor([3., 3.]) + >>> t.join() + """ + super().set_result(result) + + def set_exception(self, result: T) -> None: + r""" + Set an exception for this ``Future``, which will mark this ``Future`` as + completed with an error and trigger all attached callbacks. Note that + when calling wait()/value() on this ``Future``, the exception set here + will be raised inline. + + Args: + result (BaseException): the exception for this ``Future``. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> fut = torch.futures.Future() + >>> fut.set_exception(ValueError("foo")) + >>> fut.wait() + Traceback (most recent call last): + ... + ValueError: foo + """ + assert isinstance(result, Exception), ( + f"{result} is of type {type(result)}, not an Exception." + ) + + def raise_error(fut_result): + raise fut_result + + super()._set_unwrap_func(raise_error) + self.set_result(result) # type: ignore[arg-type] + + +def collect_all(futures: list[Future]) -> Future[list[Future]]: + r""" + Collects the provided :class:`~torch.futures.Future` objects into a single + combined :class:`~torch.futures.Future` that is completed when all of the + sub-futures are completed. + + Args: + futures (list): a list of :class:`~torch.futures.Future` objects. + + Returns: + Returns a :class:`~torch.futures.Future` object to a list of the passed + in Futures. + + Example:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> fut0 = torch.futures.Future() + >>> fut1 = torch.futures.Future() + >>> fut = torch.futures.collect_all([fut0, fut1]) + >>> fut0.set_result(0) + >>> fut1.set_result(1) + >>> fut_list = fut.wait() + >>> print(f"fut0 result = {fut_list[0].wait()}") + fut0 result = 0 + >>> print(f"fut1 result = {fut_list[1].wait()}") + fut1 result = 1 + """ + return cast( + Future[list[Future]], + torch._C._collect_all(cast(list[torch._C.Future], futures)), + ) + + +def wait_all(futures: list[Future]) -> list: + r""" + Waits for all provided futures to be complete, and returns + the list of completed values. If any of the futures encounters an error, + the method will exit early and report the error not waiting for other + futures to complete. + + Args: + futures (list): a list of :class:`~torch.futures.Future` object. + + Returns: + A list of the completed :class:`~torch.futures.Future` results. This + method will throw an error if ``wait`` on any + :class:`~torch.futures.Future` throws. + """ + return [ + fut.wait() + for fut in torch._C._collect_all(cast(list[torch._C.Future], futures)).wait() + ] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..75f0016a8dafc1a730544b994b9d5049ac1b0ec3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/__init__.py @@ -0,0 +1,116 @@ +r''' +FX is a toolkit for developers to use to transform ``nn.Module`` +instances. FX consists of three main components: a **symbolic tracer,** +an **intermediate representation**, and **Python code generation**. A +demonstration of these components in action: + +:: + + import torch + + + # Simple module for demonstration + class MyModule(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.param = torch.nn.Parameter(torch.rand(3, 4)) + self.linear = torch.nn.Linear(4, 5) + + def forward(self, x): + return self.linear(x + self.param).clamp(min=0.0, max=1.0) + + + module = MyModule() + + from torch.fx import symbolic_trace + + # Symbolic tracing frontend - captures the semantics of the module + symbolic_traced: torch.fx.GraphModule = symbolic_trace(module) + + # High-level intermediate representation (IR) - Graph representation + print(symbolic_traced.graph) + """ + graph(): + %x : [num_users=1] = placeholder[target=x] + %param : [num_users=1] = get_attr[target=param] + %add : [num_users=1] = call_function[target=operator.add](args = (%x, %param), kwargs = {}) + %linear : [num_users=1] = call_module[target=linear](args = (%add,), kwargs = {}) + %clamp : [num_users=1] = call_method[target=clamp](args = (%linear,), kwargs = {min: 0.0, max: 1.0}) + return clamp + """ + + # Code generation - valid Python code + print(symbolic_traced.code) + """ + def forward(self, x): + param = self.param + add = x + param; x = param = None + linear = self.linear(add); add = None + clamp = linear.clamp(min = 0.0, max = 1.0); linear = None + return clamp + """ + +The **symbolic tracer** performs "symbolic execution" of the Python +code. It feeds fake values, called Proxies, through the code. Operations +on these Proxies are recorded. More information about symbolic tracing +can be found in the :func:`symbolic_trace` and :class:`Tracer` +documentation. + +The **intermediate representation** is the container for the operations +that were recorded during symbolic tracing. It consists of a list of +Nodes that represent function inputs, callsites (to functions, methods, +or :class:`torch.nn.Module` instances), and return values. More information +about the IR can be found in the documentation for :class:`Graph`. The +IR is the format on which transformations are applied. + +**Python code generation** is what makes FX a Python-to-Python (or +Module-to-Module) transformation toolkit. For each Graph IR, we can +create valid Python code matching the Graph's semantics. This +functionality is wrapped up in :class:`GraphModule`, which is a +:class:`torch.nn.Module` instance that holds a :class:`Graph` as well as a +``forward`` method generated from the Graph. + +Taken together, this pipeline of components (symbolic tracing -> +intermediate representation -> transforms -> Python code generation) +constitutes the Python-to-Python transformation pipeline of FX. In +addition, these components can be used separately. For example, +symbolic tracing can be used in isolation to capture a form of +the code for analysis (and not transformation) purposes. Code +generation can be used for programmatically generating models, for +example from a config file. There are many uses for FX! + +Several example transformations can be found at the +`examples `__ +repository. +''' + +from torch.fx import immutable_collections +from torch.fx._symbolic_trace import ( # noqa: F401 + PH, + ProxyableClassMeta, + symbolic_trace, + Tracer, + wrap, +) +from torch.fx.graph import CodeGen, Graph # noqa: F401 +from torch.fx.graph_module import GraphModule +from torch.fx.interpreter import Interpreter, Transformer +from torch.fx.node import has_side_effect, map_arg, Node +from torch.fx.proxy import Proxy +from torch.fx.subgraph_rewriter import replace_pattern + + +__all__ = [ + "symbolic_trace", + "Tracer", + "wrap", + "Graph", + "GraphModule", + "Interpreter", + "Transformer", + "Node", + "Proxy", + "replace_pattern", + "has_side_effect", + "map_arg", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_compatibility.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_compatibility.py new file mode 100644 index 0000000000000000000000000000000000000000..5133d9d02f7443aae402af53bd51201482bdd017 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_compatibility.py @@ -0,0 +1,40 @@ +import textwrap +from collections.abc import Callable +from typing import Any, TypeVar + + +_BACK_COMPAT_OBJECTS: dict[Any, None] = {} +_MARKED_WITH_COMPATIBILITY: dict[Any, None] = {} + + +_T = TypeVar("_T") + + +def compatibility(is_backward_compatible: bool) -> Callable[[_T], _T]: + if is_backward_compatible: + + def mark_back_compat(fn: _T) -> _T: + docstring = textwrap.dedent(getattr(fn, "__doc__", None) or "") + docstring += """ +.. note:: + Backwards-compatibility for this API is guaranteed. +""" + fn.__doc__ = docstring + _BACK_COMPAT_OBJECTS.setdefault(fn) + _MARKED_WITH_COMPATIBILITY.setdefault(fn) + return fn + + return mark_back_compat + else: + + def mark_not_back_compat(fn: _T) -> _T: + docstring = textwrap.dedent(getattr(fn, "__doc__", None) or "") + docstring += """ +.. warning:: + This API is experimental and is *NOT* backward-compatible. +""" + fn.__doc__ = docstring + _MARKED_WITH_COMPATIBILITY.setdefault(fn) + return fn + + return mark_not_back_compat diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_graph_pickler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_graph_pickler.py new file mode 100644 index 0000000000000000000000000000000000000000..66342384fcc724907ea1162d2c5b99acd7362975 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_graph_pickler.py @@ -0,0 +1,647 @@ +import dataclasses +import importlib +import io +import pickle +from abc import abstractmethod +from collections.abc import Callable +from typing import Any, NewType, Optional, TypeVar, Union +from typing_extensions import override, Self + +import torch +import torch.utils._pytree as pytree +from torch._guards import TracingContext +from torch._inductor.standalone_compile import AOTCompiledArtifact +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode, Tensor +from torch._subclasses.meta_utils import ( + MetaConverter, + MetaTensorDesc, + MetaTensorDescriber, +) +from torch.fx.experimental.sym_node import SymNode +from torch.fx.experimental.symbolic_shapes import ShapeEnv +from torch.utils._mode_utils import no_dispatch + + +_SymNodeT = TypeVar("_SymNodeT", torch.SymInt, torch.SymFloat) + + +def _ops_filter_safe(name: str) -> bool: + """ + An ops filter which allows pickle-safe ops. Pickle-safe ops are built-in + ones where it will be possible to unpickle on any machine which has PyTorch. + """ + # TODO: This list is pretty pessimistic right now. What's the full list? + return name.startswith( + ( + "torch.ops.aten", + "torch.ops.fbgemm", + ) + ) + + +@dataclasses.dataclass +class Options: + # A filter for which ops will cause the pickler to raise a + # BypassFxGraphCache exception. If None then all ops are allowed. + ops_filter: Optional[Callable[[str], bool]] = _ops_filter_safe + + +class GraphPickler(pickle.Pickler): + """ + GraphPickler is a Pickler which helps pickling fx graph - in particular + GraphModule. + """ + + def __init__(self, file: io.BytesIO, options: Optional[Options] = None) -> None: + super().__init__(file) + self.options = options or Options() + + # This abomination is so we can pass external decoding state to the + # unpickler functions. We serialize _unpickle_state as a persistent + # external item and when we deserialize it we return the common state + # object. + self._unpickle_state = _UnpickleStateToken(object()) + + # This is used to describe tensors. It needs to be common across the + # pickle so that duplicates and views are properly handled. + self._meta_tensor_describer = MetaTensorDescriber(copy_data=False) + + @override + # pyrefly: ignore [bad-override] + def reducer_override( + self, obj: object + ) -> tuple[Callable[..., Any], tuple[Any, ...]]: + # This function is supposed to return either NotImplemented (meaning to + # do the default pickle behavior) or a pair of (unpickle callable, data + # to pass to unpickle). + + # We could instead teach individual classes how to pickle themselves but + # that has a few problems: + # + # 1. If we have some special needs (maybe for this use-case we don't + # want to fully serialize every field) then we're adding private + # details to a public interface. + # + # 2. If we need to have some common shared data (such as a + # FakeTensorMode) which is passed to each value it's harder to + # support. + + # These are the types that need special handling. See the individual + # *PickleData classes for details on pickling that particular type. + if isinstance(obj, FakeTensor): + return _TensorPickleData.reduce_helper(self, obj) + elif isinstance(obj, torch.fx.GraphModule): + return _GraphModulePickleData.reduce_helper(self, obj) + elif isinstance(obj, (torch._ops.OperatorBase, torch._ops.OpOverloadPacket)): + return _OpPickleData.reduce_helper(self, obj) + elif isinstance(obj, ShapeEnv): + return _ShapeEnvPickleData.reduce_helper(self, obj) + elif isinstance(obj, torch.SymInt): + return _SymNodePickleData.reduce_helper(self, obj) + elif isinstance(obj, torch._guards.TracingContext): + return _TracingContextPickleData.reduce_helper(self, obj) + else: + # We should never get a raw Node! + assert not isinstance(obj, torch.fx.Node) + if reduce := _TorchNumpyPickleData.reduce_helper(self, obj): + return reduce + + # returning `NotImplemented` causes pickle to revert to the default + # behavior for this object. + return NotImplemented + + @override + def persistent_id(self, obj: object) -> Optional[str]: + if obj is self._unpickle_state: + return "unpickle_state" + else: + return None + + @classmethod + def dumps(cls, obj: object, options: Optional[Options] = None) -> bytes: + """ + Pickle an object. + """ + with io.BytesIO() as stream: + pickler = cls(stream, options) + pickler.dump(obj) + return stream.getvalue() + + @staticmethod + def loads(data: bytes, fake_mode: FakeTensorMode) -> object: + """ + Unpickle an object. + """ + state = _UnpickleState(fake_mode) + with io.BytesIO(data) as stream: + unpickler = _GraphUnpickler(stream, state) + return unpickler.load() + + +class _UnpickleState: + def __init__(self, fake_mode: FakeTensorMode) -> None: + self.fake_mode = fake_mode + self.meta_converter: MetaConverter[FakeTensor] = MetaConverter() + + +# This token is passed when pickling to indicate that we want to use the +# unpickler's _UnpickleState as a parameter in that position. +_UnpickleStateToken = NewType("_UnpickleStateToken", object) + + +class _GraphUnpickler(pickle.Unpickler): + def __init__(self, stream: io.BytesIO, unpickle_state: _UnpickleState) -> None: + super().__init__(stream) + self._unpickle_state = unpickle_state + + @override + def persistent_load(self, pid: object) -> object: + if pid == "unpickle_state": + return self._unpickle_state + else: + raise pickle.UnpicklingError("Invalid persistent ID") + + +class _ShapeEnvPickleData: + data: dict[str, object] + + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, obj: ShapeEnv + ) -> tuple[ + Callable[[Self, _UnpickleState], ShapeEnv], tuple[Self, _UnpickleStateToken] + ]: + return cls.unpickle, (cls(obj), pickler._unpickle_state) + + def __init__(self, env: ShapeEnv) -> None: + # In theory pickle should recognize that a given ShapeEnv was already + # pickled and reuse the resulting _ShapeEnvPickleData (so two objects + # pointing at the same ShapeEnv get the same ShapeEnv out). + assert not env._translation_validation_enabled + self.data = env.__dict__.copy() + del self.data["tracked_fakes"] + del self.data["fake_tensor_cache"] + + def unpickle(self, unpickle_state: _UnpickleState) -> ShapeEnv: + # Fill in the existing ShapeEnv rather than creating a new one + assert unpickle_state.fake_mode + assert unpickle_state.fake_mode.shape_env + + for k, v in self.data.items(): + setattr(unpickle_state.fake_mode.shape_env, k, v) + + return unpickle_state.fake_mode.shape_env + + +class _SymNodePickleData: + @classmethod + def reduce_helper( + cls, + pickler: GraphPickler, + obj: _SymNodeT, + ) -> tuple[ + Callable[[Self, _UnpickleState], _SymNodeT], tuple[Self, _UnpickleStateToken] + ]: + args = (cls(obj.node), pickler._unpickle_state) + if isinstance(obj, torch.SymInt): + # pyrefly: ignore [bad-return] + return _SymNodePickleData.unpickle_sym_int, args + else: + raise NotImplementedError(f"Unhandled SymNode type {type(obj)}") + + def __init__(self, node: SymNode) -> None: + self.expr = node._expr + self.shape_env = node.shape_env + self.pytype = node.pytype + self.hint = node._hint + + def _to_sym_node(self) -> SymNode: + assert self.shape_env is not None + return SymNode(self.expr, self.shape_env, self.pytype, self.hint) + + def unpickle_sym_int(self, unpickle_state: _UnpickleState) -> torch.SymInt: + return torch.SymInt(self._to_sym_node()) + + +class _TensorPickleData: + metadata: MetaTensorDesc[FakeTensor] + + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, obj: FakeTensor + ) -> tuple[ + Callable[[Self, _UnpickleState], FakeTensor], tuple[Self, _UnpickleStateToken] + ]: + return cls.unpickle, ( + cls(pickler._meta_tensor_describer, obj), + pickler._unpickle_state, + ) + + def __init__(self, describer: MetaTensorDescriber, t: Tensor) -> None: + # THINGS TO WORRY ABOUT: + # 1. Need to make sure that two tensors with the same id end up with the + # same id on the other side of the wire. + + metadata = describer.describe_tensor(t) + + # view_func is fine if it's either None or a _FakeTensorViewFunc. A + # custom one (which is basically a lambda) can't be serialized. + assert not metadata.view_func or isinstance( + metadata.view_func, torch._subclasses.meta_utils._FakeTensorViewFunc + ) + self.metadata = dataclasses.replace(metadata, fake_mode=None) + + # Some debugging/verification + for k in MetaTensorDesc._UNSERIALIZABLE: + if k in ("fake_mode", "view_func"): + continue + assert getattr(self.metadata, k) is None, ( + f"not None: {k}: {getattr(self.metadata, k)}" + ) + + def unpickle(self, unpickle_state: _UnpickleState) -> FakeTensor: + # TODO: make common w/ _output_from_cache_entry() in fake_tensor.py? + metadata = dataclasses.replace( + self.metadata, + fake_mode=unpickle_state.fake_mode, + ) + + # also need to set the fake_mode on the base of a tensor if it's a view + if metadata.is_view and metadata.base is not None: + new_base = dataclasses.replace( + metadata.base, + fake_mode=unpickle_state.fake_mode, + ) + metadata = dataclasses.replace(metadata, base=new_base) + + def with_fake( + make_meta_t: Callable[[], torch.Tensor], device: Union[torch.device, str] + ) -> FakeTensor: + with no_dispatch(): + return FakeTensor( + unpickle_state.fake_mode, + make_meta_t(), + # pyrefly: ignore [bad-argument-type] + device, + ) + + return unpickle_state.meta_converter.meta_tensor( + metadata, + unpickle_state.fake_mode.shape_env, + with_fake, + None, + None, + ) + + +class _TorchNumpyPickleData: + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, obj: object + ) -> Optional[ + tuple[ + Callable[[Self, _UnpickleState], object], tuple[Self, _UnpickleStateToken] + ] + ]: + if data := cls.from_object(obj): + return (cls.unpickle, (data, pickler._unpickle_state)) + else: + return None + + def __init__(self, mod: str, name: str) -> None: + self.mod = mod + self.name = name + + def unpickle(self, unpickle_state: _UnpickleState) -> Callable[..., object]: + np = getattr(importlib.import_module(self.mod), self.name) + return torch._dynamo.variables.misc.get_np_to_tnp_map()[np] + + @classmethod + def from_object(cls, tnp: object) -> Optional[Self]: + if not callable(tnp): + return None + + tnp_to_np = torch._dynamo.variables.misc.get_tnp_to_np_map() + try: + if not (np := tnp_to_np.get(tnp)): + return None + except TypeError: + return None + + if not (mod := getattr(np, "__module__", None)): + mod = "numpy" + + if not (name := getattr(np, "__name__", None)): + return None + + # pyrefly: ignore [unbound-name] + assert np == getattr(importlib.import_module(mod), name) + # pyrefly: ignore [unbound-name] + return cls(mod, name) + + +class _GraphModulePickleData: + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, obj: torch.fx.GraphModule + ) -> tuple[ + Callable[[Self, _UnpickleState], torch.fx.GraphModule], + tuple[Self, _UnpickleStateToken], + ]: + return cls.unpickle, ( + cls(obj, pickler.options), + pickler._unpickle_state, + ) + + def __init__(self, gm: torch.fx.GraphModule, options: Options) -> None: + # Need to do this to ensure the code is created for later pickling. + if isinstance(gm, torch.fx._lazy_graph_module._LazyGraphModule): + _python_code = gm._real_recompile() + else: + _python_code = gm.recompile() + self.gm_dict = gm.__dict__.copy() + del self.gm_dict["_graph"] + self.graph = _GraphPickleData(gm._graph, options) + + def unpickle(self, unpickle_state: _UnpickleState) -> torch.fx.GraphModule: + gm = torch.fx.GraphModule.__new__(torch.fx.GraphModule) + gm.__dict__ = self.gm_dict + gm._graph = self.graph.unpickle(gm, unpickle_state) + return gm + + +class _NodePickleData: + def __init__( + self, + node: torch.fx.Node, + mapping: dict[torch.fx.Node, "_NodePickleData"], + options: Options, + ) -> None: + self.args = pytree.tree_map_only(torch.fx.Node, lambda n: mapping[n], node.args) + self.kwargs = pytree.tree_map_only( + torch.fx.Node, lambda n: mapping[n], node.kwargs + ) + # -- self.graph = node.graph + self.name = node.name + self.op = node.op + self.target = _OpPickleData.pickle(node.target, options) + # self.input_nodes = node._input_nodes + # self.users = node.users + self.type = node.type + # self.sort_key = node._sort_key + # self.repr_fn = node._repr_fn + # self.meta = node.meta + self.meta = node.meta + + def unpickle( + self, + graph: torch.fx.Graph, + mapping: dict["_NodePickleData", torch.fx.Node], + unpickle_state: _UnpickleState, + ) -> torch.fx.Node: + args = pytree.tree_map_only(_NodePickleData, lambda n: mapping[n], self.args) + kwargs = pytree.tree_map_only( + _NodePickleData, lambda n: mapping[n], self.kwargs + ) + target = self.target.unpickle(unpickle_state) + assert callable(target) or isinstance(target, str) + node = graph.create_node(self.op, target, args, kwargs, self.name, self.type) + node.meta = self.meta + return node + + +class _OpPickleData: + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, op: object + ) -> tuple[Callable[[_UnpickleState], object], tuple[_UnpickleStateToken]]: + result = cls.pickle(op, pickler.options) + return (result.unpickle, (pickler._unpickle_state,)) + + @classmethod + def pickle(cls, op: object, options: Options) -> "_OpPickleData": + if isinstance(op, str): + return _OpStrPickleData(op) + + if isinstance(getattr(op, "__wrapped__", None), AOTCompiledArtifact): + assert hasattr(op, "__wrapped__") + artifact = op.__wrapped__ + assert isinstance(artifact, AOTCompiledArtifact) + return _OpPrecompiledPickleData(artifact) + + name = torch.fx.Node._pretty_print_target(op) + + if isinstance(op, torch._ops.OpOverload): + return cls._pickle_op(name, _OpOverloadPickleData, options) + elif isinstance(op, torch._ops.OpOverloadPacket): + return cls._pickle_op(name, _OpOverloadPacketPickleData, options) + elif name.startswith(_OpFunctionPickleData.SUPPORTED_ROOTS): + root, detail = name.split(".", 1) + return _OpFunctionPickleData(root, detail) + else: + # TODO: raise a BypassFxGraphCache so we will just bypass this one... + raise NotImplementedError(f"TARGET: {type(op)} {op} {name}") + + @staticmethod + def _pickle_op( + name: str, + datacls: Union[ + type["_OpOverloadPickleData"], type["_OpOverloadPacketPickleData"] + ], + options: Options, + ) -> "_OpPickleData": + if (ops_filter := options.ops_filter) and not ops_filter(name): + from torch._inductor.codecache import BypassFxGraphCache + + raise BypassFxGraphCache(f"Unable to pickle non-standard op: {name}") + return datacls(name) + + @abstractmethod + def unpickle(self, unpickle_state: _UnpickleState) -> object: + pass + + @classmethod + def _lookup_global_by_name(cls, name: str) -> object: + """ + Like `globals()[name]` but supports dotted names. + """ + if "." in name: + mod, rest = name.split(".", 1) + root = globals()[mod] + return cls._getattr_by_name(root, rest) + else: + return globals()[name] + + @staticmethod + def _getattr_by_name(root: object, name: str) -> object: + """ + Like `getattr(root, name)` but supports dotted names. + """ + while "." in name: + mod, name = name.split(".", 1) + root = getattr(root, mod) + return getattr(root, name) + + +class _OpStrPickleData(_OpPickleData): + def __init__(self, name: str) -> None: + self.name = name + + def unpickle(self, unpickle_state: _UnpickleState) -> str: + return self.name + + +class _OpOverloadPickleData(_OpPickleData): + def __init__(self, name: str) -> None: + self.name = name + + def unpickle(self, unpickle_state: _UnpickleState) -> torch._ops.OpOverload: + obj = self._lookup_global_by_name(self.name) + assert isinstance(obj, torch._ops.OpOverload) + return obj + + +class _OpOverloadPacketPickleData(_OpPickleData): + def __init__(self, name: str) -> None: + self.name = name + + def unpickle(self, unpickle_state: _UnpickleState) -> torch._ops.OpOverloadPacket: + obj = self._lookup_global_by_name(self.name) + assert isinstance(obj, torch._ops.OpOverloadPacket) + return obj + + +class _OpPrecompiledPickleData(_OpPickleData): + def __init__(self, artifact: AOTCompiledArtifact) -> None: + self.contents = artifact.serialize() + + def unpickle(self, unpickle_state: _UnpickleState) -> object: + precompiled_artifact = AOTCompiledArtifact.deserialize(self.contents) + import functools + + @functools.wraps(precompiled_artifact) + def wrapped(*args: Any) -> Any: + return precompiled_artifact(*args) + + return wrapped + + +class _OpFunctionPickleData(_OpPickleData): + """ + Supports pickling a set of standard/common functions + These must be prefixed with the full namespace in order to properly + be pickled (i.e `einops.rearrange` and not `from einops import rearrange`) + """ + + # Static variable listing supported root names + SUPPORTED_ROOTS = ("builtins.", "math.", "torch.", "operator.", "einops.") + + def __init__(self, root: str, name: str) -> None: + self.root = root + self.name = name + + def unpickle(self, unpickle_state: _UnpickleState) -> object: + if self.root == "builtins": + return __builtins__.get(self.name) # type: ignore[attr-defined] + elif self.root == "math": + import math + + return self._getattr_by_name(math, self.name) + elif self.root == "torch": + return self._getattr_by_name(torch, self.name) + elif self.root == "operator": + import operator + + return self._getattr_by_name(operator, self.name) + elif self.root == "einops": + import einops + + return self._getattr_by_name(einops, self.name) + else: + raise NotImplementedError + + +class _GraphPickleData: + def __init__(self, graph: torch.fx.Graph, options: Options) -> None: + self.tracer_cls = graph._tracer_cls + self.tracer_extras = graph._tracer_extras + + nodes: dict[torch.fx.Node, _NodePickleData] = {} + for node in graph.nodes: + nodes[node] = _NodePickleData(node, nodes, options) + self.nodes = tuple(nodes.values()) + + # Unpickled variables: + # self._used_names = graph._used_names + # -- self._insert = self._root.prepend + # self._len = graph._len + # self._graph_namespace = graph._graph_namespace + # self._owning_module = graph._owning_module + # self._codegen = graph._codegen + # self._co_fields: Dict[str, Any] = graph._co_fields + # -- self._find_nodes_lookup_table = _FindNodesLookupTable() + + def unpickle( + self, gm: torch.fx.GraphModule, unpickle_state: _UnpickleState + ) -> torch.fx.Graph: + graph = torch.fx.Graph(gm, self.tracer_cls, self.tracer_extras) + + nodes: dict[_NodePickleData, torch.fx.Node] = {} + for nd in self.nodes: + nodes[nd] = nd.unpickle(graph, nodes, unpickle_state) + + return graph + + +class _TracingContextPickleData: + @classmethod + def reduce_helper( + cls, pickler: GraphPickler, obj: torch._guards.TracingContext + ) -> tuple[ + Callable[[Self, _UnpickleState], torch._guards.TracingContext], + tuple[Self, _UnpickleStateToken], + ]: + return ( + cls.unpickle, + ( + cls(obj), + pickler._unpickle_state, + ), + ) + + def __init__(self, context: TracingContext) -> None: + # TODO: Do we really need all of this? + self.module_context = context.module_context + self.frame_summary_stack = context.frame_summary_stack + self.loc_in_frame = context.loc_in_frame + self.aot_graph_name = context.aot_graph_name + self.params_flat = context.params_flat + self.params_flat_unwrap_subclasses = context.params_flat_unwrap_subclasses + self.params_unwrapped_to_flat_index = context.params_unwrapped_to_flat_index + self.output_strides = context.output_strides + self.force_unspec_int_unbacked_size_like = ( + context.force_unspec_int_unbacked_size_like + ) + # Not saved (because it's difficult and maybe not needed?): + # self.fw_metadata = context.fw_metadata + # self.guards_context = None + # self.global_context = None + # self.fake_mode = None + # self.fakify_first_call = None + # self.hop_dispatch_set_cache = None + # self.tensor_to_context = context.tensor_to_context + + def unpickle(self, unpickle_state: _UnpickleState) -> TracingContext: + context = TracingContext(unpickle_state.fake_mode) + context.module_context = self.module_context + context.frame_summary_stack = self.frame_summary_stack + context.loc_in_frame = self.loc_in_frame + context.aot_graph_name = self.aot_graph_name + context.params_flat = self.params_flat + context.params_flat_unwrap_subclasses = self.params_flat_unwrap_subclasses + context.params_unwrapped_to_flat_index = self.params_unwrapped_to_flat_index + context.output_strides = self.output_strides + context.force_unspec_int_unbacked_size_like = ( + self.force_unspec_int_unbacked_size_like + ) + return context diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_lazy_graph_module.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_lazy_graph_module.py new file mode 100644 index 0000000000000000000000000000000000000000..1bfb62e1a7605c8f71ebedcbbfeb2b53c56bb6a2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_lazy_graph_module.py @@ -0,0 +1,180 @@ +# mypy: allow-untyped-defs +from contextlib import contextmanager + +from torch.fx.graph_module import ( + _format_import_block, + GraphModule, + reduce_graph_module, + reduce_package_graph_module, +) +from torch.package import PackageExporter, sys_importer + +from ._compatibility import compatibility + + +_use_lazy_graph_module_flag = False +_force_skip_lazy_graph_module_flag = False + + +@compatibility(is_backward_compatible=False) +@contextmanager +def _force_skip_lazy_graph_module(): + """ + Skip using lazy graph module disregarding the setting of _use_lazy_graph_module. + Use to skip _LazyGraphModule when testing inductor torchscript related backend. + + torch.jit.script a _LazyGraphModule results in following error: + https://gist.github.com/shunting314/5143654c8084aed84ecd19b818258a69 + """ + try: + global _force_skip_lazy_graph_module_flag + prior = _force_skip_lazy_graph_module_flag + _force_skip_lazy_graph_module_flag = True + yield + finally: + _force_skip_lazy_graph_module_flag = prior + + +@compatibility(is_backward_compatible=False) +@contextmanager +def _use_lazy_graph_module(should_use: bool): + try: + global _use_lazy_graph_module_flag + prior = _use_lazy_graph_module_flag + _use_lazy_graph_module_flag = ( + should_use and not _force_skip_lazy_graph_module_flag + ) + yield + finally: + _use_lazy_graph_module_flag = prior + + +@compatibility(is_backward_compatible=False) +def _get_graph_module_cls(): + return _LazyGraphModule if _use_lazy_graph_module_flag else GraphModule + + +def _make_graph_module(*args, graph_module_cls=None, **kwargs): + if graph_module_cls is None: + graph_module_cls = _get_graph_module_cls() + + return graph_module_cls(*args, **kwargs) + + +@compatibility(is_backward_compatible=False) +class _LazyGraphModule(GraphModule): + """ + The main difference between _LazyGraphModule and GraphModule is how recompile happens. + GraphModule will do a 'recompile' call to generate python code and the forward method when it's + constructed. Later on if the graph get updated, recompile method can be called again to refresh + the saved python code and forward method. + + However in some cases especially in inductor, the recompilation can be a waste since we never + check the python code for the graph module or call its forward method. A few more concreate + examples regarding pattern matching fx passes in inductor: + 1. some passes will update the graph to be compiled and then call recompile on the GraphModule. + 2. some passes will trace small pattern function to search it in the graph being compiled and + replace the match with the traced graph of a replacement function. The pattern graph and + replacement graph are quite small but there are large amount of them. Doing GraphModule.recompile + for them in GraphModule.__init__ is also a waste of time. + + However simply skip calling GraphModule.recompile in these scenarios is also dangeruous. + People may want to check the python code or call the GraphModule's forward method for debugging purposes. + + The way _LazyGraphModule solves it is, we override the recompile method to just mark the + need for recompilation but does not do the actual recompilation. Later on if people really + access the compiled python code or call the GraphModule's forward method, we do the real + recompilation. + """ + + @classmethod + def from_graphmodule(cls, gm: GraphModule): + if isinstance(gm, _LazyGraphModule): + return gm + else: + return _LazyGraphModule(gm, gm.graph) + + @staticmethod + def force_recompile(gm): + """ + Sometimes we need force a recompile as a workaround + - we want to do the real recompilation before symbolic_trace to avoid error: + https://gist.github.com/shunting314/75549c2e82ae07ac1139c94a3583d259 + """ + if isinstance(gm, _LazyGraphModule): + gm.real_recompile() + + def real_recompile(self): + if self._needs_recompile(): + self._real_recompile() + + @classmethod + def _needs_recompile(cls): + return cls.forward is cls._lazy_forward + + def _lazy_forward(self, *args, **kwargs): + # Call self.real_recompile() rather than self._real_recompile() here. + # The _lazy_forward method may be saved and call repeatedly. + # Calling self.real_recompile can make sure we skip recompilation if + # we have already done so. + self.real_recompile() + assert not self._needs_recompile() + + # call `__call__` rather than 'forward' since recompilation may + # install a wrapper for `__call__` to provide a customized error + # message. + return self(*args, **kwargs) + + forward = _lazy_forward + + def __reduce_package__(self, exporter: PackageExporter): + """ + Follow GraphModule.__reduce__ but call 'self._real_recompile' rather + than 'self.recompile' since for a _LazyGraphModule, self.recompile just + mark the need of recompilation and does not return the PythonCode object. + """ + python_code = self._real_recompile() + dict_without_graph = self.__dict__.copy() + dict_without_graph["_graphmodule_cls_name"] = self.__class__.__name__ + del dict_without_graph["_graph"] + + generated_module_name = f"fx-generated._{exporter.get_unique_id()}" + import_block = _format_import_block(python_code.globals, exporter.importer) + module_code = import_block + self.code + exporter.save_source_string(generated_module_name, module_code) + return ( + reduce_package_graph_module, + (dict_without_graph, generated_module_name), + ) + + def __reduce__(self): + """ + Follow GraphModule.__reduce__ but call 'self._real_recompile' rather + than 'self.recompile' since for a _LazyGraphModule, self.recompile just + mark the need of recompilation and does not return the PythonCode object. + """ + python_code = self._real_recompile() + dict_without_graph = self.__dict__.copy() + import_block = _format_import_block(python_code.globals, sys_importer) + del dict_without_graph["_graph"] + return (reduce_graph_module, (dict_without_graph, import_block)) + + def _real_recompile(self): + return super().recompile() + + @classmethod + def recompile(cls): + cls.forward = cls._lazy_forward + + @property + def code(self) -> str: + self.real_recompile() + return super().code + + def __str__(self) -> str: + """ + str(GraphModule) will access the _code attribute. Make sure recompile + happens so _code attribute is available. + """ + self.real_recompile() + return super().__str__() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_pytree.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_pytree.py new file mode 100644 index 0000000000000000000000000000000000000000..4f1c1993b6b8b42a22f37af0d778fed28f2705b9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_pytree.py @@ -0,0 +1,114 @@ +from collections import namedtuple +from collections.abc import Callable +from typing import Any, Optional, TypeVar +from typing_extensions import NamedTuple + +import torch.return_types +from torch.utils._pytree import PyTree, tree_flatten, TreeSpec + + +FlattenFuncSpec = Callable[[PyTree, TreeSpec], list] +FlattenFuncExactMatchSpec = Callable[[PyTree, TreeSpec], bool] + +SUPPORTED_NODES: dict[type[Any], FlattenFuncSpec] = {} +SUPPORTED_NODES_EXACT_MATCH: dict[type[Any], Optional[FlattenFuncExactMatchSpec]] = {} + +_T = TypeVar("_T") +_K = TypeVar("_K") +_V = TypeVar("_V") + + +def register_pytree_flatten_spec( + cls: type[Any], + flatten_fn_spec: FlattenFuncSpec, + flatten_fn_exact_match_spec: Optional[FlattenFuncExactMatchSpec] = None, +) -> None: + SUPPORTED_NODES[cls] = flatten_fn_spec + SUPPORTED_NODES_EXACT_MATCH[cls] = flatten_fn_exact_match_spec + + +def _deregister_pytree_flatten_spec( + cls: type[Any], +) -> None: + del SUPPORTED_NODES[cls] + del SUPPORTED_NODES_EXACT_MATCH[cls] + + +def tree_flatten_spec( + pytree: PyTree, + spec: TreeSpec, +) -> list[Any]: + if spec.is_leaf(): + return [pytree] + # I guess these exist for BC, FC reasons. + # In general, we should be able to directly + # use pytree tree flattener to flatten them, + # as export serializes the pytree separately. + # Will remove it in follow up PR. + if spec.type in SUPPORTED_NODES: + flatten_fn_spec = SUPPORTED_NODES[spec.type] + child_pytrees = flatten_fn_spec(pytree, spec) + result = [] + for child, child_spec in zip(child_pytrees, spec.children()): + flat = tree_flatten_spec(child, child_spec) + result += flat + return result + flat_result, real_spec = tree_flatten(pytree) + if spec != real_spec: + raise RuntimeError( + f"Real spec {real_spec} of object {pytree} is different from expected spec {spec}. " + f"Please file an issue at https://github.com/pytorch/pytorch/issues/new?template=bug-report.yml" + ) + return flat_result + + +def _dict_flatten_spec(d: dict[_K, _V], spec: TreeSpec) -> list[_V]: + return [d[k] for k in spec.context] + + +def _list_flatten_spec(d: list[_T], spec: TreeSpec) -> list[_T]: + return [d[i] for i in range(spec.num_children)] + + +def _tuple_flatten_spec(d: tuple[_T, ...], spec: TreeSpec) -> list[_T]: + return [d[i] for i in range(spec.num_children)] + + +def _namedtuple_flatten_spec(d: NamedTuple, spec: TreeSpec) -> list[Any]: + return [d[i] for i in range(spec.num_children)] + + +def _dict_flatten_spec_exact_match(d: dict[_K, _V], spec: TreeSpec) -> bool: + return len(d) == spec.num_children + + +def _list_flatten_spec_exact_match(d: list[_T], spec: TreeSpec) -> bool: + return len(d) == spec.num_children + + +def _tuple_flatten_spec_exact_match(d: tuple[_T, ...], spec: TreeSpec) -> bool: + return len(d) == spec.num_children + + +def _namedtuple_flatten_spec_exact_match(d: NamedTuple, spec: TreeSpec) -> bool: + return len(d) == spec.num_children + + +register_pytree_flatten_spec(dict, _dict_flatten_spec, _dict_flatten_spec_exact_match) +register_pytree_flatten_spec(list, _list_flatten_spec, _list_flatten_spec_exact_match) +register_pytree_flatten_spec( + tuple, + _tuple_flatten_spec, + _tuple_flatten_spec_exact_match, +) +for return_type in torch.return_types.all_return_types: + register_pytree_flatten_spec( + return_type, + _tuple_flatten_spec, + _tuple_flatten_spec_exact_match, + ) +register_pytree_flatten_spec( + namedtuple, # type: ignore[arg-type] + _namedtuple_flatten_spec, + _namedtuple_flatten_spec_exact_match, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_symbolic_trace.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_symbolic_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..0937808565f41a85ac2e58cbcda6ea8ea40d5e42 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_symbolic_trace.py @@ -0,0 +1,1368 @@ +# mypy: allow-untyped-defs +import builtins +import collections +import contextlib +import copy +import functools +import inspect +import logging +import math +import os +import warnings +from collections.abc import Callable +from itertools import chain +from types import CodeType, FunctionType, ModuleType +from typing import Any, get_args, NamedTuple, Optional, TypeAlias, Union + +import torch +import torch.utils._pytree as pytree +from torch._C import ScriptObject # type: ignore[attr-defined] +from torch._library.fake_class_registry import FakeScriptObject +from torch._library.opaque_object import is_opaque_reference_type, is_opaque_type + +from ._compatibility import compatibility +from ._lazy_graph_module import _make_graph_module +from .graph import _PyTreeCodeGen, _PyTreeInfo, Graph +from .graph_module import GraphModule +from .node import Argument, base_types, map_aggregate +from .proxy import ParameterProxy, Proxy, Scope, ScopeContextManager, TracerBase + + +log = logging.getLogger(__name__) + +HAS_VARSTUFF = inspect.CO_VARARGS | inspect.CO_VARKEYWORDS + +# These need to run in global scope to handle nested calls correctly +_orig_module_call: Callable = torch.nn.Module.__call__ +_orig_module_getattr: Callable = torch.nn.Module.__getattr__ + +_proxyable_classes: dict[type, None] = {} + +_is_fx_tracing_flag = False + +_ConstantAttributeType: TypeAlias = Union[ + torch.Tensor, torch.ScriptObject, FakeScriptObject, pytree.TreeSpec +] + +_constant_attribute_types = get_args(_ConstantAttributeType) + + +# We only want to print this once to avoid flooding logs +@functools.lru_cache +def is_fx_tracing_warning(): + log.warning( + "is_fx_tracing will return true for both fx.symbolic_trace and " + "torch.export. Please use " + "is_fx_tracing_symbolic_tracing() for specifically fx.symbolic_trace " + "or torch.compiler.is_compiling() for specifically torch.export/compile." + ) + + +def is_fx_tracing(): + is_fx_tracing_warning() + return _is_fx_tracing_flag + + +def is_fx_symbolic_tracing(): + return _is_fx_tracing_flag and not torch.compiler.is_compiling() + + +@compatibility(is_backward_compatible=True) +class ProxyableClassMeta(type): + """ + ProxyableClassMeta allows you to make construction of a given Python class + symbolically traceable. For example:: + + import torch + import torch.fx + + + class TensorPair(metaclass=torch.fx.ProxyableClassMeta): + def __init__(self, left, right): + self.left, self.right = left, right + + def add(self, other): + l = self.left + other.left + r = self.right + other.right + return TensorPair(l, r) + + def mul(self, other): + l = self.left * other.left + r = self.right * other.right + return TensorPair(l, r) + + + def use_tensor_pair_ctor(x: TensorPair, y: torch.Tensor): + s = x.add(TensorPair(y, y)) + return s.mul(x) + + + x = TensorPair(torch.randn(5, 3), torch.randn(5, 3)) + y = torch.randn(5, 3) + ref_out = use_tensor_pair_ctor(x, y) + + traced = torch.fx.symbolic_trace(use_tensor_pair_ctor) + print(traced.code) + ''' + def forward(self, x : __main___TensorPair, y : torch.Tensor): + tensor_pair = __main___TensorPair(y, y); y = None + add = x.add(tensor_pair); tensor_pair = None + mul = add.mul(x); add = x = None + return mul + ''' + + From this example, we can see that construction of a class (``TensorPair``) + defined with ``ProxyableClassMeta`` as metaclass can be recorded in symbolic + tracing. + """ + + def __init__(cls, name, bases, attrs): + _proxyable_classes.setdefault(cls) + super().__init__(name, bases, attrs) + + def __call__(cls, *args, **kwargs): + instance = cls.__new__(cls) # type: ignore[call-overload] + + if not is_fx_tracing(): + cls.__init__(instance, *args, **kwargs) # type: ignore[misc] + return instance + + found_proxies = [] + + def check_proxy(a): + if isinstance(a, Proxy): + found_proxies.append(a) + + map_aggregate(args, check_proxy) + map_aggregate(kwargs, check_proxy) + + if len(found_proxies) != 0: + tracer = found_proxies[0].tracer + return tracer.create_proxy("call_function", cls, args, kwargs) + else: + cls.__init__(instance, *args, **kwargs) # type: ignore[misc] + return instance + + +def _patch_function(fn: FunctionType, nargs: int) -> FunctionType: + co = fn.__code__ + co_flags = co.co_flags & ~HAS_VARSTUFF + co_args: tuple + if hasattr(co, "co_qualname"): + # Python-3.11+ code signature + co_args = ( + nargs, + 0, + 0, + co.co_nlocals, + co.co_stacksize, + co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_qualname, # type: ignore[attr-defined] + co.co_firstlineno, + co.co_linetable, + co.co_exceptiontable, # type: ignore[attr-defined] + co.co_freevars, + co.co_cellvars, + ) + elif hasattr(co, "co_posonlyargcount"): + co_args = ( + nargs, + 0, + 0, + co.co_nlocals, + co.co_stacksize, + co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_freevars, + co.co_cellvars, + ) + else: + co_args = ( + nargs, + 0, + co.co_nlocals, + co.co_stacksize, + co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_freevars, + co.co_cellvars, + ) + new_code = CodeType(*co_args) # type: ignore[arg-type] + return FunctionType( + new_code, fn.__globals__, fn.__name__, fn.__defaults__, fn.__closure__ + ) + + # we need to insert placeholder nodes for *args and **kwargs + # we can't call this function normally, otherwise it would try to unpack them + # instead, let's make python think that args and kwargs are normal variables + + +@compatibility(is_backward_compatible=False) +class PHBase: + """ + Object representing an input placeholder to `concrete_args` + """ + + def __repr__(self): + return "PH" + + +PH = PHBase() + + +@compatibility(is_backward_compatible=False) +class PHWithMeta(PHBase): + """ + Object representing an input placeholder to `concrete_args` + """ + + def __init__(self, ph_key: Optional[str] = None): + super().__init__() + + # Provide a hey for user to identify placeholder node during analysis + self.ph_key = ph_key + + +def _transfer_attrs(fr, to): + for attr_name in dir(fr): + attr_val = getattr(fr, attr_name) + if ( + not callable(attr_val) + and not attr_name.startswith("__") + and not hasattr(to, attr_name) + ): + setattr(to, attr_name, attr_val) + + +@compatibility(is_backward_compatible=True) +class Tracer(TracerBase): + # Reference: https://github.com/pytorch/pytorch/issues/54354 + # The first line of this docstring overrides the one Sphinx generates for the + # documentation. We need it so that Sphinx doesn't leak `math`s path from the + # build environment (e.g. ` None: + # This method's signature is overridden by the first line of this class' + # docstring. If this method's signature is modified, the signature that + # overrides it also should be modified accordingly. + + """ + Construct a Tracer object. + + Args: + + autowrap_modules (Tuple[ModuleType]): defaults to `(math, )`, + Python modules whose functions should be wrapped automatically + without needing to use fx.wrap(). Backward-compatibility for + this parameter is guaranteed. + + autowrap_functions (Tuple[Callable, ...]): defaults to `()`, + Python functions that should be wrapped automatically without + needing to use fx.wrap(). Backward compatibility for this + parameter is guaranteed. + + param_shapes_constant (bool): When this flag is set, calls to shape, + size and a few other shape like attributes of a module's parameter + will be evaluated directly, rather than returning a new Proxy value + for an attribute access. Backward compatibility for this parameter + is guaranteed. + """ + + super().__init__() + + # Functions we will eagerly wrap when we see them while tracing + # this captures both `math.sqrt()` and `from math import sqrt` automatically + self._autowrap_function_ids: set[int] = { + id(value) + for name, value in chain.from_iterable( + m.__dict__.items() for m in autowrap_modules + ) + if not name.startswith("_") and callable(value) + } + self._autowrap_function_ids.update({id(f) for f in autowrap_functions}) + + # Python modules to apply autowrap to at the start, in addition to + # modules we see while tracing + self._autowrap_search: list[ModuleType] = list(autowrap_modules) + self.param_shapes_constant = param_shapes_constant + + self.submodule_paths: Optional[dict[torch.nn.Module, str]] = None + self.root_module_name: str = "" + # Maps the containing module's name to the operator name + self.scope = Scope("", None) + # Records the module call stack + self.module_stack = collections.OrderedDict() + self.num_calls: dict[str, int] = {} + # Mapping of node name to module scope + self.node_name_to_scope: dict[str, tuple[str, type]] = {} + + _qualname_counter: dict[str, int] = collections.defaultdict(int) + + @compatibility(is_backward_compatible=True) + def get_fresh_qualname(self, prefix: str) -> str: + """ + Gets a fresh name for a prefix and returns it. This function ensures + that it will not clash with an existing attribute on the graph. + """ + # The idea here is that if the module doesn't have this prefix at all we + # should reset the counter to start from the beginning + # It's a ... little bit hacky (doesn't cover all cases) but the precise + # naming of the prefixes isn't a correctness issue, just a niceness + # issue + qualname = f"{prefix}0" + if not hasattr(self.root, qualname): + self._qualname_counter[prefix] = 0 + return qualname + + i = self._qualname_counter[prefix] + while True: + qualname = f"{prefix}{i}" + i += 1 + if not hasattr(self.root, qualname): + break + self._qualname_counter[prefix] = i + + return qualname + + @compatibility(is_backward_compatible=True) + def create_arg(self, a: Any) -> "Argument": + """ + A method to specify the behavior of tracing when preparing values to + be used as arguments to nodes in the ``Graph``. + + By default, the behavior includes: + + #. Iterate through collection types (e.g. tuple, list, dict) and recursively + call ``create_args`` on the elements. + #. Given a Proxy object, return a reference to the underlying IR ``Node`` + #. Given a non-Proxy Tensor object, emit IR for various cases: + + * For a Parameter, emit a ``get_attr`` node referring to that Parameter + * For a non-Parameter Tensor, store the Tensor away in a special + attribute referring to that attribute. + + This method can be overridden to support more types. + + Args: + + a (Any): The value to be emitted as an ``Argument`` in the ``Graph``. + + + Returns: + + The value ``a`` converted into the appropriate ``Argument`` + """ + # The base tracer is used to construct Graphs when there is no associated + # module hierarchy, so it can never create parameter references. + # The default tracer adds the ability to refer to parameters when + # tracing modules. + if isinstance(a, torch.nn.Parameter): + for n, p in self.root.named_parameters(): + if a is p: + return self.create_node("get_attr", n, (), {}) + raise NameError("parameter is not a member of this module") + elif isinstance(a, torch.Tensor): + for n_, p_ in self.root.named_buffers(): + if a is p_: + return self.create_node("get_attr", n_, (), {}) + elif isinstance(a, torch.nn.Module): + for n_, p_ in self.root.named_modules(): + if a is p_: + return self.create_node("get_attr", n_, (), {}) + # For NamedTuple instances that appear literally as args, we emit + # a node to construct the NamedTuple and use that Node as the argument. + if isinstance(a, tuple) and hasattr(a, "_fields"): + args = tuple(self.create_arg(elem) for elem in a) + return self.create_node("call_function", a.__class__, args, {}) + + # Tensors do not have a reliable string repr() from which they can be + # constructed (and we probably don't want to rely on that, either), so + # for any constant Tensor values we encounter, first search for if they + # are an attribute of some module in the module hierarchy. If so, emit + # a get_attr to retrieve that tensor. Otherwise, we'll store away the + # tensor value into a special attribute on the Module s.t. we can + # retrieve it with a get_attr. + if isinstance(a, _constant_attribute_types) or ( + is_opaque_reference_type(type(a)) + ): + qualname: Optional[str] = self.tensor_attrs.get( + a + ) # pyrefly: ignore[no-matching-overload] + + # Tensor was not found in the Module hierarchy, stow it away in a + # special attribute and set the qualname to refer to that + if not qualname: + if isinstance(a, torch.Tensor): + base_name = "_tensor_constant" + elif isinstance(a, (FakeScriptObject, ScriptObject)): + base_name = "_torchbind_obj" + elif isinstance(a, pytree.TreeSpec): + base_name = "_tree_spec_constant" + elif is_opaque_type(type(a)): + base_name = "_opaque_obj" + else: + raise RuntimeError( + f"cannot create constant arg for {a} of type {type(a)}." + ) + qualname = self.get_fresh_qualname(base_name) + assert isinstance(qualname, str) + self.tensor_attrs[a] = ( # pyrefly: ignore[unsupported-operation] + qualname + ) + setattr(self.root, qualname, a) + + return self.create_node("get_attr", qualname, (), {}) + + if type(a) in _proxyable_classes: + # This is an instance of a proxyable class for which we did not + # witness its construction. Intern this as a constant attribute + + # TODO: binary search + qualname = self.get_fresh_qualname(f"_{a.__class__.__name__}_constant_") + assert isinstance(qualname, str) + setattr(self.root, qualname, a) + + return self.create_node("get_attr", qualname, (), {}) + return super().create_arg(a) + + @compatibility(is_backward_compatible=True) + def is_leaf_module(self, m: torch.nn.Module, module_qualified_name: str) -> bool: + """ + A method to specify whether a given ``nn.Module`` is a "leaf" module. + + Leaf modules are the atomic units that appear in + the IR, referenced by ``call_module`` calls. By default, + Modules in the PyTorch standard library namespace (torch.nn) + are leaf modules. All other modules are traced through and + their constituent ops are recorded, unless specified otherwise + via this parameter. + + Args: + + m (Module): The module being queried about + module_qualified_name (str): The path to root of this module. For example, + if you have a module hierarchy where submodule ``foo`` contains + submodule ``bar``, which contains submodule ``baz``, that module will + appear with the qualified name ``foo.bar.baz`` here. + """ + return ( + m.__module__.startswith("torch.nn") + or m.__module__.startswith("torch.ao.nn") + ) and not isinstance(m, torch.nn.Sequential) + + @compatibility(is_backward_compatible=True) + def path_of_module(self, mod: torch.nn.Module) -> str: + """ + Helper method to find the qualified name of ``mod`` in the Module hierarchy + of ``root``. For example, if ``root`` has a submodule named ``foo``, which has + a submodule named ``bar``, passing ``bar`` into this function will return + the string "foo.bar". + + Args: + + mod (str): The ``Module`` to retrieve the qualified name for. + """ + # Prefer the O(1) algorithm + if self.submodule_paths: + path = self.submodule_paths.get(mod) + if path is None: + raise NameError("module is not installed as a submodule") + assert isinstance(path, str) + return path + # O(N^2) fallback in the case that we didn't store the submodule + # paths. + else: + for n, p in self.root.named_modules(): + if mod is p: + return n + raise NameError("module is not installed as a submodule") + + @compatibility(is_backward_compatible=True) + def call_module( + self, + m: torch.nn.Module, + forward: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + """ + Method that specifies the behavior of this ``Tracer`` when it encounters + a call to an ``nn.Module`` instance. + + By default, the behavior is to check if the called module is a leaf module + via ``is_leaf_module``. If it is, emit a ``call_module`` node referring to + ``m`` in the ``Graph``. Otherwise, call the ``Module`` normally, tracing through + the operations in its ``forward`` function. + + This method can be overridden to--for example--create nested traced + GraphModules, or any other behavior you would want while tracing across + ``Module`` boundaries. + + Args: + + m (Module): The module for which a call is being emitted + forward (Callable): The forward() method of the ``Module`` to be invoked + args (Tuple): args of the module callsite + kwargs (Dict): kwargs of the module callsite + + Return: + + The return value from the Module call. In the case that a ``call_module`` + node was emitted, this is a ``Proxy`` value. Otherwise, it is whatever + value was returned from the ``Module`` invocation. + """ + module_qualified_name = self.path_of_module(m) + with ScopeContextManager( + self.scope, Scope(module_qualified_name, type(m)) + ) as _scope: + # module_stack is an ordered dict so writing then deleting the + # entry is equivalent to push/pop on a list + num_calls = self.num_calls.get(module_qualified_name, 0) + module_key = ( + f"{_scope.module_path}@{num_calls}" + if num_calls > 0 + else _scope.module_path + ) + self.module_stack[module_key] = (module_qualified_name, _scope.module_type) + self.num_calls[module_qualified_name] = num_calls + 1 + if not self.is_leaf_module(m, module_qualified_name): + ret_val = forward(*args, **kwargs) + else: + ret_val = self.create_proxy( + "call_module", module_qualified_name, args, kwargs + ) + key, _ = self.module_stack.popitem(last=True) + assert key == module_key, f" Unexpected key {key}" + + return ret_val + + @compatibility(is_backward_compatible=False) + def getattr(self, attr: str, attr_val: Any, parameter_proxy_cache: dict[str, Any]): + """ + Method that specifies the behavior of this ``Tracer`` when we call getattr + on a call to an ``nn.Module`` instance. + + By default, the behavior is to return a proxy value for the attribute. It + also stores the proxy value in the ``parameter_proxy_cache``, so that future + calls will reuse the proxy rather than creating a new one. + + This method can be overridden to --for example-- not return proxies when + querying parameters. + + Args: + + attr (str): The name of the attribute being queried + attr_val (Any): The value of the attribute + parameter_proxy_cache (Dict[str, Any]): A cache of attr names to proxies + + Return: + + The return value from the getattr call. + """ + + def maybe_get_proxy_for_attr( + attr_val, collection_to_search, parameter_proxy_cache + ): + for n, p in collection_to_search: + if attr_val is p: + if n not in parameter_proxy_cache: + kwargs = {} + if ( + "proxy_factory_fn" + in inspect.signature(self.create_proxy).parameters + ): + kwargs["proxy_factory_fn"] = ( + # pyrefly: ignore [unsupported-operation] + None + if not self.param_shapes_constant + else lambda node: ParameterProxy( + self, node, n, attr_val + ) + ) + val_proxy = self.create_proxy("get_attr", n, (), {}, **kwargs) # type: ignore[arg-type] + parameter_proxy_cache[n] = val_proxy + return parameter_proxy_cache[n] + return None + + if isinstance(attr_val, torch.nn.Parameter): + maybe_parameter_proxy = maybe_get_proxy_for_attr( + attr_val, self.root.named_parameters(), parameter_proxy_cache + ) + if maybe_parameter_proxy is not None: + return maybe_parameter_proxy + + if self.proxy_buffer_attributes and isinstance(attr_val, torch.Tensor): + maybe_buffer_proxy = maybe_get_proxy_for_attr( + attr_val, self.root.named_buffers(), parameter_proxy_cache + ) + if maybe_buffer_proxy is not None: + return maybe_buffer_proxy + + return attr_val + + # This method will be refactored + @compatibility(is_backward_compatible=False) + def create_args_for_root(self, root_fn, is_module, concrete_args=None): + """ + Create ``placeholder`` nodes corresponding to the signature of the ``root`` + Module. This method introspects root's signature and emits those + nodes accordingly, also supporting ``*args`` and ``**kwargs``. + """ + # In some cases, a function or method has been decorated with a wrapper + # defined via ``functools.wraps``. In this case, the outer code object + # will likely not contain the actual parameters we care about, so unwrap + # the function to get to the innermost callable. + fn_for_analysis = inspect.unwrap(root_fn) + co = fn_for_analysis.__code__ + total_args = co.co_argcount + co.co_kwonlyargcount + orig_args = list(co.co_varnames) + names_iter = iter(co.co_varnames) + args: list[Any] = [] + skip_arg_idx = 0 + if is_module: + if total_args == 0: + raise RuntimeError( + "``self`` argument cannot be part of *args expansion!" + ) + skip_arg_idx = 1 + next(names_iter) # skip self + args.append(self.root) + + sig = inspect.signature(fn_for_analysis) + + # This covers the very specific case where we are passing in flat + # concrete_args as a tuple, but our traced fn takes (*args, **kwargs). + # In this case, just take the concrete_args and pass them through. + name_idx = 0 + if ( + isinstance(concrete_args, tuple) + and len(concrete_args) > 0 + and (co.co_flags & HAS_VARSTUFF) + and total_args == 1 + ): + for concrete_arg in concrete_args: + out = self.create_proxy("placeholder", f"input_{name_idx}", (), {}) + if isinstance(concrete_arg, PHBase): + if concrete_arg != PH: + # Transfer attrs in the case where you're using a placeholder other + # than the singleton PH (PH has no attributes to transfer). + # Proxies were created out of the placeholders. + # Transfer any metadata (put on the placeholders in the form of + # attributes set by the user) from the placeholder to the + # underlying nodes (the proxy is unwrapped by the user, but + # the metadata should hold). + _transfer_attrs(fr=concrete_arg, to=out.node) + args.append(out) + name_idx += 1 + return root_fn, args + + arg_names = [next(names_iter) for idx in range(skip_arg_idx, total_args)] + if isinstance(concrete_args, tuple): + if len(arg_names) != len(concrete_args): + raise RuntimeError( + f"Tracing expected {len(arg_names)} arguments but got {len(concrete_args)} concrete arguments" + ) + concrete_args = dict(zip(arg_names, concrete_args)) + + def proxy_placeholder(name): + return self._proxy_placeholder(name, concrete_args, sig, fn_for_analysis) + + args.extend(proxy_placeholder(names) for names in arg_names) + + if co.co_kwonlyargcount > 0 or co.co_flags & HAS_VARSTUFF: + # TODO: type annotations for *args and **kwargs + if co.co_flags & inspect.CO_VARARGS: + args.append(proxy_placeholder("*" + next(names_iter))) + if co.co_flags & inspect.CO_VARKEYWORDS: + args.append(proxy_placeholder("**" + next(names_iter))) + root_fn = _patch_function(root_fn, len(args)) + + flat_args, in_spec = pytree.tree_flatten(tuple(args)) + if not all(child.is_leaf() for child in in_spec.children()): + # In the case that we have pytree-flattened inputs in + # `concrete_args`, generate a flattening wrapper around the + # original root function and return that. + self.graph._codegen = _PyTreeCodeGen( # type: ignore[has-type] + _PyTreeInfo(orig_args[:total_args], in_spec, None) + ) + + def flatten_fn(*args): + tree_args = pytree.tree_unflatten(list(args), in_spec) + tree_out = root_fn(*tree_args) + out_args, out_spec = pytree.tree_flatten(tree_out) + assert isinstance(self.graph._codegen, _PyTreeCodeGen) # type: ignore[has-type] + self.graph._codegen.pytree_info = ( + self.graph._codegen.pytree_info._replace(out_spec=out_spec) + ) + return out_args + + return flatten_fn, flat_args + return root_fn, args + + @compatibility(is_backward_compatible=True) + def trace( + self, + root: Union[torch.nn.Module, Callable[..., Any]], + concrete_args: Optional[dict[str, Any]] = None, + ) -> Graph: + """ + Trace ``root`` and return the corresponding FX ``Graph`` representation. ``root`` + can either be an ``nn.Module`` instance or a Python callable. + + Note that after this call, ``self.root`` may be different from the ``root`` passed + in here. For example, when a free function is passed to ``trace()``, we will + create an ``nn.Module`` instance to use as the root and add embedded constants + to. + + + Args: + + root (Union[Module, Callable]): Either a ``Module`` or a function to be + traced through. Backwards-compatibility for this parameter is + guaranteed. + concrete_args (Optional[Dict[str, any]]): Concrete arguments that should + not be treated as Proxies. This parameter is experimental and + its backwards-compatibility is *NOT* guaranteed. + + Returns: + + A ``Graph`` representing the semantics of the passed-in ``root``. + """ + global _is_fx_tracing_flag + old_is_fx_tracing_flag = _is_fx_tracing_flag + _is_fx_tracing_flag = True + try: + if isinstance(root, torch.nn.Module): + # do real recompilation for _LazyGraphModule before retracing since the trace + # method can not trace the _lazy_forward method. Got error: + # https://gist.github.com/shunting314/75549c2e82ae07ac1139c94a3583d259 + # without this. + from torch.fx._lazy_graph_module import _LazyGraphModule + + _LazyGraphModule.force_recompile(root) + + self.root = root + + assert hasattr(type(root), self.traced_func_name), ( + f"traced_func_name={self.traced_func_name} doesn't exist in {type(root).__name__}" + ) + + fn = getattr(type(root), self.traced_func_name) + self.root_module_name = root._get_name() + self.submodule_paths = {mod: name for name, mod in root.named_modules()} + else: + self.root = torch.nn.Module() + fn = root + + tracer_cls: Optional[type[Tracer]] = getattr(self, "__class__", None) + self.graph = Graph(tracer_cls=tracer_cls) + if hasattr(fn, "__code__"): + code = fn.__code__ + self.graph._co_fields = { + "co_name": code.co_name, + "co_filename": code.co_filename, + "co_firstlineno": code.co_firstlineno, + } + + # When we encounter a Tensor value that's not a parameter, we look if it + # is some other attribute on the model. Construct a dict mapping Tensor + # values to the qualified name here for efficiency. This is used downstream + # in create_arg + self.tensor_attrs: dict[ + _ConstantAttributeType, + str, + ] = {} + + def collect_tensor_attrs(m: torch.nn.Module, prefix_atoms: list[str]): + for k, v in m.__dict__.items(): + if isinstance(v, _constant_attribute_types): + self.tensor_attrs[v] = ".".join(prefix_atoms + [k]) + for k, v in m.named_children(): + collect_tensor_attrs(v, prefix_atoms + [k]) + + collect_tensor_attrs(self.root, []) + + assert isinstance(fn, FunctionType) + + fn_globals = fn.__globals__ # run before it gets patched + fn, args = self.create_args_for_root( + fn, isinstance(root, torch.nn.Module), concrete_args + ) + + parameter_proxy_cache: dict[ + str, Proxy + ] = {} # Reduce number of get_attr calls + + # Method dispatch on parameters is not recorded unless it's directly used. + # Thus, we need to insert a proxy when __getattr__ requests a parameter. + @functools.wraps(_orig_module_getattr) + def module_getattr_wrapper(mod, attr): + attr_val = _orig_module_getattr(mod, attr) + return self.getattr(attr, attr_val, parameter_proxy_cache) + + @functools.wraps(_orig_module_call) + def module_call_wrapper(mod, *args, **kwargs): + def forward(*args, **kwargs): + return _orig_module_call(mod, *args, **kwargs) + + _autowrap_check( + patcher, # type: ignore[has-type] + getattr(getattr(mod, "forward", mod), "__globals__", {}), + self._autowrap_function_ids, + ) + return self.call_module(mod, forward, args, kwargs) + + with _new_patcher() as patcher: + # allow duplicate patches to support the case of nested calls + patcher.patch_method( + torch.nn.Module, + "__getattr__", + module_getattr_wrapper, + deduplicate=False, + ) + patcher.patch_method( + torch.nn.Module, + "__call__", + module_call_wrapper, + deduplicate=False, + ) + _patch_wrapped_functions(patcher) + _autowrap_check(patcher, fn_globals, self._autowrap_function_ids) + for module in self._autowrap_search: + _autowrap_check( + patcher, module.__dict__, self._autowrap_function_ids + ) + ann = inspect.get_annotations(inspect.unwrap(fn)) + self.create_node( + "output", + "output", + (self.create_arg(fn(*args)),), + {}, + type_expr=ann.get("return", None), + ) + + self.submodule_paths = None + except RuntimeError as e: + if e.args and isinstance(e.args[0], str) and "data-dependent" in e.args[0]: + partial_fx_graph = self.graph.python_code( + root_module="self", + verbose=True, + ).src + e.partial_fx_graph = partial_fx_graph # type: ignore[attr-defined] + raise + + raise + finally: + _is_fx_tracing_flag = old_is_fx_tracing_flag + return self.graph + + def __deepcopy__(self, memo): + # _autowrap_search contains modules, which cannot be deepcopied. + new_tracer = Tracer.__new__(Tracer) + + for k, v in self.__dict__.items(): + if k == "_autowrap_search": + new_obj = copy.copy(v) + else: + new_obj = copy.deepcopy(v, memo) + + new_tracer.__dict__[k] = new_obj + + return new_tracer + + def _proxy_placeholder(self, name, concrete_args, sig, fn_for_analysis): + if concrete_args is not None and name in concrete_args: + cnt = 0 + + def replace_ph(x): + nonlocal cnt + cnt += 1 + param = sig.parameters[name] + default: tuple[Any, ...] = ( + () if param.default is inspect.Parameter.empty else (param.default,) + ) + out = self.create_proxy( + "placeholder", f"{name}_{str(cnt)}", default, {} + ) + if isinstance(x, PHBase): + if x != PH: + # Transfer attrs in the case where you're using a placeholder other + # than the singleton PH (PH has no attributes to transfer). + # Proxies were created out of the placeholders. + # Transfer any metadata (put on the placeholders in the form of + # attributes set by the user) from the placeholder to the + # underlying nodes (the proxy is unwrapped by the user, but + # the metadata should hold). + _transfer_attrs(fr=x, to=out.node) + + return out + # Union[int, bool] == bool in Python <= 3.6 + if ( + type(x) is bool + or type(x) in base_types + and type(x) is not torch.Tensor + ): + torch._assert( + out == x, + f"{name} has been specialized to have value {x} but got another value", + ) + elif x is None: + args = ( + out, + f"{name} has been specialized to have value None but got another value", + ) + self.create_proxy("call_function", _assert_is_none, args, {}) + else: + warnings.warn( + f"Was not able to add assertion to guarantee correct input {name} to " + f"specialized function. It is up to the user to make sure that your inputs match the " + f"inputs you specialized the function with." + ) + + return x + + return pytree.tree_map(replace_ph, concrete_args[name]) + if name[0] == "*": + default: tuple[Any, ...] = () + else: + param = sig.parameters[name] + default = ( # type: ignore[assignment] + () if param.default is inspect.Parameter.empty else (param.default,) + ) + return self.create_proxy( + "placeholder", + name, + default, + {}, + type_expr=fn_for_analysis.__annotations__.get(name, None), + ) + + +# Dictionary of (id(globals dict), function name) => globals_dict to patch for +# the purposes of the wrap() API. +# We key by the globals dict id and function name to ensure we're wrapping a given +# function only once. +_wrapped_fns_to_patch: dict[tuple[int, str], dict] = {} + +# List of methods on classes to wrap (class type, function name) +# this currently only works for Tensor.* methods that aren't traced properly +_wrapped_methods_to_patch: list[tuple[type, str]] = [] + +if os.environ.get("FX_PATCH_GETITEM") == "1": + # This change is needed to trace models like PositionalEmbedding from BERT: + # https://github.com/pytorch/benchmark/blob/master/torchbenchmark/models/BERT_pytorch/bert_pytorch/model/embedding/position.py + # but causes issues in quantization documented here: + # https://github.com/pytorch/pytorch/issues/50710 + # once that is fixed we can make this the default behavior. + _wrapped_methods_to_patch.append((torch.Tensor, "__getitem__")) + + +def _find_proxy(*objects_to_search): + """ + Recursively search a data structure for a Proxy() and return it, + return None if not found. + """ + proxy = None + + def find_proxy(x): + nonlocal proxy + if isinstance(x, Proxy): + proxy = x + + map_aggregate(objects_to_search, find_proxy) + return proxy + + +def _create_wrapped_func(orig_fn): + @functools.wraps(orig_fn) + def wrapped(*args, **kwargs): + """ + Given an closed-over ``orig_function`` to invoke, search the args and kwargs for + a Proxy object. If there is one, emit a ``call_function`` node to preserve the + call to this leaf function directly. Otherwise, just return the results of + this function call, as this function is not being traced. + """ + proxy = _find_proxy(args, kwargs) + if proxy is not None: + return_proxy = proxy.tracer.create_proxy( + "call_function", orig_fn, args, kwargs + ) + return_proxy.node.meta["is_wrapped"] = True + return return_proxy + return orig_fn(*args, **kwargs) + + return wrapped + + +def _create_wrapped_method(cls, name): + orig_fn = getattr(cls, name) + + @functools.wraps(orig_fn) + def wrapped(*args, **kwargs): + """ + Search the args and kwargs for a Proxy object. If there is one, + emit a ``call_method`` node to preserve the call to this method + directly. Otherwise, just return the results of this function + call, as this function is not being traced. + """ + proxy = _find_proxy(args, kwargs) + if proxy is not None: + return proxy.tracer.create_proxy("call_method", name, args, kwargs) + return orig_fn(*args, **kwargs) + + return wrapped + + +class _PatchedFn(NamedTuple): + frame_dict: Any + fn_name: str + orig_fn: Any + new_fn: Any + + def revert(self): + raise NotImplementedError + + def patch(self): + raise NotImplementedError + + +class _PatchedFnSetItem(_PatchedFn): + def revert(self): + self.frame_dict[self.fn_name] = self.orig_fn + + def patch(self): + self.frame_dict[self.fn_name] = self.new_fn + + +class _PatchedFnDel(_PatchedFn): + def revert(self): + del self.frame_dict[self.fn_name] + + def patch(self): + self.frame_dict[self.fn_name] = self.new_fn + + +class _PatchedFnSetAttr(_PatchedFn): + def revert(self): + setattr(self.frame_dict, self.fn_name, self.orig_fn) + + def patch(self): + setattr(self.frame_dict, self.fn_name, self.new_fn) + + +class _Patcher: + def __init__(self) -> None: + super().__init__() + self.patches_made: list[_PatchedFn] = [] + self.visited: set[int] = set() + + def patch( + self, + frame_dict: dict[str, Any], + name: str, + new_fn: Callable, + deduplicate: bool = True, + ): + """ + Replace frame_dict[name] with new_fn until we exit the context manager. + """ + new_fn.__fx_already_patched = deduplicate # type: ignore[attr-defined] + if name not in frame_dict and hasattr(builtins, name): + self.patches_made.append(_PatchedFnDel(frame_dict, name, None, new_fn)) + self.patches_made[-1].patch() + elif getattr(frame_dict[name], "__fx_already_patched", False): + return # already patched, no need to do it again + else: + self.patches_made.append( + _PatchedFnSetItem(frame_dict, name, frame_dict[name], new_fn) + ) + self.patches_made[-1].patch() + + def patch_method( + self, cls: type, name: str, new_fn: Callable, deduplicate: bool = True + ): + """ + Replace object_or_dict.name with new_fn until we exit the context manager. + """ + new_fn.__fx_already_patched = deduplicate # type: ignore[attr-defined] + orig_fn = getattr(cls, name) + if getattr(orig_fn, "__fx_already_patched", False): + return # already patched, no need to do it again + self.patches_made.append(_PatchedFnSetAttr(cls, name, orig_fn, new_fn)) + self.patches_made[-1].patch() + + def visit_once(self, thing: Any): + """Return True on the first call to with thing, otherwise false""" + idx = id(thing) + if idx in self.visited: + return False + self.visited.add(idx) + return True + + def revert_all_patches(self): + """ + Remove all the stored patcheds. It doesn't modify patches_made. + """ + for patch in self.patches_made: + patch.revert() + return self.patches_made + + def reapply_all_patches(self): + """ + Patch all the stored patcheds. It doesn't modify patches_made. + """ + for patch in self.patches_made: + patch.patch() + return self.patches_made + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Undo all the changes made via self.patch() and self.patch_method() + """ + while self.patches_made: + # unpatch in reverse order to handle duplicates correctly + self.patches_made.pop().revert() + self.visited.clear() + + +CURRENT_PATCHER: Optional[_Patcher] = None + + +@contextlib.contextmanager +def _new_patcher(): + global CURRENT_PATCHER + prior_patcher = CURRENT_PATCHER + try: + CURRENT_PATCHER = _Patcher() + yield CURRENT_PATCHER + finally: + # Clear all the patches made by when using current patcher. + assert CURRENT_PATCHER is not None + CURRENT_PATCHER.revert_all_patches() + CURRENT_PATCHER = prior_patcher + + +@contextlib.contextmanager +def _maybe_revert_all_patches(): + current_patcher = CURRENT_PATCHER + patches_made = None + patches_removed = None + try: + if current_patcher is not None: + patches_removed = current_patcher.revert_all_patches() + yield + finally: + if current_patcher is not None: + patches_made = current_patcher.reapply_all_patches() + assert patches_made == patches_removed, ( + "CURRENT_PATCHER was changed during a revert_all_patches" + ) + + +def _patch_wrapped_functions(patcher: _Patcher): + """ + Go through ``_wrapped_fn_patch_table`` and, for each frame object, wrap + the listed global functions in the `_create_wrapped_func` wrapper. + """ + for (_, name), frame_dict in _wrapped_fns_to_patch.copy().items(): + if name not in frame_dict and hasattr(builtins, name): + orig_fn = getattr(builtins, name) + else: + orig_fn = frame_dict[name] + patcher.patch(frame_dict, name, _create_wrapped_func(orig_fn)) + + for cls, name in _wrapped_methods_to_patch: + patcher.patch_method(cls, name, _create_wrapped_method(cls, name)) + + +def _autowrap_check( + patcher: _Patcher, frame_dict: dict[str, Any], function_ids: set[int] +): + """ + Some methods, like `math.sqrt` are common enough we want to automatically wrap them as we see them. + This method searches a scope for them and patches them if found. + """ + if patcher.visit_once(frame_dict): + for name, value in frame_dict.items(): + if ( + not name.startswith("_") + and callable(value) + and id(value) in function_ids + ): + patcher.patch(frame_dict, name, _create_wrapped_func(value)) + + +@compatibility(is_backward_compatible=True) +def wrap(fn_or_name: Union[str, Callable]): + """ + This function can be called at module-level scope to register fn_or_name as a "leaf function". + A "leaf function" will be preserved as a CallFunction node in the FX trace instead of being + traced through:: + + # foo/bar/baz.py + def my_custom_function(x, y): + return x * x + y * y + + + torch.fx.wrap("my_custom_function") + + + def fn_to_be_traced(x, y): + # When symbolic tracing, the below call to my_custom_function will be inserted into + # the graph rather than tracing it. + return my_custom_function(x, y) + + This function can also equivalently be used as a decorator:: + + # foo/bar/baz.py + @torch.fx.wrap + def my_custom_function(x, y): + return x * x + y * y + + A wrapped function can be thought of a "leaf function", analogous to the concept of + "leaf modules", that is, they are functions that are left as calls in the FX trace + rather than traced through. + + Args: + + fn_or_name (Union[str, Callable]): The function or name of the global function to insert into the + graph when it's called + """ + if not callable(fn_or_name) and not isinstance(fn_or_name, str): + raise RuntimeError( + "Unsupported type for global function! Must be either a callable or " + "string name" + ) + + if callable(fn_or_name): + assert not isinstance(fn_or_name, str) # to make mypy happy + fn_name = fn_or_name.__name__ + else: + assert isinstance(fn_or_name, str), ( + "fn_or_name must be a global function or string name" + ) + fn_name = fn_or_name + + currentframe = inspect.currentframe() + assert currentframe is not None + f = currentframe.f_back + assert f is not None + if f.f_code.co_name != "": + raise NotImplementedError("wrap must be called at the top level of a module") + + # consider implementing Callable version of this via _autowrap_function_ids / _autowrap_search + # semantics would be slightly different, but would add support `from x import wrapped_function` + _wrapped_fns_to_patch[(id(f.f_globals), fn_name)] = f.f_globals + return fn_or_name + + +@compatibility(is_backward_compatible=True) +def symbolic_trace( + root: Union[torch.nn.Module, Callable[..., Any]], + concrete_args: Optional[dict[str, Any]] = None, +) -> GraphModule: + """ + Symbolic tracing API + + Given an ``nn.Module`` or function instance ``root``, this function will return a ``GraphModule`` + constructed by recording operations seen while tracing through ``root``. + + ``concrete_args`` allows you to partially specialize your function, whether it's to remove control flow or data structures. + + For example:: + + def f(a, b): + if b == True: + return a + else: + return a * 2 + + FX can typically not trace through this due to the presence of control + flow. However, we can use `concrete_args` to specialize on the value of + `b` to trace through this:: + + f = fx.symbolic_trace(f, concrete_args={"b": False}) + assert f(3, False) == 6 + + Note that although you can still pass in different values of `b`, they will be ignored. + + We can also use `concrete_args` to eliminate data-structure handling from + our function. This will use pytrees to flatten your input. To avoid + overspecializing, pass in `fx.PH` for values that shouldn't be + specialized. For example:: + + def f(x): + out = 0 + for v in x.values(): + out += v + return out + + + f = fx.symbolic_trace( + f, concrete_args={"x": {"a": fx.PH, "b": fx.PH, "c": fx.PH}} + ) + assert f({"a": 1, "b": 2, "c": 4}) == 7 + + + Args: + root (Union[torch.nn.Module, Callable]): Module or function to be traced and converted + into a Graph representation. + concrete_args (Optional[Dict[str, any]]): Inputs to be partially specialized + + Returns: + GraphModule: a Module created from the recorded operations from ``root``. + """ + tracer = Tracer() + graph = tracer.trace(root, concrete_args) + name = ( + root.__class__.__name__ if isinstance(root, torch.nn.Module) else root.__name__ + ) + return _make_graph_module(tracer.root, graph, name) + + +@wrap +def _assert_is_none(value, msg): + assert value is None, msg diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eb36e081d1c25e26da25e20f722604be57db9de0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/_utils.py @@ -0,0 +1,67 @@ +# mypy: allow-untyped-defs +import sys +from typing import Optional + +import torch +from torch._logging import LazyString + + +def lazy_format_graph_code(name, gm, maybe_id=None, **kwargs): + """ + Returns a LazyString that formats the graph code. + """ + + def format_name(): + if maybe_id is not None: + return f"{name} {maybe_id}" + else: + return name + + if "print_output" not in kwargs: + kwargs["print_output"] = False + + if "colored" in kwargs: + try: + if not sys.stdout.isatty(): + kwargs["colored"] = False + except AttributeError: + kwargs["colored"] = False + + return LazyString( + lambda: _format_graph_code( + f"===== {format_name()} =====\n", + gm.forward.__code__.co_filename, + gm.print_readable(**kwargs), + ) + ) + + +def _format_graph_code(name, filename, graph_str): + """ + Returns a string that formats the graph code. + """ + return f"TRACED GRAPH\n {name} {filename} {graph_str}\n" + + +def first_call_function_nn_module_stack(graph: torch.fx.Graph) -> Optional[dict]: + """ + Returns the nn_module_stack of the first call_function node. + """ + for node in graph.nodes: + if node.op == "call_function" and "nn_module_stack" in node.meta: + return node.meta["nn_module_stack"] + return None + + +def get_node_context(node, num_nodes=2) -> str: + """ + Returns a string of the last num_nodes nodes in the graph. + """ + node_contexts = [] + cur = node + for _ in range(num_nodes): + node_contexts.append(cur.format_node()) + if cur.op == "root": + break + cur = cur.prev + return "\n".join(node_contexts[::-1]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/annotate.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/annotate.py new file mode 100644 index 0000000000000000000000000000000000000000..67d9b7feb2d81096dea84c3c05db5c7e6bcba202 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/annotate.py @@ -0,0 +1,36 @@ +# mypy: allow-untyped-defs +from torch.fx.proxy import Proxy + +from ._compatibility import compatibility + + +@compatibility(is_backward_compatible=False) +def annotate(val, type): + """ + Annotates a Proxy object with a given type. + + This function annotates a val with a given type if a type of the val is a torch.fx.Proxy object + Args: + val (object): An object to be annotated if its type is torch.fx.Proxy. + type (object): A type to be assigned to a given proxy object as val. + Returns: + The given val. + Raises: + RuntimeError: If a val already has a type in its node. + """ + if isinstance(val, Proxy): + if val.node.type: + raise RuntimeError( + f"Tried to annotate a value that already had a type on it!" + f" Existing type is {val.node.type} " + f"and new type is {type}. " + f"This could happen if you tried to annotate a function parameter " + f"value (in which case you should use the type slot " + f"on the function signature) or you called " + f"annotate on the same value twice" + ) + else: + val.node.type = type + return val + else: + return val diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/config.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/config.py new file mode 100644 index 0000000000000000000000000000000000000000..ef7dd9a15e8f9860ea37c7e6b5bbf56daa648f84 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/config.py @@ -0,0 +1,6 @@ +# Whether to disable showing progress on compilation passes +# Need to add a new config otherwise will get a circular import if dynamo config is imported here +disable_progress = True + +# If True this also shows the node names in each pass, for small models this is great but larger models it's quite noisy +verbose_progress = False diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..e1212cbbf2019732825a3b172b0e3b8a4d4fb296 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph.py @@ -0,0 +1,2316 @@ +# mypy: allow-untyped-defs +import builtins +import contextlib +import copy +import enum +import functools +import inspect +import keyword +import logging +import math +import os +import pprint +import re +import types +import typing +import warnings +from collections import defaultdict +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Literal, NamedTuple, Optional, TYPE_CHECKING + +import torch +import torch.utils._pytree as pytree +from torch._C import _fx_map_arg as map_arg, _NodeIter +from torch._library.opaque_object import is_opaque_value_type +from torch.utils._dtype_abbrs import dtype_abbrs + +from . import _pytree as fx_pytree +from ._compatibility import compatibility +from .immutable_collections import immutable_dict +from .node import _get_qualified_name, _type_repr, Argument, Node, Target + + +log = logging.getLogger(__name__) + +__all__ = ["PythonCode", "CodeGen", "Graph"] + +if TYPE_CHECKING: + from ._symbolic_trace import Tracer # noqa: F401 + from .graph_module import GraphModule # noqa: F401 + + +# Mapping of builtins to their `typing` equivalent. +# (PEP585: See D68459095 test plan) +_origin_type_map = { + list: typing.List, # noqa: UP006 + dict: typing.Dict, # noqa: UP006 + set: typing.Set, # noqa: UP006 + frozenset: typing.FrozenSet, # noqa: UP006 + tuple: typing.Tuple, # noqa: UP006 +} + +_legal_ops = dict.fromkeys( + ["call_function", "call_method", "get_attr", "call_module", "placeholder", "output"] +) + + +# Signature for functions thattransforms the body (`list[str]`) of the +# generated code +TransformCodeFunc = Callable[[list[str]], list[str]] + + +class _CustomBuiltin(NamedTuple): + """Additional objs that we add to every graph's globals. + + The repr() for some standard library objects is not valid Python code without + an import. For common objects of this sort, we bundle them in the globals of + every FX graph. + """ + + # How to import this object from the standard library. + import_str: str + # The actual object, produced from that import string. + obj: Any + + +# Combined dict of disallowed variable names so we can check with one lookup +_illegal_names = {k: object() for k in keyword.kwlist} +_illegal_names.update(builtins.__dict__) # can't shadow a builtin name + +_custom_builtins: dict[str, _CustomBuiltin] = {} + + +def _register_custom_builtin(name: str, import_str: str, obj: Any): + _custom_builtins[name] = _CustomBuiltin(import_str, obj) + _illegal_names[name] = obj + + +_register_custom_builtin("inf", "from math import inf", math.inf) +_register_custom_builtin("nan", "from math import nan", math.nan) +_register_custom_builtin("NoneType", "NoneType = type(None)", type(None)) +_register_custom_builtin("torch", "import torch", torch) +_register_custom_builtin("device", "from torch import device", torch.device) +_register_custom_builtin("fx_pytree", "import torch.fx._pytree as fx_pytree", fx_pytree) +_register_custom_builtin("pytree", "import torch.utils._pytree as pytree", pytree) + + +def _is_magic(x: str) -> bool: + return x.startswith("__") and x.endswith("__") + + +def _snake_case(s: str) -> str: + """ + Transforms the given string ``s`` to a Python-style variable name + + Examples: + ``mod.snake_case`` -> ``mod.snake_case`` + ``mod.pascalCase``-> ``mod.pascal_case`` + ``mod.ALL_CAPS`` -> ``mod.all_caps`` + """ + return _snake_case_sub(s).lower() + + +# Replace occurrences where a lowercase letter is followed by an uppercase letter +_snake_case_sub = functools.partial(re.compile(r"(?<=[a-z])([A-Z])").sub, r"_\1") + +# Find chars that can't be in a Python identifier +_illegal_char_regex = re.compile("[^0-9a-zA-Z_]+") + +# Combined check for variable names: +# 1) Checks name is not empty +# 2) Checks first character is not a digit +# 3) Checks name has no illegal characters (_illegal_char_regex) +# 3) Splits off the number suffix (if present) +_name_regex = re.compile(r"^([a-zA-Z_][0-9a-zA-Z_]*?)(?:_(\d+))?$") + +# starts with torch but does not start with torch._dynamo. or torch._inductor. +_torch_but_not_dynamo = re.compile( + r"^torch(?:\.(?!_dynamo\.|_inductor\.)[^.]+)*$" +).fullmatch + + +def _is_from_torch(obj: Any) -> bool: + module_name = getattr(obj, "__module__", None) + if module_name is not None: + return _torch_but_not_dynamo(module_name) is not None + + name = getattr(obj, "__name__", None) + # exclude torch because torch.torch.torch.torch works. idk mang + if name is not None and name != "torch": + for guess in [torch, torch.nn.functional]: + if getattr(guess, name, None) is obj: + return True + + return False + + +class _Namespace: + """A context for associating names uniquely with objects. + + The following invariants are enforced: + - Each object gets a single name. + - Each name is unique within a given namespace. + - Names generated do not shadow builtins, unless the object is indeed that builtin. + """ + + def __init__(self): + self._obj_to_name: dict[Any, str] = {} + self._used_names: set[str] = set() + self._base_count: dict[str, int] = {} + + def create_name(self, candidate: str, obj: Optional[Any]) -> str: + """Create a unique name. + + Arguments: + candidate: used as the basis for the unique name, relevant to the user. + obj: If not None, an object that will be associated with the unique name. + """ + if obj is not None and obj in self._obj_to_name: + return self._obj_to_name[obj] + + # optimistically check if candidate is already a valid name + match = _name_regex.match(candidate) + if match is None: + # delete all characters that are illegal in a Python identifier + candidate = _illegal_char_regex.sub("_", candidate) + + if not candidate: + candidate = "_unnamed" + + if candidate[0].isdigit(): + candidate = f"_{candidate}" + + match = _name_regex.match(candidate) + assert match is not None + + base, num = match.group(1, 2) + if num is None or candidate in self._used_names: + num = self._base_count.get(candidate, 0) + if _illegal_names.get(candidate, obj) is not obj: + num += 1 + candidate = f"{base}_{num}" + # assume illegal names don't end in _\d so no need to check again + else: + num = int(num) + + while candidate in self._used_names: + num += 1 + candidate = f"{base}_{num}" + + self._used_names.add(candidate) + self._base_count[base] = num + if obj is not None: + self._obj_to_name[obj] = candidate + return candidate + + def associate_name_with_obj(self, name: str, obj: Any): + """Associate a unique name with an object. + + Neither `name` nor `obj` should be associated already. + """ + maybe_existing = self._obj_to_name.setdefault(obj, name) + assert maybe_existing is name, "obj is already associated" + + def _rename_object(self, obj: Any, name: str): + assert obj in self._obj_to_name + self._obj_to_name[obj] = name + self._used_names.add(name) + + +@compatibility(is_backward_compatible=True) +@dataclass +class PythonCode: + """ + Represents all the information necessary to exec or save a graph as Python code. + """ + + # Python source code for the forward function definition. + src: str + # Values in global scope during execution of `src_def`. + globals: dict[str, Any] + # Optional mapping from the forward function's line number to + # node index. Line number starts at the prologue (i.e. forward()). + _lineno_map: Optional[dict[int, Optional[int]]] + # The line number of prologue in fn_code + _prologue_start: int = 0 + + +def _format_target(base: str, target: str) -> str: + elems = target.split(".") + r = base + for e in elems: + if not e.isidentifier(): + r = f'getattr({r}, "{e}")' + else: + r = f"{r}.{e}" + return r + + +class _InsertPoint: + def __init__(self, graph, new_insert): + self.graph = graph + self.orig_insert, graph._insert = graph._insert, new_insert + + def __enter__(self): + pass + + def __exit__(self, type, value, tb): + self.graph._insert = self.orig_insert + + +class _node_list: + def __init__(self, graph: "Graph", direction: Literal["_prev", "_next"] = "_next"): + assert direction in ("_next", "_prev") + self.graph = graph + self.direction = direction + + def __len__(self): + return self.graph._len + + def __iter__(self): + return _NodeIter(self.graph._root, self.direction == "_prev") + + def __reversed__(self): + return _node_list(self.graph, "_next" if self.direction == "_prev" else "_prev") + + +class _PyTreeInfo(NamedTuple): + """ + Contains extra info stored when we're using Pytrees + """ + + orig_args: list[str] + in_spec: pytree.TreeSpec + out_spec: Optional[pytree.TreeSpec] + + +@dataclass(frozen=True) +class _ParsedStackTrace: + """ + Represents the top-most frame of a parsed stack trace + """ + + file: str + lineno: str + name: str + code: str + + def get_summary_str(self): + return f"File: {self.file}:{self.lineno} in {self.name}, code: {self.code}" + + +# get File:lineno code from stack_trace +def _parse_stack_trace( + stack_trace: str, filter_fn: Optional[Callable[[str, str, str], bool]] = None +): + if stack_trace is None: + return None + pattern = re.compile(r"^File \"(.+)\", line (\d+), in (.+)$") + lines = stack_trace.strip().split("\n") + # stacktrace should have innermost frame last, so we + # iterate backwards to find the first line that starts + # with 'File ' + for idx in range(len(lines) - 2, -1, -1): + line = lines[idx].strip() + matches = pattern.match(line) + if matches: + file = matches.group(1) + lineno = matches.group(2) + name = matches.group(3) + # next line should be the code + code = lines[idx + 1].strip() + if filter_fn and not filter_fn(file, name, code): + continue + return _ParsedStackTrace(file, lineno, name, code) + return None + + +@compatibility(is_backward_compatible=False) +class CodeGen: + # This is an override hook so we can customize the SymNode printer. + _sym_repr: Callable[["torch.types.PySymType"], str] = lambda x: repr(x) + + def __init__(self): + self._body_transformer: Optional[TransformCodeFunc] = None + self._func_name: str = "forward" + + def _format_multiline_args(self, args: list[str]) -> str: + """Helper to format function arguments in expanded multiline format.""" + return "".join(self._format_single_arg(arg) for arg in args) + + def _format_single_arg(self, arg: str) -> str: + """Helper to format a single argument with optional comment.""" + if "#" in arg: + arg_part, comment_part = arg.split("#", 1) + return f" {arg_part.rstrip()}, # {comment_part.lstrip()}\n" + else: + return f" {arg},\n" + + def _get_delimiters(self, container) -> tuple[str, str]: + """Helper to get opening and closing delimiters for containers.""" + return ("(", ")") if isinstance(container, tuple) else ("[", "]") + + def _format_multiline_container(self, items, descs=None, prefix="") -> str: + """Helper to format containers (lists/tuples) in multiline format.""" + ldelim, rdelim = self._get_delimiters(items) + desc_trailers = self._get_desc_trailers(items, descs) + + return ( + f"{prefix}{ldelim}\n" + + "".join( + f" {item},{trailer}\n" for item, trailer in zip(items, desc_trailers) + ) + + f"{rdelim}" + ) + + def _get_desc_trailers(self, items, descs): + """Helper to generate description trailers for items.""" + if descs is None: + return [""] * len(items) + return [f" # {desc}" for desc in descs] + + def _call_method_with_signature_check(self, method, *args, **kwargs): + """Helper to call a method with optional parameters based on signature.""" + sig = inspect.signature(method) + # Filter kwargs to only include parameters that exist in the method signature + filtered_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} + return method(*args, **filtered_kwargs) + + def gen_fn_def( + self, + free_vars: list[str], + maybe_return_annotation: str, + *, + expanded_def: bool = False, + ) -> str: + """ + Given the free variables and a return annotation, generates the beginning of the FX function. + By default, `gen_fn_def(['a', 'b'], '') == 'def {self._func_name}(a, b):'` + """ + # If the original function didn't have self as its first argument, we + # would have added it. + if len(free_vars) == 0 or free_vars[0] != "self": + free_vars.insert(0, "self") + + if expanded_def: + args_formatted = self._format_multiline_args(free_vars) + return ( + f"def {self._func_name}(\n{args_formatted}){maybe_return_annotation}:" + ) + else: + return f"def {self._func_name}({', '.join(free_vars)}){maybe_return_annotation}:" + + def generate_output( + self, output_args: Argument, *, descs: Optional[Any] = None + ) -> str: + """ + Given the output arguments, generates the return statement of the FX function. + Note: The returned statement should not be indented. + """ + if descs is not None and isinstance(output_args, (list, tuple)): + return self._format_multiline_container(output_args, descs, "return ") + else: + return f"return {repr(output_args)}" + + def process_inputs(self, *args: Any) -> Any: + """ + Transforms the inputs so that the graph can take them as arguments, as + non-default codegen may result in the inputs to the function being + different from the inputs to the graph. + + If the graph was directly runnable, this invariant should hold true + `f.graph.process_outputs(f.graph(*f.graph.process_inputs(*inputs))) == f(*inputs)` + """ + return args + + def process_outputs(self, outputs: Any) -> Any: + """ + Transforms the outputs of the graph to be identical to the codegen. + + See ``process_inputs`` for more details. + """ + return outputs + + def additional_globals(self) -> list[tuple[str, Any]]: + """ + If your codegen uses extra global values, add tuples of (identifier,reference to the value) here. + For example, return ['List', typing.List] if you need ``List`` in the global context. + """ + return [] + + def _gen_python_code( + self, + nodes, + root_module: str, + namespace: _Namespace, + *, + verbose: bool = False, + include_stride: bool = False, + include_device: bool = False, + colored: bool = False, + # Render each argument on its own line + expanded_def: bool = False, + record_func: bool = False, + ) -> PythonCode: + free_vars: list[str] = [] + body: list[str] = [] + globals_: dict[str, Any] = {} + wrapped_fns: dict[str, None] = {} + + # Wrap string in list to pass by reference + maybe_return_annotation: list[str] = [""] + include_stride = include_stride or ( + os.environ.get("FX_GRAPH_SHOW_STRIDE", "0") == "1" + ) + include_device = include_device or ( + os.environ.get("FX_GRAPH_SHOW_DEVICE", "0") == "1" + ) + include_meta = os.environ.get("FX_GRAPH_SHOW_META", "0") == "1" + + def add_global(name_hint: str, obj: Any): + """Add an obj to be tracked as a global. + + We call this for names that reference objects external to the + Graph, like functions or types. + + Returns: the global name that should be used to reference 'obj' in generated source. + """ + if ( + _is_from_torch(obj) and obj != torch.device + ): # to support registering torch.device + # HACK: workaround for how torch custom ops are registered. We + # can't import them like normal modules so they must retain their + # fully qualified name. + return _get_qualified_name(obj) + + # normalize the name hint to get a proper identifier + global_name = namespace.create_name(name_hint, obj) + + if global_name in globals_: + assert globals_[global_name] == obj + return global_name + globals_[global_name] = obj + return global_name + + # Pre-fill the globals table with registered builtins. + for name, (_, obj) in _custom_builtins.items(): + add_global(name, obj) + + def type_repr(o: Any): + if o == (): + # Empty tuple is used for empty tuple type annotation Tuple[()] + return "()" + + typename = _type_repr(o) + if isinstance(o, types.UnionType) and "|" in typename: + # str | int + args = [type_repr(arg) for arg in o.__args__] + return "|".join(args) + + if origin_type := getattr(o, "__origin__", None): + # list[...], typing.List[...], TensorType[...] + + if isinstance(o, typing._GenericAlias): # type: ignore[attr-defined] + # This is a generic pre-PEP585 type, e.g. typing.List[torch.Tensor] + origin_type = _origin_type_map.get(origin_type, origin_type) + + origin_typename = add_global(_type_repr(origin_type), origin_type) + + if hasattr(o, "__args__") and o.__args__: + args = [type_repr(arg) for arg in o.__args__] + return f"{origin_typename}[{','.join(args)}]" + else: + return origin_typename + + # Common case: this is a regular module name like 'foo.bar.baz' + return add_global(typename, o) + + if colored: + red = _color_fns["red"] + dim_green = _color_fns["dim_green"] + dim = _color_fns["dim"] + dim_blue = _color_fns["dim_blue"] + blue = _color_fns["blue"] + else: + red = _identity + dim_green = _identity + dim = _identity + dim_blue = _identity + blue = _identity + + def _get_repr(arg: Any) -> str: + if isinstance(arg, Node): # first because common + return repr(arg) + elif isinstance(arg, tuple) and hasattr(arg, "_fields"): + # Handle NamedTuples (if it has `_fields`) via add_global. + qualified_name = _get_qualified_name(type(arg)) + global_name = add_global(qualified_name, type(arg)) + return f"{global_name}{repr(tuple(arg))}" + elif isinstance( + arg, (torch._ops.OpOverload, torch._ops.HigherOrderOperator) + ): + qualified_name = _get_qualified_name(arg) + global_name = add_global(qualified_name, arg) + return f"{global_name}" + elif isinstance(arg, enum.Enum): + cls = arg.__class__ + clsname = add_global(cls.__name__, cls) + return f"{clsname}.{arg.name}" + elif isinstance(arg, torch.Tensor): + size = list(arg.size()) + dtype = str(arg.dtype).split(".")[-1] + return f"torch.Tensor(size={size}, dtype={dtype})" + elif isinstance(arg, tuple): + if len(arg) == 1: + return f"({_get_repr(arg[0])},)" + else: + return "(" + ", ".join(_get_repr(a) for a in arg) + ")" + elif isinstance(arg, list): + return "[" + ", ".join(_get_repr(a) for a in arg) + "]" + elif isinstance(arg, slice): + return f"slice({_get_repr(arg.start)}, {_get_repr(arg.stop)}, {_get_repr(arg.step)})" + elif is_opaque_value_type(type(arg)): + arg_type = type(arg) + add_global(arg_type.__name__, arg_type) + return repr(arg) + else: + return blue(repr(arg)) + + def _format_args( + args: tuple[Argument, ...], kwargs: dict[str, Argument] + ) -> str: + res = [_get_repr(a) for a in args] + res.extend([f"{k} = {_get_repr(v)}" for k, v in kwargs.items()]) + return ", ".join(res) + + # Run through reverse nodes and record the first instance of a use + # of a given node. This represents the *last* use of the node in the + # execution order of the program, which we will use to free unused + # values + node_to_last_use: dict[Node, Node] = {} + user_to_last_uses: dict[Node, list[Node]] = {} + + def register_last_uses(n: Node, user: Node): + if n not in node_to_last_use: + node_to_last_use[n] = user + user_to_last_uses.setdefault(user, []).append(n) + + for node in reversed(nodes): + for input_node in node._input_nodes: + register_last_uses(input_node, node) + + def delete_unused_values(user: Node): + """ + Delete values after their last use. This ensures that values that are + not used in the remainder of the code are freed and the memory usage + of the code is optimal. + """ + if user.op == "placeholder": + return + if user.op == "output": + body.append("\n") + return + nodes_to_delete = user_to_last_uses.get(user, []) + + if len(user.users.keys()) == 0: + # This node is not used by any others. however it's also not + # removed by DCE since side-effect. We want to free it's outputs + # right after its execution done to save memory. + nodes_to_delete.append(user) + + if len(nodes_to_delete): + to_delete_str = " = ".join( + [repr(n) for n in nodes_to_delete] + ["None"] + ) + body.append(f"; {dim(to_delete_str)}\n") + else: + body.append("\n") + + prev_summary_str = None + + def append_stacktrace_summary(node: Node): + """ + Append a summary of the stacktrace to the generated code. This is + useful for debugging. + """ + nonlocal prev_summary_str + + if node.op not in {"placeholder", "output"}: + annotation_str = "" + annotation = node.meta.get("custom", {}) + if annotation: + annotation_str = f" Annotation: {annotation}" + + stack_trace_str = "No stacktrace found for following nodes" + if stack_trace := node.stack_trace: + if parsed_stack_trace := _parse_stack_trace(stack_trace): + stack_trace_str = parsed_stack_trace.get_summary_str() + + maybe_recompute_info = "" + if hasattr(node, "meta") and node.meta: + # recompute tags are generated by torch.compile and put in the joint graph. + # These tags are load bearing enough that we want them to show up by default + # in tlparse, when you run torch.compile. + recompute = node.meta.get("recompute", None) + ac_graph_id = node.meta.get("ac_graph_id", None) + + if recompute is not None and ac_graph_id is not None: + maybe_recompute_info = f" # ac_graph_id: {str(ac_graph_id)} - {str(recompute.name)}" + elif recompute is not None: + maybe_recompute_info = f" # recompute: {str(recompute.name)}" + elif ac_graph_id is not None: + maybe_recompute_info = f" # ac_graph_id: {str(ac_graph_id)}" + + summary_str = f"\n{dim(f'#{annotation_str}{maybe_recompute_info} {stack_trace_str}')}\n" + + if summary_str != prev_summary_str: + prev_summary_str = summary_str + body.append(summary_str) + + def stringify_shape(shape: Iterable) -> str: + return f"[{', '.join([str(x) for x in shape])}]" + + def emit_node(node: Node): + maybe_type_annotation = ( + "" if node.type is None else f" : {type_repr(node.type)}" + ) + maybe_comment = "" + + if verbose: + # override annotation with more detailed information + try: + from torch.distributed.tensor._api import DTensor, DTensorSpec + + dtensorspec_format_shard_order_str = ( + DTensorSpec.format_shard_order_str + ) + except ModuleNotFoundError: + DTensor = None # type: ignore[assignment,misc] + dtensorspec_format_shard_order_str = None + from torch.fx.experimental.proxy_tensor import py_sym_types + from torch.fx.passes.shape_prop import TensorMetadata + + meta_val = node.meta.get( + "val", + node.meta.get("tensor_meta", node.meta.get("example_value", None)), + ) + + def _tensor_annotation(t: torch.Tensor) -> str: + stride = stringify_shape(t.stride()) if include_stride else "" + device = f"{t.device}" if include_device else "" + return ( + f"{red(dtype_abbrs[t.dtype])}" + f"{blue(stringify_shape(t.shape))}" + f"{dim_blue(stride)}" + f"{dim_green(device)}" + ) + + # use string as annotation, to make it valid python code + if isinstance(meta_val, torch.Tensor) and meta_val.layout not in ( + torch.sparse_csc, + torch.sparse_csr, + ): + # Fake tensors cause tests to wobble, so do not custom print them. + is_plain = type(meta_val) is torch.Tensor or isinstance( + meta_val, torch._subclasses.FakeTensor + ) + core = _tensor_annotation(meta_val) + if is_plain: + maybe_type_annotation = f': "{core}"' + elif type(meta_val) is DTensor: + assert dtensorspec_format_shard_order_str is not None + dtensor_meta = dtensorspec_format_shard_order_str( + meta_val._spec.placements, # type: ignore[attr-defined] + meta_val._spec.shard_order, # type: ignore[attr-defined] + ) + cls = meta_val.__class__.__name__ + maybe_type_annotation = ( + f': "{cls}({core}, {dim_green(dtensor_meta)})"' + ) + else: + cls = meta_val.__class__.__name__ + maybe_type_annotation = f': "{cls}({core})"' + + elif isinstance(meta_val, py_sym_types): + val_str = CodeGen._sym_repr(meta_val) + maybe_type_annotation = f': "Sym({val_str})"' + + elif isinstance(meta_val, TensorMetadata): + maybe_type_annotation = f': "{dtype_abbrs[meta_val.dtype]}{stringify_shape(meta_val.shape)}"' + + desc = None + if expanded_def: + desc = node.meta.get("desc", None) + if desc is not None and node.op == "placeholder": + maybe_comment += f" # {desc}" + # output is handled specially + + if include_meta and hasattr(node, "meta") and node.meta: + body.append('"""\n') + for k, v in node.meta.items(): + # use str over repr since repr is susceptible to sympy + # errors such as "cannot determine truth value of Relational" + # Pretty print the high-level dict with str() for values + body.append( + f"{k}: {pprint.pformat(str(v), width=80, compact=True)}\n" + ) + body.append('"""\n') + + if node.op == "placeholder": + assert isinstance(node.target, str) + maybe_default_arg = ( + "" if not node.args else f" = {_get_repr(node.args[0])}" + ) + free_vars.append( + f"{node.target}{maybe_type_annotation}{maybe_default_arg}{maybe_comment}" + ) + raw_name = node.target.replace("*", "") + if raw_name != repr(node): + body.append(f"{repr(node)} = {raw_name}\n") + return + elif node.op == "call_method": + assert isinstance(node.target, str) + body.append( + f"{repr(node)}{maybe_type_annotation} = {_format_target(_get_repr(node.args[0]), node.target)}" + f"({_format_args(node.args[1:], node.kwargs)})" + ) + return + elif node.op == "call_function": + assert callable(node.target) + # pretty print operators + if ( + getattr(node.target, "__module__", "") == "_operator" + and node.target.__name__ in magic_methods + ): + assert isinstance(node.args, tuple) + body.append( + f"{repr(node)}{maybe_type_annotation} = " + f"{magic_methods[node.target.__name__].format(*(_get_repr(a) for a in node.args))}" + ) + return + + # pretty print inplace operators; required for jit.script to work properly + # not currently supported in normal FX graphs, but generated by torchdynamo + if ( + getattr(node.target, "__module__", "") == "_operator" + and node.target.__name__ in inplace_methods + ): + body.append( + f"{inplace_methods[node.target.__name__].format(*(_get_repr(a) for a in node.args))}; " + f"{repr(node)}{maybe_type_annotation} = {_get_repr(node.args[0])}" + ) + return + + qualified_name = _get_qualified_name(node.target) + global_name = add_global(qualified_name, node.target) + # special case for getattr: node.args could be 2-argument or 3-argument + # 2-argument: attribute access; 3-argument: fall through to attrib function call with default value + if ( + global_name == "getattr" + and isinstance(node.args, tuple) + and isinstance(node.args[1], str) + and node.args[1].isidentifier() + and len(node.args) == 2 + ): + body.append( + f"{repr(node)}{maybe_type_annotation} = {_format_target(_get_repr(node.args[0]), node.args[1])}" + ) + return + body.append( + f"{repr(node)}{maybe_type_annotation} = {global_name}({_format_args(node.args, node.kwargs)})" + ) + if node.meta.get("is_wrapped", False): + wrapped_fns.setdefault(global_name) + return + elif node.op == "call_module": + assert isinstance(node.target, str) + body.append( + f"{repr(node)}{maybe_type_annotation} = " + f"{_format_target(root_module, node.target)}({_format_args(node.args, node.kwargs)})" + ) + return + elif node.op == "get_attr": + assert isinstance(node.target, str) + body.append( + f"{repr(node)}{maybe_type_annotation} = {_format_target(root_module, node.target)}" + ) + return + elif node.op == "output": + if node.type is not None: + maybe_return_annotation[0] = f" -> {type_repr(node.type)}" + body.append( + self._call_method_with_signature_check( + self.generate_output, + node.args[0], + descs=desc if expanded_def else None, + ) + ) + return + raise NotImplementedError(f"node: {node.op} {node.target}") + + if record_func: + body.append( + "_rf = torch._C._profiler._RecordFunctionFast('## ENTER_GRAPH_PLACEHOLDER_KEY ##'); _rf.__enter__()\n" + ) + for i, node in enumerate(nodes): + # NOTE: emit_node does not emit a string with newline. It depends + # on delete_unused_values to append one + if verbose: + append_stacktrace_summary(node) + # emit a counter comment to keep track of + # node index, which will be deleted later + # after going through _body_transformer + body.append(f"# COUNTER: {i}\n") + do_record = record_func and node.op in ( + "call_function", + "call_method", + "call_module", + ) + if do_record: + # The double hash ## convention is used by post-processing to find the fx markers + body.append( + f"_rf_{node.name} = torch._C._profiler._RecordFunctionFast('## {i} ##'); _rf_{node.name}.__enter__()\n" + ) + emit_node(node) + delete_unused_values(node) + if do_record: + body.append(f"_rf_{node.name}.__exit__(None, None, None)\n") + if record_func: + body.append("_rf.__exit__(None, None, None)\n") + + if len(body) == 0: + # If the Graph has no non-placeholder nodes, no lines for the body + # have been emitted. To continue to have valid Python code, emit a + # single pass statement + body.append("pass\n") + + if len(wrapped_fns) > 0: + wrap_name = add_global("wrap", torch.fx.wrap) + wrap_stmts = "\n".join([f'{wrap_name}("{name}")' for name in wrapped_fns]) + else: + wrap_stmts = "" + + if self._body_transformer: + body = self._body_transformer(body) + + for name, value in self.additional_globals(): + add_global(name, value) + + prologue = self._call_method_with_signature_check( + self.gen_fn_def, + free_vars, + maybe_return_annotation[0], + expanded_def=expanded_def, + ) + + # remove counter and generate lineno to node index mapping + lineno_map: dict[int, Optional[int]] = {} + prologue_len = prologue.count("\n") + 1 + new_lines: list[str] = [] + cur_idx = None + for line in "".join(body).split("\n"): + counter = _counter_regexp.search(line) + if counter is not None: + cur_idx = int(counter.group(1)) + else: + lineno_map[len(new_lines) + prologue_len] = cur_idx + new_lines.append(line) + + code = "\n".join(new_lines).lstrip("\n") + code = "\n".join(" " + line for line in code.split("\n")) + + fn_code = f""" +{wrap_stmts} + +{prologue} +{code}""" + # The +4 accounts for the empty lines before prologue in fn_code + prologue_start = wrap_stmts.count("\n") + 4 + return PythonCode( + fn_code, + globals_, + _lineno_map=lineno_map, + _prologue_start=prologue_start, + ) + + +# Ideally, we'd like to refactor all of the pytree logic into this codegen +# class. Unfortunately, there are 3 areas we currently need extra logic in FX. +# 1. In the initial symbolic trace, the pytree logic is tied up with `concrete_args`. +# 2. In the FX graph, we need to access 2 attributes - in_spec and out_spec. +# Since we can't access .graph within the FX forward, we need to copy the attribute to the module. +# 3. We currently can't register the pytree imports with `add_global` - not sure why. +class _BoxedCodeGen(CodeGen): + """ + CodeGen subclass that generates code using the "boxed" calling convention. + + The boxed calling convention takes a single list argument and clears it + after extracting the arguments, which allows for early deallocation of + input tensors. + """ + + def gen_fn_def( + self, free_vars, maybe_return_annotation, *, expanded_def: bool = False + ): + """ + Generate function definition for boxed calling convention. + + Instead of taking individual arguments, the generated function takes + a single 'args_list' parameter, extracts placeholder values from it, + and clears the list. + """ + # Generate the function signature with args_list parameter + fn_def = f"def {self._func_name}(self, args_list){maybe_return_annotation}:" + + if free_vars: + # This is horribly manual but we don't get the "raw" free vars + # without a bigger refactor. + placeholder_vars = [ + v.split(":")[0].split("=")[0].strip() for v in free_vars if v != "self" + ] + + if placeholder_vars: + fn_def += "\n args_iter = iter(args_list)" + for var in placeholder_vars: + fn_def += f"\n {var} = next(args_iter)" + fn_def += "\n args_list.clear()" + + return fn_def + + +class _PyTreeCodeGen(CodeGen): + def __init__(self, pytree_info: _PyTreeInfo): + super().__init__() + self.pytree_info: _PyTreeInfo = pytree_info + + def process_inputs(self, *inputs: Any) -> Any: + flat_args = pytree.arg_tree_leaves(*inputs) + return flat_args + + def process_outputs(self, out: Any) -> Any: + if self.pytree_info is None or self.pytree_info.out_spec is None: + return out + if not isinstance(out, (list, tuple)): + out = [out] + assert self.pytree_info.out_spec is not None + return pytree.tree_unflatten(out, self.pytree_info.out_spec) + + def _format_annotations(self, free_vars: list[str], expanded_def: bool) -> str: + """Helper to format annotations for variables in pytree codegen.""" + if not free_vars: + return "" + + has_annotation = [x for x in free_vars if ":" in x] + if not has_annotation: + return "" + + if expanded_def: + return "\n " + "\n ".join(has_annotation) + else: + return "\n " + "".join(x + "; " for x in has_annotation) + "\n" + + def gen_var_bindings(self, fn_args, free_vars, expanded_def) -> str: + in_spec = self.pytree_info.in_spec + # when kwargs is present, in_spec is tuple(args, kwargs) + has_args_kwargs_tuple = ( + in_spec.type is tuple + and in_spec.num_children == 2 + and in_spec.child(0).type is tuple + and in_spec.child(1).type is dict + ) + fn_kwargs = "{}" + fn_signature = f"[{', '.join(fn_args)}], self._in_spec" + if has_args_kwargs_tuple: + count_args = in_spec.child(0).num_children + fn_args = self.pytree_info.orig_args[:count_args] + fn_kwargs = ( + "{" + + ", ".join( + f"'{k}':{v}" + for k, v in zip( + in_spec.child(1).context, + self.pytree_info.orig_args[count_args:], + ) + ) + + "}" + ) + fn_signature = f"([{', '.join(fn_args)}], {fn_kwargs}), self._in_spec" + + # in Python, `var1: annotation1, var2: annotation2 = function_call()` is invalid. + # we need to split it to two lines: + # one for annotation: `var1: annotation1; var2: annotation2;` (note the semicolon) + # one for code: `var1, var2, = function_call()` + without_annotation = [x.split(":")[0].split("#")[0] for x in free_vars] + bindings = self._format_annotations(free_vars, expanded_def) + bindings += f""" + {", ".join(without_annotation)}, = fx_pytree.tree_flatten_spec({fn_signature})""" + return bindings + + def gen_fn_def( + self, free_vars, maybe_return_annotation, *, expanded_def: bool = False + ): + # Given a user function/model: + # myargs = [myargs0, myargs1] + # mykwargs = {'mykwargs0': ..., 'mykwargs1': ...} + # def forward(self, mypos, *myargs, mykey=None, **mykwargs): + # + # The generated code flattens all keywords into positional arguments for `forward()` + # e.g forward(self, mypos, myargs0, myargs1, mykey, mykwargs0, mykwargs1): + # + # Within `forward`, `tree_flatten_spec``still parses args and kwargs separately + # e.g. tree_flatten_spec(([mypos, myargs0, myargs1], + # {'mykey':mykey, 'mykwargs0':mykwargs0, 'mykwargs1':mykwargs1}), + # self._in_spec) + # + # If the user function/model does not have keywords, the dict is suppressed from tree_flatten_spec + # e.g. tree_flatten_spec([mypos, myargs0, myargs1]), self._in_spec) + if self.pytree_info is None: + return super().gen_fn_def( + free_vars, maybe_return_annotation, expanded_def=expanded_def + ) + + fn_args = self.pytree_info.orig_args + has_orig_self = (fn_args[0] == "self") if len(fn_args) > 0 else False + if has_orig_self: + free_vars.insert(0, "self") + fn_definition = super().gen_fn_def( + fn_args[:], maybe_return_annotation, expanded_def=expanded_def + ) + + if len(free_vars) > 0: # pytree has placeholders in it + fn_definition += self.gen_var_bindings(fn_args, free_vars, expanded_def) + return fn_definition + + def generate_output(self, output_args, *, descs: Optional[Any] = None): + if self.pytree_info and self.pytree_info.out_spec: + if descs is not None and isinstance(output_args, (list, tuple)): + return ( + self._format_multiline_container( + output_args, descs, "return pytree.tree_unflatten(" + ) + + ", self._out_spec)" + ) + else: + return ( + f"return pytree.tree_unflatten({repr(output_args)}, self._out_spec)" + ) + else: + return super().generate_output(output_args, descs=descs) + + +class _ExportCodeGen(_PyTreeCodeGen): + def __init__( + self, + pytree_info: _PyTreeInfo, + in_shuffle_graph: "GraphModule", + out_shuffle_graph: "GraphModule", + tree_leaf_names: list[str], + root: Optional[torch.nn.Module], + ): + super().__init__(pytree_info) + self.in_shuffle_graph = in_shuffle_graph + self.out_shuffle_graph = out_shuffle_graph + self.tree_leaf_names = tree_leaf_names + self.root = root + + def process_inputs(self, *inputs: Any) -> Any: + flat_args = super().process_inputs(*inputs) + if self.root is not None: + flat_args = (self.root, *flat_args) + self.flat_args = flat_args + return self.in_shuffle_graph(*flat_args) + + def process_outputs(self, out: Any) -> Any: + flat_outs = self.out_shuffle_graph(*self.flat_args, *out) + del self.flat_args + ret = super().process_outputs(flat_outs) + return ret + + def gen_fn_def(self, *args, **kwargs) -> str: + fn_def = super().gen_fn_def(*args, **kwargs) + return fn_def + + def gen_var_bindings(self, fn_args, free_vars, expanded_def) -> str: + without_annotation = [x.split(":")[0].split("#")[0] for x in free_vars] + fn_signature: str = f"{', '.join(fn_args)}" + if self.root is not None: + fn_signature = f"self, {fn_signature}" + return f""" + {", ".join(self.tree_leaf_names)}, = pytree.tree_leaves(({fn_signature},)) + {", ".join(without_annotation)}, = self._in_shuffle_graph({", ".join(self.tree_leaf_names)})""" + + def generate_output(self, output_args, *args, **kwargs) -> str: + output = f"self._out_shuffle_graph({', '.join(self.tree_leaf_names)}, {', '.join([str(a) for a in output_args])})" + return f"return pytree.tree_unflatten({output}, self._out_spec)" + + +class _FindNodesLookupTable: + """ + Side table for the graph for the purpose of doing fast queries + """ + + def __init__(self): + self.table: dict[tuple[str, Optional[Target]], dict[Node, None]] = defaultdict( + dict + ) + + def _key(self, node) -> tuple[str, Optional[Target]]: + return (node.op, node.target if node.op == "call_function" else None) + + def __contains__(self, node) -> bool: + return node in self.table[self._key(node)] + + def insert(self, node: Node) -> None: + self.table[self._key(node)][node] = None + + def remove(self, node: Node) -> None: + self.table[self._key(node)].pop(node) + + def find_nodes(self, *, op: str, target: Optional["Target"] = None): + if op == "call_function": + assert target is not None + return [*self.table[(op, target)].keys()] + + if target is None: + return [*self.table[(op, None)].keys()] + + # op is call_method, get_attr, call_module + return [node for node in self.table[(op, None)] if node.target == target] + + +@compatibility(is_backward_compatible=True) +class Graph: + """ + ``Graph`` is the main data structure used in the FX Intermediate Representation. + It consists of a series of ``Node`` s, each representing callsites (or other + syntactic constructs). The list of ``Node`` s, taken together, constitute a + valid Python function. + + For example, the following code + + .. code-block:: python + + import torch + import torch.fx + + + class MyModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.param = torch.nn.Parameter(torch.rand(3, 4)) + self.linear = torch.nn.Linear(4, 5) + + def forward(self, x): + return torch.topk( + torch.sum(self.linear(x + self.linear.weight).relu(), dim=-1), 3 + ) + + + m = MyModule() + gm = torch.fx.symbolic_trace(m) + + Will produce the following Graph:: + + print(gm.graph) + + .. code-block:: text + + graph(x): + %linear_weight : [num_users=1] = self.linear.weight + %add_1 : [num_users=1] = call_function[target=operator.add](args = (%x, %linear_weight), kwargs = {}) + %linear_1 : [num_users=1] = call_module[target=linear](args = (%add_1,), kwargs = {}) + %relu_1 : [num_users=1] = call_method[target=relu](args = (%linear_1,), kwargs = {}) + %sum_1 : [num_users=1] = call_function[target=torch.sum](args = (%relu_1,), kwargs = {dim: -1}) + %topk_1 : [num_users=1] = call_function[target=torch.topk](args = (%sum_1, 3), kwargs = {}) + return topk_1 + + For the semantics of operations represented in the ``Graph``, please see :class:`Node`. + """ + + @compatibility(is_backward_compatible=True) + def __init__( + self, + owning_module: Optional["GraphModule"] = None, + tracer_cls: Optional[type["Tracer"]] = None, + tracer_extras: Optional[dict[str, Any]] = None, + ): + """ + Construct an empty Graph. + """ + self._root: Node = Node(self, "", "root", "", (), {}) + self._used_names: dict[str, int] = {} # base name -> number + self._insert = self._root.prepend + self._len = 0 + self._graph_namespace = _Namespace() + self._owning_module = owning_module + self._tracer_cls = tracer_cls + self._tracer_extras = tracer_extras + self._codegen = CodeGen() + self._co_fields: dict[str, Any] = {} + self._find_nodes_lookup_table = _FindNodesLookupTable() + + @property + def owning_module(self): + return self._owning_module + + @owning_module.setter + def owning_module(self, mod: Optional["GraphModule"]): + self._owning_module = mod + + @property + def nodes(self) -> _node_list: + """ + Get the list of Nodes that constitute this Graph. + + Note that this ``Node`` list representation is a doubly-linked list. Mutations + during iteration (e.g. delete a Node, add a Node) are safe. + + Returns: + + A doubly-linked list of Nodes. Note that ``reversed`` can be called on + this list to switch iteration order. + """ + return _node_list(self) + + @compatibility(is_backward_compatible=False) + def output_node(self) -> Node: + output_node = next(iter(reversed(self.nodes))) + assert output_node.op == "output" + return output_node + + @compatibility(is_backward_compatible=False) + def find_nodes( + self, *, op: str, target: Optional["Target"] = None, sort: bool = True + ): + """ + Allows for fast query of nodes + + Args: + + op (str): the name of the operation + + target (Optional[Target]): the target of the node. For call_function, + the target is required. For other ops, the target is optional. + + sort (bool): whether to return nodes in the order they appear on + on the graph. + + Returns: + + Iterable of nodes with the requested op and target. + """ + node_list = self._find_nodes_lookup_table.find_nodes(op=op, target=target) + if sort: + return sorted(node_list) + return node_list + + @compatibility(is_backward_compatible=True) + def graph_copy( + self, g: "Graph", val_map: dict[Node, Node], return_output_node=False + ) -> "Optional[Argument]": + """ + Copy all nodes from a given graph into ``self``. + + Args: + + g (Graph): The source graph from which to copy Nodes. + + val_map (Dict[Node, Node]): a dictionary that will be populated with a mapping + from nodes in ``g`` to nodes in ``self``. Note that ``val_map`` can be passed + in with values in it already to override copying of certain values. + + Returns: + + The value in ``self`` that is now equivalent to the output value in ``g``, + if ``g`` had an ``output`` node. ``None`` otherwise. + """ + for node in g.nodes: + if node in val_map: + continue + if node.op == "output": + rv = map_arg(node.args[0], lambda n: val_map[n]) + return rv if not return_output_node else (rv, node) + val_map[node] = self.node_copy(node, lambda n: val_map[n]) + return None + + def __deepcopy__(self, memo=None) -> "Graph": + """ + Explicitly implement __deepcopy__ to prevent excessive recursion depth + from the default implementation. This uses graph_copy to copy the nodes + in an iterative way, rather than recursive. It also populates the + memoization table to prevent unnecessary copies (e.g. references to + nodes or other parts of the Graph from a custom GraphModule implementation. + """ + memo = memo if memo else {} + g = Graph(tracer_cls=self._tracer_cls) + output_vals = g.graph_copy(self, val_map=memo, return_output_node=True) + g._codegen = copy.deepcopy(self._codegen) + if output_vals is not None: + assert isinstance(output_vals, tuple) + output_val, old_output_node = output_vals + new_output_node = g.output( + output_val, type_expr=getattr(old_output_node, "type", None) + ) + new_output_node.meta = copy.copy(old_output_node.meta) + return g + + @compatibility(is_backward_compatible=True) + def create_node( + self, + op: str, + target: "Target", + args: Optional[tuple["Argument", ...]] = None, + kwargs: Optional[dict[str, "Argument"]] = None, + name: Optional[str] = None, + type_expr: Optional[Any] = None, + ) -> Node: + """ + Create a ``Node`` and add it to the ``Graph`` at the current insert-point. + Note that the current insert-point can be set via :meth:`Graph.inserting_before` + and :meth:`Graph.inserting_after`. + + Args: + op (str): the opcode for this Node. One of 'call_function', 'call_method', 'get_attr', + 'call_module', 'placeholder', or 'output'. The semantics of these opcodes are + described in the ``Graph`` docstring. + + args (Optional[Tuple[Argument, ...]]): is a tuple of arguments to this node. + + kwargs (Optional[Dict[str, Argument]]): the kwargs of this Node + + name (Optional[str]): an optional string name for the ``Node``. + This will influence the name of the value assigned to in the + Python generated code. + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + Returns: + + The newly-created and inserted node. + """ + # `target in _legal_ops` is checked in Node.__init__ + if not args: + args = () + else: + assert isinstance(args, tuple), "args must be a tuple" + if not kwargs: + kwargs = immutable_dict() + else: + assert isinstance(kwargs, dict), "kwargs must be a dict" + + candidate = name if name is not None else self._target_to_str(target) + name = self._graph_namespace.create_name(candidate, None) + n = Node(self, name, op, target, args, kwargs, type_expr) + + if ( + self.owning_module is not None + and getattr(self.owning_module, "_create_node_hooks", None) is not None + ): + for f in self.owning_module._create_node_hooks: + f(n) + + self._graph_namespace.associate_name_with_obj(name, n) + + self._insert(n) + self._find_nodes_lookup_table.insert(n) + self._len += 1 + return n + + @compatibility(is_backward_compatible=False) + def process_inputs(self, *args): + """ + Processes args so that they can be passed to the FX graph. + """ + return self._codegen.process_inputs(*args) + + @compatibility(is_backward_compatible=False) + def process_outputs(self, out): + return self._codegen.process_outputs(out) + + @compatibility(is_backward_compatible=True) + def erase_node(self, to_erase: Node) -> None: + """ + Erases a ``Node`` from the ``Graph``. Throws an exception if + there are still users of that node in the ``Graph``. + + Args: + + to_erase (Node): The ``Node`` to erase from the ``Graph``. + """ + if len(to_erase.users) > 0: + raise RuntimeError( + f"Tried to erase Node {to_erase} but it still had {len(to_erase.users)} " + f"users in the graph: {to_erase.users}!" + ) + if to_erase.graph != self: + raise RuntimeError(f"Attempting to remove {to_erase} from wrong graph!") + if to_erase._erased: + warnings.warn(f"erase_node({to_erase}) on an already erased node") + return + + if ( + self.owning_module is not None + and getattr(self.owning_module, "_erase_node_hooks", None) is not None + ): + for f in self.owning_module._erase_node_hooks: + f(to_erase) + + self._find_nodes_lookup_table.remove(to_erase) + # pyrefly: ignore [missing-attribute] + to_erase._remove_from_list() + to_erase._erased = True # iterators may retain handles to erased nodes + self._len -= 1 + + # Null out this Node's argument nodes so that the Nodes referred to + # can update their ``users`` accordingly + to_erase._update_args_kwargs( + map_arg(to_erase._args, lambda n: None), + map_arg(to_erase._kwargs, lambda n: None), + ) + + @compatibility(is_backward_compatible=True) + def inserting_before(self, n: Optional[Node] = None): + """Set the point at which create_node and companion methods will insert into the graph. + When used within a 'with' statement, this will temporary set the insert point and + then restore it when the with statement exits:: + + with g.inserting_before(n): + ... # inserting before node n + ... # insert point restored to what it was previously + g.inserting_before(n) # set the insert point permanently + + Args: + + n (Optional[Node]): The node before which to insert. If None this will insert before + the beginning of the entire graph. + + Returns: + A resource manager that will restore the insert point on ``__exit__``. + """ + if n is None: + return self.inserting_after(self._root) + assert n.graph == self, "Node to insert before is not in graph." + return _InsertPoint(self, n.prepend) + + @compatibility(is_backward_compatible=True) + def inserting_after(self, n: Optional[Node] = None): + """Set the point at which create_node and companion methods will insert into the graph. + When used within a 'with' statement, this will temporary set the insert point and + then restore it when the with statement exits:: + + with g.inserting_after(n): + ... # inserting after node n + ... # insert point restored to what it was previously + g.inserting_after(n) # set the insert point permanently + + Args: + + n (Optional[Node]): The node before which to insert. If None this will insert after + the beginning of the entire graph. + + Returns: + A resource manager that will restore the insert point on ``__exit__``. + """ + if n is None: + return self.inserting_before(self._root) + assert n.graph == self, "Node to insert after is not in graph." + return _InsertPoint(self, n.append) + + @compatibility(is_backward_compatible=True) + def placeholder( + self, + name: str, + type_expr: Optional[Any] = None, + default_value: Any = inspect.Signature.empty, + ) -> Node: + """ + Insert a ``placeholder`` node into the Graph. A ``placeholder`` represents + a function input. + + Args: + + name (str): A name for the input value. This corresponds to the name + of the positional argument to the function this ``Graph`` represents. + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. This is needed in some + cases for proper code generation (e.g. when the function is used + subsequently in TorchScript compilation). + + default_value (Any): The default value this function argument should take + on. NOTE: to allow for `None` as a default value, `inspect.Signature.empty` + should be passed as this argument to specify that the parameter does _not_ + have a default value. + + .. note:: + The same insertion point and type expression rules apply for this method + as ``Graph.create_node``. + """ + args = () if default_value is inspect.Signature.empty else (default_value,) + return self.create_node("placeholder", name, args=args, type_expr=type_expr) + + @compatibility(is_backward_compatible=True) + def get_attr(self, qualified_name: str, type_expr: Optional[Any] = None) -> Node: + """ + Insert a ``get_attr`` node into the Graph. A ``get_attr`` ``Node`` represents the + fetch of an attribute from the ``Module`` hierarchy. + + Args: + + qualified_name (str): the fully-qualified name of the attribute to be retrieved. + For example, if the traced Module has a submodule named ``foo``, which has a + submodule named ``bar``, which has an attribute named ``baz``, the qualified + name ``foo.bar.baz`` should be passed as ``qualified_name``. + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + + Returns: + + The newly-created and inserted ``get_attr`` node. + + .. note:: + The same insertion point and type expression rules apply for this method + as ``Graph.create_node``. + """ + + def _get_attr_reference_exists( + mod: torch.nn.Module, qualified_name: str + ) -> bool: + module_path, _, name = qualified_name.rpartition(".") + + try: + submod: torch.nn.Module = mod.get_submodule(module_path) + except AttributeError: + warnings.warn(f"Failed to fetch module {module_path}!") + return False + + if not hasattr(submod, name): + return False + + res = getattr(submod, name) + + if ( + not isinstance(res, torch.nn.Module) + and not isinstance(res, torch.nn.Parameter) + and name not in submod._buffers + ): + return False + + return True + + if self.owning_module and not _get_attr_reference_exists( + self.owning_module, qualified_name + ): + warnings.warn( + "Attempted to insert a get_attr Node with no " + "underlying reference in the owning " + "GraphModule! Call " + "GraphModule.add_submodule to add the " + "necessary submodule, " + "GraphModule.add_parameter to add the " + "necessary Parameter, or " + "nn.Module.register_buffer to add the " + "necessary buffer", + stacklevel=2, + ) + return self.create_node("get_attr", qualified_name, type_expr=type_expr) + + @compatibility(is_backward_compatible=True) + def call_module( + self, + module_name: str, + args: Optional[tuple["Argument", ...]] = None, + kwargs: Optional[dict[str, "Argument"]] = None, + type_expr: Optional[Any] = None, + ) -> Node: + """ + Insert a ``call_module`` ``Node`` into the ``Graph``. A ``call_module`` node + represents a call to the forward() function of a ``Module`` in the ``Module`` + hierarchy. + + Args: + + module_name (str): The qualified name of the ``Module`` in the ``Module`` + hierarchy to be called. For example, if the traced ``Module`` has a + submodule named ``foo``, which has a submodule named ``bar``, the + qualified name ``foo.bar`` should be passed as ``module_name`` to + call that module. + + args (Optional[Tuple[Argument, ...]]): The positional arguments to be passed + to the called method. Note that this should *not* include a ``self`` argument. + + kwargs (Optional[Dict[str, Argument]]): The keyword arguments to be passed + to the called method + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + Returns: + + The newly-created and inserted ``call_module`` node. + + .. note:: + The same insertion point and type expression rules apply for this method + as :meth:`Graph.create_node`. + """ + if self.owning_module and self.owning_module.get_submodule(module_name) is None: + warnings.warn( + "Attempted to insert a call_module Node with " + "no underlying reference in the owning " + "GraphModule! Call " + "GraphModule.add_submodule to add the " + "necessary submodule" + ) + return self.create_node( + "call_module", module_name, args, kwargs, type_expr=type_expr + ) + + @compatibility(is_backward_compatible=True) + def call_method( + self, + method_name: str, + args: Optional[tuple["Argument", ...]] = None, + kwargs: Optional[dict[str, "Argument"]] = None, + type_expr: Optional[Any] = None, + ) -> Node: + """ + Insert a ``call_method`` ``Node`` into the ``Graph``. A ``call_method`` node + represents a call to a given method on the 0th element of ``args``. + + Args: + + method_name (str): The name of the method to apply to the self argument. + For example, if args[0] is a ``Node`` representing a ``Tensor``, + then to call ``relu()`` on that ``Tensor``, pass ``relu`` to ``method_name``. + + args (Optional[Tuple[Argument, ...]]): The positional arguments to be passed + to the called method. Note that this *should* include a ``self`` argument. + + kwargs (Optional[Dict[str, Argument]]): The keyword arguments to be passed + to the called method + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + Returns: + + The newly created and inserted ``call_method`` node. + + .. note:: + The same insertion point and type expression rules apply for this method + as :meth:`Graph.create_node`. + """ + return self.create_node( + "call_method", method_name, args, kwargs, type_expr=type_expr + ) + + @compatibility(is_backward_compatible=True) + def call_function( + self, + the_function: Callable[..., Any], + args: Optional[tuple["Argument", ...]] = None, + kwargs: Optional[dict[str, "Argument"]] = None, + type_expr: Optional[Any] = None, + name: Optional[str] = None, + ) -> Node: + """ + Insert a ``call_function`` ``Node`` into the ``Graph``. A ``call_function`` node + represents a call to a Python callable, specified by ``the_function``. + + Args: + + the_function (Callable[..., Any]): The function to be called. Can be any PyTorch + operator, Python function, or member of the ``builtins`` or ``operator`` + namespaces. + + args (Optional[Tuple[Argument, ...]]): The positional arguments to be passed + to the called function. + + kwargs (Optional[Dict[str, Argument]]): The keyword arguments to be passed + to the called function + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + name (Optional[str]): The name of the node. If not specified, set to None + + Returns: + + The newly created and inserted ``call_function`` node. + + .. note:: + The same insertion point and type expression rules apply for this method + as :meth:`Graph.create_node`. + """ + return self.create_node( + "call_function", the_function, args, kwargs, name=name, type_expr=type_expr + ) + + @compatibility(is_backward_compatible=True) + def node_copy( + self, node: Node, arg_transform: Callable[[Node], "Argument"] = lambda x: x + ) -> Node: + """ + Copy a node from one graph into another. ``arg_transform`` needs to transform arguments from + the graph of node to the graph of self. Example:: + + # Copying all the nodes in `g` into `new_graph` + g: torch.fx.Graph = ... + new_graph = torch.fx.graph() + value_remap = {} + for node in g.nodes: + value_remap[node] = new_graph.node_copy(node, lambda n: value_remap[n]) + + Args: + + node (Node): The node to copy into ``self``. + + arg_transform (Callable[[Node], Argument]): A function that transforms + ``Node`` arguments in node's ``args`` and ``kwargs`` into the + equivalent argument in ``self``. In the simplest case, this should + retrieve a value out of a table mapping Nodes in the original + graph to ``self``. + """ + args = map_arg(node.args, arg_transform) + kwargs = map_arg(node.kwargs, arg_transform) + assert isinstance(args, tuple) + assert isinstance(kwargs, dict) + result_node = self.create_node( + node.op, node.target, args, kwargs, node.name, node.type + ) + result_node.meta = copy.copy(node.meta) + return result_node + + @compatibility(is_backward_compatible=True) + def output(self, result: "Argument", type_expr: Optional[Any] = None): + """ + Insert an ``output`` ``Node`` into the ``Graph``. An ``output`` node represents + a ``return`` statement in Python code. ``result`` is the value that should + be returned. + + Args: + + result (Argument): The value to be returned. + + type_expr (Optional[Any]): an optional type annotation representing the + Python type the output of this node will have. + + .. note:: + + The same insertion point and type expression rules apply for this method + as ``Graph.create_node``. + """ + return self.create_node( + op="output", target="output", args=(result,), type_expr=type_expr + ) + + def _target_to_str(self, target: Optional[Target]) -> str: + if callable(target): + op = target.__name__ + else: + assert isinstance(target, str) + op = target + if _is_magic(op): + op = op[2:-2] + op = _snake_case(op) + return op + + @compatibility(is_backward_compatible=True) + def python_code( + self, + root_module: str, + *, + verbose: bool = False, + include_stride: bool = False, + include_device: bool = False, + colored: bool = False, + expanded_def: bool = False, + record_func: bool = False, + ) -> PythonCode: + """ + Turn this ``Graph`` into valid Python code. + + Args: + + root_module (str): The name of the root module on which to look-up + qualified name targets. This is usually 'self'. + + Returns: + + A PythonCode object, consisting of two fields: + src: the Python source code representing the object + globals: a dictionary of global names in `src` -> the objects that they reference. + """ + # NOTE: [Graph Namespaces] + # + # There are two types of symbols in generated Python source code: + # locals and globals. + # Locals are locally defined by the output of a node in the Graph. + # Globals are references to external objects, like functions or types. + # + # When generating Python code, we need to make sure to name things + # appropriately. In particular: + # - All names should be unique, to avoid weird shadowing bugs. + # - These names need to be consistent, e.g. a object should always be + # referenced by the same name. + # + # To do this, we create a new namespace just for this source. All names + # that get printed must come from this namespace. + # + # Why can't we reuse node.name? Because it was generated within the + # namespace `self._graph_namespace`. In order to provide uniqueness + # over both locals (node.name) *and* globals, we create a completely + # new namespace to put all identifiers in. + namespace = _Namespace() + + # Override Node's repr to generate a valid name within our namespace. + # Since repr() is designed to produce a valid Python expression, it + # makes sense to reuse it. This way, it's easy to print something like + # Tuple[Node, Node] by simply calling repr() on it. Node's __repr__ is + # implemented cooperatively to allow this. + def node_repr(n: Node): + return namespace.create_name(n.name, n) + + @contextmanager + def override_node_repr(graph: Graph): + orig_repr_fns = {} + for node in graph.nodes: + orig_repr_fns[node] = node._repr_fn + node._repr_fn = node_repr + try: + yield None + finally: + # restore the original repr functions + for node in graph.nodes: + node._repr_fn = orig_repr_fns[node] + + with override_node_repr(self): + return self._python_code( + root_module, + namespace, + verbose=verbose, + include_stride=include_stride, + include_device=include_device, + colored=colored, + expanded_def=expanded_def, + record_func=record_func, + ) + + def _python_code( + self, + root_module: str, + namespace: _Namespace, + *, + verbose: bool = False, + include_stride: bool = False, + include_device: bool = False, + colored: bool = False, + expanded_def: bool = False, + record_func: bool = False, + ) -> PythonCode: + return self._codegen._gen_python_code( + self.nodes, + root_module, + namespace, + verbose=verbose, + include_stride=include_stride, + include_device=include_device, + colored=colored, + expanded_def=expanded_def, + record_func=record_func, + ) + + def __str__(self) -> str: + """ + Return a human-readable (not machine-readable) string representation + of this Graph + """ + placeholder_names: list[str] = [] + # This is a one-element array just so ``format_node`` can modify the closed + # over value + maybe_return_typename: list[str] = [""] + + node_strs = [node.format_node(placeholder_names) for node in self.nodes] + param_str = ", ".join(placeholder_names) + s = f"graph({param_str}){maybe_return_typename[0]}:" + for node_str in node_strs: + if node_str: + s += "\n " + node_str + return s + + @compatibility(is_backward_compatible=True) + def print_tabular(self): + """ + Prints the intermediate representation of the graph in tabular + format. Note that this API requires the ``tabulate`` module to be + installed. + """ + try: + from tabulate import tabulate + except ImportError: + print( + "`print_tabular` relies on the library `tabulate`, " + "which could not be found on this machine. Run `pip " + "install tabulate` to install the library." + ) + raise + + node_specs = [[n.op, n.name, n.target, n.args, n.kwargs] for n in self.nodes] + print( + tabulate(node_specs, headers=["opcode", "name", "target", "args", "kwargs"]) + ) + + @compatibility(is_backward_compatible=True) + def lint(self): + """ + Runs various checks on this Graph to make sure it is well-formed. In + particular: + - Checks Nodes have correct ownership (owned by this graph) + - Checks Nodes appear in topological order + - If this Graph has an owning GraphModule, checks that targets + exist in that GraphModule + """ + + # Check topo order + def check_arg(arg: Node, n: Optional[Node] = None) -> None: + context_str = f" of Node '{n}' " if n else " " + if arg.graph is not self: + raise RuntimeError( + f"Argument '{arg}'{context_str}does not belong to this Graph, " + f"but was used as an argument! If you are copying nodes from another graph, make " + f"sure to use ``arg_transform`` on node_copy() to remap values\n{self}" + ) + if arg not in seen_values: + raise RuntimeError( + f"Argument '{arg}'{context_str}was used before it has been " + f"defined! Please check that Nodes in the graph are topologically ordered\n{self}" + ) + + seen_names: set[str] = set() + seen_values: set[Node] = set() + for node in self.nodes: + if node.op not in _legal_ops: + raise RuntimeError(f"Node {node} had unknown opcode {node.op}!") + if node.graph is not self: + raise RuntimeError(f"Node '{node}' does not belong to this Graph!") + if node not in self._find_nodes_lookup_table: + raise RuntimeError(f"Node '{node}' is not added to the side table") + for arg in node._input_nodes: + check_arg(arg, node) + seen_values.add(node) + + if node.name in seen_names: + raise RuntimeError(f"Node redefined name {node.name}!") + seen_names.add(node.name) + + # Check targets are legit + if self.owning_module: + for node in self.nodes: + if node.op == "call_function": + if not callable(node.target): + raise ValueError( + f"Node {node} target {node.target} has type {torch.typename(node.target)} but " + "a Callable is expected" + ) + else: + if not isinstance(node.target, str): + raise ValueError( + f"Node {node} target {node.target} has type {torch.typename(node.target)} but " + "a str is expected" + ) + if node.op in ["get_attr", "call_module"]: + # pyrefly: ignore [missing-attribute] + target_atoms = node.target.split(".") + m_itr = self.owning_module + for i, atom in enumerate(target_atoms): + new_m_itr = getattr(m_itr, atom, None) + seen_qualname = ".".join(target_atoms[:i]) + if new_m_itr is None: + raise RuntimeError( + f"Node {node} target {node.target} references nonexistent attribute " + f"{atom} of {seen_qualname}" + ) + if node.op == "call_module" and not isinstance( + new_m_itr, torch.nn.Module + ): + raise RuntimeError( + f"Node {node} target {node.target} {atom} of {seen_qualname} does " + "not reference an nn.Module" + ) + + m_itr = new_m_itr + + @compatibility(is_backward_compatible=True) + def eliminate_dead_code( + self, is_impure_node: Optional[Callable[[Node], bool]] = None + ) -> bool: + """ + Remove all dead code from the graph, based on each node's number of + users, and whether the nodes have any side effects. The graph must be + topologically sorted before calling. + + Args: + is_impure_node (Optional[Callable[[Node], bool]]): A function that returns + whether a node is impure. If this is None, then the default behavior is to + use Node.is_impure. + + Returns: + bool: Whether the graph was changed as a result of the pass. + + Example: + + Before dead code is eliminated, `a` from `a = x + 1` below has no users + and thus can be eliminated from the graph without having an effect. + + .. code-block:: python + + def forward(self, x): + a = x + 1 + return x + self.attr_1 + + After dead code is eliminated, `a = x + 1` has been removed, and the rest + of `forward` remains. + + .. code-block:: python + + def forward(self, x): + return x + self.attr_1 + + .. warning:: + + Dead code elimination has some heuristics to avoid removing + side-effectful nodes (see Node.is_impure) but in general coverage + is very bad, so you should assume that this method is not sound + to call unless you know that your FX graph consists entirely + of functional operations or you supply your own custom + function for detecting side-effectful nodes. + """ + from torch.utils._ordered_set import OrderedSet + + # Lint the graph first to make sure its topologically sorted, otherwise + # DCE below will not behave as expected. + self.lint() + + impure_random = True + if torch._guards.TracingContext.try_get(): + impure_random = torch._inductor.config.fallback_random + + def has_side_effect(node): + if is_impure_node is not None: + return is_impure_node(node) + return node.is_impure(impure_random) + + # Reverse iterate so that when we remove a node, any nodes used as an + # input to that node have an updated user count that no longer reflects + # the removed node. + removed_nodes = set() + for node in reversed(self.nodes): + if not has_side_effect(node) and len(node.users) == 0: + self.erase_node(node) + removed_nodes.add(node.name) + + changed = len(removed_nodes) > 0 + if changed: + log.info("The following nodes were dead code eliminated: %s", removed_nodes) + + # Call DCE on the subgraphs + if self.owning_module is not None: + subgraph_names = OrderedSet( + x.target for x in self.find_nodes(op="get_attr") + ) + for child_name, child_module in self.owning_module.named_children(): + # Sometimes an owning_module can have unused children. Skip them + # by checking them from get_attr node targets. + if child_name in subgraph_names and isinstance( + child_module, torch.fx.GraphModule + ): + changed |= child_module.graph.eliminate_dead_code() + child_module.recompile() + + return changed + + @compatibility(is_backward_compatible=False) + def set_codegen(self, codegen: CodeGen): + self._codegen = codegen + + @compatibility(is_backward_compatible=False) + def on_generate_code( + self, + make_transformer: Callable[[Optional[TransformCodeFunc]], TransformCodeFunc], + ): + """Register a transformer function when python code is generated + + Args: + make_transformer (Callable[[Optional[TransformCodeFunc]], TransformCodeFunc]): + a function that returns a code transformer to be registered. + This function is called by `on_generate_code` to obtain the + code transformer. + + This function is also given as its input the currently + registered code transformer (or None if nothing is registered), + in case it is not desirable to overwrite it. This is useful to + chain code transformers together. + + Returns: + a context manager that when used in a `with` statement, to automatically + restore the previously registered code transformer. + + Example: + + .. code-block:: python + + + gm: fx.GraphModule = ... + + + # This is a code transformer we want to register. This code + # transformer prepends a pdb import and trace statement at the very + # beginning of the generated torch.fx code to allow for manual + # debugging with the PDB library. + def insert_pdb(body): + return ["import pdb; pdb.set_trace()\\n", *body] + + + # Registers `insert_pdb`, and overwrites the current registered + # code transformer (given by `_` to the lambda): + gm.graph.on_generate_code(lambda _: insert_pdb) + + # Or alternatively, registers a code transformer which first + # runs `body` through existing registered transformer, then + # through `insert_pdb`: + gm.graph.on_generate_code( + lambda current_trans: ( + lambda body: insert_pdb( + current_trans(body) if current_trans else body + ) + ) + ) + + gm.recompile() + gm(*inputs) # drops into pdb + + + This function can also be used as a context manager, with the benefit to + automatically restores the previously registered code transformer: + + .. code-block:: python + + # ... continue from previous example + + with gm.graph.on_generate_code(lambda _: insert_pdb): + # do more stuff with `gm`... + gm.recompile() + gm(*inputs) # drops into pdb + + # now previous code transformer is restored (but `gm`'s code with pdb + # remains - that means you can run `gm` with pdb here too, until you + # run next `recompile()`). + """ + on_gen_code_old = self._codegen._body_transformer + self._codegen._body_transformer = make_transformer(on_gen_code_old) + + @contextlib.contextmanager + def on_generate_code_context_manager(): + try: + yield + finally: + self._codegen._body_transformer = on_gen_code_old + + return on_generate_code_context_manager() + + def _clear_nodes(self) -> None: + for node in reversed(self.nodes): + node.meta.clear() + self.erase_node(node) + + +@contextmanager +def _override_sym_repr( + override: Callable[["torch.types.PySymType"], str], +) -> Iterator[None]: + tmp = CodeGen._sym_repr + try: + CodeGen._sym_repr = override + yield + finally: + CodeGen._sym_repr = tmp + + +def _identity(x): + return x + + +def _make_color_fn(code): + def f(s): + reset = "\033[0m" + return f"{code}{s}{reset}" + + return f + + +_color_codes = { + "yellow": "\033[33m", + "cyan": "\033[36m", + "green": "\033[32m", + "blue": "\033[34m", + "red": "\033[31m", + "dim": "\033[2m", + "dim_blue": "\033[2m\033[34m", + "dim_green": "\033[2m\033[32m", +} +_color_fns = {k: _make_color_fn(v) for k, v in _color_codes.items()} +_counter_regexp = re.compile(r"# COUNTER: (\d+)") + + +reflectable_magic_methods = { + "add": "{} + {}", + "sub": "{} - {}", + "mul": "{} * {}", + "floordiv": "{} // {}", + "truediv": "{} / {}", + "div": "{} / {}", + "mod": "{} % {}", + "pow": "{} ** {}", + "lshift": "{} << {}", + "rshift": "{} >> {}", + "and_": "{} & {}", + "or_": "{} | {}", + "xor": "{} ^ {}", + "getitem": "{}[{}]", + "matmul": "{} @ {}", +} + +magic_methods = { + "eq": "{} == {}", + "ne": "{} != {}", + "lt": "{} < {}", + "gt": "{} > {}", + "le": "{} <= {}", + "ge": "{} >= {}", + "pos": "+{}", + "neg": "-{}", + "invert": "~{}", + **reflectable_magic_methods, +} + +inplace_methods = { + "iadd": "{} += {}", + "iand": "{} &= {}", + "ifloordiv": "{} //= {}", + "ilshift": "{} <<= {}", + "imod": "{} %= {}", + "imul": "{} *= {}", + "imatmul": "{} @= {}", + "ior": "{} |= {}", + "ipow": "{} **= {}", + "irshift": "{} >>= {}", + "isub": "{} -= {}", + "itruediv": "{} /= {}", + "ixor": "{} ^= {}", + "setitem": "{}[{}] = {}", +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph_module.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph_module.py new file mode 100644 index 0000000000000000000000000000000000000000..a986f2b085e6b14a3a8bc76a66935a6029b36ddb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/graph_module.py @@ -0,0 +1,1183 @@ +# mypy: allow-untyped-defs +import base64 +import contextlib +import copy +import hashlib +import itertools +import linecache +import os +import sys +import traceback +import warnings +from collections.abc import Callable +from pathlib import Path +from typing import Any, Optional, Union + +import torch +import torch.nn as nn +import torch.overrides +from torch.nn.modules.module import _addindent +from torch.package import Importer, PackageExporter, PackageImporter, sys_importer + +from ._compatibility import compatibility +from .experimental import _config as fx_experimental_config +from .graph import ( + _BoxedCodeGen, + _custom_builtins, + _is_from_torch, + _override_sym_repr, + _PyTreeCodeGen, + Graph, + PythonCode, +) + + +__all__ = [ + "reduce_graph_module", + "reduce_package_graph_module", + "GraphModule", +] + +_USER_PRESERVED_ATTRIBUTES_KEY = "_user_preserved_attributes" +FX_GRAPH_MODULE_FILE_PREFIX = "fx_generated_" + + +# Normal exec loses the source code, however we can work with +# the linecache module to recover it. +# Using _exec_with_source will add it to our local cache +# and then tools like TorchScript will be able to get source info. +class _EvalCacheLoader: + def __init__(self): + self.eval_cache = {} + self.next_id = 0 + + def cache(self, src: str, globals: dict[str, Any], co_fields=None): + """Store the source in a private cache, and add a lazy entry in linecache + that allows the source to be retrieved by 'filename'. + + Args: + src (str): The module source to cache + globals (dict): The module globals + + Returns: + str: The cache key (and dummy filename) generated for src. + """ + + key = self._get_key() + if co_fields: + if "co_filename" in co_fields: + # If only co_filename is provided, use it directly as the key + if "co_firstlineno" not in co_fields or "co_name" not in co_fields: + key = co_fields["co_filename"] + else: + # Full co_fields with all three components + key += f" from {co_fields['co_filename']}:{co_fields['co_firstlineno']} in {co_fields['co_name']}" + self.eval_cache[key] = src + + # Don't mutate globals so that this loader is only used + # to populate linecache, and doesn't interact with other modules + # that might check `__loader__` + globals_copy = globals.copy() + globals_copy["__file__"] = key + globals_copy["__name__"] = key + globals_copy["__loader__"] = self + linecache.lazycache(key, globals_copy) + + return key + + # Part of the loader protocol (PEP 302) + # linecache will use this method when trying to find source code + def get_source(self, module_name) -> Optional[str]: + if module_name in self.eval_cache: + return self.eval_cache[module_name] + return None + + def _get_key(self): + key = f".{self.next_id}" + self.next_id += 1 + return key + + +_loader = _EvalCacheLoader() + + +def _exec_with_source(src: str, globals: dict[str, Any], co_fields=None): + key = _loader.cache(src, globals, co_fields) + exec(compile(src, key, "exec"), globals) + + +def _forward_from_src(src: str, globals: dict[str, Any], co_fields=None): + return _method_from_src( + method_name="forward", src=src, globals=globals, co_fields=co_fields + ) + + +def _method_from_src( + method_name: str, src: str, globals: dict[str, Any], co_fields=None +) -> Callable: + # avoid mutating the passed in dict + globals_copy = globals.copy() + _exec_with_source(src, globals_copy, co_fields) + fn = globals_copy[method_name] + del globals_copy[method_name] + return fn + + +def _format_import_statement(name: str, obj: Any, importer: Importer) -> str: + if name in _custom_builtins: + return _custom_builtins[name].import_str + if _is_from_torch(name): + return "import torch" + module_name, attr_name = importer.get_name(obj) + return f"from {module_name} import {attr_name} as {name}" + + +def _format_import_block(globals: dict[str, Any], importer: Importer): + import_strs: set[str] = { + _format_import_statement(name, obj, importer) for name, obj in globals.items() + } + # Sort the imports so we have a stable import block that allows us to + # hash the graph module and get a consistent key for use in a cache. + return "\n".join(sorted(import_strs)) + + +@compatibility(is_backward_compatible=True) +def reduce_graph_module(body: dict[Any, Any], import_block: str) -> torch.nn.Module: + # BC: attribute name was changed from `code` to `_code` to facilitate + # making `code` into a property and adding a docstring to it + fn_src = body.get("_code") or body["code"] + forward = _forward_from_src(import_block + fn_src, {}) + return _deserialize_graph_module(forward, body) + + +@compatibility(is_backward_compatible=True) +def reduce_package_graph_module( + importer: PackageImporter, body: dict[Any, Any], generated_module_name: str +) -> torch.nn.Module: + forward = importer.import_module(generated_module_name).forward + return _deserialize_graph_module(forward, body) + + +# We create a dummy class here because symbolic_trace pulls the forward() +# function off of the class, rather than the instance. This class is used +# in _deserialize_graph_module() below. +class _CodeOnlyModule(torch.nn.Module): + def __init__(self, body): + super().__init__() + self.__dict__ = body + + +def _deserialize_graph_module( + forward, body: dict[Any, Any], graph_module_cls=None +) -> torch.nn.Module: + """ + Deserialize a GraphModule given the dictionary of the original module, + using the code to reconstruct the graph. We delete the actual graph before + saving the dictionary so that changes to the in-memory graph format do not + get serialized. + """ + + # Try to retrieve the forward source in a backward-compatible way + _CodeOnlyModule.forward = forward + + tracer_cls = body.get("_tracer_cls") + if tracer_cls is None: + from ._symbolic_trace import Tracer + + tracer_cls = Tracer + + graphmodule_cls_name = body.get("_graphmodule_cls_name", "GraphModule") + + # This is a workaround for a mypy linter issue related to + # passing base class as an argument - https://github.com/python/mypy/issues/5865. + cls_tracer: Any = tracer_cls + + class KeepModules(cls_tracer): + # we shouldn't trace into any of the submodules, + # because they were not traced in the original GraphModule + def is_leaf_module(self, _: torch.nn.Module, __: str) -> bool: + return True + + com = _CodeOnlyModule(body) + + tracer_extras = body.get("_tracer_extras", {}) + graph = KeepModules().trace(com, **tracer_extras) + + # Recover node.meta["stack_trace"] after re-tracing + node_meta_stack_trace = body.get("_graphmodule_graph_node_meta_stack_trace") + if node_meta_stack_trace is not None: + del body["_graphmodule_graph_node_meta_stack_trace"] + for node in graph.nodes: + if node_meta_stack_trace.get(node.name, None) is not None: + node.meta["stack_trace"] = node_meta_stack_trace[node.name] + + # Manually set Tracer class on the reconstructed Graph, to avoid + # referencing the private local subclass KeepModules. + graph._tracer_cls = tracer_cls + from ._lazy_graph_module import _make_graph_module + + gm = _make_graph_module( + com, graph, class_name=graphmodule_cls_name, graph_module_cls=graph_module_cls + ) + + # The GraphModule constructor only retains attributes referenced by the graph. + # In this case, our goal is return a GraphModule as close to identical as the one + # put into the package. If any additional attributes were present in body, + # we should keep them. + for k, v in body.items(): + if not hasattr(gm, k): + setattr(gm, k, v) + return gm + + +# copy an attribute value with qualified name 'target' from 'from_module' to 'to_module' +# This installs empty Modules where none exist yet if they are subpaths of target +def _copy_attr(from_module: torch.nn.Module, to_module: torch.nn.Module, target: str): + *prefix, field = target.split(".") + for item in prefix: + f = getattr(from_module, item) + t = getattr(to_module, item, None) + if f is t: + # we have already installed one of its parents + # (e.g. target = root.linear.weight, but we have already installed root.linear) + # once we install a parent, we no longer need to copy the children + # since all the needed properties will already be present + return + + if t is None: + t = torch.nn.Module() + setattr(to_module, item, t) + from_module, to_module = f, t + + orig = getattr(from_module, field) + # If it is a tensor and not a parameter attribute of a module, it should be a named buffer. + # So, we register it as a named buffer in the target module. + if isinstance(orig, torch.Tensor) and not isinstance(orig, torch.nn.Parameter): + to_module.register_buffer(field, orig) + else: + setattr(to_module, field, orig) + + +# Assign attribute 'from_obj' to the qualified name 'target' on 'to_module +# This installs empty Modules where none exist yet if they are subpaths of target +def _assign_attr(from_obj: Any, to_module: torch.nn.Module, target: str): + *prefix, field = target.split(".") + for item in prefix: + t = getattr(to_module, item, None) + + if t is None: + t = torch.nn.Module() + setattr(to_module, item, t) + to_module = t + + # If it is a tensor and not a parameter attribute of a module, it should be a named buffer. + # So, we register it as a named buffer in the target module. + if isinstance(from_obj, torch.Tensor) and not isinstance( + from_obj, torch.nn.Parameter + ): + to_module.register_buffer(field, from_obj) + else: + setattr(to_module, field, from_obj) + + +# Recursively look up target from a graph module. +def _get_attr(model: torch.nn.Module, attr_name: str): + return _get_attr_via_attr_list(model, attr_name.split(".")) + + +def _del_attr(model: torch.nn.Module, attr_name: str): + attr_names = attr_name.split(".") + t = _get_attr_via_attr_list(model, attr_names[:-1]) + return delattr(t, attr_names[-1]) + + +def _get_attr_via_attr_list(model: torch.nn.Module, attr_list: list[str]): + if len(attr_list) == 0: + return model + *prefix, field = attr_list + t = model + for item in prefix: + t = getattr(t, item, None) # type: ignore[assignment] + assert t is not None + + return getattr(t, field) + + +def _has_attr(model: torch.nn.Module, attr_name: str): + *prefix, field = attr_name.split(".") + t = model + for item in prefix: + t = hasattr(t, item) # type: ignore[assignment] + if t is False: + return False + + return hasattr(t, field) + + +def _print_readable( + module, + module_name, + print_output=True, + include_stride=False, + include_device=False, + colored=False, + expanded_def=False, +): + graph = module.graph + assert graph is not None and isinstance(graph, torch.fx.Graph), ( + "print_readable must be used on a module with a graph" + ) + + verbose_python_code = graph.python_code( + root_module="self", + verbose=True, + include_stride=include_stride, + include_device=include_device, + colored=colored, + expanded_def=expanded_def, + ) + module_code = verbose_python_code.src + module_code = module_code.lstrip("\n") + module_code = f"class {module_name}(torch.nn.Module):\n" + module_code + module_code = _addindent(module_code, 4) + + submodule_code_list = [""] + for submodule_name, submodule in module.named_children(): + if hasattr(submodule, "graph"): + submodule_code_list.append( + _print_readable( + submodule, + submodule_name, + print_output=False, + include_stride=include_stride, + include_device=include_device, + colored=colored, + ) + ) + submodule_code = "\n".join(submodule_code_list) + submodule_code = _addindent(submodule_code, 4) + + output = module_code + submodule_code + if print_output: + print(module_code + submodule_code) + return output + + +def _metadata_hash(code: str, node_metadata: dict) -> str: + """ + Create a content-addressed hash from code and metadata. + + Args: + code: The source code string + lineno_map: Mapping from line numbers to node indices + node_metadata: Metadata for each node + + Returns: + A 51-character base32-encoded hash + """ + import json + + # Create a deterministic string representation of all components + # We use JSON to ensure consistent serialization + hash_data = { + "code": code, + "node_metadata": node_metadata, + } + hashing_str = json.dumps(hash_data).encode("utf-8") + + # [:51] to strip off the "Q====" suffix common to every hash value. + return ( + base64.b32encode(hashlib.sha256(hashing_str).digest())[:51] + .decode("utf-8") + .lower() + ) + + +class _WrappedCall: + def __init__(self, cls, cls_call): + self.cls = cls + self.cls_call = cls_call + + # Previously, if an error occurred when valid + # symbolically-traced code was run with an invalid input, the + # user would see the source of the error as coming from + # `File "`, where N is some number. We use + # this function to generate a more informative error message. We + # return the traceback itself, a message explaining that the + # error occurred in a traced Module's generated forward + # function, and five lines of context surrounding the faulty + # line + @staticmethod + def _generate_error_message(frame_summary: traceback.FrameSummary) -> str: + # auxiliary variables (for readability) + err_lineno = frame_summary.lineno + assert err_lineno is not None + line = frame_summary.line + assert line is not None + err_line_len = len(line) + all_src_lines = linecache.getlines(frame_summary.filename) + + # constituent substrings of the error message + tb_repr = torch._dynamo.disable( + traceback.format_exc, + reason="do not trace into traceback.format_exc when generating error message", + )() + custom_msg = ( + "Call using an FX-traced Module, " + f"line {err_lineno} of the traced Module's " + "generated forward function:" + ) + before_err = "".join(all_src_lines[err_lineno - 2 : err_lineno]) + marker = "~" * err_line_len + "~~~ <--- HERE" + err_and_after_err = "\n".join(all_src_lines[err_lineno : err_lineno + 2]) + + # joined message + return "\n".join([tb_repr, custom_msg, before_err, marker, err_and_after_err]) + + def __call__(self, obj, *args, **kwargs): + try: + if self.cls_call is not None: + return self.cls_call(obj, *args, **kwargs) + else: + return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] + except Exception as e: + assert e.__traceback__ + topmost_framesummary: traceback.FrameSummary = ( + traceback.StackSummary.extract(traceback.walk_tb(e.__traceback__))[-1] + ) + if "eval_with_key" in topmost_framesummary.filename: + print( + _WrappedCall._generate_error_message(topmost_framesummary), + file=sys.stderr, + ) + raise e.with_traceback(None) # noqa: B904 + else: + raise e + + +@compatibility(is_backward_compatible=True) +class GraphModule(torch.nn.Module): + """ + GraphModule is an nn.Module generated from an fx.Graph. Graphmodule has a + ``graph`` attribute, as well as ``code`` and ``forward`` attributes generated + from that ``graph``. + + .. warning:: + + When ``graph`` is reassigned, ``code`` and ``forward`` will be automatically + regenerated. However, if you edit the contents of the ``graph`` without reassigning + the ``graph`` attribute itself, you must call ``recompile()`` to update the generated + code. + """ + + def __new__(cls: "type[GraphModule]", *args, **kwargs): + # each instance of a graph module needs its own forward method + # so create a new singleton class for each instance. + # it is a subclass of the user-defined class, the only difference + # is an extra layer to install the forward method + + # address issue described at https://github.com/pytorch/pytorch/issues/63883 + # in other words, traverse class hierarchy to fix the redundant class definition problem + for t in cls.__mro__: + c = t.__qualname__.split(".")[-1] + if c != "GraphModuleImpl": + cls = t + break + + class GraphModuleImpl(cls): # type: ignore[misc, valid-type] + pass + + return super().__new__(GraphModuleImpl) + + @compatibility(is_backward_compatible=True) + def __init__( + self, + root: Union[torch.nn.Module, dict[str, Any]], + graph: Graph, + class_name: str = "GraphModule", + ): + """ + Construct a GraphModule. + + Args: + + root (Union[torch.nn.Module, Dict[str, Any]): + ``root`` can either be an nn.Module instance or a Dict mapping strings to any attribute type. + In the case that ``root`` is a Module, any references to Module-based objects (via qualified + name) in the Graph's Nodes' ``target`` field will be copied over from the respective place + within ``root``'s Module hierarchy into the GraphModule's module hierarchy. + In the case that ``root`` is a dict, the qualified name found in a Node's ``target`` will be + looked up directly in the dict's keys. The object mapped to by the Dict will be copied + over into the appropriate place within the GraphModule's module hierarchy. + + graph (Graph): ``graph`` contains the nodes this GraphModule should use for code generation + + class_name (str): ``name`` denotes the name of this GraphModule for debugging purposes. If it's unset, all + error messages will report as originating from ``GraphModule``. It may be helpful to set this + to ``root``'s original name or a name that makes sense within the context of your transform. + """ + super().__init__() + self.__class__.__name__ = class_name + if isinstance(root, torch.nn.Module): + if hasattr(root, "training"): + self.training = root.training + + # When we pickle/unpickle graph module, we don't want to drop any module or attributes. + if isinstance(root, _CodeOnlyModule): + for k, _ in root.named_children(): + _copy_attr(root, self, k) + + for k, _ in root.named_buffers(): + _copy_attr(root, self, k) + + for k, _ in root.named_parameters(): + _copy_attr(root, self, k) + + for node in graph.nodes: + if node.op in ["get_attr", "call_module"]: + assert isinstance(node.target, str) + _copy_attr(root, self, node.target) + elif isinstance(root, dict): + targets_to_copy = [] + for node in graph.nodes: + if node.op in ["get_attr", "call_module"]: + assert isinstance(node.target, str) + if node.target not in root: + raise RuntimeError( + "Node " + + str(node) + + " referenced target " + + node.target + + " but that target was not provided in ``root``!" + ) + targets_to_copy.append(node.target) + # Sort targets in ascending order of the # of atoms. + # This will ensure that less deeply nested attributes are assigned + # before more deeply nested attributes. For example, foo.bar + # will be assigned before foo.bar.baz. Otherwise, we might assign + # the user-provided ``foo.bar`` and wipe out the previously-assigned + # ``foo.bar.baz`` + targets_to_copy.sort(key=lambda t: t.count(".")) + for target_to_copy in targets_to_copy: + _assign_attr(root[target_to_copy], self, target_to_copy) + else: + raise RuntimeError("Unsupported type " + str(root) + " passed for root!") + + self.graph = graph + + # Store the Tracer class responsible for creating a Graph separately as part of the + # GraphModule state, except when the Tracer is defined in a local namespace. + # Locally defined Tracers are not pickleable. This is needed because torch.package will + # serialize a GraphModule without retaining the Graph, and needs to use the correct Tracer + # to re-create the Graph during deserialization. + self._tracer_cls = None + if ( + self.graph._tracer_cls + and "" not in self.graph._tracer_cls.__qualname__ + ): + # pyrefly: ignore [bad-assignment] + self._tracer_cls = self.graph._tracer_cls + + self._tracer_extras = {} + if self.graph._tracer_extras: + self._tracer_extras = self.graph._tracer_extras + + # Dictionary to store metadata + self.meta: dict[str, Any] = {} + self._replace_hooks: list[Callable] = [] + self._create_node_hooks: list[Callable] = [] + self._erase_node_hooks: list[Callable] = [] + # Used to remove hooks from deepcopied graph modules within a context manager. + self._deepcopy_hooks: list[Callable] = [] + self.shape_env = None # optional not always set even when dynamic shapes exist. + + # TorchScript breaks trying to compile the graph setter because of the + # continued string literal. Issue here: https://github.com/pytorch/pytorch/issues/44842 + # + # Shouldn't be an issue since these methods shouldn't be used in TorchScript anyway + __jit_unused_properties__ = ["graph", "_boxed_call"] + + @property + def _boxed_call(self) -> bool: + return isinstance(self._graph._codegen, _BoxedCodeGen) + + @property + def graph(self) -> Graph: + """ + Return the ``Graph`` underlying this ``GraphModule`` + """ + return self._graph + + @graph.setter + def graph(self, g: Graph) -> None: + """ + Set the underlying ``Graph`` for this ``GraphModule``. This will internally + recompile the ``GraphModule`` so that the generated ``forward()`` function + corresponds to ``g`` + """ + assert isinstance(g, Graph), f"Expected a Graph instance, but got {type(g)}" + self._graph = g + g.owning_module = self + self.recompile() + + @compatibility(is_backward_compatible=False) + def to_folder(self, folder: Union[str, os.PathLike], module_name: str = "FxModule"): + """Dumps out module to ``folder`` with ``module_name`` so that it can be + imported with ``from import `` + + Args: + + folder (Union[str, os.PathLike]): The folder to write the code out to + + module_name (str): Top-level name to use for the ``Module`` while + writing out the code + """ + folder = Path(folder) + Path(folder).mkdir(exist_ok=True) + torch.save(self.state_dict(), folder / "state_dict.pt") + tab = " " * 4 + custom_builtins = "\n".join([v.import_str for v in _custom_builtins.values()]) + model_str = f""" +import torch +{custom_builtins} + +from torch.nn import * +class {module_name}(torch.nn.Module): + def __init__(self): + super().__init__() +""" + + def _gen_model_repr(module_name: str, module: torch.nn.Module) -> Optional[str]: + safe_reprs = [ + nn.Linear, + nn.Conv1d, + nn.Conv2d, + nn.Conv3d, + nn.BatchNorm1d, + nn.BatchNorm2d, + nn.BatchNorm3d, + ] + if type(module) in safe_reprs: + return f"{module.__repr__()}" + else: + return None + + blobified_modules = [] + for module_name, module in self.named_children(): + module_str = _gen_model_repr(module_name, module) + if module_str is None: + module_file = folder / f"{module_name}.pt" + torch.save(module, module_file) + blobified_modules.append(module_name) + module_repr = module.__repr__().replace("\r", " ").replace("\n", " ") + # weights_only=False as this is legacy code that saves the model + module_str = ( + f"torch.load(r'{module_file}', weights_only=False) # {module_repr}" + ) + model_str += f"{tab * 2}self.{module_name} = {module_str}\n" + + for buffer_name, buffer in self._buffers.items(): + if buffer is None: + continue + model_str += f"{tab * 2}self.register_buffer('{buffer_name}', torch.empty({list(buffer.shape)}, dtype={buffer.dtype}))\n" # noqa: B950 + + for param_name, param in self._parameters.items(): + if param is None: + continue + model_str += f"{tab * 2}self.{param_name} = torch.nn.Parameter(torch.empty({list(param.shape)}, dtype={param.dtype}))\n" # noqa: B950 + + model_str += ( + f"{tab * 2}self.load_state_dict(torch.load(r'{folder}/state_dict.pt'))\n" + ) + model_str += f"{_addindent(self.code, 4)}\n" + + module_file = folder / "module.py" + module_file.write_text(model_str) + + init_file = folder / "__init__.py" + init_file.write_text("from .module import *") + + if len(blobified_modules) > 0: + warnings.warn( + "Was not able to save the following children modules as reprs -" + f"saved as pickled files instead: {blobified_modules}" + ) + + @compatibility(is_backward_compatible=True) + def add_submodule(self, target: str, m: torch.nn.Module) -> bool: + """ + Adds the given submodule to ``self``. + + This installs empty Modules where none exist yet if they are + subpaths of ``target``. + + Args: + target: The fully-qualified string name of the new submodule + (See example in ``nn.Module.get_submodule`` for how to + specify a fully-qualified string.) + m: The submodule itself; the actual object we want to + install in the current Module + + Return: + bool: Whether or not the submodule could be inserted. For + this method to return True, each object in the chain + denoted by ``target`` must either a) not exist yet, + or b) reference an ``nn.Module`` (not a parameter or + other attribute) + """ + *prefix, field = target.split(".") + mod: torch.nn.Module = self + + for item in prefix: + submod = getattr(mod, item, None) + + if submod is None: + submod = torch.nn.Module() + setattr(mod, item, submod) + + if not isinstance(submod, torch.nn.Module): + return False + + mod = submod + + mod.add_module(field, m) + return True + + @compatibility(is_backward_compatible=True) + def delete_submodule(self, target: str) -> bool: + """ + Deletes the given submodule from ``self``. + + The module will not be deleted if ``target`` is not a valid + target. + + Args: + target: The fully-qualified string name of the new submodule + (See example in ``nn.Module.get_submodule`` for how to + specify a fully-qualified string.) + + Returns: + bool: Whether or not the target string referenced a + submodule we want to delete. A return value of ``False`` + means that the ``target`` was not a valid reference to + a submodule. + """ + atoms = target.split(".") + path, target_submod = atoms[:-1], atoms[-1] + mod: torch.nn.Module = self + + # Get the parent module + for item in path: + if not hasattr(mod, item): + return False + + mod = getattr(mod, item) + + if not isinstance(mod, torch.nn.Module): + return False + + if not hasattr(mod, target_submod): + return False + + if not isinstance(getattr(mod, target_submod), torch.nn.Module): + return False + + delattr(mod, target_submod) + return True + + @compatibility(is_backward_compatible=True) + def delete_all_unused_submodules(self) -> None: + """ + Deletes all unused submodules from ``self``. + + A Module is considered "used" if any one of the following is + true: + 1. It has children that are used + 2. Its forward is called directly via a ``call_module`` node + 3. It has a non-Module attribute that is used from a + ``get_attr`` node + + This method can be called to clean up an ``nn.Module`` without + manually calling ``delete_submodule`` on each unused submodule. + """ + used: list[str] = [] + + for node in self.graph.nodes: + if node.op == "call_module" or node.op == "get_attr": + # A list of strings representing the different parts + # of the path. For example, `foo.bar.baz` gives us + # ["foo", "bar", "baz"] + fullpath = node.target.split(".") + + # If we're looking at multiple parts of a path, join + # join them with a dot. Otherwise, return that single + # element without doing anything to it. + def join_fn(x: str, y: str) -> str: + return ".".join([x, y] if y else [x]) + + # Progressively collect all the names of intermediate + # modules. For example, if we have the target + # `foo.bar.baz`, we'll add `foo`, `foo.bar`, and + # `foo.bar.baz` to the list. + used.extend(itertools.accumulate(fullpath, join_fn)) + + # For a `call_module` node, also register all recursive submodules + # as used + if node.op == "call_module": + try: + submod = self.get_submodule(node.target) + + for submod_name, _ in submod.named_modules(): + if submod_name != "": + used.append(".".join([node.target, submod_name])) + except AttributeError: + # Node referenced nonexistent submodule, don't need to + # worry about GCing anything + pass + + to_delete = [name for name, _ in self.named_modules() if name not in used] + + for name in to_delete: + self.delete_submodule(name) + + @property + def code(self) -> str: + """ + Return the Python code generated from the ``Graph`` underlying this + ``GraphModule``. + """ + if not hasattr(self, "_code"): + raise RuntimeError( + "Code has not been generated! Please report a bug to PyTorch" + ) + return self._code + + @compatibility(is_backward_compatible=True) + def recompile(self) -> PythonCode: + """ + Recompile this GraphModule from its ``graph`` attribute. This should be + called after editing the contained ``graph``, otherwise the generated + code of this ``GraphModule`` will be out of date. + """ + # Do not import anything inside recompile, it might slow down the + # function and cause perf regression. Import outside of the method instead. + if isinstance(self._graph._codegen, _PyTreeCodeGen): + self._in_spec = self._graph._codegen.pytree_info.in_spec + self._out_spec = self._graph._codegen.pytree_info.out_spec + + python_code = self._graph.python_code( + root_module="self", + record_func=fx_experimental_config.enrich_profiler_metadata, + ) + self._code = python_code.src + self._lineno_map = python_code._lineno_map + self._prologue_start = python_code._prologue_start + + cls = type(self) + co_fields = self._graph._co_fields if hasattr(self._graph, "_co_fields") else {} + + if fx_experimental_config.enrich_profiler_metadata: + # Generate metadata and register for profiler augmentation + node_metadata: dict[int, dict[str, Any]] = {} + for i, node in enumerate(self._graph.nodes): + node_metadata[i] = { + "name": node.name, + "op": node.op, + "target": str(node.target), + "stack_trace": node.meta.get("stack_trace", None), + } + + # Generate a content-addressed filename based on hash of code and metadata + # This ensures the same code+metadata always generates the same filename + hash_value = _metadata_hash(self._code, node_metadata) + file_stem = f"{FX_GRAPH_MODULE_FILE_PREFIX}_{hash_value}" + filename = f"{file_stem}.py" + + # Only include co_filename to use it directly as the cache key + co_fields = { + "co_filename": filename, + } + + # Store metadata in global in-memory registry + metadata = { + "lineno_map": python_code._lineno_map, + "prologue_start": python_code._prologue_start, + "node_metadata": node_metadata, + } + + # Register metadata in the global registry + from torch.fx.traceback import _register_fx_metadata + + _register_fx_metadata(filename, metadata) + + # Replace the placeholder in generated code with actual filename + # The double hash ## convention is used by post-processing to find the fx markers + self._code = self._code.replace( + "torch._C._profiler._RecordFunctionFast('## ENTER_GRAPH_PLACEHOLDER_KEY ##')", + f"torch._C._profiler._RecordFunctionFast('## {filename} ##')", + ) + + cls.forward = _forward_from_src(self._code, python_code.globals, co_fields) + + # Determine whether this class explicitly defines a __call__ implementation + # to wrap. If it does, save it in order to have wrapped_call invoke it. + # If it does not, wrapped_call can use a dynamic call to super() instead. + # In most cases, super().__call__ should be torch.nn.Module.__call__. + # We do not want to hold a reference to Module.__call__ here; doing so will + # bypass patching of torch.nn.Module.__call__ done while symbolic tracing. + cls_call = cls.__call__ if "__call__" in vars(cls) else None + + if "_wrapped_call" not in vars(cls): + cls._wrapped_call = _WrappedCall(cls, cls_call) # type: ignore[attr-defined] + + self._recompile_submodules() + + def call_wrapped(self, *args, **kwargs): + return self._wrapped_call(self, *args, **kwargs) + + cls.__call__ = call_wrapped # type: ignore[method-assign] + + return python_code + + def _recompile_submodules(self) -> list[tuple[str, PythonCode]]: + """ + Recompile all submodules of this graph module, returning their respective PythonCodes + in a similar format to named_children() + """ + results: list[tuple[str, PythonCode]] = [] + for name, mod in self.named_children(): + if isinstance(mod, GraphModule): + results.append((name, mod.recompile())) + return results + + # Passing Tracer as argument allows subclasses extending fx.GraphModule + # define their own Tracer (extending fx.Tracer). + + def __reduce_package__(self, exporter: PackageExporter): + dict_without_graph = self.__dict__.copy() + dict_without_graph["_graphmodule_cls_name"] = self.__class__.__name__ + del dict_without_graph["_graph"] + + # Store node.meta["stack_trace"] so we can recover them after re-tracing during deserialization + node_meta_stack_trace = { + node.name: node.meta["stack_trace"] + for node in self.graph.nodes + if "stack_trace" in node.meta + } + dict_without_graph["_graphmodule_graph_node_meta_stack_trace"] = ( + node_meta_stack_trace + ) + + generated_module_name = f"fx-generated._{exporter.get_unique_id()}" + python_code = self.recompile() + import_block = _format_import_block(python_code.globals, exporter.importer) + module_code = import_block + self.code + exporter.save_source_string(generated_module_name, module_code) + return ( + reduce_package_graph_module, + (dict_without_graph, generated_module_name), + ) + + def __reduce__(self): + """ + Serialization of GraphModule. We serialize only the generated code, not + the underlying ``Graph``. This is because ``Graph`` does not have on-disk + backward-compatibility guarantees, whereas Python source code does. + On the deserialization side, we symbolically trace through the generated + code to regenerate the underlying ``Graph`` + """ + dict_without_graph = self.__dict__.copy() + + python_code = self.recompile() + import_block = _format_import_block(python_code.globals, sys_importer) + del dict_without_graph["_graph"] + return (reduce_graph_module, (dict_without_graph, import_block)) + + def _deepcopy_init(self): + return GraphModule.__init__ + + # because __reduce__ is defined for serialization, + # we need to define deepcopy otherwise it will call __reduce__ + # and cause symbolic tracing to occur every time we try to copy the object + def __deepcopy__(self, memo): + res = type(self).__new__(type(self)) + memo[id(self)] = res + fake_mod = _CodeOnlyModule(copy.deepcopy(self.__dict__, memo)) + self._deepcopy_init()(res, fake_mod, fake_mod.__dict__["_graph"]) + # hooks are lost during `GraphModule.__init__`, so we need to copy over + # them explicitly, note right now we are only copying state_dict related + # hooks, to reduce bc-related issues, we can copy forward/backward related + # hooks in the future as well if needed + extra_preserved_attrs = [ + "_state_dict_hooks", + "_load_state_dict_pre_hooks", + "_load_state_dict_post_hooks", + "_replace_hooks", + "_create_node_hooks", + "_erase_node_hooks", + "_deepcopy_hooks", + ] + for attr in extra_preserved_attrs: + if attr in self.__dict__: + setattr(res, attr, copy.deepcopy(self.__dict__[attr], memo)) + res.meta = copy.deepcopy(getattr(self, "meta", {}), memo) + if _USER_PRESERVED_ATTRIBUTES_KEY in res.meta: + for attr_name, attr in res.meta[_USER_PRESERVED_ATTRIBUTES_KEY].items(): + setattr(res, attr_name, attr) + if hasattr(self, "_deepcopy_hooks"): + for hook in self._deepcopy_hooks: + hook(res) + return res + + def __copy__(self): + from ._lazy_graph_module import _make_graph_module + + res = _make_graph_module(self, self.graph) + res.meta = getattr(self, "meta", {}) + return res + + @compatibility(is_backward_compatible=False) + def print_readable( + self, + print_output=True, + include_stride=False, + include_device=False, + colored=False, + *, + # If `fast_sympy_print` is True then we use a sympy printer which is faster + # but may result in less-readable output. + fast_sympy_print: bool = False, + expanded_def: bool = False, + ): + """ + Return the Python code generated for current GraphModule and its children GraphModules + """ + ctx_mgr = contextlib.ExitStack() + with ctx_mgr: + if fast_sympy_print: + from torch._inductor.utils import sympy_str + + def fast_repr(expr: torch.types.PySymType) -> str: + return sympy_str(expr.node.expr) + + ctx_mgr.enter_context(_override_sym_repr(fast_repr)) + + r = _print_readable( + self, + self._get_name(), + print_output, + include_stride, + include_device, + colored, + expanded_def, + ) + return r + + def __str__(self) -> str: + orig_str = super().__str__() + print_readable_reminder = ( + "# To see more debug info, please use `graph_module.print_readable()`" + ) + return "\n".join([orig_str, self._code, print_readable_reminder]) + + def _replicate_for_data_parallel(self): + new_gm = self.__copy__() + new_gm._is_replica = True + return new_gm + + @contextlib.contextmanager + def _set_replace_hook(self, f): + """ + Takes a callable which will be called every time when we replace a node + to a new node, or change the node's name. Callable takes three arguments: + the old node we're changing, and NAME of the new node, followed by the + user node which consumes the old node to be replaced. + """ + assert callable(f), "Replace hook must be a callable." + self._register_replace_node_hook(f) + try: + yield + finally: + self._unregister_replace_node_hook(f) + + def _register_replace_node_hook(self, f): + """ + Takes a callable which will be called every time when we replace a node + to a new node, or change the node's name. Callable takes three arguments: + the old node we're changing, and NAME of the new node, followed by the + user node which consumes the old node to be replaced. + """ + assert callable(f), "create_node hook must be a callable." + self._replace_hooks.append(f) + + def _unregister_replace_node_hook(self, f): + """ + Takes a callable which was previously registered to be called every time when we replace a node. + This function will unregister that callable so it is no longer invoked on node replacement. + """ + assert callable(f), "create_node hook must be a callable." + self._replace_hooks.remove(f) + + def _register_create_node_hook(self, f): + """ + Takes a callable which will be called after we create a new node. The + callable takes the newly created node as input and returns None. + """ + assert callable(f), "create_node hook must be a callable." + self._create_node_hooks.append(f) + + def _unregister_create_node_hook(self, f): + """ + Takes a callable which was previously registered to be called after we create a node. + This function will unregister that callable so it is no longer invoked on node creation. + """ + assert callable(f), "create_node hook must be a callable." + self._create_node_hooks.remove(f) + + def _register_erase_node_hook(self, f): + """ + Takes a callable which will be called after we erase a node. The + callable takes the node that is being erased as input and returns None. + """ + assert callable(f), "erase_node hook must be a callable." + self._erase_node_hooks.append(f) + + def _unregister_erase_node_hook(self, f): + """ + Takes a callable which was previously registered to be called after we erase a node. + This function will unregister that callable so it is no longer invoked on node erasure. + """ + assert callable(f), "erase_node hook must be a callable." + self._erase_node_hooks.remove(f) + + def _register_deepcopy_hook(self, f): + """ + Takes a callable which will be called when we deepcopy this graph module. The + callable takes the resulting deepcopied graph module. + """ + assert callable(f), "deepcopy hook must be a callable." + self._deepcopy_hooks.append(f) + + def _unregister_deepcopy_hook(self, f): + """ + Takes a callable which was previously registered to be called after deepcopy. + This function will unregister that callable so it is no longer invoked on deepcopy. + """ + assert callable(f), "deepcopy hook must be a callable." + self._deepcopy_hooks.remove(f) + + +# workarounds for issues in __torch_function__ + +# WAR for __torch_function__ not handling tensor lists, +# fix is in https://github.com/pytorch/pytorch/pull/34725 +# orig_cat = torch.cat +# def patched_cat(*args, **kwargs): +# tensors = args[0] +# for t in tensors: +# if isinstance(t, Proxy): +# return t.__torch_function__(patched_cat, (), args, kwargs) +# return orig_cat(*args, **kwargs) +# patched_cat.__module__ = 'torch' +# patched_cat.__name__ = 'cat' +# torch.cat = patched_cat diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/immutable_collections.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/immutable_collections.py new file mode 100644 index 0000000000000000000000000000000000000000..4f4bd21180361942ed397667d739dc269b28bfd9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/immutable_collections.py @@ -0,0 +1,122 @@ +from collections.abc import Iterable +from typing import Any, NoReturn, TypeVar +from typing_extensions import Self + +from torch.utils._pytree import ( + _dict_flatten, + _dict_flatten_with_keys, + _dict_unflatten, + _list_flatten, + _list_flatten_with_keys, + _list_unflatten, + Context, + register_pytree_node, +) + +from ._compatibility import compatibility + + +__all__ = ["immutable_list", "immutable_dict"] + + +_help_mutation = """ +If you are attempting to modify the kwargs or args of a torch.fx.Node object, +instead create a new copy of it and assign the copy to the node: + + new_args = ... # copy and mutate args + node.args = new_args +""".strip() + + +_T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + + +def _no_mutation(self: Any, *args: Any, **kwargs: Any) -> NoReturn: + raise TypeError( + f"{type(self).__name__!r} object does not support mutation. {_help_mutation}", + ) + + +@compatibility(is_backward_compatible=True) +class immutable_list(list[_T]): + """An immutable version of :class:`list`.""" + + __delitem__ = _no_mutation + __iadd__ = _no_mutation + __imul__ = _no_mutation + __setitem__ = _no_mutation + append = _no_mutation + clear = _no_mutation + extend = _no_mutation + insert = _no_mutation + pop = _no_mutation + remove = _no_mutation + reverse = _no_mutation + sort = _no_mutation + + def __hash__(self) -> int: # type: ignore[override] + return hash(tuple(self)) + + def __reduce__(self) -> tuple[type[Self], tuple[tuple[_T, ...]]]: + return (type(self), (tuple(self),)) + + +@compatibility(is_backward_compatible=True) +class immutable_dict(dict[_KT, _VT]): + """An immutable version of :class:`dict`.""" + + __delitem__ = _no_mutation + __ior__ = _no_mutation + __setitem__ = _no_mutation + clear = _no_mutation + pop = _no_mutation + popitem = _no_mutation + setdefault = _no_mutation + update = _no_mutation # type: ignore[assignment] + + def __hash__(self) -> int: # type: ignore[override] + return hash(frozenset(self.items())) + + def __reduce__(self) -> tuple[type[Self], tuple[tuple[tuple[_KT, _VT], ...]]]: + return (type(self), (tuple(self.items()),)) + + +# Register immutable collections for PyTree operations +def _immutable_list_flatten(d: immutable_list[_T]) -> tuple[list[_T], Context]: + return _list_flatten(d) + + +def _immutable_list_unflatten( + values: Iterable[_T], + context: Context, +) -> immutable_list[_T]: + return immutable_list(_list_unflatten(values, context)) + + +def _immutable_dict_flatten(d: immutable_dict[Any, _VT]) -> tuple[list[_VT], Context]: + return _dict_flatten(d) + + +def _immutable_dict_unflatten( + values: Iterable[_VT], + context: Context, +) -> immutable_dict[Any, _VT]: + return immutable_dict(_dict_unflatten(values, context)) + + +register_pytree_node( + immutable_list, + _immutable_list_flatten, + _immutable_list_unflatten, + serialized_type_name="torch.fx.immutable_collections.immutable_list", + flatten_with_keys_fn=_list_flatten_with_keys, +) +register_pytree_node( + immutable_dict, + _immutable_dict_flatten, + _immutable_dict_unflatten, + serialized_type_name="torch.fx.immutable_collections.immutable_dict", + flatten_with_keys_fn=_dict_flatten_with_keys, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/interpreter.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..b28270a099048faf38d68e5e926a43207d30a075 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/interpreter.py @@ -0,0 +1,656 @@ +# mypy: allow-untyped-defs +import inspect +import logging +from contextlib import contextmanager +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +import torch.fx.traceback as fx_traceback +from torch._logging import LazyString, trace_structured +from torch.hub import tqdm + +from . import config +from ._compatibility import compatibility +from ._lazy_graph_module import _make_graph_module +from ._symbolic_trace import Tracer +from .graph import Graph +from .graph_module import GraphModule +from .node import Argument, map_aggregate, map_arg, Node, Target +from .proxy import Proxy + + +if TYPE_CHECKING: + from collections.abc import Iterator + +log = logging.getLogger(__name__) + +__all__ = ["Interpreter", "Transformer"] + + +def _format_fx_node(n): + """ + Format a torch.fx.Node into a human-readable string for debug logging. + + Args: + n (torch.fx.Node): The FX node being executed. + + Returns: + str: A formatted string describing the node operation, including its + name, target, positional arguments, and keyword arguments. + """ + module_prefix = getattr(n.target, "__module__", "") + module_prefix = f"{module_prefix}." if module_prefix else "" + + # Handle positional and keyword arguments + args = ", ".join(map(str, n.args)) + kwargs = ", ".join(f"{k}={v}" for k, v in n.kwargs.items()) + joined = ", ".join(filter(None, [args, kwargs])) + + return ( + f"{n.name} = {module_prefix}{getattr(n.target, '__name__', n.target)}({joined})" + ) + + +@compatibility(is_backward_compatible=True) +class Interpreter: + """ + An Interpreter executes an FX graph Node-by-Node. This pattern + can be useful for many things, including writing code + transformations as well as analysis passes. + + Methods in the Interpreter class can be overridden to customize + the behavior of execution. The map of overridable methods + in terms of call hierarchy:: + + run() + +-- run_node + +-- placeholder() + +-- get_attr() + +-- call_function() + +-- call_method() + +-- call_module() + +-- output() + + Example: + + Suppose we want to swap all instances of ``torch.neg`` with + ``torch.sigmoid`` and vice versa (including their ``Tensor`` + method equivalents). We could subclass Interpreter like so:: + + class NegSigmSwapInterpreter(Interpreter): + def call_function( + self, target: Target, args: Tuple, kwargs: Dict + ) -> Any: + if target is torch.sigmoid: + return torch.neg(*args, **kwargs) + return super().call_function(target, args, kwargs) + + def call_method(self, target: Target, args: Tuple, kwargs: Dict) -> Any: + if target == "neg": + call_self, *args_tail = args + return call_self.sigmoid(*args_tail, **kwargs) + return super().call_method(target, args, kwargs) + + + def fn(x): + return torch.sigmoid(x).neg() + + + gm = torch.fx.symbolic_trace(fn) + input = torch.randn(3, 4) + result = NegSigmSwapInterpreter(gm).run(input) + torch.testing.assert_close(result, torch.neg(input).sigmoid()) + + Args: + module (torch.nn.Module): The module to be executed + garbage_collect_values (bool): Whether to delete values after their last + use within the Module's execution. This ensures optimal memory usage during + execution. This can be disabled to, for example, examine all of the intermediate + values in the execution by looking at the ``Interpreter.env`` attribute. + graph (Optional[Graph]): If passed, the interpreter will execute this + graph instead of `module.graph`, using the provided `module` + argument to satisfy any requests for state. + """ + + @compatibility(is_backward_compatible=True) + def __init__( + self, + module: torch.nn.Module, + garbage_collect_values: bool = True, + graph: Optional[Graph] = None, + ): + self.module = module + self.submodules = dict(self.module.named_modules()) + if graph is not None: + self.graph = graph + else: + self.graph = self.module.graph # type: ignore[assignment] + self.env: dict[Node, Any] = {} + self.name = "Interpreter" + self.garbage_collect_values = garbage_collect_values + self.extra_traceback = True + + if self.garbage_collect_values: + # Run through reverse nodes and record the first instance of a use + # of a given node. This represents the *last* use of the node in the + # execution order of the program, which we will use to free unused + # values + node_to_last_use: dict[Node, Node] = {} + self.user_to_last_uses: dict[Node, list[Node]] = {} + + def register_last_uses(n: Node, user: Node): + if n not in node_to_last_use: + node_to_last_use[n] = user + self.user_to_last_uses.setdefault(user, []).append(n) + + for node in reversed(self.graph.nodes): + for n in node._input_nodes: + register_last_uses(n, node) + + @compatibility(is_backward_compatible=True) + def run( + self, + *args, + initial_env: Optional[dict[Node, Any]] = None, + enable_io_processing: bool = True, + ) -> Any: + """ + Run `module` via interpretation and return the result. + + Args: + *args: The arguments to the Module to run, in positional order + initial_env (Optional[Dict[Node, Any]]): An optional starting environment for execution. + This is a dict mapping `Node` to any value. This can be used, for example, to + pre-populate results for certain `Nodes` so as to do only partial evaluation within + the interpreter. + enable_io_processing (bool): If true, we process the inputs and outputs with graph's process_inputs and + process_outputs function first before using them. + + Returns: + Any: The value returned from executing the Module + """ + self.env = initial_env if initial_env is not None else {} + + # Positional function args are consumed left-to-right by + # `placeholder` nodes. Use an iterator to keep track of + # position and extract those values. + if enable_io_processing: + args = self.graph.process_inputs(*args) + self.args_iter: Iterator[Any] = iter(args) + pbar = tqdm( + total=len(self.graph.nodes), + desc=f"{self.name}: {str(list(self.graph.nodes)) if config.verbose_progress else ''}", + initial=0, + position=0, + leave=True, + disable=config.disable_progress, + delay=0, + ) + + for node in self.graph.nodes: + pbar.update(1) + if node in self.env: + # Short circuit if we have this value. This could + # be used, for example, for partial evaluation + # where the caller has pre-populated `env` with + # values for a subset of the program. + continue + + try: + self.env[node] = self.run_node(node) + except Exception as e: + if self.extra_traceback: + msg = f"While executing {node.format_node()}" + msg = f"{e.args[0]}\n\n{msg}" if e.args else str(msg) + msg += f"\nOriginal traceback:\n{node.stack_trace}" + if ( + isinstance(self.module, GraphModule) + and self.module.graph is not None + and isinstance(self.module.graph, torch.fx.Graph) + ): + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "fx_interpreter_error", + "encoding": "string", + }, + payload_fn=lambda: ( + f"{msg}\nGraphModule: " + f"{self.module.print_readable(print_output=False, include_stride=True)}" # type: ignore[operator] + ), + ) + + msg += "\nUse tlparse to see full graph. " + msg += "(https://github.com/pytorch/tlparse?tab=readme-ov-file#tlparse-parse-structured-pt2-logs)" + e.args = (msg,) + e.args[1:] + if isinstance(e, KeyError): + raise RuntimeError(*e.args) from e + raise + + if self.garbage_collect_values: + for to_delete in self.user_to_last_uses.get(node, []): + del self.env[to_delete] + + if node.op == "output": + output_val = self.env[node] + return ( + self.graph.process_outputs(output_val) + if enable_io_processing + else output_val + ) + + @compatibility(is_backward_compatible=True) + def boxed_run(self, args_list): + """ + Run `module` via interpretation and return the result. This uses the "boxed" + calling convention, where you pass a list of arguments, which will be cleared + by the interpreter. This ensures that input tensors are promptly deallocated. + """ + # Collect placeholder nodes first + placeholder_nodes = [n for n in self.graph.nodes if n.op == "placeholder"] + + # Check argument count + if len(args_list) != len(placeholder_nodes): + detail = ( + "extra arguments" + if len(args_list) > len(placeholder_nodes) + else "missing arguments" + ) + raise RuntimeError( + f"Interpreter.boxed_run expected {len(placeholder_nodes)} arguments for placeholders " + f"but received {len(args_list)} ({detail})" + ) + + # Assign arguments to placeholders + env = dict(zip(placeholder_nodes, args_list)) + args_list.clear() + return self.run(initial_env=env) + + @contextmanager + def _set_current_node(self, node): + with fx_traceback.set_current_meta( + node, f"Interpreter_{self.__class__.__name__}" + ): + yield + + @compatibility(is_backward_compatible=True) + def run_node(self, n: Node) -> Any: + """ + Run a specific node ``n`` and return the result. + Calls into placeholder, get_attr, call_function, + call_method, call_module, or output depending + on ``node.op`` + + Args: + n (Node): The Node to execute + + Returns: + Any: The result of executing ``n`` + """ + log.debug("run_node %s", LazyString(lambda: _format_fx_node(n))) + with self._set_current_node(n): + args, kwargs = self.fetch_args_kwargs_from_env(n) + assert isinstance(args, tuple) + assert isinstance(kwargs, dict) + return getattr(self, n.op)(n.target, args, kwargs) + + # Main Node running APIs + @compatibility(is_backward_compatible=True) + def placeholder( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute a ``placeholder`` node. Note that this is stateful: + ``Interpreter`` maintains an internal iterator over + arguments passed to ``run`` and this method returns + next() on that iterator. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Returns: + Any: The argument value that was retrieved. + """ + assert isinstance(target, str) + if target.startswith("*"): + # For a starred parameter e.g. `*args`, retrieve all + # remaining values from the args list. + return list(self.args_iter) + else: + try: + return next(self.args_iter) + except StopIteration as si: + if len(args) > 0: + return args[0] + else: + raise RuntimeError( + f"Expected positional argument for parameter {target}, but one was not passed in!" + ) from si + + @compatibility(is_backward_compatible=True) + def get_attr( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute a ``get_attr`` node. Will retrieve an attribute + value from the ``Module`` hierarchy of ``self.module``. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Return: + Any: The value of the attribute that was retrieved + """ + assert isinstance(target, str) + return self.fetch_attr(target) + + @compatibility(is_backward_compatible=True) + def call_function( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute a ``call_function`` node and return the result. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Return + Any: The value returned by the function invocation + """ + assert not isinstance(target, str) + + # Execute the function and return the result + return target(*args, **kwargs) + + @compatibility(is_backward_compatible=True) + def call_method( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute a ``call_method`` node and return the result. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Return + Any: The value returned by the method invocation + """ + # args[0] is the `self` object for this method call + self_obj, *args_tail = args + + # Execute the method and return the result + assert isinstance(target, str) + return getattr(self_obj, target)(*args_tail, **kwargs) + + @compatibility(is_backward_compatible=True) + def call_module( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute a ``call_module`` node and return the result. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Return + Any: The value returned by the module invocation + """ + # Retrieve executed args and kwargs values from the environment + + # Execute the method and return the result + assert isinstance(target, str) + submod = self.fetch_attr(target) + + return submod(*args, **kwargs) + + @compatibility(is_backward_compatible=True) + def output( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + """ + Execute an ``output`` node. This really just retrieves + the value referenced by the ``output`` node and returns it. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + + Return: + Any: The return value referenced by the output node + """ + return args[0] + + # Helper methods + @compatibility(is_backward_compatible=True) + def fetch_attr(self, target: str): + """ + Fetch an attribute from the ``Module`` hierarchy of ``self.module``. + + Args: + target (str): The fully-qualified name of the attribute to fetch + + Return: + Any: The value of the attribute. + """ + target_atoms = target.split(".") + attr_itr = self.module + for i, atom in enumerate(target_atoms): + if not hasattr(attr_itr, atom): + raise RuntimeError( + f"Node referenced nonexistent target {'.'.join(target_atoms[: i + 1])}" + ) + attr_itr = getattr(attr_itr, atom) + return attr_itr + + @compatibility(is_backward_compatible=True) + def fetch_args_kwargs_from_env(self, n: Node) -> tuple[tuple, dict]: + """ + Fetch the concrete values of ``args`` and ``kwargs`` of node ``n`` + from the current execution environment. + + Args: + n (Node): The node for which ``args`` and ``kwargs`` should be fetched. + + Return: + Tuple[Tuple, Dict]: ``args`` and ``kwargs`` with concrete values for ``n``. + """ + args = self.map_nodes_to_values(n.args, n) + assert isinstance(args, tuple) + kwargs = self.map_nodes_to_values(n.kwargs, n) + assert isinstance(kwargs, dict) + return args, kwargs + + @compatibility(is_backward_compatible=True) + def map_nodes_to_values(self, args: Argument, n: Node) -> Argument: + """ + Recursively descend through ``args`` and look up the concrete value + for each ``Node`` in the current execution environment. + + Args: + args (Argument): Data structure within which to look up concrete values + + n (Node): Node to which ``args`` belongs. This is only used for error reporting. + """ + + def load_arg(n_arg: Node) -> Any: + if n_arg not in self.env: + raise RuntimeError( + f"Node {n} referenced nonexistent value {n_arg}! Run Graph.lint() " + f"to diagnose such issues" + ) + return self.env[n_arg] + + return map_arg(args, load_arg) + + +@compatibility(is_backward_compatible=True) +class Transformer(Interpreter): + """ + ``Transformer`` is a special type of interpreter that produces a + new ``Module``. It exposes a ``transform()`` method that returns + the transformed ``Module``. ``Transformer`` does not require + arguments to run, as ``Interpreter`` does. ``Transformer`` works + entirely symbolically. + + Example: + + Suppose we want to swap all instances of ``torch.neg`` with + ``torch.sigmoid`` and vice versa (including their ``Tensor`` + method equivalents). We could subclass ``Transformer`` like so:: + + class NegSigmSwapXformer(Transformer): + def call_function( + self, + target: "Target", + args: Tuple[Argument, ...], + kwargs: Dict[str, Any], + ) -> Any: + if target is torch.sigmoid: + return torch.neg(*args, **kwargs) + return super().call_function(target, args, kwargs) + + def call_method( + self, + target: "Target", + args: Tuple[Argument, ...], + kwargs: Dict[str, Any], + ) -> Any: + if target == "neg": + call_self, *args_tail = args + return call_self.sigmoid(*args_tail, **kwargs) + return super().call_method(target, args, kwargs) + + + def fn(x): + return torch.sigmoid(x).neg() + + + gm = torch.fx.symbolic_trace(fn) + + transformed: torch.nn.Module = NegSigmSwapXformer(gm).transform() + input = torch.randn(3, 4) + torch.testing.assert_close(transformed(input), torch.neg(input).sigmoid()) + + Args: + module (GraphModule): The ``Module`` to be transformed. + """ + + @compatibility(is_backward_compatible=True) + def __init__(self, module): + super().__init__(module) + self.new_graph = Graph() + self.new_graph.set_codegen(module.graph._codegen) + + class TransformerTracer(Tracer): + def __init__(self, graph: Graph): + super().__init__() + self.graph = graph + self.tensor_attrs: dict[torch.Tensor, str] = {} # type: ignore[assignment] + + def is_leaf_module(self, _, __) -> bool: + return True + + self.tracer = TransformerTracer(self.new_graph) + self.tracer.root = module + + @compatibility(is_backward_compatible=True) + def placeholder( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Proxy: + """ + Execute a ``placeholder`` node. In ``Transformer``, this is + overridden to insert a new ``placeholder`` into the output + graph. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + """ + assert isinstance(target, str) + default_value = next(iter(args)) if args else inspect.Signature.empty + return Proxy( + self.new_graph.placeholder(target, default_value=default_value), self.tracer + ) + + @compatibility(is_backward_compatible=True) + def get_attr( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Proxy: + """ + Execute a ``get_attr`` node. In ``Transformer``, this is + overridden to insert a new ``get_attr`` node into the output + graph. + + Args: + target (Target): The call target for this node. See + `Node `__ for + details on semantics + args (Tuple): Tuple of positional args for this invocation + kwargs (Dict): Dict of keyword arguments for this invocation + """ + assert isinstance(target, str) + return self.tracer.create_proxy("get_attr", target, args, kwargs) + + @compatibility(is_backward_compatible=True) + def call_module( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + # Override so that the leaf module policy from `self.tracer` is respected. + assert isinstance(target, str) + submod = self.fetch_attr(target) + return self.tracer.call_module(submod, submod.forward, args, kwargs) + + @compatibility(is_backward_compatible=True) + def call_function( + self, target: "Target", args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + # Override so that functions that were wrapped are still wrapped. + return self.tracer.create_proxy("call_function", target, args, kwargs) + + @compatibility(is_backward_compatible=True) + def transform(self) -> GraphModule: + """ + Transform ``self.module`` and return the transformed + ``GraphModule``. + """ + with fx_traceback.preserve_node_meta(): + result = super().run(enable_io_processing=False) + if result is not None: + + def strip_proxy(a: Union[Argument, Proxy]) -> Any: + return a.node if isinstance(a, Proxy) else a + + new_output_node = self.new_graph.output(map_aggregate(result, strip_proxy)) + # also preserve the metadata from the old output node, if it exists + old_output_node = list(self.graph.nodes)[-1] + assert old_output_node.op == "output" + for k, v in old_output_node.meta.items(): + new_output_node.meta[k] = v + + return _make_graph_module(self.module, self.new_graph) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/node.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/node.py new file mode 100644 index 0000000000000000000000000000000000000000..5365cd1f4f9ad488153e992a2bdd6db0fd828296 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/node.py @@ -0,0 +1,893 @@ +# Nodes represent a definition of a value in our graph of operators. +import builtins +import inspect +import logging +import operator +import types +import typing +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import Any, Optional, TYPE_CHECKING, TypeAlias, Union +from typing_extensions import ParamSpec, TypeVar + +import torch +from torch._C import _fx_map_aggregate, _fx_map_arg, _NodeBase +from torch.fx.operator_schemas import ( + ArgsKwargsPair, + normalize_function, + normalize_module, +) +from torch.utils._dtype_abbrs import dtype_abbrs + +from .._ops import ops as _ops +from ._compatibility import compatibility + + +if TYPE_CHECKING: + from .graph import Graph + +__all__ = ["Node", "map_arg", "map_aggregate", "has_side_effect"] + +log = logging.getLogger(__name__) + +BaseArgumentTypes = Union[ + str, + int, + float, + bool, + complex, + torch.dtype, + torch.Tensor, + torch.device, + torch.memory_format, + torch.layout, + torch._ops.OpOverload, + torch.SymInt, + torch.SymBool, + torch.SymFloat, +] +base_types = typing.get_args(BaseArgumentTypes) + +Target: TypeAlias = Union[Callable[..., Any], str] + +Argument = Optional[ + Union[ + tuple["Argument", ...], + Sequence["Argument"], + Mapping[str, "Argument"], + slice, # Slice[Argument, Argument, Argument], but slice is not a templated type in typing + range, + "Node", + BaseArgumentTypes, + ] +] +# pyrefly: ignore [invalid-annotation] +ArgumentT = TypeVar("ArgumentT", bound=Argument) +_P = ParamSpec("_P") +_R = TypeVar("_R") + +_legal_ops = dict.fromkeys( + [ + "placeholder", + "call_method", + "call_module", + "call_function", + "get_attr", + "output", + "root", + ] +) + +# Dynamo is unable to trace global set[Callable].__contains__. +# See https://github.com/pytorch/pytorch/issues/145761. Since we only have +# a handful of ops so switch to list of callables. +_side_effectful_need_to_be_preserved_pre_dispatch: list[Callable[..., Any]] = [ + torch._C._set_grad_enabled, + torch.amp._enter_autocast, + torch.amp._exit_autocast, +] + +# TODO: Either refactor this into 2 functions 1 dce for functional graphs and 1 dce for all graphs, +# or add logic to correctly mark all inplace ops as side effectful. +# +# NOTE: For new operators, please do not add to this set! +# Instead, consider using the effects system via +# torch.library._register_effectful_op() for operators. +# +# This _side_effectful_functions set is only for: +# - Legacy functions that aren't operators (e.g., profiler ops, asserts) +# - Things that cannot be marked via the normal effects system +_side_effectful_functions: set[Callable[..., Any]] = { + torch._assert, + torch._assert_async, + _ops.aten._assert_async.msg, + _ops.aten._assert_scalar.default, + _ops.aten._assert_tensor_metadata.default, + _ops.aten.sym_constrain_range.default, + _ops.aten.sym_constrain_range_for_size.default, + _ops.profiler._record_function_enter, + _ops.profiler._record_function_enter_new, + _ops.profiler._record_function_exit, + _ops.inductor.accumulate_grad_.default, + operator.setitem, + *_side_effectful_need_to_be_preserved_pre_dispatch, +} + +if hasattr(_ops.inductor, "resize_storage_bytes_"): + _side_effectful_functions.add(_ops.inductor.resize_storage_bytes_.default) + + +@compatibility(is_backward_compatible=False) +def has_side_effect(fn: Callable[_P, _R]) -> Callable[_P, _R]: + """ + Registers a function to not be dead code eliminated by + fx.graph.eliminate_dead_code + + NOTE: For new operators, please do not add to this set! + Instead, consider using the effects system via + torch.library._register_effectful_op() for operators. + + This _side_effectful_functions set is only for: + - Legacy functions that aren't operators (e.g., profiler ops, asserts) + - Things that cannot be marked via the normal effects system + """ + _side_effectful_functions.add(fn) + return fn + + +# this is fixed on master, WAR for 1.5 +def _find_module_of_method(orig_method: Callable[..., Any]) -> str: + name = orig_method.__name__ + module = orig_method.__module__ + if module is not None: + return module + for guess in [torch, torch.nn.functional]: + if getattr(guess, name, None) is orig_method: + return guess.__name__ + raise RuntimeError(f"cannot find module for {orig_method}") + + +# Borrowed from CPython typing module +# https://github.com/python/cpython/blob/f90dc36c15d7fee0efaf6d39e97be0bdf2683e93/Lib/typing.py#L156 +def _type_repr(obj: object) -> str: + """Return the repr() of an object, special-casing types (internal helper). + If obj is a type, we return a shorter version than the default + type.__repr__, based on the module and qualified name, which is + typically enough to uniquely identify a type. For everything + else, we fall back on repr(obj). + """ + # Extension: If we don't ignore GenericAlias then `list[int]` will print + # simply "list". + if isinstance(obj, type) and not isinstance(obj, types.GenericAlias): + if obj.__module__ == "builtins": + return obj.__qualname__ + return f"{obj.__module__}.{obj.__qualname__}" + if obj is ...: + return "..." + if isinstance(obj, types.FunctionType): + return obj.__name__ + return repr(obj) + + +def _get_qualified_name(func: Callable[..., Any]) -> str: + # things like getattr just appear in builtins + if getattr(builtins, func.__name__, None) is func: + return func.__name__ + # torch.Tensor.{fn} + if ( + isinstance(func, (types.MethodDescriptorType, types.WrapperDescriptorType)) + and func is getattr(torch.Tensor, func.__name__, None) + ) or ( + func.__module__ == torch._tensor.__name__ + and func.__qualname__ == f"Tensor.{func.__name__}" + ): + return f"torch.Tensor.{func.__name__}" + name = func.__name__ + if name == "": + # For lambdas, try to get their defining name in the module + try: + name = inspect.getsource(func).split("=")[0].strip() + except Exception as e: + raise RuntimeError("Unable to represent lambda") from e + module = _find_module_of_method(func) + module = module.replace( + "torch._ops", "torch.ops" + ) # WAR for bug in how torch.ops assigns module + # Fixup segment_reduce mismatch + if module == "torch" and name == "segment_reduce": + name = "_" + name + if module == "torch.nn.functional" and name in ("_ScalingType", "_SwizzleType"): + name = name.removeprefix("_") + return f"{module}.{name}" + + +def _format_arg(arg: object, max_list_len: float = float("inf")) -> str: + if hasattr(arg, "_custom_fx_repr_fn"): + return arg._custom_fx_repr_fn() + elif isinstance(arg, list): + items = ", ".join( + _format_arg(a) for idx, a in enumerate(arg) if idx < max_list_len + ) + maybe_len = ( + "" if len(arg) < max_list_len + 1 else f", ...[total_len={len(arg)}]" + ) + return f"[{items}{maybe_len}]" + elif isinstance(arg, tuple): + items = ", ".join( + _format_arg(a) for idx, a in enumerate(arg) if idx < max_list_len + ) + maybe_len = ( + "" if len(arg) < max_list_len + 1 else f", ...[total_len={len(arg)}]" + ) + maybe_comma = "," if len(arg) == 1 else "" + return f"({items}{maybe_comma}{maybe_len})" + elif isinstance(arg, dict): + items_str = ", ".join(f"{k}: {_format_arg(v)}" for k, v in arg.items()) + return f"{{{items_str}}}" + + if isinstance(arg, Node): + return "%" + str(arg) + else: + return str(arg) + + +@compatibility(is_backward_compatible=True) +class Node(_NodeBase): + """ + ``Node`` is the data structure that represents individual operations within + a ``Graph``. For the most part, Nodes represent callsites to various entities, + such as operators, methods, and Modules (some exceptions include nodes that + specify function inputs and outputs). Each ``Node`` has a function specified + by its ``op`` property. The ``Node`` semantics for each value of ``op`` are as follows: + + - ``placeholder`` represents a function input. The ``name`` attribute specifies the name this value will take on. + ``target`` is similarly the name of the argument. ``args`` holds either: 1) nothing, or 2) a single argument + denoting the default parameter of the function input. ``kwargs`` is don't-care. Placeholders correspond to + the function parameters (e.g. ``x``) in the graph printout. + - ``get_attr`` retrieves a parameter from the module hierarchy. ``name`` is similarly the name the result of the + fetch is assigned to. ``target`` is the fully-qualified name of the parameter's position in the module hierarchy. + ``args`` and ``kwargs`` are don't-care + - ``call_function`` applies a free function to some values. ``name`` is similarly the name of the value to assign + to. ``target`` is the function to be applied. ``args`` and ``kwargs`` represent the arguments to the function, + following the Python calling convention + - ``call_module`` applies a module in the module hierarchy's ``forward()`` method to given arguments. ``name`` is + as previous. ``target`` is the fully-qualified name of the module in the module hierarchy to call. + ``args`` and ``kwargs`` represent the arguments to invoke the module on, *excluding the self argument*. + - ``call_method`` calls a method on a value. ``name`` is as similar. ``target`` is the string name of the method + to apply to the ``self`` argument. ``args`` and ``kwargs`` represent the arguments to invoke the module on, + *including the self argument* + - ``output`` contains the output of the traced function in its ``args[0]`` attribute. This corresponds to the "return" statement + in the Graph printout. + """ + + _args: tuple["Argument", ...] + _kwargs: dict[str, "Argument"] + graph: "Graph" + # unique name of value being created + name: str + # the kind of operation = placeholder|call_method|call_module|call_function|get_attr + op: str + # for method/module/function, the name of the method/module/function/attr + # being invoked, e.g add, layer1, or torch.add + target: "Target" + # All `Node`-valued inputs. Key is the Node, value is don't-care. + # The public API for this is `all_input_nodes`, this private attribute + # should not be accessed directly. + _input_nodes: dict["Node", None] + # All of the nodes that use the value produced by this Node + # Note one user may correspond to several uses, e.g. the node for ``x + x`` + # would appear once here, but represents two uses. + # Is a dict to act as an "ordered set". Keys are significant, value dont-care + users: dict["Node", None] + # Type expression representing the output value of this node. + # This should contain the same class of Type objects that would appear + # as type annotations for function inputs/outputs. + # + # For placeholder nodes, this value will be used to type-annotate the + # generated function parameters. + # For the return node, this value will be used to type-annotate the + # generated function return type. (Note this is a special case. ``return`` + # does not produce a value, it's more of a notation. Thus, this value + # describes the type of args[0] in the ``return`` node. + type: Optional[Any] + _sort_key: Any + # If set, use this fn to print this node + _repr_fn: Optional[Callable[["Node"], str]] + # Dictionary to store metadata passes need to do their + # transformations. This metadata is preserved across node copies + meta: dict[str, Any] + + @compatibility(is_backward_compatible=True) + def __init__( + self, + graph: "Graph", + name: str, + op: str, + target: "Target", + args: tuple["Argument", ...], + kwargs: dict[str, "Argument"], + return_type: Optional[Any] = None, + ) -> None: + """ + Instantiate an instance of ``Node``. Note: most often, you want to use the + Graph APIs, i.e. ``Graph.call_module``, ``Graph.call_method``, etc. rather + than instantiating a ``Node`` directly. + + Args: + graph (Graph): The ``Graph`` to which this ``Node`` should belong. + + name (str): The name to which the output of this ``Node`` should be assigned + + op (str): The opcode for this ``Node``. Can be one of 'placeholder', + 'call_method', 'call_module', 'call_function', 'get_attr', + 'output' + + target ('Target'): The target this op should call. See the broader + ``Node`` docstring for more details. + + args (Tuple['Argument']): The args to be passed to ``target`` + + kwargs (Dict[str, 'Argument']): The kwargs to be passed to ``target`` + + return_type (Optional[Any]): The python type expression representing the + type of the output of this node. This field can be used for + annotation of values in the generated code or for other types + of analyses. + """ + if op == "call_function": + if not callable(target): + raise ValueError( + f"Node [graph = {graph}, name = '{name}'] target {target} has type {torch.typename(target)} " + "but a Callable is expected" + ) + else: + assert op in _legal_ops + if not isinstance(target, str): + raise ValueError( + f"Node [graph = {graph}, name = '{name}'] target {target} has type {torch.typename(target)} " + "but a str is expected" + ) + super().__init__(graph, name, op, target, return_type) + self._update_args_kwargs(args, kwargs) + + def __getstate__(self) -> dict[str, Any]: + return { + **self.__dict__, + "graph": self.graph, + "name": self.name, + "op": self.op, + "target": self.target, + "type": self.target, + "_sort_key": self._sort_key, + "_args": self._args, + "_kwargs": self._kwargs, + "_erased": self._erased, + "_prev": self._prev, + "_next": self._next, + "_input_nodes": self._input_nodes, + "users": self.users, + "_repr_fn": self._repr_fn, + "meta": self.meta, + } + + def __setstate__(self, state: dict[str, Any]) -> None: + for k, v in state.items(): + setattr(self, k, v) + + @property + def next(self) -> "Node": + """ + Returns the next ``Node`` in the linked list of Nodes. + + Returns: + + The next ``Node`` in the linked list of Nodes. + """ + return self._next + + @property + def prev(self) -> "Node": + """ + Returns the previous ``Node`` in the linked list of Nodes. + + Returns: + + The previous ``Node`` in the linked list of Nodes. + """ + return self._prev + + @compatibility(is_backward_compatible=True) + def prepend(self, x: "Node") -> None: + """ + Insert x before this node in the list of nodes in the graph. Example:: + + Before: p -> self + bx -> x -> ax + After: p -> x -> self + bx -> ax + + Args: + x (Node): The node to put before this node. Must be a member of the same graph. + """ + # pyrefly: ignore [missing-attribute] + self._prepend(x) + + @compatibility(is_backward_compatible=True) + def append(self, x: "Node") -> None: + """ + Insert ``x`` after this node in the list of nodes in the graph. + Equivalent to ``self.next.prepend(x)`` + + Args: + x (Node): The node to put after this node. Must be a member of the same graph. + """ + # pyrefly: ignore [missing-attribute] + self._next._prepend(x) + + @property + def args(self) -> tuple[Argument, ...]: + """ + The tuple of arguments to this ``Node``. The interpretation of arguments + depends on the node's opcode. See the :class:`Node` docstring for more + information. + + Assignment to this property is allowed. All accounting of uses and users + is updated automatically on assignment. + """ + return self._args + + @args.setter + def args(self, a: tuple[Argument, ...]) -> None: + """ + Set the tuple of arguments to this Node. The interpretation of arguments + depends on the node's opcode. See the ``fx.Graph`` docstring for more + information. + """ + # DO NOT CALL `_update_args_kwargs` directly. The correct way to + # set `args` is via direct assignment, i.e. `node.args = new_args` + self._update_args_kwargs(a, self._kwargs) + + @property + def kwargs(self) -> dict[str, Argument]: + """ + The dict of keyword arguments to this ``Node``. The interpretation of arguments + depends on the node's opcode. See the :class:`Node` docstring for more + information. + + Assignment to this property is allowed. All accounting of uses and users + is updated automatically on assignment. + """ + return self._kwargs + + @kwargs.setter + def kwargs(self, k: dict[str, Argument]) -> None: + """ + Set the dict of kwargs to this Node. The interpretation of arguments + depends on the node's opcode. See the ``fx.Graph`` docstring for more + information. + """ + # DO NOT CALL `_update_args_kwargs` directly. The correct way to + # set `args` is via direct assignment, i.e. `node.kwargs = new_kwargs` + self._update_args_kwargs(self._args, k) + + @property + def all_input_nodes(self) -> list["Node"]: + """ + Return all Nodes that are inputs to this Node. This is equivalent to + iterating over ``args`` and ``kwargs`` and only collecting the values that + are Nodes. + + Returns: + + List of ``Nodes`` that appear in the ``args`` and ``kwargs`` of this + ``Node``, in that order. + """ + return list(self._input_nodes.keys()) + + @compatibility(is_backward_compatible=True) + def update_arg(self, idx: int, arg: Argument) -> None: + """ + Update an existing positional argument to contain the new value + ``arg``. After calling, ``self.args[idx] == arg``. + + Args: + + idx (int): The index into ``self.args`` of the element to update + arg (Argument): The new argument value to write into ``args`` + """ + args = list(self.args) + args[idx] = arg + self.args = tuple(args) + + @compatibility(is_backward_compatible=True) + def insert_arg(self, idx: int, arg: Argument) -> None: + """ + Insert an positional argument to the argument list with given index. + + Args: + + idx (int): The index of the element in ``self.args`` to be inserted before. + arg (Argument): The new argument value to insert into ``args`` + """ + assert 0 <= idx <= len(self.args), ( + "insert_args index must be between 0 and len(self.args)" + ) + args_left = self.args[:idx] + args_right = self.args[idx:] + + self._args = args_left + (arg,) + args_right + + _new_input_nodes: dict[Node, None] = {} + _fx_map_arg(arg, _new_input_nodes.setdefault) + + for new_use in _new_input_nodes: + if new_use not in self._input_nodes: + self._input_nodes.setdefault(new_use) + new_use.users.setdefault(self) + + @compatibility(is_backward_compatible=True) + def update_kwarg(self, key: str, arg: Argument) -> None: + """ + Update an existing keyword argument to contain the new value + ``arg``. After calling, ``self.kwargs[key] == arg``. + + Args: + + key (str): The key in ``self.kwargs`` of the element to update + arg (Argument): The new argument value to write into ``kwargs`` + """ + self.kwargs = {**self.kwargs, key: arg} + + @property + def stack_trace(self) -> Optional[str]: + """ + Return the Python stack trace that was recorded during tracing, if any. + When traced with fx.Tracer, this property is usually populated by + `Tracer.create_proxy`. To record stack traces during tracing for debug purposes, + set `record_stack_traces = True` on the `Tracer` instance. + When traced with dynamo, this property will be populated by default by + `OutputGraph.create_proxy`. + + stack_trace would have the innermost frame at the end of the string. + """ + return self.meta.get("stack_trace", None) + + @stack_trace.setter + def stack_trace(self, trace: Optional[str]) -> None: + self.meta["stack_trace"] = trace + + def __repr__(self) -> str: + if self._repr_fn: + return self._repr_fn(self) + return self.name + + @staticmethod + def _pretty_print_target(target: object) -> str: + """ + Make target printouts more user-friendly. + 1) builtins will be printed as `builtins.xyz` + 2) operators will be printed as `operator.xyz` + 3) other callables will be printed with qualified name, e.g. torch.add + """ + if isinstance(target, str): + return target + if hasattr(target, "__module__"): + name = getattr(target, "__name__", None) + if name is None: + # Just to be defensive, if we don't have `__name__`, get the + # qualname. Not sure if this happens for any members of `operator` + # or `builtins`. This fallback path is not as good, since e.g. + # things in `operator` have `_operator` as their __module__. + # TODO: THIS IS BROKEN: _get_qualified_name calls `__name__` + return _get_qualified_name(target) # type: ignore[arg-type] + if target.__module__ == "builtins": + return f"builtins.{name}" + elif target.__module__ == "_operator": + return f"operator.{name}" + return _get_qualified_name(target) # type: ignore[arg-type] + + @compatibility(is_backward_compatible=True) + def format_node( + self, + placeholder_names: Optional[list[str]] = None, + maybe_return_typename: Optional[list[str]] = None, + *, + include_tensor_metadata: bool = False, + ) -> Optional[str]: + """ + Return a descriptive string representation of ``self``. + + This method can be used with no arguments as a debugging + utility. + + This function is also used internally in the ``__str__`` method + of ``Graph``. Together, the strings in ``placeholder_names`` + and ``maybe_return_typename`` make up the signature of the + autogenerated ``forward`` function in this Graph's surrounding + GraphModule. ``placeholder_names`` and ``maybe_return_typename`` + should not be used otherwise. + + Args: + placeholder_names: A list that will store formatted strings + representing the placeholders in the generated + ``forward`` function. Internal use only. + maybe_return_typename: A single-element list that will store + a formatted string representing the output of the + generated ``forward`` function. Internal use only. + include_tensor_metadata: Whether to include tensor metadata + + Returns: + str: If 1) we're using ``format_node`` as an internal helper + in the ``__str__`` method of ``Graph``, and 2) ``self`` + is a placeholder Node, return ``None``. Otherwise, + return a descriptive string representation of the + current Node. + """ + if self.op == "placeholder": + assert isinstance(self.target, str) + arg_str = self.target + arg_str += arg_str + f": {_type_repr(self.type)}" if self.type else "" + if placeholder_names: + placeholder_names.append(arg_str) + return None + maybe_typename = f"{_type_repr(self.type)} " if self.type else "" + default_val = "(default=" + str(self.args[0]) + ")" if self.args else "" + return f"%{self.name} : {maybe_typename}[num_users={len(self.users)}] = {self.op}[target={self.target}]{default_val}" + elif self.op == "get_attr": + maybe_typename = ( + f"{_type_repr(self.type)} " if self.type is not None else "" + ) + return ( + f"%{self.name} : {maybe_typename}[num_users={len(self.users)}] = " + f"{self.op}[target={self._pretty_print_target(self.target)}]" + ) + elif self.op == "output": + if self.type and maybe_return_typename: + maybe_return_typename[0] = f" -> {_type_repr(self.type)}" + return f"return {self.args[0]}" + else: + + def stringify_shape(shape: Iterable) -> str: + return f"[{', '.join([str(x) for x in shape])}]" + + meta_val = self.meta.get( + "val", + self.meta.get("tensor_meta", self.meta.get("example_value", None)), + ) + type_annotation = "" + if ( + include_tensor_metadata + and isinstance(meta_val, torch.Tensor) + and meta_val.layout + not in ( + torch.sparse_csc, + torch.sparse_csr, + ) + ): + stride_annotation = f"{stringify_shape(meta_val.stride())}" + device_annotation = f"{meta_val.device}" + type_annotation = ( + f'Tensor "{dtype_abbrs[meta_val.dtype]}{stringify_shape(meta_val.shape)}' + f'{stride_annotation}{device_annotation}"' + ) + else: + type_annotation = ( + f"{_type_repr(self.type)} " if self.type is not None else "" + ) + return ( + f"%{self.name} : {type_annotation}[num_users={len(self.users)}] = " + f"{self.op}[target={self._pretty_print_target(self.target)}](" + f"args = {_format_arg(self.args)}, kwargs = {_format_arg(self.kwargs)})" + ) + + @compatibility(is_backward_compatible=True) + def replace_all_uses_with( + self, + replace_with: "Node", + delete_user_cb: Optional[Callable[["Node"], bool]] = None, + *, + propagate_meta: bool = False, + ) -> list["Node"]: + """ + Replace all uses of ``self`` in the Graph with the Node ``replace_with``. + + Args: + + replace_with (Node): The node to replace all uses of ``self`` with. + delete_user_cb (Callable): Callback that is called to determine + whether a given user of the self node should be removed. + propagate_meta (bool): Whether or not to copy all properties + on the .meta field of the original node onto the replacement node. + For safety, this is only valid to do if the replacement node + doesn't already have an existing .meta field. + + Returns: + + The list of Nodes on which this change was made. + """ + if propagate_meta: + assert len(replace_with.meta) == 0, ( + "Called node.replace_all_uses_with(replace_with, propagate_meta=True), " + "but replace_with already has .meta keys" + ) + for k, v in self.meta.items(): + replace_with.meta[k] = v + to_process = [*self.users] + replace_hooks = getattr(self.graph.owning_module, "_replace_hooks", None) + result = [] + for use_node in to_process: + if delete_user_cb is not None and not delete_user_cb(use_node): + continue + result.append(use_node) + if replace_hooks: + for replace_hook in replace_hooks: + replace_hook(old=self, new=replace_with.name, user=use_node) + # pyrefly: ignore [missing-attribute] + use_node._replace_input_with(self, replace_with) # type: ignore[attr-defined] + return result + + @compatibility(is_backward_compatible=False) + def is_impure(self, impure_random: bool = True) -> bool: + """ + Returns whether this op is impure, i.e. if its op is a placeholder or + output, or if a call_function or call_module which is impure. + + Args: + impure_random (bool): Whether to treat rand op as impure. + + Returns: + + bool: If the op is impure or not. + """ + # Placeholders and outputs are always impure for DCE purposes + if self.op in {"placeholder", "output"}: + return True + + # Check if an impure module. + if self.op == "call_module": + assert self.graph.owning_module is not None, ( + "self.graph.owning_module not set for purity check" + ) + target_mod = self.graph.owning_module.get_submodule(self.target) + assert target_mod is not None, ( + f"Did not find expected submodule target {self.target}" + ) + # NOTE: here we can end up considering GraphModule submodules pure, + # even if they contain impure ops. It may not be safe to change + # because this function is used by graph.eliminate_dead_code, + # and some users depend on current elimination behavior. + return getattr(target_mod, "_is_impure", False) + + # For call_function, delegate to the unified has_side_effects function + if self.op == "call_function": + from torch._library.utils import is_impure + + return is_impure( + self.target, # pyrefly: ignore[bad-argument-type] + args=self.args, + kwargs=self.kwargs, + impure_random=impure_random, + ) + + return False + + @compatibility(is_backward_compatible=False) + def normalized_arguments( + self, + root: torch.nn.Module, + arg_types: Optional[tuple[Any]] = None, + kwarg_types: Optional[dict[str, Any]] = None, + normalize_to_only_use_kwargs: bool = False, + ) -> Optional[ArgsKwargsPair]: + """ + Returns normalized arguments to Python targets. This means that + `args/kwargs` will be matched up to the module/functional's + signature and return exclusively kwargs in positional order + if `normalize_to_only_use_kwargs` is true. + Also populates default values. Does not support positional-only + parameters or varargs parameters. + + Supports module calls. + + May require `arg_types` and `kwarg_types` in order to disambiguate overloads. + + Args: + root (torch.nn.Module): Module upon which to resolve module targets. + arg_types (Optional[Tuple[Any]]): Tuple of arg types for the args + kwarg_types (Optional[Dict[str, Any]]): Dict of arg types for the kwargs + normalize_to_only_use_kwargs (bool): Whether to normalize to only use kwargs. + + Returns: + + Returns NamedTuple ArgsKwargsPair, or `None` if not successful. + """ + if self.op == "call_function": + assert callable(self.target) + return normalize_function( + self.target, + self.args, # type: ignore[arg-type] + self.kwargs, + arg_types, + kwarg_types, + normalize_to_only_use_kwargs=normalize_to_only_use_kwargs, + ) + elif self.op == "call_module": + assert isinstance(self.target, str) + return normalize_module( + root, + self.target, + self.args, # type: ignore[arg-type] + self.kwargs, + normalize_to_only_use_kwargs=normalize_to_only_use_kwargs, + ) + + return None + + @compatibility(is_backward_compatible=True) + def replace_input_with(self, old_input: "Node", new_input: "Node") -> None: + """ + Loop through input nodes of ``self``, and replace all instances of + ``old_input`` with ``new_input``. + + Args: + + old_input (Node): The old input node to be replaced. + new_input (Node): The new input node to replace ``old_input``. + """ + + m = self.graph.owning_module + if getattr(m, "_replace_hooks", None): + for replace_hook in m._replace_hooks: + replace_hook(old=old_input, new=new_input.name, user=self) + + # pyrefly: ignore [missing-attribute] + self._replace_input_with(old_input, new_input) # type: ignore[attr-defined] + + def _rename(self, candidate: str) -> None: + if candidate == self.name: + return + name = self.graph._graph_namespace.create_name(candidate, None) + self.name = name + self.graph._graph_namespace._rename_object(self, name) + + def __setattr__(self, name: str, value: Any) -> None: + if name == "name" and hasattr(self, "name"): + m = self.graph.owning_module + if getattr(m, "_replace_hooks", None): + assert isinstance(value, str) + for user in self.users: + for replace_hook in m._replace_hooks: + replace_hook(old=self, new=value, user=user) + update = False + if ( + hasattr(self, name) + and hasattr(self.graph, "_find_nodes_lookup_table") + and self in self.graph._find_nodes_lookup_table + ): + update = True + self.graph._find_nodes_lookup_table.remove(self) + object.__setattr__(self, name, value) + if update: + self.graph._find_nodes_lookup_table.insert(self) + + +@compatibility(is_backward_compatible=True) +def map_arg(a: ArgumentT, fn: Callable[[Node], Argument]) -> ArgumentT: + """ + Apply fn recursively to each Node appearing in arg. + + arg may be a list, tuple, slice, or dict with string keys: the return value will + have the same type and structure. + """ + assert callable(fn), "torch.fx.map_arg(a, fn): fn must be a callable" + return _fx_map_arg(a, fn) + + +@compatibility(is_backward_compatible=True) +def map_aggregate(a: ArgumentT, fn: Callable[[Argument], Argument]) -> ArgumentT: + """ + Apply fn recursively to each object appearing in arg. + + arg may be a list, tuple, slice, or dict with string keys: the return value will + have the same type and structure. + """ + return _fx_map_aggregate(a, fn) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/operator_schemas.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/operator_schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..d655814d20a2659d74126151287cad811b7af230 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/operator_schemas.py @@ -0,0 +1,570 @@ +# mypy: allow-untyped-defs +import enum +import inspect +import numbers +import types +import typing +import warnings +from collections.abc import Callable +from typing import Any, cast, NamedTuple, Optional, TYPE_CHECKING + +import torch +from torch._jit_internal import boolean_dispatched +from torch._ops import OpOverload, OpOverloadPacket + +from ._compatibility import compatibility + + +if TYPE_CHECKING: + from .node import Argument + +__all__ = [ + "ArgsKwargsPair", + "check_for_mutable_operation", + "get_signature_for_torch_op", + "create_type_hint", + "type_matches", + "normalize_function", + "normalize_module", +] + + +@compatibility(is_backward_compatible=False) +class ArgsKwargsPair(NamedTuple): + """ + Simple named tuple for wrapping args/kwargs pairs. + """ + + args: tuple[Any, ...] + kwargs: dict[str, Any] + + +_manual_overrides: dict[Callable, list[inspect.Signature]] = {} + + +def _nonzero_schemas(): + signatures = [] + + def nonzero(self): + pass + + signatures.append(inspect.signature(nonzero)) + + def nonzero(self, *, as_tuple: bool): # type: ignore[no-redef] + pass + + signatures.append(inspect.signature(nonzero)) + + return signatures + + +_manual_overrides[torch.nonzero] = _nonzero_schemas() + + +class _FakeGlobalNamespace: + def __getattr__(self, name): + if name == "torch": + return torch + raise RuntimeError("Expected a torch namespace lookup") + + +_type_eval_globals = { + "Tensor": torch.Tensor, + "Device": torch.device, + "Layout": torch.layout, + "number": numbers.Number, + "Future": torch.jit.Future, + "AnyEnumType": enum.Enum, + "QScheme": torch.qscheme, + "__torch__": _FakeGlobalNamespace(), + "NoneType": type(None), + "Storage": torch.UntypedStorage, + "t": typing.TypeVar("t"), + "PyObject": Any, +} +for k in dir(typing): + _type_eval_globals[k] = getattr(typing, k) + + +def _torchscript_type_to_python_type(ts_type: "torch._C.JitType") -> Any: + """ + Convert a TorchScript type to a Python type (including subtypes) via + eval'ing the annotation_str. _type_eval_globals sets up expressions + like "List" and "Future" to map to actual types (typing.List and jit.Future) + """ + return eval(ts_type.annotation_str, _type_eval_globals) + + +def _torchscript_schema_to_signature_impl( + ts_schema: torch._C.FunctionSchema, +) -> inspect.Signature: + from inspect import Parameter + + parameters: list[Parameter] = [] + for arg in ts_schema.arguments: + arg_type = _torchscript_type_to_python_type(arg.type) + default = arg.default_value if arg.has_default_value() else Parameter.empty + # TODO: Figure out if this is safe. It seems like when generating the type signatures for + # PythonArgParser, we emit signatures with `input` instead of `self` as the first tensor + # argument name. Downstream, if someone converts that positional argument to a keyword + # argument, the name mismatch will break things, so here we're going to normalize the + # name to "input" + name = arg.name if arg.name != "self" else "input" + kind = ( + Parameter.KEYWORD_ONLY + if arg.kwarg_only + else Parameter.POSITIONAL_OR_KEYWORD + ) + # "from" is a keyword therefore it must be a POSITIONAL_ONLY argument + if name == "from": + assert kind == Parameter.POSITIONAL_OR_KEYWORD + # ParameterKind type is internal implementation detail to inspec package + # which makes it hard to do type annotation + kind = Parameter.POSITIONAL_ONLY # type: ignore[assignment] + # This renders all previous arguments to positional only + + for idx, p in enumerate(parameters): + assert p.kind == Parameter.POSITIONAL_OR_KEYWORD + parameters[idx] = Parameter( + name=p.name, + kind=Parameter.POSITIONAL_ONLY, + default=p.default, + annotation=p.annotation, + ) + + parameters.append( + Parameter(name=name, kind=kind, default=default, annotation=arg_type) + ) + return_types = [ + _torchscript_type_to_python_type(ret.type) for ret in ts_schema.returns + ] + if len(return_types) == 0: + return_type = None + elif len(return_types) == 1: + return_type = return_types[0] + else: + return_type = tuple(return_types) + + return inspect.Signature(parameters, return_annotation=return_type) + + +_SCHEMA_TO_SIGNATURE_CACHE: dict[tuple[str, str], inspect.Signature] = {} + + +def _torchscript_schema_to_signature( + ts_schema: torch._C.FunctionSchema, +) -> inspect.Signature: + # Cached as it's called in the hot path of FakeTensor dispatch + cache_key = ts_schema.name, ts_schema.overload_name + cache_val = _SCHEMA_TO_SIGNATURE_CACHE.get(cache_key) + if cache_val is not None: + return cache_val + + res = _torchscript_schema_to_signature_impl(ts_schema) + _SCHEMA_TO_SIGNATURE_CACHE[cache_key] = res + return res + + +@compatibility(is_backward_compatible=False) +def check_for_mutable_operation( + target: Callable, args: tuple["Argument", ...], kwargs: dict[str, "Argument"] +): + signatures, schemas = get_signature_for_torch_op(target, return_schemas=True) + + if signatures and schemas: + matched_schemas = [] + + # Iterate through all of the schema until we find one that matches + # If one matches, populate `new_args_and_kwargs` with the new args/kwargs + # values. If none matches, `new_args_and_kwargs` will be None + for candidate_signature, schema in zip(signatures, schemas): + try: + candidate_signature.bind(*args, **kwargs) + matched_schemas.append((candidate_signature, schema)) + except TypeError: + continue + + def throw_if_mutable(schema): + if schema.is_mutable: + raise RuntimeError( + f"Tried to trace mutable operation {schema}. FX only supports functional " + f"code, so operations that mutate operands in-place (e.g. via `out` arguments) " + f"are not supported" + ) + + if len(matched_schemas) == 0: + # Did not match any schema. Cannot check for mutation + pass + elif len(matched_schemas) == 1: + # Matched exactly one schema, unambiguous + _, schema_to_check = matched_schemas[0] + throw_if_mutable(schema_to_check) + else: + # Ambiguous schema match. Since mutability checking is best effort, + # do nothing. + pass + + +@compatibility(is_backward_compatible=False) +def get_signature_for_torch_op(op: Callable, return_schemas: bool = False): + """ + Given an operator on the `torch` namespace, return a list of `inspect.Signature` + objects corresponding to the overloads of that op.. May return `None` if a signature + could not be retrieved. + + Args: + op (Callable): An operator on the `torch` namespace to look up a signature for + + Returns: + Optional[List[inspect.Signature]]: A list of signatures for the overloads of this + operator, or None if the operator signatures could not be retrieved. If + return_schemas=True, returns a tuple containing the optional Python signatures + and the optional TorchScript Function signature + """ + if isinstance(op, OpOverload): + schemas = [op._schema] + elif isinstance(op, OpOverloadPacket): + schemas = [getattr(op, overload)._schema for overload in op.overloads()] + else: + override = _manual_overrides.get(op) + if override: + return (override, None) if return_schemas else None + + aten_fn = torch.jit._builtins._find_builtin(op) + + if aten_fn is None: + return (None, None) if return_schemas else None + schemas = torch._C._jit_get_schemas_for_operator(aten_fn) + + signatures = [_torchscript_schema_to_signature(schema) for schema in schemas] + return (signatures, schemas) if return_schemas else signatures + + +@compatibility(is_backward_compatible=False) +def create_type_hint(x): + """ + Produces a type hint for the given argument. + + The :func:`create_type_hint` looks for a type hint compatible with the input argument `x`. + + If `x` is a `list` or `tuple`, it looks for an object in the list whose type is a superclass + of the rest, and uses that as `base_type` for the `List` or `Tuple` to be returned. + If no such object is found, it defaults to `List[Any]`. + + If `x` is neither a `list` nor a `tuple`, it returns `x`. + """ + try: + if isinstance(x, (list, tuple)): + # todo(chilli): Figure out the right way for mypy to handle this + if isinstance(x, list): + + def ret_type(x): + return list[x] # type: ignore[valid-type] + + else: + + def ret_type(x): + return tuple[x, ...] # type: ignore[valid-type] + + if len(x) == 0: + return ret_type(Any) + base_type = x[0] + for t in x: + if issubclass(t, base_type): + continue + elif issubclass(base_type, t): + base_type = t + else: + return ret_type(Any) + return ret_type(base_type) + except Exception: + # We tried to create a type hint for list but failed. + warnings.warn( + f"We were not able to successfully create type hint from the type {x}" + ) + return x + + +@compatibility(is_backward_compatible=False) +def type_matches(signature_type: Any, argument_type: Any): + sig_origin_type = getattr(signature_type, "__origin__", signature_type) + + if signature_type is argument_type: + return True + + # Union types in signature. Given type needs to match one of the + # contained types in the Union + if sig_origin_type is typing.Union and signature_type != argument_type: + sig_contained = signature_type.__args__ + return any(type_matches(c, argument_type) for c in sig_contained) + + if getattr(signature_type, "__origin__", None) is list: + sig_el_type = signature_type.__args__[0] + + # int can be promoted to list[int] + if argument_type is int and sig_el_type is int: + return True + + if not inspect.isclass(sig_el_type): + warnings.warn( + f"Does not support nested parametric types, got {signature_type}. Please file a bug." + ) + return False + if getattr(argument_type, "__origin__", None) is list: + return issubclass(argument_type.__args__[0], sig_el_type) + + def is_homogeneous_tuple(t): + if getattr(t, "__origin__", None) is not tuple: + return False + contained = t.__args__ + if t.__args__ == ((),): # Tuple[()].__args__ == ((),) for some reason + return True + return all((c is Ellipsis) or issubclass(c, sig_el_type) for c in contained) + + # Tuple[T] is accepted for List[T] parameters + return is_homogeneous_tuple(argument_type) + + # Dtype is an int in schemas + if signature_type is int and argument_type is torch.dtype: + return True + + if signature_type is numbers.Number and argument_type in {int, float}: + return True + if inspect.isclass(argument_type) and inspect.isclass(signature_type): + return issubclass(argument_type, signature_type) + + return False + + +@compatibility(is_backward_compatible=False) +def normalize_function( + target: Callable, + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, + arg_types: Optional[tuple[Any]] = None, + kwarg_types: Optional[dict[str, Any]] = None, + normalize_to_only_use_kwargs: bool = False, +) -> Optional[ArgsKwargsPair]: + """ + Returns normalized arguments to PyTorch functions. This means that + `args/kwargs` will be matched up to the functional's + signature and return exclusively kwargs in positional order if + `normalize_to_only_use_kwargs` is True. + Also populates default values. Does not support positional-only + parameters or varargs parameters (*args, **kwargs). Does not support modules. + + May require `arg_types` and `kwarg_types` in order to disambiguate overloads. + + Args: + target (Callable): Function that we are normalizing + args (Tuple[Any]): Tuple of args to the function + kwargs (Optional[Dict[str, Any]]): Dict of kwargs to the function + arg_types (Optional[Tuple[Any]]): Tuple of arg types for the args + kwarg_types (Optional[Dict[str, Any]]): Dict of arg types for the kwargs + normalize_to_only_use_kwargs (bool): Whether to normalize to only use kwargs. + + Returns: + + Returns normalized_args_and_kwargs, or `None` if not successful. + """ + if kwargs is None: + kwargs = {} + new_args_and_kwargs = None + if ( + not isinstance(target, types.BuiltinFunctionType) + and not (isinstance(target, (OpOverloadPacket, OpOverload))) + and hasattr(target, "_op") + ): + # ExecuTorch's EdgeOpOverload are a wrapper around PyTorch's OpOverload, + # so we can unwrap it here to get its schema + # Can't import EdgeOpOverload directly because of a circular dependency, + # so checking for "_op" existing is the next best thing. + target = target._op + + # Repeat the condition after checking for the inner _op field. + if not isinstance(target, types.BuiltinFunctionType) and not ( + isinstance(target, (OpOverloadPacket, OpOverload)) + ): + target_for_analysis = target + if target in boolean_dispatched: + # HACK: `boolean_dispatch` as used in `torch.nn.functional` makes it so that we have + # a 2-way dispatch based on a boolean value. Here we check that the `true` and `false` + # branches of the dispatch have exactly the same signature. If they do, use the `true` + # branch signature for analysis. Otherwise, leave this un-normalized + assert not isinstance(target, str) + dispatched = boolean_dispatched[target] + if_true, if_false = dispatched["if_true"], dispatched["if_false"] + if ( + inspect.signature(if_true).parameters + != inspect.signature(if_false).parameters + ): + return None + target_for_analysis = if_true + + assert callable(target_for_analysis) + sig = inspect.signature(inspect.unwrap(target_for_analysis)) + new_args_and_kwargs = _args_kwargs_to_normalized_args_kwargs( + sig, args, kwargs, normalize_to_only_use_kwargs + ) + else: + assert callable(target) + torch_op_schemas = get_signature_for_torch_op(target) + matched_schemas = [] + if torch_op_schemas: + # Iterate through all of the schema until we find one that matches + # If one matches, populate `new_args_and_kwargs` with the new args/kwargs + # values. If none matches, `new_args_and_kwargs` will be None + for candidate_signature in torch_op_schemas: + try: + candidate_signature.bind(*args, **kwargs) + matched_schemas.append(candidate_signature) + except TypeError: + continue + + if len(matched_schemas) == 0: + # Did not match any schema. Cannot normalize + pass + elif len(matched_schemas) == 1: + # Matched exactly one schema, unambiguous + new_args_and_kwargs = _args_kwargs_to_normalized_args_kwargs( + matched_schemas[0], args, kwargs, normalize_to_only_use_kwargs + ) + else: + if arg_types is not None or kwarg_types is not None: + arg_types = arg_types if arg_types else cast(tuple[Any], ()) + kwarg_types = kwarg_types if kwarg_types else {} + for candidate_signature in torch_op_schemas: + sig_matches = True + try: + bound_types = candidate_signature.bind( + *arg_types, **kwarg_types + ) + for arg_name, arg_type in bound_types.arguments.items(): + param = candidate_signature.parameters[arg_name] + sig_matches = sig_matches and type_matches( + param.annotation, arg_type + ) + except TypeError: + sig_matches = False + if sig_matches: + new_args_and_kwargs = ( + _args_kwargs_to_normalized_args_kwargs( + candidate_signature, + args, + kwargs, + normalize_to_only_use_kwargs, + ) + ) + break + else: + # Matched more than one schema. In this situation, the caller must provide the types of + # the arguments of the overload they expect. + schema_printouts = "\n".join( + str(schema) for schema in matched_schemas + ) + raise RuntimeError( + f"Tried to normalize arguments to {torch.typename(target)} but " + f"the schema match was ambiguous! Please provide argument types to " + f"the normalize_arguments() call. Available schemas:\n{schema_printouts}" + ) + + return new_args_and_kwargs + + +@compatibility(is_backward_compatible=False) +def normalize_module( + root: torch.nn.Module, + target: str, + args: tuple[Any], + kwargs: Optional[dict[str, Any]] = None, + normalize_to_only_use_kwargs: bool = False, +) -> Optional[ArgsKwargsPair]: + """ + Returns normalized arguments to PyTorch modules. This means that + `args/kwargs` will be matched up to the functional's + signature and return exclusively kwargs in positional order if + `normalize_to_only_use_kwargs` is True. + Also populates default values. Does not support positional-only + parameters or varargs parameters (*args, **kwargs). + + Args: + root (nn.Module): root module upon which we query modules + target (Callable): Function that we are normalizing + args (Tuple[Any]): Tuple of args to the function + kwargs (Optional[Dict[str, Any]]): Dict of kwargs to the function + normalize_to_only_use_kwargs (bool): Whether to normalize to only use kwargs. + + Returns: + + Returns normalized_args_and_kwargs, or `None` if not successful. + """ + try: + submod = root.get_submodule(target) + except AttributeError as e: + raise RuntimeError( + f"Tried to normalize node with target {target} but root did not " + f"have that target!" + ) from e + if hasattr(submod.__class__, "__name__"): + classname = submod.__class__.__name__ + if getattr(torch.nn, classname, None) == submod.__class__: + sig = inspect.signature(inspect.unwrap(submod.forward)) + if kwargs is None: + kwargs = {} + new_args_and_kwargs = _args_kwargs_to_normalized_args_kwargs( + sig, args, kwargs, normalize_to_only_use_kwargs + ) + return new_args_and_kwargs + return None + + +def _args_kwargs_to_normalized_args_kwargs( + sig: inspect.Signature, + args: tuple[Any, ...], + kwargs: dict[str, Any], + normalize_to_only_use_kwargs: bool, +) -> Optional[ArgsKwargsPair]: + """ + Given a call target, args, and kwargs, return the arguments normalized into + an ArgsKwargsPair, or None if the type signature is not supported by + this normalization. + + Args: + + sig (inspect.Signature): Signature object for the target + args (Tuple): Arguments that appear at the callsite for `target` + kwargs (Dict): Keyword arguments that appear at the callsite for `target` + normalize_to_only_use_kwargs (bool): Whether to normalize to only use kwargs. + + Returns: + + Optional[ArgsKwargsPair]: Normalized args and kwargs for `target`, or `None` if + this target is not supported. + """ + + # Don't currently support positional-only + # or varargs (*args, **kwargs) signatures + supported_parameter_types = { + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + } + if any(p.kind not in supported_parameter_types for p in sig.parameters.values()): + # Add an exception for one signature, which is common for random/uniform, i.e.: + # Tensor(a!) self, float from=0, float to=1, *, Generator? generator=None + # `from` is Python keyword and as such functions with that signature should have + # positional-only args, but at the same time they could be dispatched as kwargs + if list(sig.parameters.keys()) != ["input", "from", "to", "generator"]: + return None + + bound_args = sig.bind(*args, **kwargs) + bound_args.apply_defaults() + + new_kwargs: dict[str, Any] = {} + new_args: list[Any] = [] + for i, param in enumerate(sig.parameters): + if not normalize_to_only_use_kwargs and i < len(args): + new_args.append(bound_args.arguments[param]) + else: + new_kwargs[param] = bound_args.arguments[param] + + return ArgsKwargsPair(tuple(new_args), new_kwargs) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/proxy.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..b27c2c705de1127ad6ae47ecace96a2d5fdf792e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/proxy.py @@ -0,0 +1,850 @@ +# mypy: ignore-errors + +import collections +import copy +import dis +import enum +import inspect +import logging +import operator +import sys +import traceback +from collections import OrderedDict +from collections.abc import Callable, Iterator +from dataclasses import fields, is_dataclass +from typing import Any, Optional + +import torch +import torch.fx.traceback as fx_traceback +from torch._C import _fx_map_aggregate as map_aggregate, _fx_map_arg as map_arg +from torch._library.opaque_object import is_opaque_value_type +from torch._logging import getArtifactLogger +from torch.utils._traceback import CapturedTraceback + +from ._compatibility import compatibility +from .graph import Graph, magic_methods, reflectable_magic_methods +from .immutable_collections import immutable_dict, immutable_list +from .node import Argument, base_types, Node, Target +from .operator_schemas import check_for_mutable_operation + + +__all__ = [ + "TracerBase", + "GraphAppendingTracer", + "TraceError", + "Proxy", + "MetaProxy", + "Attribute", + "ParameterProxy", + "Scope", + "ScopeContextManager", +] + + +log = logging.getLogger(__name__) +annotation_log = getArtifactLogger(__name__, "annotation") + + +@compatibility(is_backward_compatible=False) +class Scope: + """Scope object that records the module path and the module type + of a module. Scope is used to track the information of the module + that contains a Node in a Graph of GraphModule. For example:: + + class Sub(torch.nn.Module): + def forward(self, x): + # This will be a call_method Node in GraphModule, + # scope for this would be (module_path="sub", module_type=Sub) + return x.transpose(1, 2) + + + class M(torch.nn.Module): + def __init__(self) -> None: + self.sub = Sub() + + def forward(self, x): + # This will be a call_method Node as well, + # scope for this would be (module_path="", None) + x = x.transpose(1, 2) + x = self.sub(x) + return x + + """ + + def __init__(self, module_path: str, module_type: Any): + super().__init__() + self.module_path = module_path + self.module_type = module_type + + +@compatibility(is_backward_compatible=False) +class ScopeContextManager: + """A context manager to track the Scope of Node during symbolic tracing. + When entering a forward function of a Module, we'll update the scope information of + the current module, and when we exit, we'll restore the previous scope information. + """ + + def __init__( + self, + scope: Scope, + current_scope: Scope, + ): + super().__init__() + # Keep a copy of prev scope to restore on exit + self._prev_scope = copy.copy(scope) + # Update scope to current scope + scope.module_path = current_scope.module_path + scope.module_type = current_scope.module_type + # Save a reference so we can restore it + self._scope = scope + + def __enter__(self): + return self._scope + + def __exit__(self, *args): + self._scope.module_path = self._prev_scope.module_path + self._scope.module_type = self._prev_scope.module_type + return + + +_COPY_META_FIELDS = [ + "nn_module_stack", + "torch_fn", + "source_fn_stack", + "original_aten", + "recompute", + "ac_graph_id", + "has_backward_hook", + "from_node", + "quantization_tag", # TODO deprecated + "_numeric_debug_handle", # TODO deprecated + "custom", + "partitioner_tag", +] + + +@compatibility(is_backward_compatible=True) +class TracerBase: + graph: Graph + record_stack_traces: bool = False + # When record_stack_traces is True, only reocrd stack traces + # with forward function names. + # This helps when we want stack trace back to model code + _record_forward_stack_traces_only: bool = False + # Feature flag for mutable schema checking + # Enableby default in 1.12 + check_mutable_operations: bool = False + # Feature flag for assert tracing + trace_asserts: bool = False + # Feature flag for proxying accesses to buffer values + proxy_buffer_attributes: bool = False + + # Name of the function to be traced. It will only be used when + # ``root`` is an instance of ``nn.Module`` + traced_func_name: str = "forward" + + # Maps the containing module's name to the operator name + scope: Scope + + # Records the module call stack + module_stack: OrderedDict[str, tuple[str, Any]] + + # Mapping of node name to module scope + node_name_to_scope: dict[str, tuple[str, type]] + + @compatibility(is_backward_compatible=True) + def create_node( + self, + kind: str, + target: Target, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + name: Optional[str] = None, + type_expr: Optional[Any] = None, + ) -> Node: + """ + Inserts a graph node given target, args, kwargs, and name. + + This method can be overridden to do extra checking, validation, or + modification of values used in node creation. For example, one might + want to disallow in-place operations from being recorded. + """ + + if kind == "call_function" and self.check_mutable_operations: + check_for_mutable_operation(target, args, kwargs) + + node = self.graph.create_node(kind, target, args, kwargs, name, type_expr) + # TODO node_name_to_scope will be depreciated in favor of + # node.meta['nn_module_stack'] + self.node_name_to_scope[node.name] = ( + self.scope.module_path, + self.scope.module_type, + ) + + # Optionally set stack trace on the created Node for debugging purposes + if fx_traceback.has_preserved_node_meta(): + current_meta: dict[str, Any] = fx_traceback.get_current_meta() + + stack_trace = current_meta.get("stack_trace") + if stack_trace: + node.stack_trace = stack_trace + + if fx_traceback.GRADIENT_ACC_SPECIAL_STACK in stack_trace: + node.meta["is_gradient_acc"] = True + + # Explicitly set the stack_trace, nn_module_stack and source_fn on the node.meta + # If other meta fields are needed, they can be added here + for field in _COPY_META_FIELDS: + if field in current_meta: + node.meta[field] = copy.copy(current_meta[field]) + + # Here we decrement to account for the sequence_nr having + # just been incremented while tracing this lowered aten op. + new_seq_nr = torch.autograd._get_sequence_nr() - 1 + # The sequence_nr increments every time a new autograd Node + # is created. During the FWD pass we store the sequence_nr + # corresponding to the last autograd Node created on this fx + # node's meta. A single aten op can create multiple autograd + # nodes as is the case with in-place foreach ops. During the + # BWD pass we retrieve the sequence_nr stored on the current + # executing autograd Node. See NOTE [ Sequence Number ]. + if current_meta.get("in_grad_fn", 0) > 0: + annotation_log.debug("seq_nr from current_meta") + new_seq_nr = current_meta["grad_fn_seq_nr"][-1] + + # See Note [Functionalization View Replay Annotation] + # Overriding some node meta with the original node meta of the + # regenerated node. + replay_node: Node = fx_traceback.get_current_replay_node() + if replay_node is not None: + node.meta["is_functional_regenerated"] = True + if "seq_nr" in replay_node.meta: + annotation_log.debug("seq_nr from replay_node") + new_seq_nr = replay_node.meta["seq_nr"] + if "custom" in replay_node.meta: + node.meta["custom"] = replay_node.meta.get("custom") + if "stack_trace" in replay_node.meta: + node.stack_trace = replay_node.meta.get("stack_trace") + + annotation_log.debug("Assigning new_seq_nr %s to %s", new_seq_nr, node.name) + node.meta["seq_nr"] = new_seq_nr + + elif self.module_stack: + node.meta["nn_module_stack"] = copy.copy(self.module_stack) + + if self.record_stack_traces and not node.stack_trace: + user_stack_summary = CapturedTraceback.extract().summary() + if user_stack_summary: + user_stack_summary = self._filter_traceback_frames(user_stack_summary) + if user_stack_summary: + node.stack_trace = "".join(user_stack_summary.format()).strip() + + log.debug("create_node %s", node) + return node + + def _filter_traceback_frames( + self, user_stack_summary: traceback.StackSummary + ) -> traceback.StackSummary: + # This method can be overridden to customize the frame filtering logic + # for the recorded stack trace + user_frames = [] + if self._record_forward_stack_traces_only: + user_frames = [ + frame + for frame in user_stack_summary + if ( + frame.name == "forward" + or frame.filename.endswith("torch/__init__.py") + ) + ] + else: + first_forward = -1 + for i, frame in enumerate(user_stack_summary): + if frame.name == "forward": + user_frames = user_stack_summary[i:] + first_forward = i + break + + # Not having a "forward" call in the stacktrace implies the + # stacktrace will probably be irrelevant + if first_forward == -1: + user_frames = [] + + from torch.fx.experimental.symbolic_shapes import uninteresting_files + + user_frames = [ + frame + for frame in user_frames + if frame.filename not in uninteresting_files() + ] + + return traceback.StackSummary.from_list(user_frames) + + @compatibility(is_backward_compatible=True) + def proxy(self, node: Node) -> "Proxy": + return Proxy(node, self) + + @compatibility(is_backward_compatible=True) + def create_proxy( + self, + kind: str, + target: Target, + args: tuple[Any, ...], + kwargs: dict[str, Any], + name: Optional[str] = None, + type_expr: Optional[Any] = None, + # fix noqa when updating bc tests + proxy_factory_fn: Callable[[Node], "Proxy"] = None, # noqa: RUF013 + ): + """ + Create a Node from the given arguments, then return the Node + wrapped in a Proxy object. + + If kind = 'placeholder', then we're creating a Node that + represents the parameter of a function. If we need to encode + a default parameter, we use the ``args`` tuple. ``args`` is + otherwise empty for ``placeholder`` Nodes. + """ + + args_ = self.create_arg(args) + kwargs_ = self.create_arg(kwargs) + assert isinstance(args_, tuple) + assert isinstance(kwargs_, dict) + + node = self.create_node(kind, target, args_, kwargs_, name, type_expr) + + if not proxy_factory_fn: + proxy = self.proxy(node) + else: + proxy = proxy_factory_fn(node) + + return proxy + + def _find_user_frame(self): + """ + Find the Python stack frame executing the user code during + symbolic tracing. + """ + # We have to do a little dance here. Basically, walk up the callstack and + # record the first frame not in the pytorch source. This is the frame executing + # the user code during tracing. + frame = inspect.currentframe() + + pt_files = [ + "torch/fx/proxy.py", + "torch/fx/_symbolic_trace.py", + "torch/fx/experimental/proxy_tensor.py", + "torch/_ops.py", + "torch/_tensor.py", + "torch/utils/_python_dispatch.py", + "torch/_prims_common/wrappers.py", + "torch/_refs/__init__.py", + "torch/_refs/nn/functional/__init__.py", + "torch/utils/_stats.py", + ] + while frame: + frame = frame.f_back + if frame and all( + not frame.f_code.co_filename.endswith(file) for file in pt_files + ): + break + + if not frame: + return None + + return frame + + @compatibility(is_backward_compatible=True) + def create_arg(self, a: Any) -> Argument: + """ + A method that lowers the objects seen as arguments during symbolic evaluation + into Argument types that can be stored in IR. + + Can be override to support more trace-specific types. + """ + # IMPORTANT: Are you here because you are trying to proxy a new type into + # the graph? Please Please Please contact someone on the PyTorch Compiler team; + # the considerations are subtle. + # + # 1) When you add a new type, all of the downstream consumers and pass writers + # need to handle the new type. torch.fx is intended to be easy to write + # passes for, so we will push back against new types. + # 2) In torch.compile's IR, there are only specific operations that go + # into the graph. In particular, Tensor operations should go into the graph, + # but non-Tensor operations shouldn't. What that means is that constructors + # for new types *SHOULD NOT* become nodes in the FX graph. + handler = _create_arg_bypass.get(type(a)) + if handler is not None: + # this is just a performance optimization and can be removed if needed + # for common types, we have a fast path to avoid isinstance() overhead + # this doesn't remove the checks below since we need to handle subclasses + return handler(self, a) + + if isinstance(a, Proxy): + return a.node # most common arg type goes first + elif hasattr(a, "__fx_create_arg__"): + return a.__fx_create_arg__(self) + # aggregates + elif isinstance(a, tuple): + if hasattr(a, "_fields"): + # NamedTuple constructors don't seem to like getting a generator + # expression as an argument to their constructor, so build this + # intermediate tuple and unpack it into the NamedTuple constructor + args = [self.create_arg(elem) for elem in a] + return type(a)(*args) # type: ignore[arg-type] + return type(a)([self.create_arg(elem) for elem in a]) + elif isinstance(a, list): + return [self.create_arg(elem) for elem in a] + elif isinstance(a, dict): + return _create_arg_dict(self, a) + elif isinstance(a, slice): + return slice( + self.create_arg(a.start), + self.create_arg(a.stop), + self.create_arg(a.step), + ) + + elif isinstance(a, range): + return range( + self.create_arg(a.start), + self.create_arg(a.stop), + self.create_arg(a.step), + ) + + elif isinstance(a, (torch._ops.OpOverload, torch._ops.HigherOrderOperator)): + return a + + elif is_opaque_value_type(type(a)): + return a + + elif is_dataclass(a): + kwargs = { + field.name: self.create_arg(getattr(a, field.name)) + for field in fields(a) + } + return self.create_node("call_function", a.__class__, (), kwargs) + + elif isinstance(a, (*base_types, enum.Enum)) or a is None or a is ...: + return a + + raise NotImplementedError(f"argument of type: {type(a)}") + + @compatibility(is_backward_compatible=True) + def to_bool(self, obj: "Proxy") -> bool: + """Called when a proxy object is being converted to a boolean, such as + when used in control flow. Normally we don't know what to do because + we don't know the value of the proxy, but a custom tracer can attach more + information to the graph node using create_node and can choose to return a value. + """ + raise TraceError( + "symbolically traced variables cannot be used as inputs to control flow" + ) + + @compatibility(is_backward_compatible=True) + def iter(self, obj: "Proxy") -> Iterator: + """Called when a proxy object is being iterated over, such as + when used in control flow. Normally we don't know what to do because + we don't know the value of the proxy, but a custom tracer can attach more + information to the graph node using create_node and can choose to return an iterator. + """ + raise TraceError( + "Proxy object cannot be iterated. This can be " + "attempted when the Proxy is used in a loop or" + " as a *args or **kwargs function argument. " + "See the torch.fx docs on pytorch.org for a " + "more detailed explanation of what types of " + "control flow can be traced, and check out the" + " Proxy docstring for help troubleshooting " + "Proxy iteration errors" + ) + + @compatibility(is_backward_compatible=True) + def keys(self, obj: "Proxy") -> Any: + """Called when a proxy object is has the keys() method called. + This is what happens when ** is called on a proxy. This should return an + iterator it ** is suppose to work in your custom tracer. + """ + return Attribute(obj, "keys")() + + +# used in Proxy object when just appending to the graph while not tracing. +@compatibility(is_backward_compatible=True) +class GraphAppendingTracer(TracerBase): + def __init__(self, graph: Graph): + super().__init__() + self.graph = graph + self.scope = Scope("", None) + self.module_stack = collections.OrderedDict() + self.node_name_to_scope = {} + + +@compatibility(is_backward_compatible=False) +def assert_fn(x): + assert x + + +@compatibility(is_backward_compatible=True) +class TraceError(ValueError): + pass + + +@compatibility(is_backward_compatible=True) +class Proxy: + """ + ``Proxy`` objects are ``Node`` wrappers that flow through the + program during symbolic tracing and record all the operations + (``torch`` function calls, method calls, operators) that they touch + into the growing FX Graph. + + If you're doing graph transforms, you can wrap your own ``Proxy`` + method around a raw ``Node`` so that you can use the overloaded + operators to add additional things to a ``Graph``. + + ``Proxy`` objects cannot be iterated. In other words, the symbolic + tracer will throw an error if a ``Proxy`` is used in a loop or as + an ``*args``/``**kwargs`` function argument. + + There are two main ways around this: + 1. Factor out the untraceable logic into a top-level function and + use ``fx.wrap`` on it. + 2. If the control flow is static (i.e. the loop trip count is + based on some hyperparameter), the code can be kept in its original + position and refactored into something like:: + + for i in range(self.some_hyperparameter): + indexed_item = proxied_value[i] + + For a more detailed description into the Proxy internals, check out + the "Proxy" section in `torch/fx/README.md` + """ + + @compatibility(is_backward_compatible=True) + def __init__(self, node: Node, tracer: "Optional[TracerBase]" = None): + if tracer is None: + # This allows you to create a Proxy object around a raw Node + tracer = GraphAppendingTracer(node.graph) + self.tracer = tracer + self.node = node + + def __repr__(self) -> str: + return f"Proxy({self.node.name})" + + def __getattr__(self, k) -> "Attribute": + # note: not added to the graph yet, if this is a method call + # we peephole optimize to the method invocation + return Attribute(self, k) + + def __getstate__(self) -> dict: + return self.__dict__ + + def __deepcopy__(self, memo) -> dict: + # We have to explicitly override this method, because otherwise deepcopy + # will go to __getattr__(self, "__deepcopy__") and return a + # Attribute(__deepcopy__), and may go into an infinite loop in some cases. + import copy + + new_dict = {} + for k, v in self.__dict__.items(): + try: + new_obj = copy.deepcopy(v, memo) + except Exception: + log.warning( + "Shallow copy %s of Proxy because it cannot be deepcopied. " + "Proxy is created for node %s", + k, + self.node.name, + ) + new_obj = copy.copy(v) + new_dict[k] = new_obj + assert "node" in new_dict + assert "tracer" in new_dict + new_proxy = Proxy(new_dict["node"], new_dict["tracer"]) + for k, v in new_dict.items(): + new_proxy.__dict__[k] = v + return new_proxy + + def __setstate__(self, d): + # This is called when being unpickled/loaded. + self.__dict__ = d + + def __call__(self, *args, **kwargs) -> "Proxy": + return self.tracer.create_proxy( + "call_method", "__call__", (self,) + args, kwargs + ) + + def __iter__(self) -> Iterator["Proxy"]: + frame = inspect.currentframe() + assert frame is not None + calling_frame = frame.f_back + assert calling_frame is not None + inst_list = list(dis.get_instructions(calling_frame.f_code)) + if sys.version_info >= (3, 11): + from bisect import bisect_left + + inst_idx = bisect_left( + inst_list, calling_frame.f_lasti, key=lambda x: x.offset + ) + else: + inst_idx = calling_frame.f_lasti // 2 + inst = inst_list[inst_idx] + if inst.opname == "UNPACK_SEQUENCE": + return (self[i] for i in range(inst.argval)) # type: ignore[index] + + return self.tracer.iter(self) + + def __abs__(self): + return self.tracer.create_proxy("call_function", operator.abs, (self,), {}) + + def __bool__(self) -> bool: + if self.tracer.trace_asserts: + # check if this boolean is used in an assertion, bytecode pattern for assertions + # is pretty stable for Python 3.7--3.9 + frame = inspect.currentframe() + assert frame is not None + calling_frame = frame.f_back + assert calling_frame is not None + insts = list(dis.get_instructions(calling_frame.f_code)) + if sys.version_info >= (3, 11): + from bisect import bisect_left + + cur = bisect_left(insts, calling_frame.f_lasti, key=lambda x: x.offset) + else: + cur = calling_frame.f_lasti // 2 + inst = insts[cur] + + if inst.opname == "POP_JUMP_IF_TRUE": + first = insts[cur + 1] + assert inst.arg is not None + last = insts[inst.arg // 2 - 1] + starts_with_assert = ( + first.opname == "LOAD_GLOBAL" + and first.argval == "AssertionError" + or first.opname == "LOAD_ASSERTION_ERROR" + ) + if starts_with_assert and last.opname == "RAISE_VARARGS": + self.tracer.create_proxy("call_function", assert_fn, (self,), {}) + return True + + return self.tracer.to_bool(self) + + @compatibility(is_backward_compatible=True) + def keys(self): + return self.tracer.keys(self) + + def __len__(self): + raise RuntimeError( + "'len' is not supported in symbolic tracing by default. If you want " + "this call to be recorded, please call torch.fx.wrap('len') at " + "module scope" + ) + + @classmethod + def __torch_function__(cls, orig_method, types, args=None, kwargs=None): + args = args if args else () + kwargs = kwargs if kwargs else {} + + tracers: dict[Any, None] = {} + + def find_tracer(a): + if isinstance(a, cls): + tracers[a.tracer] = None + + map_aggregate(args, find_tracer) + map_aggregate(kwargs, find_tracer) + + if len(tracers) > 1: + raise RuntimeError( + f"Found multiple different tracers {list(tracers.keys())} while " + f"trying to trace operations {orig_method}" + ) + tracer = next(iter(tracers.keys())) + + if isinstance(orig_method, torch._C.ScriptMethod): + args = (orig_method.owner,) + args + return tracer.create_proxy("call_method", orig_method.name, args, kwargs) + if torch.overrides.is_tensor_method_or_property(orig_method): + return tracer.create_proxy( + "call_method", orig_method.__name__, args, kwargs + ) + else: + if isinstance(orig_method, torch._ops.HigherOrderOperator): + # TODO: Define how to symbolically trace HigherOrderOperators + raise RuntimeError("Unable to symbolically trace HigherOrderOperators") + return tracer.create_proxy( + "call_function", + orig_method, + args, + kwargs, + name=tracer.graph._target_to_str(orig_method.__name__), + ) + + +@compatibility(is_backward_compatible=False) +class MetaProxy(Proxy): + """ + A Proxy subclass that propagates metadata (meta['val']) during graph tracing. + """ + + def __init__( + self, node: Node, tracer: "Optional[TracerBase]" = None, fake_mode=None + ): + super().__init__(node, tracer) + self.fake_mode = fake_mode + + def __repr__(self) -> str: + return f"MetaProxy({self.node.name})" + + @classmethod + def __torch_function__(cls, orig_method, types, args=None, kwargs=None): + args = args if args else () + kwargs = kwargs if kwargs else {} + + meta_proxy = None + for arg in args: + if isinstance(arg, MetaProxy): + meta_proxy = arg + break + + assert meta_proxy is not None, ( + "No MetaProxy found in arguments, but one is expected." + ) + + proxy = super().__torch_function__(orig_method, types, args, kwargs) + with meta_proxy.fake_mode: + proxy.node.meta["val"] = orig_method( + *[a.node.meta["val"] if isinstance(a, Proxy) else a for a in args], + **kwargs, + ) + return MetaProxy(proxy.node, proxy.tracer, meta_proxy.fake_mode) + + +@compatibility(is_backward_compatible=True) +class Attribute(Proxy): + @compatibility(is_backward_compatible=True) + def __init__(self, root: Proxy, attr: str): + self.root = root + self.attr = attr + self.tracer = root.tracer + self._node: Optional[Node] = None + + @property + def node(self): + # the node for attributes is added lazily, since most will just be method calls + # which do not rely on the getitem call + if self._node is None: + self._node = self.tracer.create_proxy( + "call_function", getattr, (self.root, self.attr), {} + ).node + return self._node + + def __call__(self, *args, **kwargs): + return self.tracer.create_proxy( + "call_method", self.attr, (self.root,) + args, kwargs + ) + + +@compatibility(is_backward_compatible=False) +class ParameterProxy(Proxy): + """ + A special proxy which lets "shape", "size", "dim", and a few other + attribute accesses pass through to the underlying module parameter object, + so that conditional tests on these attributes will not throw exception during tracing + """ + + def __init__(self, tracer: TracerBase, node: Node, name, param): + super().__init__(node, tracer) + assert isinstance(param, torch.nn.Parameter) + self.param = param + self.name = name + + def __repr__(self) -> str: + return f"ParameterProxy({self.name})" + + @property + def shape(self): + return self.param.shape + + def size(self): + return self.param.size() + + def dim(self): + return self.param.dim() + + @property + def ndim(self): + return self.param.ndim + + def numel(self): + return self.param.numel() + + def nelement(self): + return self.param.nelement() + + +for method in magic_methods: + + def _scope(method): + def impl(*args, **kwargs): + tracer = args[0].tracer + target = getattr(operator, method) + return tracer.create_proxy("call_function", target, args, kwargs) + + impl.__name__ = method + as_magic = f"__{method.strip('_')}__" + setattr(Proxy, as_magic, impl) + + _scope(method) + + +def _define_reflectable(orig_method_name): + method_name = f"__r{orig_method_name.strip('_')}__" + + def impl(self, rhs): + target = getattr(operator, orig_method_name) + return self.tracer.create_proxy("call_function", target, (rhs, self), {}) + + impl.__name__ = method_name + impl.__qualname__ = method_name + setattr(Proxy, method_name, impl) + + +for orig_method_name in reflectable_magic_methods: + _define_reflectable(orig_method_name) + + +def _no_nodes_error(arg): + raise RuntimeError( + "Keys for dictionaries used as an argument cannot contain a " + f"Node. Got key: {arg}" + ) + + +def _create_arg_dict(self, a): + r = {} + for k, v in a.items(): + if not isinstance(k, str): + # Check for invalid dict keys. We do not want a Proxy to appear + # anywhere within the key. Since keys can be collection types, + # we iterate through the key with map_arg + k = self.create_arg(k) + map_arg(k, _no_nodes_error) + r[k] = self.create_arg(v) + return r + + +_create_arg_bypass = { + t: lambda self, a: a + for t in [ + *base_types, + type(None), + type(...), + torch._ops.OpOverload, + torch._ops.HigherOrderOperator, + ] +} +_create_arg_bypass[Proxy] = lambda self, a: a.node +_create_arg_bypass[tuple] = lambda self, a: tuple(self.create_arg(elem) for elem in a) +_create_arg_bypass[list] = lambda self, a: [self.create_arg(elem) for elem in a] +_create_arg_bypass[dict] = _create_arg_dict +_create_arg_bypass[immutable_list] = _create_arg_bypass[list] +_create_arg_bypass[immutable_dict] = _create_arg_bypass[dict] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/subgraph_rewriter.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/subgraph_rewriter.py new file mode 100644 index 0000000000000000000000000000000000000000..6fcc06d0cb302fb0119ba8a554c51bf6d89c198c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/subgraph_rewriter.py @@ -0,0 +1,440 @@ +import copy +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, NamedTuple, Optional, TYPE_CHECKING, Union + +import torch + +from ._compatibility import compatibility +from ._symbolic_trace import symbolic_trace +from .graph import Graph +from .graph_module import GraphModule +from .node import Node + + +if TYPE_CHECKING: + from .passes.utils.matcher_with_name_node_map_utils import InternalMatch + +__all__ = [ + "Match", + "replace_pattern", + "replace_pattern_with_filters", + "ReplacedPatterns", +] + + +@compatibility(is_backward_compatible=True) +class Match(NamedTuple): + # Node from which the match was found + anchor: Node + # Maps nodes in the pattern subgraph to nodes in the larger graph + nodes_map: dict[Node, Node] + + +@compatibility(is_backward_compatible=False) +@dataclass +class ReplacedPatterns: + # Node from which the match was found + anchor: Node + # Maps nodes in the pattern subgraph to nodes in the larger graph + nodes_map: dict[Node, Node] + # List of nodes that were added into the graph + replacements: list[Node] + + +def _replace_attributes(gm: GraphModule, replacement: torch.nn.Module) -> None: + gm.delete_all_unused_submodules() + + if isinstance(replacement, GraphModule): + replacement.graph.lint() + + def try_get_attr(gm: torch.nn.Module, target: str) -> Optional[Any]: + module_path, _, attr_name = target.rpartition(".") + try: + mod: torch.nn.Module = gm.get_submodule(module_path) + except AttributeError: + return None + attr = getattr(mod, attr_name, None) + return attr + + for node in gm.graph.nodes: + if node.op == "call_module" or node.op == "get_attr": + gm_attr = try_get_attr(gm, node.target) + replacement_attr = try_get_attr(replacement, node.target) + + # CASE 1: This target already exists as an attribute in our + # result GraphModule. Whether or not it exists in + # `replacement`, the existing submodule takes precedence. + if gm_attr is not None: + continue + + # CASE 2: The target exists as an attribute in `replacement` + # only, so we need to copy it over. + elif replacement_attr is not None: + new_attr = copy.deepcopy(replacement_attr) + if isinstance(replacement_attr, torch.nn.Module): + gm.add_submodule(node.target, new_attr) + else: + setattr(gm, node.target, new_attr) + + # CASE 3: The target doesn't exist as an attribute in `gm` + # or `replacement` + else: + raise RuntimeError( + 'Attempted to create a "', + node.op, + '" node during subgraph rewriting ' + f"with target {node.target}, but " + "the referenced attribute does not " + "exist in the replacement GraphModule", + ) + + gm.graph.lint() + + +@compatibility(is_backward_compatible=True) +def replace_pattern( + gm: GraphModule, + pattern: Union[Callable, GraphModule], + replacement: Union[Callable, GraphModule], +) -> list[Match]: + """ + Matches all possible non-overlapping sets of operators and their + data dependencies (``pattern``) in the Graph of a GraphModule + (``gm``), then replaces each of these matched subgraphs with another + subgraph (``replacement``). + + Args: + ``gm``: The GraphModule that wraps the Graph to operate on + ``pattern``: The subgraph to match in ``gm`` for replacement + ``replacement``: The subgraph to replace ``pattern`` with + + Returns: + List[Match]: A list of ``Match`` objects representing the places + in the original graph that ``pattern`` was matched to. The list + is empty if there are no matches. ``Match`` is defined as: + + .. code-block:: python + + class Match(NamedTuple): + # Node from which the match was found + anchor: Node + # Maps nodes in the pattern subgraph to nodes in the larger graph + nodes_map: Dict[Node, Node] + + Examples: + + .. code-block:: python + + import torch + from torch.fx import symbolic_trace, subgraph_rewriter + + + class M(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, x, w1, w2): + m1 = torch.cat([w1, w2]).sum() + m2 = torch.cat([w1, w2]).sum() + return x + torch.max(m1) + torch.max(m2) + + + def pattern(w1, w2): + return torch.cat([w1, w2]) + + + def replacement(w1, w2): + return torch.stack([w1, w2]) + + + traced_module = symbolic_trace(M()) + + subgraph_rewriter.replace_pattern(traced_module, pattern, replacement) + + The above code will first match ``pattern`` in the ``forward`` + method of ``traced_module``. Pattern-matching is done based on + use-def relationships, not node names. For example, if you had + ``p = torch.cat([a, b])`` in ``pattern``, you could match + ``m = torch.cat([a, b])`` in the original ``forward`` function, + despite the variable names being different (``p`` vs ``m``). + + The ``return`` statement in ``pattern`` is matched based on its + value only; it may or may not match to the ``return`` statement in + the larger graph. In other words, the pattern doesn't have to extend + to the end of the larger graph. + + When the pattern is matched, it will be removed from the larger + function and replaced by ``replacement``. If there are multiple + matches for ``pattern`` in the larger function, each non-overlapping + match will be replaced. In the case of a match overlap, the first + found match in the set of overlapping matches will be replaced. + ("First" here being defined as the first in a topological ordering + of the Nodes' use-def relationships. In most cases, the first Node + is the parameter that appears directly after ``self``, while the + last Node is whatever the function returns.) + + One important thing to note is that the parameters of the + ``pattern`` Callable must be used in the Callable itself, + and the parameters of the ``replacement`` Callable must match + the pattern. The first rule is why, in the above code block, the + ``forward`` function has parameters ``x, w1, w2``, but the + ``pattern`` function only has parameters ``w1, w2``. ``pattern`` + doesn't use ``x``, so it shouldn't specify ``x`` as a parameter. + As an example of the second rule, consider replacing + + .. code-block:: python + + def pattern(x, y): + return torch.neg(x) + torch.relu(y) + + with + + .. code-block:: python + + def replacement(x, y): + return torch.relu(x) + + In this case, ``replacement`` needs the same number of parameters + as ``pattern`` (both ``x`` and ``y``), even though the parameter + ``y`` isn't used in ``replacement``. + + After calling ``subgraph_rewriter.replace_pattern``, the generated + Python code looks like this: + + .. code-block:: python + + def forward(self, x, w1, w2): + stack_1 = torch.stack([w1, w2]) + sum_1 = stack_1.sum() + stack_2 = torch.stack([w1, w2]) + sum_2 = stack_2.sum() + max_1 = torch.max(sum_1) + add_1 = x + max_1 + max_2 = torch.max(sum_2) + add_2 = add_1 + max_2 + return add_2 + """ + match_and_replacements = _replace_pattern(gm, pattern, replacement) + return [ + Match(anchor=m.anchor, nodes_map=m.nodes_map) for m in match_and_replacements + ] + + +# Experimental API, not backward compatible +@compatibility(is_backward_compatible=False) +def replace_pattern_with_filters( + gm: GraphModule, + pattern: Union[Callable, Graph, GraphModule], + replacement: Union[Callable, Graph, GraphModule, None] = None, + match_filters: Optional[ + list[Callable[["InternalMatch", Graph, Graph], bool]] + ] = None, + ignore_literals: bool = False, + # Placed at the end to avoid breaking backward compatibility + replacement_callback: Optional[ + Callable[["InternalMatch", Graph, Graph], Graph] + ] = None, + node_name_match: str = "", +) -> list[ReplacedPatterns]: + """ + See replace_pattern for documentation. This function is an overload with an additional match_filter argument. + + Args: + ``match_filters``: A list of functions that take in + (match: InternalMatch, original_graph: Graph, pattern_graph: Graph) and return a boolean indicating + whether the match satisfies the condition. + See matcher_utils.py for definition of InternalMatch. + ``replacement_callback``: A function that takes in a match and returns a + Graph to be used as the replacement. This allows you to construct a + replacement graph based on the match. + ``replacement_callback``: Node name to match. If not empty, it will try to match the node name. + """ + + return _replace_pattern( + gm, + pattern, + replacement, + match_filters, + ignore_literals, + replacement_callback, + node_name_match, + ) + + +def _replace_pattern( + gm: GraphModule, + pattern: Union[Callable, Graph, GraphModule], + replacement: Union[Callable, Graph, GraphModule, None] = None, + match_filters: Optional[ + list[Callable[["InternalMatch", Graph, Graph], bool]] + ] = None, + ignore_literals: bool = False, + # Placed at the end to avoid breaking backward compatibility + replacement_callback: Optional[ + Callable[["InternalMatch", Graph, Graph], Graph] + ] = None, + node_name_match: str = "", +) -> list[ReplacedPatterns]: + from torch.fx.passes.utils.matcher_utils import InternalMatch, SubgraphMatcher + + if match_filters is None: + match_filters = [] + + # Get the graphs for `gm`, `pattern`, `replacement` + original_graph: Graph = gm.graph + + if isinstance(pattern, GraphModule): + pattern_graph = pattern.graph + elif isinstance(pattern, Graph): + pattern_graph = pattern + else: + pattern_graph = symbolic_trace(pattern).graph # type: ignore[arg-type] + + matcher = SubgraphMatcher( + pattern_graph, + match_output=False, + match_placeholder=False, + remove_overlapping_matches=True, + ignore_literals=ignore_literals, + ) + _matches: list[InternalMatch] = matcher.match( + original_graph, node_name_match=node_name_match + ) + + # Filter out matches that don't match the filter + _matches = [ + m + for m in _matches + if all( + match_filter(m, original_graph, pattern_graph) + for match_filter in match_filters + ) + ] + + if isinstance(replacement, GraphModule): + common_replacement_graph = replacement.graph + elif isinstance(replacement, Graph): + common_replacement_graph = replacement + elif callable(replacement): + common_replacement_graph = symbolic_trace(replacement).graph + else: + assert replacement_callback is not None, ( + "Must provide either a replacement GraphModule or a replacement callback" + ) + common_replacement_graph = None # type: ignore[assignment] + + # As we progressively replace nodes, we'll need to keep track of how the match results should change + match_changed_node: dict[Node, Node] = {} + + match_and_replacements = [] + for match in _matches: + if replacement_callback is not None: + replacement_graph = replacement_callback( + match, original_graph, pattern_graph + ) + else: + assert common_replacement_graph is not None, ( + "Must provide either a replacement GraphModule or a replacement callback" + ) + replacement_graph = common_replacement_graph + replacement_placeholders = [ + n for n in replacement_graph.nodes if n.op == "placeholder" + ] + + # Build connecting between replacement graph's input and original graph input producer node + + # Initialize `val_map` with mappings from placeholder nodes in + # `replacement` to their corresponding node in `original_graph` + assert len(match.placeholder_nodes) == len(replacement_placeholders) + val_map: dict[Node, Node] = {} + for rn, gn in zip(replacement_placeholders, match.placeholder_nodes): + if isinstance(gn, Node): + val_map[rn] = match_changed_node.get(gn, gn) + if gn != val_map[rn]: + # Update match.placeholder_nodes and match.nodes_map with the node that replaced gn + gn_ind = match.placeholder_nodes.index(gn) + match.placeholder_nodes[gn_ind] = match_changed_node[gn] + map_key = list(match.nodes_map.keys())[ + list(match.nodes_map.values()).index(gn) + ] + match.nodes_map[map_key] = match_changed_node[gn] + else: + val_map[rn] = gn + + # Copy the replacement graph over + user_nodes: set[Node] = set() + for n in match.returning_nodes: + user_nodes.update(n.users) + + first_user_node = None + if len(user_nodes) == 0: + first_user_node = None + elif len(user_nodes) == 1: + first_user_node = next(iter(user_nodes)) + else: + # If there are multiple user nodes, we need to find the first user node + # in the current execution order of the `original_graph` + for n in original_graph.nodes: + if n in user_nodes: + first_user_node = n + break + + first_next_node = None + if first_user_node is None: + # no users, so we insert the replacement graph before the first next + # node of returning nodes + next_node = None + for n in reversed(original_graph.nodes): + if n in match.returning_nodes: + first_next_node = next_node + break + else: + next_node = n + insert_point = ( + first_user_node if first_user_node is not None else first_next_node + ) + assert insert_point is not None, "The insert point can't be None" + with original_graph.inserting_before(insert_point): + copied_returning_nodes = original_graph.graph_copy( + replacement_graph, val_map + ) + + if isinstance(copied_returning_nodes, Node): + copied_returning_nodes = (copied_returning_nodes,) + + # Get a list of nodes that have been replaced into the graph + replacement_nodes: list[Node] = [ + v for v in val_map.values() if v not in match.placeholder_nodes + ] + + # Hook the output Node of the replacement subgraph in to the + # original Graph at the correct location + assert len(match.returning_nodes) == len(copied_returning_nodes) # type: ignore[arg-type] + for gn, copied_node in zip(match.returning_nodes, copied_returning_nodes): # type: ignore[arg-type] + gn.replace_all_uses_with(copied_node) + match_changed_node[gn] = copied_node + # Remove the original nodes + for node in reversed(pattern_graph.nodes): + if node.op != "placeholder" and node.op != "output": + gn = match.nodes_map[node] + gm.graph.erase_node(gn) + + match_and_replacements.append( + ReplacedPatterns( + anchor=match.anchors[0], + nodes_map=match.nodes_map, + replacements=replacement_nodes, + ) + ) + + # Update the passed-in GraphModule to reflect the new state of + # `original_graph` + gm.recompile() + + # If `replacement` was an nn.Module, we'll need to make sure that + # all the submodules have been copied over correctly + if isinstance(replacement, torch.nn.Module): + _replace_attributes(gm, replacement) + + return match_and_replacements diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/tensor_type.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/tensor_type.py new file mode 100644 index 0000000000000000000000000000000000000000..a388ae90206b3d3881dfb81b85a05ca3950314b7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/tensor_type.py @@ -0,0 +1,111 @@ +# mypy: allow-untyped-defs +from torch.fx.experimental.unification import Var # type: ignore[attr-defined] + +from ._compatibility import compatibility + + +@compatibility(is_backward_compatible=False) +class TensorType: + """ + TensorType defines a type for tensors, which consists of a list of dimensions. + Example: + class M(torch.nn.Module): + def forward(self, x:TensorType((1,2,3, Dyn)), y:TensorType((1,2,3, Dyn))): + return torch.add(x, y) + """ + + def __init__(self, dim): + self.__origin__ = TensorType + self.__args__ = dim + + def __repr__(self): + return f"TensorType[{self.__args__}]" + + def __eq__(self, other): + if isinstance(other, self.__class__): + return list(self.__args__) == list(other.__args__) + else: + return False + + @staticmethod + def __class_getitem__(*args): + if len(args) == 1 and isinstance(args[0], tuple): + args = args[0] + return TensorType(tuple(args)) + + +class _DynType: + """ + _DynType defines a type which stands for the absence of type information. + """ + + def __init__(self) -> None: + self.__name__ = "_DynType" + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __str__(self): + return "Dyn" + + def __repr__(self): + return "Dyn" + + +Dyn = _DynType() + + +@compatibility(is_backward_compatible=False) +def is_consistent(t1, t2): + """ + A binary relation denoted by ~ that determines if t1 is consistent with t2. + The relation is reflexive, symmetric but not transitive. + returns True if t1 and t2 are consistent and False otherwise. + Example: + Dyn ~ TensorType((1,2,3)) + int ~ Dyn + int ~ int + TensorType((1,Dyn,3)) ~ TensorType((1,2,3)) + """ + + if t1 == t2: + return True + + if t1 == Dyn or t2 == Dyn or isinstance(t1, Var) or isinstance(t2, Var): + return True + + if isinstance(t1, TensorType) and isinstance(t2, TensorType): + return len(t1.__args__) == len(t2.__args__) and all( + is_consistent(elem1, elem2) + for elem1, elem2 in zip(t1.__args__, t2.__args__) + ) + else: + return False + + +@compatibility(is_backward_compatible=False) +def is_more_precise(t1, t2): + """ + A binary relation denoted by <= that determines if t1 is more precise than t2. + The relation is reflexive and transitive. + returns True if t1 is more precise than t2 and False otherwise. + Example: + Dyn >= TensorType((1,2,3)) + int >= Dyn + int >= int + TensorType((1,Dyn,3)) <= TensorType((1,2,3)) + """ + if t1 == t2: + return True + + if isinstance(t2, _DynType): + return True + + if isinstance(t1, TensorType) and isinstance(t2, TensorType): + return len(t1.__args__) == len(t2.__args__) and all( + is_more_precise(elem1, elem2) + for elem1, elem2 in zip(t1.__args__, t2.__args__) + ) + + else: + return False diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/traceback.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/traceback.py new file mode 100644 index 0000000000000000000000000000000000000000..e221cae558eb2653d2686b4952c4e97543fb01be --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/fx/traceback.py @@ -0,0 +1,502 @@ +# mypy: allow-untyped-defs +import copy +import logging +import traceback +from contextlib import contextmanager +from enum import Enum +from typing import Any, Optional, Union + +from torch._utils_internal import signpost_event + +from ._compatibility import compatibility +from .graph import Graph +from .graph_module import GraphModule +from .node import Node + + +log = logging.getLogger(__name__) + +__all__ = [ + "annotate", + "annotate_fn", + "preserve_node_meta", + "has_preserved_node_meta", + "set_stack_trace", + "set_grad_fn_seq_nr", + "reset_grad_fn_seq_nr", + "format_stack", + "set_current_meta", + "get_current_meta", + "NodeSource", + "NodeSourceAction", + "get_graph_provenance_json", + "set_current_replay_node", + "get_current_replay_node", +] + +current_meta: dict[str, Any] = {} +current_replay_node: Optional[Node] = None +should_preserve_node_meta = False + +GRADIENT_ACC_SPECIAL_STACK = ( + "Gradient addition node due to multiple use of tensor around:" +) +# ============================================================================= +# FX Metadata Registry for Memory Profiler +# ============================================================================= +# Global in-memory registry for FX metadata +# Maps module_name -> metadata dict containing lineno_map and node_metadata +_FX_METADATA_REGISTRY: dict[str, dict[str, Any]] = {} + + +def _register_fx_metadata(module_name: str, metadata: dict[str, Any]) -> None: + """ + Register FX metadata in the global in-memory registry. + + This is called automatically during graph module compilation to store metadata + for later use by memory profiler augmentation. + + Args: + module_name: The module identifier (content-addressed filename) + metadata: Metadata dict containing lineno_map, node_metadata, and source_code + """ + # TODO: add logging to tlparse + _FX_METADATA_REGISTRY[module_name] = metadata + + +@compatibility(is_backward_compatible=False) +class NodeSourceAction(Enum): + CREATE = "create" + REPLACE = "replace" + + +@compatibility(is_backward_compatible=False) +class NodeSource: + """ + NodeSource is a data structure that contains the provenance information of a node. + If node `a` is created from node `b`, then `a.meta["from_node"]` may contain NodeSource(b). + """ + + class NodeInfo: + def __init__(self, name: str, target: str, graph_id: int): + self.name = name + self.target = target + self.graph_id = graph_id + + pass_name: str + action: list["NodeSourceAction"] + from_node: list["NodeSource"] + node_info: Optional["NodeInfo"] + _dict: Optional[dict[str, Any]] + _action_string: Optional[str] + + def __init__( + self, + node: Optional[Node], + pass_name: str = "", + action: Optional[Union["NodeSourceAction", list["NodeSourceAction"]]] = None, + ): + self.pass_name = pass_name + + if action is None: + action = [] + elif not isinstance(action, list): + action = [action] + for a in action: + assert isinstance(a, NodeSourceAction) + self.action = action + if node: + self.node_info = self.NodeInfo( + name=node.name, target=str(node.target), graph_id=id(node.graph) + ) + self.from_node = ( + copy.deepcopy(node.meta["from_node"]) + if "from_node" in node.meta + else [] + ) + else: + self.node_info = None + self.from_node = [] + + # cache the action string and dict representation for performance. + self._action_string: Optional[str] = None + self._dict: Optional[dict[str, Any]] = None + + @property + def name(self) -> str: + return self.node_info.name if self.node_info else "" + + @property + def target(self) -> str: + return self.node_info.target if self.node_info else "" + + @property + def graph_id(self) -> int: + return self.node_info.graph_id if self.node_info else -1 + + def __repr__(self): + return self.print_readable() + + def _get_action_string(self): + if self._action_string is None: + self._action_string = "+".join([a.name.lower() for a in self.action]) + return self._action_string + + def print_readable(self, indent=0): + if indent > 9: + return "" + result = "" + action_string = self._get_action_string() + result += ( + " " * indent * 4 + + f"(name={self.name}, pass_name={self.pass_name}, action={action_string}, graph_id={self.graph_id})\n" + ) + for item in self.from_node: + result += item.print_readable(indent + 1) + return result + + def to_dict(self) -> dict: + if self._dict is None: + # Convert the object to a dictionary + action_string = self._get_action_string() + self._dict = { + "name": self.name, + "target": self.target, + "graph_id": self.graph_id, + "pass_name": self.pass_name, + "action": action_string, + "from_node": [node.to_dict() for node in self.from_node], + } + + assert self._dict is not None + return self._dict + + def __eq__(self, other: object): + if not isinstance(other, NodeSource): + return False + return self.to_dict() == other.to_dict() + + def __hash__(self): + # Create a hash based on the dictionary representation + # We need to convert the dict to a hashable form + def _make_hashable(obj): + if isinstance(obj, dict): + return tuple(sorted((k, _make_hashable(v)) for k, v in obj.items())) + elif isinstance(obj, list): + return tuple(_make_hashable(item) for item in obj) + else: + return obj + + return hash(_make_hashable(self.to_dict())) + + @classmethod + def _from_dict(cls, d: Optional[dict]) -> Optional["NodeSource"]: + """ + Recursively deserialize from_node metadata from dictionary data. + It is used to deserialize the from_node field from serialized metadata. + Please use constructor NodeSource(node, ...) to create a NodeSource object. + """ + if d is None: + return None + + assert isinstance(d, dict), f"Expected a dict, got {type(d)}" + + # Create a NodeSource object directly without going through the constructor + # to avoid issues with graph ID and node creation + node_source = NodeSource.__new__(NodeSource) + + # Reset the cached properties + node_source._action_string = None + node_source._dict = None + + # Set the basic attributes + node_source.pass_name = d.get("pass_name", "") + + # Parse action string back to NodeSourceAction enum list + action_str = d.get("action", "") + actions = [] + if action_str: + for action_name in action_str.split("+"): + if action_name.upper() == "CREATE": + actions.append(NodeSourceAction.CREATE) + elif action_name.upper() == "REPLACE": + actions.append(NodeSourceAction.REPLACE) + node_source.action = actions + + # Create the NodeInfo object directly + if "name" in d and "target" in d and "graph_id" in d: + node_info = NodeSource.NodeInfo( + d.get("name", ""), d.get("target", ""), d.get("graph_id", -1) + ) + node_source.node_info = node_info + else: + node_source.node_info = None + + # Recursively deserialize nested from_node + if d.get("from_node", None) is not None: + node_source.from_node = [ + result + for fn in d.get("from_node", []) + if (result := cls._from_dict(fn)) is not None + ] + else: + node_source.from_node = [] + return node_source + + +@compatibility(is_backward_compatible=False) +@contextmanager +def preserve_node_meta(enable=True): + global should_preserve_node_meta + global current_meta + saved_should_preserve_node_meta = should_preserve_node_meta + # Shallow copy is OK since fields of current_meta are not mutated + saved_current_meta = current_meta.copy() + try: + should_preserve_node_meta = enable + yield + finally: + should_preserve_node_meta = saved_should_preserve_node_meta + current_meta = saved_current_meta + + +@compatibility(is_backward_compatible=False) +def set_stack_trace(stack: list[str]): + global current_meta + + if should_preserve_node_meta and stack: + current_meta["stack_trace"] = "".join(stack) + + +@compatibility(is_backward_compatible=False) +@contextmanager +def annotate(annotation_dict: dict): + """ + Temporarily adds custom annotations to the current tracing context. + The fx_node produced from this tracing context will have the + custom annotations in node.metadata["custom"] field. + + This context manager allows you to insert arbitrary metadata into the PT2 + tracing system by updating the global `current_meta["custom"]` dictionary. + The annotations are automatically reverted after the context exits. + + Gradient accumulation nodes will not be annotated. + + This is intended for advanced users who need to attach additional metadata to the fx nodes + (e.g., for debugging, analysis, or external tooling) during export tracing. + + Note: + This API is **not backward compatible** and may evolve in future releases. + + Note: + This API is not compatible with fx.symbolic_trace or jit.trace. It's intended + to be used with PT2 family of tracers, e.g. torch.export and dynamo. + + Args: + annotation_dict (dict): A dictionary of custom key-value pairs to inject + into the FX trace metadata. + + Example: + After exiting the context, custom annotations are removed. + + >>> with annotate({"source": "custom_pass", "tag": 42}): + ... pass # Your computation here + """ + + global current_meta + + has_custom = "custom" in current_meta + old_custom = copy.copy(current_meta.get("custom", {})) + + try: + if not has_custom: + current_meta["custom"] = {} + + # Update with all key-value pairs from the input dict + current_meta["custom"].update(annotation_dict) + yield + finally: + if has_custom: + # Restore the original custom dict + current_meta["custom"] = old_custom + else: + del current_meta["custom"] + + +@compatibility(is_backward_compatible=False) +def annotate_fn(annotation_dict: dict): + """ + A decorator that wraps a function with the annotate context manager. + Use this when you want to annotate an entire function instead of a specific code block. + + Note: + This API is **not backward compatible** and may evolve in future releases. + + Note: + This API is not compatible with fx.symbolic_trace or jit.trace. It's intended + to be used with PT2 family of tracers, e.g. torch.export and dynamo. + + Args: + annotation_dict (dict): A dictionary of custom key-value pairs to inject + into the FX trace metadata for all operations in the function. + + Example: + All operations in my_function will have {"pp_stage": 1} in their metadata. + + >>> @annotate_fn({"pp_stage": 1}) + ... def my_function(x): + ... return x + 1 + """ + from functools import wraps + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + with annotate(annotation_dict): + return func(*args, **kwargs) + + return wrapper + + return decorator + + +@compatibility(is_backward_compatible=False) +def set_grad_fn_seq_nr(seq_nr): + global current_meta + + if should_preserve_node_meta: + # The seq_nr is captured by eager mode in the grad_fn during forward + current_meta["grad_fn_seq_nr"] = current_meta.get("grad_fn_seq_nr", []) + [ + seq_nr + ] + current_meta["in_grad_fn"] = current_meta.get("in_grad_fn", 0) + 1 + + +@compatibility(is_backward_compatible=False) +def reset_grad_fn_seq_nr(): + # NB: reset state properly, this would be helpful towards supporting + # reentrant autograd if we actually wanted to do that. + global current_meta + if should_preserve_node_meta: + current_level = current_meta.get("in_grad_fn", 0) + assert current_level > 0 + if current_level == 1: + del current_meta["in_grad_fn"] + del current_meta["grad_fn_seq_nr"] + else: + current_meta["in_grad_fn"] = current_level - 1 + current_meta["grad_fn_seq_nr"] = current_meta["grad_fn_seq_nr"][:-1] + + +@compatibility(is_backward_compatible=False) +def format_stack() -> list[str]: + if should_preserve_node_meta: + return [current_meta.get("stack_trace", "")] + else: + # fallback to traceback.format_stack() + return traceback.format_list(traceback.extract_stack()[:-1]) + + +@compatibility(is_backward_compatible=False) +def has_preserved_node_meta() -> bool: + return should_preserve_node_meta + + +@compatibility(is_backward_compatible=False) +@contextmanager +def set_current_meta(node, pass_name=""): + global current_meta + if should_preserve_node_meta and node.meta: + saved_meta = current_meta + try: + current_meta = node.meta.copy() + + # Update the "from_node" field in current_meta for provenance tracking. + # Instead of appending, overwrite the "from_node" field because current_meta + # will be assigned to the new node. The new NodeSource(node, ...) will + # include the information from the previous current_meta["from_node"]. + current_meta["from_node"] = [ + NodeSource(node, pass_name, NodeSourceAction.CREATE) + ] + yield + finally: + current_meta = saved_meta + else: + yield + + +@compatibility(is_backward_compatible=False) +def get_current_meta() -> dict[str, Any]: + return current_meta + + +@compatibility(is_backward_compatible=False) +@contextmanager +def set_current_replay_node(node): + """ + Set the currently replay node. If `current_replay_node` is not None, + then we're re-generating the `current_replay_node` in FunctionalTensorMode. + """ + # See [Note] annotation for more details. + global current_replay_node + saved_current_replay_node = current_replay_node + try: + current_replay_node = node + yield + finally: + current_replay_node = saved_current_replay_node + + +@compatibility(is_backward_compatible=False) +def get_current_replay_node(): + """ + Get the currently replay node + """ + return current_replay_node + + +@compatibility(is_backward_compatible=False) +def get_graph_provenance_json(graph: Graph) -> dict[str, Any]: + """ + Given an fx.Graph, return a json that contains the provenance information of each node. + """ + try: + provenance_tracking_json = {} + for node in graph.nodes: + if node.op == "call_function": + provenance_tracking_json[node.name] = ( + [source.to_dict() for source in node.meta["from_node"]] + if "from_node" in node.meta + else [] + ) + return provenance_tracking_json + except Exception as e: + # Since this is just debugging, it should never interfere with regular + # program execution, so we use this try-except to guard against any error + signpost_event( + "inductor", + "provenance_tracking_error", + { + "function": "get_graph_provenance_json", + "error_msg": str(e), + "stack_trace": traceback.format_exc(), + }, + ) + return {} + + +def _get_custom_metadata(gm: GraphModule) -> str: + assert isinstance(gm, GraphModule) + + def helper(gm: GraphModule): + custom_metadata = [] + for node in gm.graph.nodes: + if hasattr(node, "meta") and node.meta.get("custom", None): + custom_metadata.append((node.op, node.name, node.meta["custom"])) + if node.op == "get_attr" and isinstance( + getattr(gm, node.target), GraphModule + ): + custom_metadata.append(helper(getattr(gm, node.target))) + return custom_metadata + + return "\n".join(str(x) for x in helper(gm)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/advisor-annotate.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/advisor-annotate.h new file mode 100644 index 0000000000000000000000000000000000000000..bad0972c97578b1d72dc3e858352229a33fc1b3a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/advisor-annotate.h @@ -0,0 +1,525 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + * Copyright (C) 2005-2019 Intel Corporation + * SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + */ + +/* This file defines macros and inline functions used by + * the Intel(R) Advisor XE "Dependencies Modeling" and + * "Suitability Modeling" analysis, and are in described + * in the "Annotations" section of the help. + * + * Expansion Options + * + * There are several options you can used to control how advisor-annotate.h + * is included. To use these, define the option prior to including + * advisor-annotate.h. e.g. + * #define ANNOTATE_DECLARE + * #include "advisor-annotate.h" + * + * Controlling inclusion of windows.h + * + * windows.h is included for declarations for LoadLibrary, GetProcSymbol, + * but this can have interactions with user code, such as conflicting + * definitions of types. There are two general approaches to work around + * this if this triggers problems building your application: + * + * 1. Reduce the amount declared by windows.h by using the following: + * #define NOMINMAX + * #define WIN32_LEAN_AND_MEAN + * prior to including advisor-annotate.h in your code. + * The first avoids problems with STL min/max in particular + * This is sufficient in some cases, and may be the easiest. + * + * 2. Use a declaration/definition approach, where all uses of advisor-annotate.h + * other than one, generate a set of declarations, and windows.h is only + * needed in a single implementation module. In this model, all includes + * of advisor-annotate.h except one specify ANNOTATE_DECLARE, which causes + * advisor-annotate.h to declare an external routine, and not include + * windows.h. A final include of advisor-annotate.h than specifies + * ANNOTATE_DEFINE, to actually define the global routine to resolve + * the external reference. This one include is the only one that winds up + * using windows.h. If necessary, this can be placed in a file by itself. + * + * An example using this mechanism: + * + * ... + * // Some header(s) used in places in your system where you want + * // to be able to use annotations + * #define ANNOTATE_DECLARE + * #include "advisor-annotate.h" + * ... + * // annotation uses + * ANNOTATE_SITE_BEGIN(MySite1) + * ... + * ANNOTATE_SITE_END(MySite1) + * ... + * + * ... + * // A single implementation file (.cpp/.cxx) causes windows.h + * // to be included, and the support routine to be defined as a + * // global routine called from the various annotation uses. + * #define ANNOTATE_DEFINE + * #include "advisor-annotate.h" + * ... + * + * Null expansion of annotations + * + * Some people may find it useful to have no expansion for annotations, + * if you have a project that you want to build without any annotation + * effects at all. (e.g. if you have a project where you want to have + * some annotations in a shared source pool, but only particular + * developers are actually building with the annotations enabled.) + * Defining ANNOTATE_EXPAND_NULL avoids declaring comdat routines, + * and avoids any textual expansion for annotation macros. + */ + +#ifndef _ADVISOR_ANNOTATE_H_ +#define _ADVISOR_ANNOTATE_H_ + +/* Version of the annotations. + * The presence of this macro serves to idetify the annotation definition + * file and the form of annotations. + */ +#define INTEL_ADVISOR_ANNOTATION_VERSION 1.0 + +#ifdef ANNOTATE_EXPAND_NULL + +#define ANNOTATE_SITE_BEGIN(_SITE) +#define ANNOTATE_SITE_END(...) +#define ANNOTATE_TASK_BEGIN(_TASK) +#define ANNOTATE_TASK_END(...) +#define ANNOTATE_ITERATION_TASK(_TASK) +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) +#define ANNOTATE_LOCK_RELEASE(_ADDR) +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) +#define ANNOTATE_CLEAR_USES(_ADDR) +#define ANNOTATE_DISABLE_OBSERVATION_PUSH +#define ANNOTATE_DISABLE_OBSERVATION_POP +#define ANNOTATE_DISABLE_COLLECTION_PUSH +#define ANNOTATE_DISABLE_COLLECTION_POP +#define ANNOTATE_AGGREGATE_TASK(_COUNT) + +#else /* ANNOTATE_EXPAND_NULL */ + +#if defined(WIN32) || defined(_WIN32) + +#define ANNOTATEAPI __cdecl + +#ifndef ANNOTATE_DECLARE +#include + +typedef HMODULE lib_t; + +#define __itt_get_proc(lib, name) GetProcAddress(lib, name) +#define __itt_load_lib(name) LoadLibraryA(name) +#define __itt_unload_lib(handle) FreeLibrary(handle) +#define __itt_system_error() (int)GetLastError() +#endif /* ANNOTATE_DECLARE */ + +#else /* defined(WIN32) || defined(_WIN32) */ + +#if defined _M_IX86 || __i386__ +# define ANNOTATEAPI __attribute__ ((cdecl)) +#else +# define ANNOTATEAPI /* actual only on x86 platform */ +#endif + + +#ifndef ANNOTATE_DECLARE +#include +#include +#include + +typedef void* lib_t; + +#define __itt_get_proc(lib, name) dlsym(lib, name) +#define __itt_load_lib(name) dlopen(name, RTLD_LAZY) +#define __itt_unload_lib(handle) dlclose(handle) +#define __itt_system_error() errno +#endif /* ANNOTATE_DECLARE */ + +#endif /* defined(WIN32) || defined(_WIN32) */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef _ITTNOTIFY_H_ + +/* Handles for sites and tasks. + */ +typedef void* __itt_model_site; /* handle for lexical site */ +typedef void* __itt_model_site_instance; /* handle for dynamic instance */ +typedef void* __itt_model_task; /* handle for lexical site */ +typedef void* __itt_model_task_instance; /* handle for dynamic instance */ + +typedef enum { + __itt_model_disable_observation, + __itt_model_disable_collection +} __itt_model_disable; + +#endif /* _ITTNOTIFY_H_ */ + +/*** Use the routines in libittnotify.dll. ***/ + +/* Basic approach: + * For the case of calling the dll, there is an __annotate_routine function + * declared as a comdat in each compilation unit with annotations present. + * That routine in turn has an internal static structure that is initialized + * once to contain the address of functions occuring in libittnotify.dll. + * Each time an annotation macro is invoked, that causes a call to the + * __annotate_routine function to get addresses of the routines, followed + * by calling the specific routine, provided the address is non-null. + */ + +/* This set of macros generates calls that are part of application images, + * which call the __itt_model_xxx routines in the dynamically loaded + * libittnotify.dll. + */ +#ifndef _ITTNOTIFY_H_ +#define ITT_NOTIFY_DECL(_text) _text +#else +#define ITT_NOTIFY_DECL(_text) +#endif + +/* For C++, a static initialization is used */ +#if defined(__cplusplus) && defined(WIN32) +#define _ANNOTATE_ROUTINES_ADDR __annotate_routines_s +#else +#define _ANNOTATE_ROUTINES_ADDR __annotate_routines_init( __annotate_routines() ) +#endif /* __cplusplus */ + + +#define _ANNOTATE_DECLARE_0(_BASENAME) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop() { }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(); ) + +#define _ANNOTATE_DECLARE_0_INT(_BASENAME) \ +typedef int (ANNOTATEAPI * __annotate_##_BASENAME##_t)(); \ +static __inline int ANNOTATEAPI __annotate_##_BASENAME##_t_nop() { return 0; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(); ) + +#define _ANNOTATE_CALL_0(_BASENAME) { _ANNOTATE_ROUTINES_ADDR->_BASENAME(); } + +#define _ANNOTATE_DECLARE_1(_BASENAME, _P1TYPE) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(_P1TYPE p1); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop(_P1TYPE p1) { (void)p1; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(_P1TYPE p1); ) + +#define _ANNOTATE_CALL_1(_BASENAME, _P1) { _ANNOTATE_ROUTINES_ADDR->_BASENAME(_P1); } + +#define _ANNOTATE_DECLARE_2(_BASENAME, _P1TYPE, _P2TYPE) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(_P1TYPE p1, _P2TYPE p2); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop(_P1TYPE p1, _P2TYPE p2) { (void)p1; (void)p2; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(_P1TYPE p1, _P2TYPE p2); ) + +#define _ANNOTATE_CALL_2(_BASENAME, _P1, _P2) { _ANNOTATE_ROUTINES_ADDR->_BASENAME((_P1), (_P2)); } + +/*** Declare routines appropriately based on usage style ***/ + +/* Depending on above, this will either expand to comdats that are + * used directly, or comdats that call routines in libittnotify.dll + */ +_ANNOTATE_DECLARE_1(site_beginA, const char *) +_ANNOTATE_DECLARE_0(site_end_2) +_ANNOTATE_DECLARE_1(task_beginA, const char *) +_ANNOTATE_DECLARE_0(task_end_2) +_ANNOTATE_DECLARE_1(iteration_taskA, const char *) +_ANNOTATE_DECLARE_1(lock_acquire_2, void *) +_ANNOTATE_DECLARE_1(lock_release_2, void *) +_ANNOTATE_DECLARE_2(record_allocation, void *, size_t) +_ANNOTATE_DECLARE_1(record_deallocation, void *) +_ANNOTATE_DECLARE_2(induction_uses, void *, size_t) +_ANNOTATE_DECLARE_2(reduction_uses, void *, size_t) +_ANNOTATE_DECLARE_2(observe_uses, void *, size_t) +_ANNOTATE_DECLARE_1(clear_uses, void *) +_ANNOTATE_DECLARE_1(disable_push, __itt_model_disable) +_ANNOTATE_DECLARE_0(disable_pop) +_ANNOTATE_DECLARE_1(aggregate_task, size_t) +_ANNOTATE_DECLARE_0_INT(is_collection_disabled) + +/* All of the symbols potentially in the library + */ +struct __annotate_routines { + volatile int initialized; + __annotate_site_beginA_t site_beginA; + __annotate_site_end_2_t site_end_2; + __annotate_task_beginA_t task_beginA; + __annotate_task_end_2_t task_end_2; + __annotate_iteration_taskA_t iteration_taskA; + __annotate_lock_acquire_2_t lock_acquire_2; + __annotate_lock_release_2_t lock_release_2; + __annotate_record_allocation_t record_allocation; + __annotate_record_deallocation_t record_deallocation; + __annotate_induction_uses_t induction_uses; + __annotate_reduction_uses_t reduction_uses; + __annotate_observe_uses_t observe_uses; + __annotate_clear_uses_t clear_uses; + __annotate_disable_push_t disable_push; + __annotate_disable_pop_t disable_pop; + __annotate_aggregate_task_t aggregate_task; + __annotate_is_collection_disabled_t is_collection_disabled; +}; + +/* This comdat-ed routine means there is a single instance of the function pointer + * structure per image + */ +static __inline struct __annotate_routines* __annotate_routines() +{ + static struct __annotate_routines __annotate_routines; + return &__annotate_routines; +} + +/* This routine is called to get the address of an initialized + * set of function pointers for the annotation routines. + */ + +#ifdef ANNOTATE_DECLARE +extern struct __annotate_routines* ANNOTATEAPI __annotate_routines_init(struct __annotate_routines* itt); +#else +#ifdef ANNOTATE_DEFINE + /* */ +#else + static __inline +#endif +struct __annotate_routines* +ANNOTATEAPI +__annotate_routines_init(struct __annotate_routines* itt) { + + if (itt->initialized) { + return itt; + } else { + + /* Initialized by first invocation + * This assumes that the code here can be executed successfully + * by multiple threads, should that ever happen. + */ + int do_disable_pop = 0; + char* lib_name = NULL; + lib_t itt_notify = 0; + + if (sizeof(void*) > 4) { + lib_name = getenv("INTEL_LIBITTNOTIFY64"); + } else { + lib_name = getenv("INTEL_LIBITTNOTIFY32"); + } + + if (lib_name) { + itt_notify = __itt_load_lib(lib_name); + } else { +#if defined(WIN32) || defined(_WIN32) + itt_notify = __itt_load_lib("libittnotify.dll"); +#elif defined(__APPLE__) + itt_notify = __itt_load_lib("libittnotify.dylib"); +#else + itt_notify = __itt_load_lib("libittnotify.so"); +#endif + } + + if (itt_notify != NULL) { + /* The static variables initialized and itt are reported as race conditions + * or inconsistent lock usage by Dependencies Modeling in some obscure cases + * involving multiple dlls. Ignoring this initialization phase gets rid of + * this problem. + */ + __annotate_disable_push_t disable_push; + __annotate_is_collection_disabled_t is_collection_disabled; + disable_push = (__annotate_disable_push_t) __itt_get_proc(itt_notify, "__itt_model_disable_push"); + is_collection_disabled = (__annotate_is_collection_disabled_t) __itt_get_proc(itt_notify, "__itt_model_is_collection_disabled"); + if (disable_push) { + if ( ! (is_collection_disabled && is_collection_disabled()) ) + { + // disable collection only if it is not disabled already (for example, started paused) + disable_push(__itt_model_disable_observation); + do_disable_pop = 1; + } + } + itt->site_beginA = (__annotate_site_beginA_t) __itt_get_proc(itt_notify, "__itt_model_site_beginA"); + itt->site_end_2 = (__annotate_site_end_2_t) __itt_get_proc(itt_notify, "__itt_model_site_end_2"); + itt->task_beginA = (__annotate_task_beginA_t) __itt_get_proc(itt_notify, "__itt_model_task_beginA"); + itt->task_end_2 = (__annotate_task_end_2_t) __itt_get_proc(itt_notify, "__itt_model_task_end_2"); + itt->iteration_taskA = (__annotate_iteration_taskA_t) __itt_get_proc(itt_notify, "__itt_model_iteration_taskA"); + itt->lock_acquire_2 = (__annotate_lock_acquire_2_t) __itt_get_proc(itt_notify, "__itt_model_lock_acquire_2"); + itt->lock_release_2 = (__annotate_lock_release_2_t) __itt_get_proc(itt_notify, "__itt_model_lock_release_2"); + itt->record_allocation = (__annotate_record_allocation_t) __itt_get_proc(itt_notify, "__itt_model_record_allocation"); + itt->record_deallocation = (__annotate_record_deallocation_t)__itt_get_proc(itt_notify, "__itt_model_record_deallocation"); + itt->induction_uses = (__annotate_induction_uses_t) __itt_get_proc(itt_notify, "__itt_model_induction_uses"); + itt->reduction_uses = (__annotate_reduction_uses_t) __itt_get_proc(itt_notify, "__itt_model_reduction_uses"); + itt->observe_uses = (__annotate_observe_uses_t) __itt_get_proc(itt_notify, "__itt_model_observe_uses"); + itt->clear_uses = (__annotate_clear_uses_t) __itt_get_proc(itt_notify, "__itt_model_clear_uses"); + itt->disable_push = disable_push; + itt->disable_pop = (__annotate_disable_pop_t) __itt_get_proc(itt_notify, "__itt_model_disable_pop"); + itt->aggregate_task = (__annotate_aggregate_task_t) __itt_get_proc(itt_notify, "__itt_model_aggregate_task"); + itt->is_collection_disabled = is_collection_disabled; + } + /* No-op routine for any that didn't get resolved */ + if (!itt->site_beginA) itt->site_beginA = __annotate_site_beginA_t_nop; + if (!itt->site_end_2) itt->site_end_2 = __annotate_site_end_2_t_nop; + if (!itt->task_beginA) itt->task_beginA = __annotate_task_beginA_t_nop; + if (!itt->task_end_2) itt->task_end_2 = __annotate_task_end_2_t_nop; + if (!itt->iteration_taskA) itt->iteration_taskA = __annotate_iteration_taskA_t_nop; + if (!itt->lock_acquire_2) itt->lock_acquire_2 = __annotate_lock_acquire_2_t_nop; + if (!itt->lock_release_2) itt->lock_release_2 = __annotate_lock_release_2_t_nop; + if (!itt->record_allocation) itt->record_allocation = __annotate_record_allocation_t_nop; + if (!itt->record_deallocation) itt->record_deallocation=__annotate_record_deallocation_t_nop; + if (!itt->induction_uses) itt->induction_uses = __annotate_induction_uses_t_nop; + if (!itt->reduction_uses) itt->reduction_uses = __annotate_reduction_uses_t_nop; + if (!itt->observe_uses) itt->observe_uses = __annotate_observe_uses_t_nop; + if (!itt->clear_uses) itt->clear_uses = __annotate_clear_uses_t_nop; + if (!itt->disable_push) itt->disable_push = __annotate_disable_push_t_nop; + if (!itt->disable_pop) itt->disable_pop = __annotate_disable_pop_t_nop; + if (!itt->aggregate_task) itt->aggregate_task = __annotate_aggregate_task_t_nop; + if (!itt->is_collection_disabled) itt->is_collection_disabled = __annotate_is_collection_disabled_t_nop; + + itt->initialized = 1; + + if (do_disable_pop) { + itt->disable_pop(); + } + } + return itt; +} +#endif /* ANNOTATE_DECLARE */ + +/* For C++ only, use a class to force initialization */ + +#if defined(__cplusplus) && defined(WIN32) +/* Force one-shot initialization so individual calls don't need it */ +static struct __annotate_routines* __annotate_routines_s = __annotate_routines_init( __annotate_routines() ); +#endif + +/* For C++, allow the Annotate::SiteBegin(x) form. For Windows CLR, this is the default + * expansion for the macros (with no-inline) to get the best call stacks in the tools. */ +#if defined(__cplusplus) +/* Ensure this code is managed and non-inlinable */ +#if defined(WIN32) && defined(__CLR_VER) +#pragma managed(push, on) +#define ANNOTATE_CLR_NOINLINE __declspec(noinline) +#else +#define ANNOTATE_CLR_NOINLINE +#endif +class Annotate { +public: + static ANNOTATE_CLR_NOINLINE void SiteBegin(const char* site) { _ANNOTATE_ROUTINES_ADDR->site_beginA(site); } + static ANNOTATE_CLR_NOINLINE void SiteEnd() { _ANNOTATE_ROUTINES_ADDR->site_end_2(); } + static ANNOTATE_CLR_NOINLINE void TaskBegin(const char* task) { _ANNOTATE_ROUTINES_ADDR->task_beginA(task); } + static ANNOTATE_CLR_NOINLINE void TaskEnd() { _ANNOTATE_ROUTINES_ADDR->task_end_2(); } + static ANNOTATE_CLR_NOINLINE void IterationTask(const char* task) { _ANNOTATE_ROUTINES_ADDR->iteration_taskA(task); } + static ANNOTATE_CLR_NOINLINE void LockAcquire(void* lockId) { _ANNOTATE_ROUTINES_ADDR->lock_acquire_2(lockId); } + static ANNOTATE_CLR_NOINLINE void LockRelease(void* lockId) { _ANNOTATE_ROUTINES_ADDR->lock_release_2(lockId); } + static ANNOTATE_CLR_NOINLINE void RecordAllocation(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->record_allocation(p, s); } + static ANNOTATE_CLR_NOINLINE void RecordDeallocation(void *p) { _ANNOTATE_ROUTINES_ADDR->record_deallocation(p); } + static ANNOTATE_CLR_NOINLINE void InductionUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->induction_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ReductionUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->reduction_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ObserveUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->observe_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ClearUses(void *p) { _ANNOTATE_ROUTINES_ADDR->clear_uses(p); } + static ANNOTATE_CLR_NOINLINE void DisablePush(__itt_model_disable d) { _ANNOTATE_ROUTINES_ADDR->disable_push(d); } + static ANNOTATE_CLR_NOINLINE void DisablePop() { _ANNOTATE_ROUTINES_ADDR->disable_pop(); } + static ANNOTATE_CLR_NOINLINE void AggregateTask(size_t c) { _ANNOTATE_ROUTINES_ADDR->aggregate_task(c); } +}; +#if defined(WIN32) && defined(__CLR_VER) +#pragma managed(pop) +#endif +#undef ANNOTATE_CLR_NOINLINE +#endif + +#if defined(__cplusplus) && defined(WIN32) && defined(__CLR_VER) + +#define ANNOTATE_SITE_BEGIN(_SITE) Annotate::SiteBegin(#_SITE) +#define ANNOTATE_SITE_END(...) Annotate::SiteEnd() +#define ANNOTATE_TASK_BEGIN(_TASK) Annotate::TaskBegin(#_TASK) +#define ANNOTATE_TASK_END(...) Annotate::TaskEnd() +#define ANNOTATE_ITERATION_TASK(_TASK) Annotate::IterationTask(#_TASK) +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) Annotate::LockAcquire(_ADDR) +#define ANNOTATE_LOCK_RELEASE(_ADDR) Annotate::LockRelease(_ADDR) +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) Annotate::RecordAllocation((_ADDR), (_SIZE)) +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) Annotate::RecordDeallocation(_ADDR) +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) Annotate::InductionUses((_ADDR), (_SIZE)) +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) Annotate::ReductionUses((_ADDR), (_SIZE)) +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) Annotate::ObserveUses((_ADDR), (_SIZE)) +#define ANNOTATE_CLEAR_USES(_ADDR) Annotate::ClearUses(_ADDR) +#define ANNOTATE_DISABLE_OBSERVATION_PUSH Annotate::DisablePush(itt_model_disable_observation) +#define ANNOTATE_DISABLE_OBSERVATION_POP Annotate::DisablePop() +#define ANNOTATE_DISABLE_COLLECTION_PUSH Annotate::DisablePush(__itt_model_disable_collection) +#define ANNOTATE_DISABLE_COLLECTION_POP Annotate::DisablePop() +#define ANNOTATE_AGGREGATE_TASK(_COUNT) Annotate::AggregateTask(_COUNT) + +#else + +/* Mark the start of a site (region) to be analyzed by the tool */ +#define ANNOTATE_SITE_BEGIN(_SITE) _ANNOTATE_CALL_1(site_beginA, #_SITE) + +/* Mark the end of a site (region) to be analyzed by the tool and + * indicate a WaitForAll task synchronization */ +#define ANNOTATE_SITE_END(...) _ANNOTATE_CALL_0(site_end_2) + +/* Mark the beginning of a region of code that constitutes a task */ +#define ANNOTATE_TASK_BEGIN(_TASK) _ANNOTATE_CALL_1(task_beginA, #_TASK) + +/* Mark the end of a region of code that constitutes a task */ +#define ANNOTATE_TASK_END(...) _ANNOTATE_CALL_0(task_end_2) + +/* Mark the break between one task and the next task (a "split" description model + * rather than a "begin/end" description model. */ +#define ANNOTATE_ITERATION_TASK(_TASK) _ANNOTATE_CALL_1(iteration_taskA, #_TASK) + +/* Acquire a lock identified by lockId */ +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) _ANNOTATE_CALL_1(lock_acquire_2, (_ADDR)) + +/* Release a lock identified by lockId */ +#define ANNOTATE_LOCK_RELEASE(_ADDR) _ANNOTATE_CALL_1(lock_release_2, (_ADDR)) + +/* Record user allocation of memory */ +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) _ANNOTATE_CALL_2(record_allocation, (_ADDR), (_SIZE)) + +/* Record user deallocation of memory */ +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) _ANNOTATE_CALL_1(record_deallocation, (_ADDR)) + +/* Denote storage as an inductive value */ +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(induction_uses, (_ADDR), (_SIZE)) + +/* Denote storage as a reduction */ +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(reduction_uses, (_ADDR), (_SIZE)) + +/* Record all observations of uses */ +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(observe_uses, (_ADDR), (_SIZE)) + +/* Clear handling of values */ +#define ANNOTATE_CLEAR_USES(_ADDR) _ANNOTATE_CALL_1(clear_uses, (_ADDR)) + +/* Push disable of observations */ +#define ANNOTATE_DISABLE_OBSERVATION_PUSH _ANNOTATE_CALL_1(disable_push, __itt_model_disable_observation) + +/* Pop disable of observations */ +#define ANNOTATE_DISABLE_OBSERVATION_POP _ANNOTATE_CALL_0(disable_pop) + +/* Push disable of collection */ +#define ANNOTATE_DISABLE_COLLECTION_PUSH _ANNOTATE_CALL_1(disable_push, __itt_model_disable_collection) + +/* Pop disable of collection */ +#define ANNOTATE_DISABLE_COLLECTION_POP _ANNOTATE_CALL_0(disable_pop) + +/* Task aggregation */ +#define ANNOTATE_AGGREGATE_TASK(_COUNT) _ANNOTATE_CALL_1(aggregate_task, (_COUNT)) + +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* ANNOTATE_EXPAND_NULL */ + +#endif /* _ADVISOR_ANNOTATE_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/cpuinfo.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/cpuinfo.h new file mode 100644 index 0000000000000000000000000000000000000000..8c6ec9a87299d3d5fd694b7c9090da902cc44b60 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/cpuinfo.h @@ -0,0 +1,2366 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef CPUINFO_H +#define CPUINFO_H + +#ifndef __cplusplus +#include +#endif + +#ifdef __APPLE__ +#include +#endif + +#include + +/* Identify architecture and define corresponding macro */ + +#if defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) +#define CPUINFO_ARCH_X86 1 +#endif + +#if defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define CPUINFO_ARCH_X86_64 1 +#endif + +#if defined(__arm__) || defined(_M_ARM) +#define CPUINFO_ARCH_ARM 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define CPUINFO_ARCH_ARM64 1 +#endif + +#if defined(__PPC64__) || defined(__powerpc64__) || defined(_ARCH_PPC64) +#define CPUINFO_ARCH_PPC64 1 +#endif + +#if defined(__asmjs__) +#define CPUINFO_ARCH_ASMJS 1 +#endif + +#if defined(__wasm__) +#if defined(__wasm_simd128__) +#define CPUINFO_ARCH_WASMSIMD 1 +#else +#define CPUINFO_ARCH_WASM 1 +#endif +#endif + +#if defined(__riscv) +#if (__riscv_xlen == 32) +#define CPUINFO_ARCH_RISCV32 1 +#elif (__riscv_xlen == 64) +#define CPUINFO_ARCH_RISCV64 1 +#endif +#endif + +/* Define other architecture-specific macros as 0 */ + +#ifndef CPUINFO_ARCH_X86 +#define CPUINFO_ARCH_X86 0 +#endif + +#ifndef CPUINFO_ARCH_X86_64 +#define CPUINFO_ARCH_X86_64 0 +#endif + +#ifndef CPUINFO_ARCH_ARM +#define CPUINFO_ARCH_ARM 0 +#endif + +#ifndef CPUINFO_ARCH_ARM64 +#define CPUINFO_ARCH_ARM64 0 +#endif + +#ifndef CPUINFO_ARCH_PPC64 +#define CPUINFO_ARCH_PPC64 0 +#endif + +#ifndef CPUINFO_ARCH_ASMJS +#define CPUINFO_ARCH_ASMJS 0 +#endif + +#ifndef CPUINFO_ARCH_WASM +#define CPUINFO_ARCH_WASM 0 +#endif + +#ifndef CPUINFO_ARCH_WASMSIMD +#define CPUINFO_ARCH_WASMSIMD 0 +#endif + +#ifndef CPUINFO_ARCH_RISCV32 +#define CPUINFO_ARCH_RISCV32 0 +#endif + +#ifndef CPUINFO_ARCH_RISCV64 +#define CPUINFO_ARCH_RISCV64 0 +#endif + +#if CPUINFO_ARCH_X86 && defined(_MSC_VER) +#define CPUINFO_ABI __cdecl +#elif CPUINFO_ARCH_X86 && defined(__GNUC__) +#define CPUINFO_ABI __attribute__((__cdecl__)) +#else +#define CPUINFO_ABI +#endif + +#define CPUINFO_CACHE_UNIFIED 0x00000001 +#define CPUINFO_CACHE_INCLUSIVE 0x00000002 +#define CPUINFO_CACHE_COMPLEX_INDEXING 0x00000004 + +struct cpuinfo_cache { + /** Cache size in bytes */ + uint32_t size; + /** Number of ways of associativity */ + uint32_t associativity; + /** Number of sets */ + uint32_t sets; + /** Number of partitions */ + uint32_t partitions; + /** Line size in bytes */ + uint32_t line_size; + /** + * Binary characteristics of the cache (unified cache, inclusive cache, + * cache with complex indexing). + * + * @see CPUINFO_CACHE_UNIFIED, CPUINFO_CACHE_INCLUSIVE, + * CPUINFO_CACHE_COMPLEX_INDEXING + */ + uint32_t flags; + /** Index of the first logical processor that shares this cache */ + uint32_t processor_start; + /** Number of logical processors that share this cache */ + uint32_t processor_count; +}; + +struct cpuinfo_trace_cache { + uint32_t uops; + uint32_t associativity; +}; + +#define CPUINFO_PAGE_SIZE_4KB 0x1000 +#define CPUINFO_PAGE_SIZE_1MB 0x100000 +#define CPUINFO_PAGE_SIZE_2MB 0x200000 +#define CPUINFO_PAGE_SIZE_4MB 0x400000 +#define CPUINFO_PAGE_SIZE_16MB 0x1000000 +#define CPUINFO_PAGE_SIZE_1GB 0x40000000 + +struct cpuinfo_tlb { + uint32_t entries; + uint32_t associativity; + uint64_t pages; +}; + +/** Vendor of processor core design */ +enum cpuinfo_vendor { + /** Processor vendor is not known to the library, or the library failed + to get vendor information from the OS. */ + cpuinfo_vendor_unknown = 0, + + /* Active vendors of modern CPUs */ + + /** + * Intel Corporation. Vendor of x86, x86-64, IA64, and ARM processor + * microarchitectures. + * + * Sold its ARM design subsidiary in 2006. The last ARM processor design + * was released in 2004. + */ + cpuinfo_vendor_intel = 1, + /** Advanced Micro Devices, Inc. Vendor of x86 and x86-64 processor + microarchitectures. */ + cpuinfo_vendor_amd = 2, + /** ARM Holdings plc. Vendor of ARM and ARM64 processor + microarchitectures. */ + cpuinfo_vendor_arm = 3, + /** Qualcomm Incorporated. Vendor of ARM and ARM64 processor + microarchitectures. */ + cpuinfo_vendor_qualcomm = 4, + /** Apple Inc. Vendor of ARM and ARM64 processor microarchitectures. */ + cpuinfo_vendor_apple = 5, + /** Samsung Electronics Co., Ltd. Vendir if ARM64 processor + microarchitectures. */ + cpuinfo_vendor_samsung = 6, + /** Nvidia Corporation. Vendor of ARM64-compatible processor + microarchitectures. */ + cpuinfo_vendor_nvidia = 7, + /** MIPS Technologies, Inc. Vendor of MIPS processor microarchitectures. + */ + cpuinfo_vendor_mips = 8, + /** International Business Machines Corporation. Vendor of PowerPC + processor microarchitectures. */ + cpuinfo_vendor_ibm = 9, + /** Ingenic Semiconductor. Vendor of MIPS processor microarchitectures. + */ + cpuinfo_vendor_ingenic = 10, + /** + * VIA Technologies, Inc. Vendor of x86 and x86-64 processor + * microarchitectures. + * + * Processors are designed by Centaur Technology, a subsidiary of VIA + * Technologies. + */ + cpuinfo_vendor_via = 11, + /** Cavium, Inc. Vendor of ARM64 processor microarchitectures. */ + cpuinfo_vendor_cavium = 12, + /** Broadcom, Inc. Vendor of ARM processor microarchitectures. */ + cpuinfo_vendor_broadcom = 13, + /** Applied Micro Circuits Corporation (APM). Vendor of ARM64 processor + microarchitectures. */ + cpuinfo_vendor_apm = 14, + /** + * Huawei Technologies Co., Ltd. Vendor of ARM64 processor + * microarchitectures. + * + * Processors are designed by HiSilicon, a subsidiary of Huawei. + */ + cpuinfo_vendor_huawei = 15, + /** + * Hygon (Chengdu Haiguang Integrated Circuit Design Co., Ltd), Vendor + * of x86-64 processor microarchitectures. + * + * Processors are variants of AMD cores. + */ + cpuinfo_vendor_hygon = 16, + /** SiFive, Inc. Vendor of RISC-V processor microarchitectures. */ + cpuinfo_vendor_sifive = 17, + + /* Active vendors of embedded CPUs */ + + /** Texas Instruments Inc. Vendor of ARM processor microarchitectures. + */ + cpuinfo_vendor_texas_instruments = 30, + /** Marvell Technology Group Ltd. Vendor of ARM processor + * microarchitectures. + */ + cpuinfo_vendor_marvell = 31, + /** RDC Semiconductor Co., Ltd. Vendor of x86 processor + microarchitectures. */ + cpuinfo_vendor_rdc = 32, + /** DM&P Electronics Inc. Vendor of x86 processor microarchitectures. */ + cpuinfo_vendor_dmp = 33, + /** Motorola, Inc. Vendor of PowerPC and ARM processor + microarchitectures. */ + cpuinfo_vendor_motorola = 34, + + /* Defunct CPU vendors */ + + /** + * Transmeta Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 2004. + * Transmeta processors implemented VLIW ISA and used binary translation + * to execute x86 code. + */ + cpuinfo_vendor_transmeta = 50, + /** + * Cyrix Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1996. + */ + cpuinfo_vendor_cyrix = 51, + /** + * Rise Technology. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1999. + */ + cpuinfo_vendor_rise = 52, + /** + * National Semiconductor. Vendor of x86 processor microarchitectures. + * + * Sold its x86 design subsidiary in 1999. The last processor design was + * released in 1998. + */ + cpuinfo_vendor_nsc = 53, + /** + * Silicon Integrated Systems. Vendor of x86 processor + * microarchitectures. + * + * Sold its x86 design subsidiary in 2001. The last processor design was + * released in 2001. + */ + cpuinfo_vendor_sis = 54, + /** + * NexGen. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1994. + * NexGen designed the first x86 microarchitecture which decomposed x86 + * instructions into simple microoperations. + */ + cpuinfo_vendor_nexgen = 55, + /** + * United Microelectronics Corporation. Vendor of x86 processor + * microarchitectures. + * + * Ceased x86 in the early 1990s. The last processor design was released + * in 1991. Designed U5C and U5D processors. Both are 486 level. + */ + cpuinfo_vendor_umc = 56, + /** + * Digital Equipment Corporation. Vendor of ARM processor + * microarchitecture. + * + * Sold its ARM designs in 1997. The last processor design was released + * in 1997. + */ + cpuinfo_vendor_dec = 57, +}; + +/** + * Processor microarchitecture + * + * Processors with different microarchitectures often have different instruction + * performance characteristics, and may have dramatically different pipeline + * organization. + */ +enum cpuinfo_uarch { + /** Microarchitecture is unknown, or the library failed to get + information about the microarchitecture from OS */ + cpuinfo_uarch_unknown = 0, + + /** Pentium and Pentium MMX microarchitecture. */ + cpuinfo_uarch_p5 = 0x00100100, + /** Intel Quark microarchitecture. */ + cpuinfo_uarch_quark = 0x00100101, + + /** Pentium Pro, Pentium II, and Pentium III. */ + cpuinfo_uarch_p6 = 0x00100200, + /** Pentium M. */ + cpuinfo_uarch_dothan = 0x00100201, + /** Intel Core microarchitecture. */ + cpuinfo_uarch_yonah = 0x00100202, + /** Intel Core 2 microarchitecture on 65 nm process. */ + cpuinfo_uarch_conroe = 0x00100203, + /** Intel Core 2 microarchitecture on 45 nm process. */ + cpuinfo_uarch_penryn = 0x00100204, + /** Intel Nehalem and Westmere microarchitectures (Core i3/i5/i7 1st + gen). */ + cpuinfo_uarch_nehalem = 0x00100205, + /** Intel Sandy Bridge microarchitecture (Core i3/i5/i7 2nd gen). */ + cpuinfo_uarch_sandy_bridge = 0x00100206, + /** Intel Ivy Bridge microarchitecture (Core i3/i5/i7 3rd gen). */ + cpuinfo_uarch_ivy_bridge = 0x00100207, + /** Intel Haswell microarchitecture (Core i3/i5/i7 4th gen). */ + cpuinfo_uarch_haswell = 0x00100208, + /** Intel Broadwell microarchitecture. */ + cpuinfo_uarch_broadwell = 0x00100209, + /** Intel Sky Lake microarchitecture (14 nm, including + Kaby/Coffee/Whiskey/Amber/Comet/Cascade/Cooper Lake). */ + cpuinfo_uarch_sky_lake = 0x0010020A, + /** DEPRECATED (Intel Kaby Lake microarchitecture). */ + cpuinfo_uarch_kaby_lake = 0x0010020A, + /** Intel Palm Cove microarchitecture (10 nm, Cannon Lake). */ + cpuinfo_uarch_palm_cove = 0x0010020B, + /** Intel Sunny Cove microarchitecture (10 nm, Ice Lake). */ + cpuinfo_uarch_sunny_cove = 0x0010020C, + /** Intel Willow Cove microarchitecture (10 nm, Tiger Lake). */ + cpuinfo_uarch_willow_cove = 0x0010020D, + + /** Pentium 4 with Willamette, Northwood, or Foster cores. */ + cpuinfo_uarch_willamette = 0x00100300, + /** Pentium 4 with Prescott and later cores. */ + cpuinfo_uarch_prescott = 0x00100301, + + /** Intel Atom on 45 nm process. */ + cpuinfo_uarch_bonnell = 0x00100400, + /** Intel Atom on 32 nm process. */ + cpuinfo_uarch_saltwell = 0x00100401, + /** Intel Silvermont microarchitecture (22 nm out-of-order Atom). */ + cpuinfo_uarch_silvermont = 0x00100402, + /** Intel Airmont microarchitecture (14 nm out-of-order Atom). */ + cpuinfo_uarch_airmont = 0x00100403, + /** Intel Goldmont microarchitecture (Denverton, Apollo Lake). */ + cpuinfo_uarch_goldmont = 0x00100404, + /** Intel Goldmont Plus microarchitecture (Gemini Lake). */ + cpuinfo_uarch_goldmont_plus = 0x00100405, + /** Intel Airmont microarchitecture (10 nm out-of-order Atom). */ + cpuinfo_uarch_tremont = 0x00100406, + /** Intel Gracemont microarchitecture (AlderLake N). */ + cpuinfo_uarch_gracemont = 0x00100407, + /** Intel Crestmont microarchitecture (Sierra Forest). */ + cpuinfo_uarch_crestmont = 0x00100408, + /** Intel Darkmont microarchitecture (e-core used in Clearwater Forest). */ + cpuinfo_uarch_darkmont = 0x00100409, + + /** Intel Knights Ferry HPC boards. */ + cpuinfo_uarch_knights_ferry = 0x00100500, + /** Intel Knights Corner HPC boards (aka Xeon Phi). */ + cpuinfo_uarch_knights_corner = 0x00100501, + /** Intel Knights Landing microarchitecture (second-gen MIC). */ + cpuinfo_uarch_knights_landing = 0x00100502, + /** Intel Knights Hill microarchitecture (third-gen MIC). */ + cpuinfo_uarch_knights_hill = 0x00100503, + /** Intel Knights Mill Xeon Phi. */ + cpuinfo_uarch_knights_mill = 0x00100504, + + /** Intel/Marvell XScale series. */ + cpuinfo_uarch_xscale = 0x00100600, + + /** AMD K5. */ + cpuinfo_uarch_k5 = 0x00200100, + /** AMD K6 and alike. */ + cpuinfo_uarch_k6 = 0x00200101, + /** AMD Athlon and Duron. */ + cpuinfo_uarch_k7 = 0x00200102, + /** AMD Athlon 64, Opteron 64. */ + cpuinfo_uarch_k8 = 0x00200103, + /** AMD Family 10h (Barcelona, Istambul, Magny-Cours). */ + cpuinfo_uarch_k10 = 0x00200104, + /** + * AMD Bulldozer microarchitecture + * Zambezi FX-series CPUs, Zurich, Valencia and Interlagos Opteron CPUs. + */ + cpuinfo_uarch_bulldozer = 0x00200105, + /** + * AMD Piledriver microarchitecture + * Vishera FX-series CPUs, Trinity and Richland APUs, Delhi, Seoul, Abu + * Dhabi Opteron CPUs. + */ + cpuinfo_uarch_piledriver = 0x00200106, + /** AMD Steamroller microarchitecture (Kaveri APUs). */ + cpuinfo_uarch_steamroller = 0x00200107, + /** AMD Excavator microarchitecture (Carizzo APUs). */ + cpuinfo_uarch_excavator = 0x00200108, + /** AMD Zen microarchitecture (12/14 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen = 0x00200109, + /** AMD Zen 2 microarchitecture (7 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen2 = 0x0020010A, + /** AMD Zen 3 microarchitecture. */ + cpuinfo_uarch_zen3 = 0x0020010B, + /** AMD Zen 4 microarchitecture. */ + cpuinfo_uarch_zen4 = 0x0020010C, + /** AMD Zen 5 microarchitecture. */ + cpuinfo_uarch_zen5 = 0x0020010D, + + /** NSC Geode and AMD Geode GX and LX. */ + cpuinfo_uarch_geode = 0x00200200, + /** AMD Bobcat mobile microarchitecture. */ + cpuinfo_uarch_bobcat = 0x00200201, + /** AMD Jaguar mobile microarchitecture. */ + cpuinfo_uarch_jaguar = 0x00200202, + /** AMD Puma mobile microarchitecture. */ + cpuinfo_uarch_puma = 0x00200203, + + /** ARM7 series. */ + cpuinfo_uarch_arm7 = 0x00300100, + /** ARM9 series. */ + cpuinfo_uarch_arm9 = 0x00300101, + /** ARM 1136, ARM 1156, ARM 1176, or ARM 11MPCore. */ + cpuinfo_uarch_arm11 = 0x00300102, + + /** ARM Cortex-A5. */ + cpuinfo_uarch_cortex_a5 = 0x00300205, + /** ARM Cortex-A7. */ + cpuinfo_uarch_cortex_a7 = 0x00300207, + /** ARM Cortex-A8. */ + cpuinfo_uarch_cortex_a8 = 0x00300208, + /** ARM Cortex-A9. */ + cpuinfo_uarch_cortex_a9 = 0x00300209, + /** ARM Cortex-A12. */ + cpuinfo_uarch_cortex_a12 = 0x00300212, + /** ARM Cortex-A15. */ + cpuinfo_uarch_cortex_a15 = 0x00300215, + /** ARM Cortex-A17. */ + cpuinfo_uarch_cortex_a17 = 0x00300217, + + /** ARM Cortex-A32. */ + cpuinfo_uarch_cortex_a32 = 0x00300332, + /** ARM Cortex-A35. */ + cpuinfo_uarch_cortex_a35 = 0x00300335, + /** ARM Cortex-A53. */ + cpuinfo_uarch_cortex_a53 = 0x00300353, + /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities + compared to revision 1+). */ + cpuinfo_uarch_cortex_a55r0 = 0x00300354, + /** ARM Cortex-A55. */ + cpuinfo_uarch_cortex_a55 = 0x00300355, + /** ARM Cortex-A57. */ + cpuinfo_uarch_cortex_a57 = 0x00300357, + /** ARM Cortex-A65. */ + cpuinfo_uarch_cortex_a65 = 0x00300365, + /** ARM Cortex-A72. */ + cpuinfo_uarch_cortex_a72 = 0x00300372, + /** ARM Cortex-A73. */ + cpuinfo_uarch_cortex_a73 = 0x00300373, + /** ARM Cortex-A75. */ + cpuinfo_uarch_cortex_a75 = 0x00300375, + /** ARM Cortex-A76. */ + cpuinfo_uarch_cortex_a76 = 0x00300376, + /** ARM Cortex-A77. */ + cpuinfo_uarch_cortex_a77 = 0x00300377, + /** ARM Cortex-A78. */ + cpuinfo_uarch_cortex_a78 = 0x00300378, + + /** ARM Neoverse N1. */ + cpuinfo_uarch_neoverse_n1 = 0x00300400, + /** ARM Neoverse E1. */ + cpuinfo_uarch_neoverse_e1 = 0x00300401, + /** ARM Neoverse V1. */ + cpuinfo_uarch_neoverse_v1 = 0x00300402, + /** ARM Neoverse N2. */ + cpuinfo_uarch_neoverse_n2 = 0x00300403, + /** ARM Neoverse V2. */ + cpuinfo_uarch_neoverse_v2 = 0x00300404, + + /** ARM Cortex-X1. */ + cpuinfo_uarch_cortex_x1 = 0x00300501, + /** ARM Cortex-X2. */ + cpuinfo_uarch_cortex_x2 = 0x00300502, + /** ARM Cortex-X3. */ + cpuinfo_uarch_cortex_x3 = 0x00300503, + /** ARM Cortex-X4. */ + cpuinfo_uarch_cortex_x4 = 0x00300504, + /** ARM Cortex-X925. */ + cpuinfo_uarch_cortex_x925 = 0x00300505, + + /** ARM Cortex-A510. */ + cpuinfo_uarch_cortex_a510 = 0x00300551, + /** ARM Cortex-A520. */ + cpuinfo_uarch_cortex_a520 = 0x00300552, + /** ARM Cortex-A710. */ + cpuinfo_uarch_cortex_a710 = 0x00300571, + /** ARM Cortex-A715. */ + cpuinfo_uarch_cortex_a715 = 0x00300572, + /** ARM Cortex-A720. */ + cpuinfo_uarch_cortex_a720 = 0x00300573, + /** ARM Cortex-A725. */ + cpuinfo_uarch_cortex_a725 = 0x00300574, + + /** Qualcomm Scorpion. */ + cpuinfo_uarch_scorpion = 0x00400100, + /** Qualcomm Krait. */ + cpuinfo_uarch_krait = 0x00400101, + /** Qualcomm Kryo. */ + cpuinfo_uarch_kryo = 0x00400102, + /** Qualcomm Falkor. */ + cpuinfo_uarch_falkor = 0x00400103, + /** Qualcomm Saphira. */ + cpuinfo_uarch_saphira = 0x00400104, + /** Qualcomm Oryon. */ + cpuinfo_uarch_oryon = 0x00400105, + + /** Nvidia Denver. */ + cpuinfo_uarch_denver = 0x00500100, + /** Nvidia Denver 2. */ + cpuinfo_uarch_denver2 = 0x00500101, + /** Nvidia Carmel. */ + cpuinfo_uarch_carmel = 0x00500102, + + /** Samsung Exynos M1 (Exynos 8890 big cores). */ + cpuinfo_uarch_exynos_m1 = 0x00600100, + /** Samsung Exynos M2 (Exynos 8895 big cores). */ + cpuinfo_uarch_exynos_m2 = 0x00600101, + /** Samsung Exynos M3 (Exynos 9810 big cores). */ + cpuinfo_uarch_exynos_m3 = 0x00600102, + /** Samsung Exynos M4 (Exynos 9820 big cores). */ + cpuinfo_uarch_exynos_m4 = 0x00600103, + /** Samsung Exynos M5 (Exynos 9830 big cores). */ + cpuinfo_uarch_exynos_m5 = 0x00600104, + + /* Deprecated synonym for Cortex-A76 */ + cpuinfo_uarch_cortex_a76ae = 0x00300376, + /* Deprecated names for Exynos. */ + cpuinfo_uarch_mongoose_m1 = 0x00600100, + cpuinfo_uarch_mongoose_m2 = 0x00600101, + cpuinfo_uarch_meerkat_m3 = 0x00600102, + cpuinfo_uarch_meerkat_m4 = 0x00600103, + + /** Apple A6 and A6X processors. */ + cpuinfo_uarch_swift = 0x00700100, + /** Apple A7 processor. */ + cpuinfo_uarch_cyclone = 0x00700101, + /** Apple A8 and A8X processor. */ + cpuinfo_uarch_typhoon = 0x00700102, + /** Apple A9 and A9X processor. */ + cpuinfo_uarch_twister = 0x00700103, + /** Apple A10 and A10X processor. */ + cpuinfo_uarch_hurricane = 0x00700104, + /** Apple A11 processor (big cores). */ + cpuinfo_uarch_monsoon = 0x00700105, + /** Apple A11 processor (little cores). */ + cpuinfo_uarch_mistral = 0x00700106, + /** Apple A12 processor (big cores). */ + cpuinfo_uarch_vortex = 0x00700107, + /** Apple A12 processor (little cores). */ + cpuinfo_uarch_tempest = 0x00700108, + /** Apple A13 processor (big cores). */ + cpuinfo_uarch_lightning = 0x00700109, + /** Apple A13 processor (little cores). */ + cpuinfo_uarch_thunder = 0x0070010A, + /** Apple A14 / M1 processor (big cores). */ + cpuinfo_uarch_firestorm = 0x0070010B, + /** Apple A14 / M1 processor (little cores). */ + cpuinfo_uarch_icestorm = 0x0070010C, + /** Apple A15 / M2 processor (big cores). */ + cpuinfo_uarch_avalanche = 0x0070010D, + /** Apple A15 / M2 processor (little cores). */ + cpuinfo_uarch_blizzard = 0x0070010E, + /** Apple A16 processor (big cores). */ + cpuinfo_uarch_everest = 0x00700200, + /** Apple A16 processor (little cores). */ + cpuinfo_uarch_sawtooth = 0x00700201, + /** Apple A17 processor (big cores). */ + cpuinfo_uarch_coll_everest = 0x00700202, + /** Apple A17 processor (little cores). */ + cpuinfo_uarch_coll_sawtooth = 0x00700203, + /** Apple A18 processor (big cores). */ + cpuinfo_uarch_tupai_everest = 0x00700204, + /** Apple A18 processor (little cores). */ + cpuinfo_uarch_tupai_sawtooth = 0x00700205, + /** Apple A18 pro processor (big cores). */ + cpuinfo_uarch_tahiti_everest = 0x00700206, + /** Apple A18 pro processor (little cores). */ + cpuinfo_uarch_tahiti_sawtooth = 0x00700207, + + /** Cavium ThunderX. */ + cpuinfo_uarch_thunderx = 0x00800100, + /** Cavium ThunderX2 (originally Broadcom Vulkan). */ + cpuinfo_uarch_thunderx2 = 0x00800200, + + /** Marvell PJ4. */ + cpuinfo_uarch_pj4 = 0x00900100, + + /** Broadcom Brahma B15. */ + cpuinfo_uarch_brahma_b15 = 0x00A00100, + /** Broadcom Brahma B53. */ + cpuinfo_uarch_brahma_b53 = 0x00A00101, + + /** Applied Micro X-Gene. */ + cpuinfo_uarch_xgene = 0x00B00100, + + /* Hygon Dhyana (a modification of AMD Zen for Chinese market). */ + cpuinfo_uarch_dhyana = 0x01000100, + + /** HiSilicon TaiShan v110 (Huawei Kunpeng 920 series processors). */ + cpuinfo_uarch_taishan_v110 = 0x00C00100, +}; + +struct cpuinfo_processor { + /** SMT (hyperthread) ID within a core */ + uint32_t smt_id; + /** Core containing this logical processor */ + const struct cpuinfo_core* core; + /** Cluster of cores containing this logical processor */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this logical processor */ + const struct cpuinfo_package* package; +#if defined(__linux__) + /** + * Linux-specific ID for the logical processor: + * - Linux kernel exposes information about this logical processor in + * /sys/devices/system/cpu/cpu/ + * - Bit in the cpu_set_t identifies this logical processor + */ + int linux_id; +#endif +#if defined(_WIN32) || defined(__CYGWIN__) + /** Windows-specific ID for the group containing the logical processor. + */ + uint16_t windows_group_id; + /** + * Windows-specific ID of the logical processor within its group: + * - Bit in the KAFFINITY mask identifies this + * logical processor within its group. + */ + uint16_t windows_processor_id; +#endif +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** APIC ID (unique x86-specific ID of the logical processor) */ + uint32_t apic_id; +#endif + struct { + /** Level 1 instruction cache */ + const struct cpuinfo_cache* l1i; + /** Level 1 data cache */ + const struct cpuinfo_cache* l1d; + /** Level 2 unified or data cache */ + const struct cpuinfo_cache* l2; + /** Level 3 unified or data cache */ + const struct cpuinfo_cache* l3; + /** Level 4 unified or data cache */ + const struct cpuinfo_cache* l4; + } cache; +}; + +struct cpuinfo_core { + /** Index of the first logical processor on this core. */ + uint32_t processor_start; + /** Number of logical processors on this core */ + uint32_t processor_count; + /** Core ID within a package */ + uint32_t core_id; + /** Cluster containing this core */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this core. */ + const struct cpuinfo_package* package; + /** Vendor of the CPU microarchitecture for this core */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture for this core */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for this core */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for this core */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the core, in Hz */ + uint64_t frequency; +}; + +struct cpuinfo_cluster { + /** Index of the first logical processor in the cluster */ + uint32_t processor_start; + /** Number of logical processors in the cluster */ + uint32_t processor_count; + /** Index of the first core in the cluster */ + uint32_t core_start; + /** Number of cores on the cluster */ + uint32_t core_count; + /** Cluster ID within a package */ + uint32_t cluster_id; + /** Physical package containing the cluster */ + const struct cpuinfo_package* package; + /** CPU microarchitecture vendor of the cores in the cluster */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture of the cores in the cluster */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register of the cores in the cluster */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) of the cores in the cluster */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the cores in the cluster, in Hz */ + uint64_t frequency; +}; + +#define CPUINFO_PACKAGE_NAME_MAX 64 + +struct cpuinfo_package { + /** SoC or processor chip model name */ + char name[CPUINFO_PACKAGE_NAME_MAX]; + /** Index of the first logical processor on this physical package */ + uint32_t processor_start; + /** Number of logical processors on this physical package */ + uint32_t processor_count; + /** Index of the first core on this physical package */ + uint32_t core_start; + /** Number of cores on this physical package */ + uint32_t core_count; + /** Index of the first cluster of cores on this physical package */ + uint32_t cluster_start; + /** Number of clusters of cores on this physical package */ + uint32_t cluster_count; +}; + +struct cpuinfo_uarch_info { + /** Type of CPU microarchitecture */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for the microarchitecture */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for the microarchitecture */ + uint32_t midr; +#endif + /** Number of logical processors with the microarchitecture */ + uint32_t processor_count; + /** Number of cores with the microarchitecture */ + uint32_t core_count; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +bool CPUINFO_ABI cpuinfo_initialize(void); + +void CPUINFO_ABI cpuinfo_deinitialize(void); + +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +/* This structure is not a part of stable API. Use cpuinfo_has_x86_* functions + * instead. */ +struct cpuinfo_x86_isa { +#if CPUINFO_ARCH_X86 + bool rdtsc; +#endif + bool rdtscp; + bool rdpid; + bool sysenter; +#if CPUINFO_ARCH_X86 + bool syscall; +#endif + bool msr; + bool clzero; + bool clflush; + bool clflushopt; + bool mwait; + bool mwaitx; +#if CPUINFO_ARCH_X86 + bool emmx; +#endif + bool fxsave; + bool xsave; +#if CPUINFO_ARCH_X86 + bool fpu; + bool mmx; + bool mmx_plus; +#endif + bool three_d_now; + bool three_d_now_plus; +#if CPUINFO_ARCH_X86 + bool three_d_now_geode; +#endif + bool prefetch; + bool prefetchw; + bool prefetchwt1; +#if CPUINFO_ARCH_X86 + bool daz; + bool sse; + bool sse2; +#endif + bool sse3; + bool ssse3; + bool sse4_1; + bool sse4_2; + bool sse4a; + bool misaligned_sse; + bool avx; + bool avxvnni; + bool fma3; + bool fma4; + bool xop; + bool f16c; + bool avx2; + bool avx512f; + bool avx512pf; + bool avx512er; + bool avx512cd; + bool avx512dq; + bool avx512bw; + bool avx512vl; + bool avx512ifma; + bool avx512vbmi; + bool avx512vbmi2; + bool avx512bitalg; + bool avx512vpopcntdq; + bool avx512vnni; + bool avx512bf16; + bool avx512fp16; + bool avx512vp2intersect; + bool avx512_4vnniw; + bool avx512_4fmaps; + bool avx10_1; + bool avx10_2; + bool amx_bf16; + bool amx_tile; + bool amx_int8; + bool amx_fp16; + bool avx_vnni_int8; + bool avx_vnni_int16; + bool avx_ne_convert; + bool hle; + bool rtm; + bool xtest; + bool mpx; +#if CPUINFO_ARCH_X86 + bool cmov; + bool cmpxchg8b; +#endif + bool cmpxchg16b; + bool clwb; + bool movbe; +#if CPUINFO_ARCH_X86_64 + bool lahf_sahf; +#endif + bool fs_gs_base; + bool lzcnt; + bool popcnt; + bool tbm; + bool bmi; + bool bmi2; + bool adx; + bool aes; + bool vaes; + bool pclmulqdq; + bool vpclmulqdq; + bool gfni; + bool rdrand; + bool rdseed; + bool sha; + bool rng; + bool ace; + bool ace2; + bool phe; + bool pmm; + bool lwp; +}; + +extern struct cpuinfo_x86_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_x86_rdtsc(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.rdtsc; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdtscp(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdtscp; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdpid(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdpid; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_clzero(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clzero; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mwait(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwait; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mwaitx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwaitx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fxsave(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fxsave; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xsave(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xsave; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fpu(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.fpu; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mmx(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.mmx; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mmx_plus(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.mmx_plus; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow_plus(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now_plus; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow_geode(void) { +#if CPUINFO_ARCH_X86_64 + return false; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return false; +#else + return cpuinfo_isa.three_d_now_geode; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetch(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetch; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetchw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetchwt1(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchwt1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_daz(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.daz; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse2(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse2; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse3; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_ssse3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.ssse3; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4_1(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse4_1; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4_2(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse4_2; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4a(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sse4a; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_misaligned_sse(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.misaligned_sse; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avxvnni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avxvnni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fma3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma3; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fma4(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma4; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xop(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xop; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_f16c(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.f16c; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512f(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512f; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512pf(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512pf; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512er(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512er; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512cd(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512cd; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512dq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512dq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vl(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vl; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512ifma(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512ifma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bitalg(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bitalg; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vpopcntdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vpopcntdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vnni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vnni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bf16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512fp16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vp2intersect(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vp2intersect; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512_4vnniw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4vnniw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512_4fmaps(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4fmaps; +#else + return false; +#endif +} + +/* [NOTE] Intel Advanced Matrix Extensions (AMX) detection + * + * I. AMX is a new extensions to the x86 ISA to work on matrices, consists of + * 1) 2-dimentional registers (tiles), hold sub-matrices from larger matrices in memory + * 2) Accelerator called Tile Matrix Multiply (TMUL), contains instructions operating on tiles + * + * II. Platforms that supports AMX: + * +-----------------+-----+----------+----------+----------+----------+ + * | Platforms | Gen | amx-bf16 | amx-tile | amx-int8 | amx-fp16 | + * +-----------------+-----+----------+----------+----------+----------+ + * | Sapphire Rapids | 4th | YES | YES | YES | NO | + * +-----------------+-----+----------+----------+----------+----------+ + * | Emerald Rapids | 5th | YES | YES | YES | NO | + * +-----------------+-----+----------+----------+----------+----------+ + * | Granite Rapids | 6th | YES | YES | YES | YES | + * +-----------------+-----+----------+----------+----------+----------+ + * + * Reference: https://www.intel.com/content/www/us/en/products/docs + * /accelerator-engines/advanced-matrix-extensions/overview.html + */ +static inline bool cpuinfo_has_x86_amx_bf16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_tile(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_tile; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_int8(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_int8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_fp16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_fp16; +#else + return false; +#endif +} + +/* + * Intel AVX Vector Neural Network Instructions (VNNI) INT8 + * Supported Platfroms: Sierra Forest, Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_vnni_int8(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_vnni_int8; +#else + return false; +#endif +} + +/* + * Intel AVX Vector Neural Network Instructions (VNNI) INT16 + * Supported Platfroms: Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_vnni_int16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_vnni_int16; +#else + return false; +#endif +} + +/* + * A new set of instructions, which can convert low precision floating point + * like BF16/FP16 to high precision floating point FP32, as well as convert FP32 + * elements to BF16. This instruction allows the platform to have improved AI + * capabilities and better compatibility. + * + * Supported Platforms: Sierra Forest, Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_ne_convert(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_ne_convert; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx10_1(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx10_1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx10_2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx10_2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_hle(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.hle; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rtm(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rtm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xtest(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xtest; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mpx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mpx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmov(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmov; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmpxchg8b(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmpxchg8b; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmpxchg16b(void) { +#if CPUINFO_ARCH_X86_64 + return cpuinfo_isa.cmpxchg16b; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_clwb(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clwb; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_movbe(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.movbe; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_lahf_sahf(void) { +#if CPUINFO_ARCH_X86 + return true; +#elif CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lahf_sahf; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_lzcnt(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lzcnt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_popcnt(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.popcnt; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.popcnt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_tbm(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.tbm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_bmi(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_bmi2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_adx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.adx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_aes(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.aes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_vaes(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vaes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_pclmulqdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.pclmulqdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_vpclmulqdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vpclmulqdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_gfni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.gfni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdrand(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdrand; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdseed(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdseed; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sha(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sha; +#else + return false; +#endif +} + +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 +/* This structure is not a part of stable API. Use cpuinfo_has_arm_* functions + * instead. */ +struct cpuinfo_arm_isa { +#if CPUINFO_ARCH_ARM + bool thumb; + bool thumb2; + bool thumbee; + bool jazelle; + bool armv5e; + bool armv6; + bool armv6k; + bool armv7; + bool armv7mp; + bool armv8; + bool idiv; + + bool vfpv2; + bool vfpv3; + bool d32; + bool fp16; + bool fma; + + bool wmmx; + bool wmmx2; + bool neon; +#endif +#if CPUINFO_ARCH_ARM64 + bool atomics; + bool bf16; + bool sve; + bool sve2; + bool i8mm; + bool sme; + bool sme2; + bool sme2p1; + bool sme_i16i32; + bool sme_bi32i32; + bool sme_b16b16; + bool sme_f16f16; + uint32_t svelen; + uint32_t smelen; +#endif + bool rdm; + bool fp16arith; + bool dot; + bool jscvt; + bool fcma; + bool fhm; + + bool aes; + bool sha1; + bool sha2; + bool pmull; + bool crc32; +}; + +extern struct cpuinfo_arm_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_arm_thumb(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_thumb2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v5e(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv5e; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v6(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v6k(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6k; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v7(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v7mp(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7mp; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v8(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.armv8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_idiv(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.idiv; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16 && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv4(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv4_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fp16_arith(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_wmmx(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_wmmx2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fp16(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fma(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_v8(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.armv8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_atomics(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.atomics; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_rdm(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.rdm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fp16_arith(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16arith; +#elif CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fhm(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fhm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_dot(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.dot; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_jscvt(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.jscvt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fcma(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fcma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_i8mm(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.i8mm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_aes(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.aes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sha1(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sha2(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_pmull(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.pmull; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_crc32(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.crc32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve && cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve2(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve2; +#else + return false; +#endif +} + +// Function to get the max SVE vector length on ARM CPU's which support SVE. +static inline uint32_t cpuinfo_get_max_arm_sve_length(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.svelen * 8; // bytes * 8 = bit length(vector length) +#else + return 0; +#endif +} + +// Function to get the max SME vector length on ARM CPU's which support SME. +static inline uint32_t cpuinfo_get_max_arm_sme_length(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.smelen * 8; // bytes * 8 = bit length(vector length) +#else + return 0; +#endif +} + +static inline bool cpuinfo_has_arm_sme(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme2(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme2p1(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme2p1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_i16i32(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_i16i32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_bi32i32(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_bi32i32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_b16b16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_b16b16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_f16f16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_f16f16; +#else + return false; +#endif +} + +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 +/* This structure is not a part of stable API. Use cpuinfo_has_riscv_* functions + * instead. */ +struct cpuinfo_riscv_isa { + /** + * Keep fields in line with the canonical order as defined by + * Section 27.11 Subset Naming Convention. + */ + /* RV32I/64I/128I Base ISA. */ + bool i; +#if CPUINFO_ARCH_RISCV32 + /* RV32E Base ISA. */ + bool e; +#endif + /* Integer Multiply/Divide Extension. */ + bool m; + /* Atomic Extension. */ + bool a; + /* Single-Precision Floating-Point Extension. */ + bool f; + /* Double-Precision Floating-Point Extension. */ + bool d; + /* Compressed Extension. */ + bool c; + /* Vector Extension. */ + bool v; +}; + +extern struct cpuinfo_riscv_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_riscv_i(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.i; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_e(void) { +#if CPUINFO_ARCH_RISCV32 + return cpuinfo_isa.e; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_m(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.m; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_a(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.a; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_f(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.f; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_d(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.d; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_g(void) { + // The 'G' extension is simply shorthand for 'IMAFD'. + return cpuinfo_has_riscv_i() && cpuinfo_has_riscv_m() && cpuinfo_has_riscv_a() && cpuinfo_has_riscv_f() && + cpuinfo_has_riscv_d(); +} + +static inline bool cpuinfo_has_riscv_c(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.c; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_v(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.v; +#else + return false; +#endif +} + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processors(void); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_cores(void); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_clusters(void); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_packages(void); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarchs(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_caches(void); + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processor(uint32_t index); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_core(uint32_t index); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_cluster(uint32_t index); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_package(uint32_t index); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarch(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_cache(uint32_t index); + +uint32_t CPUINFO_ABI cpuinfo_get_processors_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_cores_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_clusters_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_packages_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_uarchs_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l3_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void); + +/** + * Returns upper bound on cache size. + */ +uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void); + +/** + * Identify the logical processor that executes the current thread. + * + * There is no guarantee that the thread will stay on the same logical processor + * for any time. Callers should treat the result as only a hint, and be prepared + * to handle NULL return value. + */ +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void); + +/** + * Identify the core that executes the current thread. + * + * There is no guarantee that the thread will stay on the same core for any + * time. Callers should treat the result as only a hint, and be prepared to + * handle NULL return value. + */ +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void); + +/** + * Identify the microarchitecture index of the core that executes the current + * thread. If the system does not support such identification, the function + * returns 0. + * + * There is no guarantee that the thread will stay on the same type of core for + * any time. Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void); + +/** + * Identify the microarchitecture index of the core that executes the current + * thread. If the system does not support such identification, the function + * returns the user-specified default value. + * + * There is no guarantee that the thread will stay on the same type of core for + * any time. Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index_with_default(uint32_t default_uarch_index); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPUINFO_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.h new file mode 100644 index 0000000000000000000000000000000000000000..70f0687fa0f610873c0b1211f6007ed31daf04a8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_H +#define DNNL_H + +#include "oneapi/dnnl/dnnl.h" + +#endif /* DNNL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.hpp b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..225ea4860ceb6b68574162668b491b3170be4ee5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_HPP +#define DNNL_HPP + +#include "oneapi/dnnl/dnnl.hpp" + +#endif /* DNNL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_config.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_config.h new file mode 100644 index 0000000000000000000000000000000000000000..a8a36850d5dab208fa7786a4ae185f9e86edd2a5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_config.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_CONFIG_H +#define DNNL_CONFIG_H + +#include "oneapi/dnnl/dnnl_config.h" + +#endif /* DNNL_CONFIG_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_debug.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_debug.h new file mode 100644 index 0000000000000000000000000000000000000000..b6a46ccd1c5ddd63ca24aebecd9244274f208193 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_debug.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_DEBUG_H +#define DNNL_DEBUG_H + +#include "oneapi/dnnl/dnnl_debug.h" + +#endif /* DNNL_DEBUG_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.h new file mode 100644 index 0000000000000000000000000000000000000000..f51beae792a6a4f6dca7de9a9d0945255f185b59 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_OCL_H +#define DNNL_OCL_H + +#include "oneapi/dnnl/dnnl_ocl.h" + +#endif /* DNNL_OCL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.hpp b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..1349678ac6456d84f8ea869d95a3f2a61a7d7205 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_ocl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_OCL_HPP +#define DNNL_OCL_HPP + +#include "oneapi/dnnl/dnnl_ocl.hpp" + +#endif /* DNNL_OCL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.h new file mode 100644 index 0000000000000000000000000000000000000000..09741d39990dbe318af531d5e7cf2a2715302444 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_H +#define DNNL_SYCL_H + +#include "oneapi/dnnl/dnnl_sycl.h" + +#endif /* DNNL_SYCL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.hpp b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..5c3758da99817e9ecf828ce07b0328d19340bd39 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_HPP +#define DNNL_SYCL_HPP + +#include "oneapi/dnnl/dnnl_sycl.hpp" + +#endif /* DNNL_SYCL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl_types.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl_types.h new file mode 100644 index 0000000000000000000000000000000000000000..199a51cac0542ebf1cee6a02436e12de901ffb87 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_sycl_types.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_TYPES_H +#define DNNL_SYCL_TYPES_H + +#include "oneapi/dnnl/dnnl_sycl_types.h" + +#endif /* DNNL_SYCL_TYPES_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.h new file mode 100644 index 0000000000000000000000000000000000000000..f58b08c805e2f4550ff6e4d596d36b75f3cfafe9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_H +#define DNNL_THREADPOOL_H + +#include "oneapi/dnnl/dnnl_threadpool.h" + +#endif /* DNNL_THREADPOOL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.hpp b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.hpp new file mode 100644 index 0000000000000000000000000000000000000000..e1fbdb0356b169da7ff9a433a947e056a7f3e78f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_HPP +#define DNNL_THREADPOOL_HPP + +#include "oneapi/dnnl/dnnl_threadpool.hpp" + +#endif /* DNNL_THREADPOOL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool_iface.hpp b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool_iface.hpp new file mode 100644 index 0000000000000000000000000000000000000000..e270b69329d889c2773aca920bf39ea15328efd7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_threadpool_iface.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_IFACE_HPP +#define DNNL_THREADPOOL_IFACE_HPP + +#include "oneapi/dnnl/dnnl_threadpool_iface.hpp" + +#endif /* DNNL_THREADPOOL_IFACE_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_types.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_types.h new file mode 100644 index 0000000000000000000000000000000000000000..643ab4cafcb0376cfab4007c33b39f0a2b81191f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_types.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_TYPES_H +#define DNNL_TYPES_H + +#include "oneapi/dnnl/dnnl_types.h" + +#endif /* DNNL_TYPES_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_version.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_version.h new file mode 100644 index 0000000000000000000000000000000000000000..4873281b5abd74e4da45c2308a6f01ce4a630dbb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/dnnl_version.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_VERSION_H +#define DNNL_VERSION_H + +#include "oneapi/dnnl/dnnl_version.h" + +#endif /* DNNL_VERSION_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/experiments-config.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/experiments-config.h new file mode 100644 index 0000000000000000000000000000000000000000..ed123d2284a5fb9d139e6bb62721e6836578ca7e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/experiments-config.h @@ -0,0 +1,30 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright 2023 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct xnn_experiment_config { + int dummy; // C requires that a struct or union has at least one member +}; + +struct xnn_experiment_config* xnn_get_experiment_config(); + +void xnn_experiment_enable_adaptive_avx_optimization(); + + +#ifdef __cplusplus +} // extern "C" +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fp16.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fp16.h new file mode 100644 index 0000000000000000000000000000000000000000..4cdcaecb99a6a48e4b6f9643e51630c1ca975e0c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fp16.h @@ -0,0 +1,16 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef FP16_H +#define FP16_H + +#include + +#if defined(PSIMD_H) +#include +#endif + +#endif /* FP16_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fxdiv.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fxdiv.h new file mode 100644 index 0000000000000000000000000000000000000000..7027522b47dfa9d78488fd226e47cbfb27fe7aac --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/fxdiv.h @@ -0,0 +1,430 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef FXDIV_H +#define FXDIV_H + +#if defined(__cplusplus) && (__cplusplus >= 201103L) + #include + #include + #include +#elif !defined(__OPENCL_VERSION__) + #include + #include + #include +#endif + +#if defined(_MSC_VER) + #include + #if defined(_M_IX86) || defined(_M_X64) + #include + #endif +#endif + +#ifndef FXDIV_USE_INLINE_ASSEMBLY + #define FXDIV_USE_INLINE_ASSEMBLY 0 +#endif + +static inline uint64_t fxdiv_mulext_uint32_t(uint32_t a, uint32_t b) { +#if defined(_MSC_VER) && defined(_M_IX86) + return (uint64_t) __emulu((unsigned int) a, (unsigned int) b); +#else + return (uint64_t) a * (uint64_t) b; +#endif +} + +static inline uint32_t fxdiv_mulhi_uint32_t(uint32_t a, uint32_t b) { +#if defined(__OPENCL_VERSION__) + return mul_hi(a, b); +#elif defined(__CUDA_ARCH__) + return (uint32_t) __umulhi((unsigned int) a, (unsigned int) b); +#elif defined(_MSC_VER) && defined(_M_IX86) + return (uint32_t) (__emulu((unsigned int) a, (unsigned int) b) >> 32); +#elif defined(_MSC_VER) && defined(_M_ARM) + return (uint32_t) _MulUnsignedHigh((unsigned long) a, (unsigned long) b); +#else + return (uint32_t) (((uint64_t) a * (uint64_t) b) >> 32); +#endif +} + +static inline uint64_t fxdiv_mulhi_uint64_t(uint64_t a, uint64_t b) { +#if defined(__OPENCL_VERSION__) + return mul_hi(a, b); +#elif defined(__CUDA_ARCH__) + return (uint64_t) __umul64hi((unsigned long long) a, (unsigned long long) b); +#elif defined(_MSC_VER) && defined(_M_X64) + return (uint64_t) __umulh((unsigned __int64) a, (unsigned __int64) b); +#elif defined(__GNUC__) && defined(__SIZEOF_INT128__) + return (uint64_t) (((((unsigned __int128) a) * ((unsigned __int128) b))) >> 64); +#else + const uint32_t a_lo = (uint32_t) a; + const uint32_t a_hi = (uint32_t) (a >> 32); + const uint32_t b_lo = (uint32_t) b; + const uint32_t b_hi = (uint32_t) (b >> 32); + + const uint64_t t = fxdiv_mulext_uint32_t(a_hi, b_lo) + + (uint64_t) fxdiv_mulhi_uint32_t(a_lo, b_lo); + return fxdiv_mulext_uint32_t(a_hi, b_hi) + (t >> 32) + + ((fxdiv_mulext_uint32_t(a_lo, b_hi) + (uint64_t) (uint32_t) t) >> 32); +#endif +} + +static inline size_t fxdiv_mulhi_size_t(size_t a, size_t b) { +#if SIZE_MAX == UINT32_MAX + return (size_t) fxdiv_mulhi_uint32_t((uint32_t) a, (uint32_t) b); +#elif SIZE_MAX == UINT64_MAX + return (size_t) fxdiv_mulhi_uint64_t((uint64_t) a, (uint64_t) b); +#else + #error Unsupported platform +#endif +} + +struct fxdiv_divisor_uint32_t { + uint32_t value; + uint32_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_uint32_t { + uint32_t quotient; + uint32_t remainder; +}; + +struct fxdiv_divisor_uint64_t { + uint64_t value; + uint64_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_uint64_t { + uint64_t quotient; + uint64_t remainder; +}; + +struct fxdiv_divisor_size_t { + size_t value; + size_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_size_t { + size_t quotient; + size_t remainder; +}; + +static inline struct fxdiv_divisor_uint32_t fxdiv_init_uint32_t(uint32_t d) { + struct fxdiv_divisor_uint32_t result = { d }; + if (d == 1) { + result.m = UINT32_C(1); + result.s1 = 0; + result.s2 = 0; + } else { + #if defined(__OPENCL_VERSION__) + const uint32_t l_minus_1 = 31 - clz(d - 1); + #elif defined(__CUDA_ARCH__) + const uint32_t l_minus_1 = 31 - __clz((int) (d - 1)); + #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) + unsigned long l_minus_1; + _BitScanReverse(&l_minus_1, (unsigned long) (d - 1)); + #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) && FXDIV_USE_INLINE_ASSEMBLY + uint32_t l_minus_1; + __asm__("BSRL %[d_minus_1], %[l_minus_1]" + : [l_minus_1] "=r" (l_minus_1) + : [d_minus_1] "r" (d - 1) + : "cc"); + #elif defined(__GNUC__) + const uint32_t l_minus_1 = 31 - __builtin_clz(d - 1); + #else + /* Based on Algorithm 2 from Hacker's delight */ + + uint32_t l_minus_1 = 0; + uint32_t x = d - 1; + uint32_t y = x >> 16; + if (y != 0) { + l_minus_1 += 16; + x = y; + } + y = x >> 8; + if (y != 0) { + l_minus_1 += 8; + x = y; + } + y = x >> 4; + if (y != 0) { + l_minus_1 += 4; + x = y; + } + y = x >> 2; + if (y != 0) { + l_minus_1 += 2; + x = y; + } + if ((x & 2) != 0) { + l_minus_1 += 1; + } + #endif + uint32_t u_hi = (UINT32_C(2) << (uint32_t) l_minus_1) - d; + + /* Division of 64-bit number u_hi:UINT32_C(0) by 32-bit number d, 32-bit quotient output q */ + #if defined(__GNUC__) && defined(__i386__) && FXDIV_USE_INLINE_ASSEMBLY + uint32_t q; + __asm__("DIVL %[d]" + : "=a" (q), "+d" (u_hi) + : [d] "r" (d), "a" (0) + : "cc"); + #elif (defined(_MSC_VER) && _MSC_VER >= 1920) && !defined(__clang__) && !defined(__INTEL_COMPILER) && (defined(_M_IX86) || defined(_M_X64)) + unsigned int remainder; + const uint32_t q = (uint32_t) _udiv64((unsigned __int64) ((uint64_t) u_hi << 32), (unsigned int) d, &remainder); + #else + const uint32_t q = ((uint64_t) u_hi << 32) / d; + #endif + + result.m = q + UINT32_C(1); + result.s1 = 1; + result.s2 = (uint8_t) l_minus_1; + } + return result; +} + +static inline struct fxdiv_divisor_uint64_t fxdiv_init_uint64_t(uint64_t d) { + struct fxdiv_divisor_uint64_t result = { d }; + if (d == 1) { + result.m = UINT64_C(1); + result.s1 = 0; + result.s2 = 0; + } else { + #if defined(__OPENCL_VERSION__) + const uint32_t nlz_d = clz(d); + const uint32_t l_minus_1 = 63 - clz(d - 1); + #elif defined(__CUDA_ARCH__) + const uint32_t nlz_d = __clzll((long long) d); + const uint32_t l_minus_1 = 63 - __clzll((long long) (d - 1)); + #elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long l_minus_1; + _BitScanReverse64(&l_minus_1, (unsigned __int64) (d - 1)); + unsigned long bsr_d; + _BitScanReverse64(&bsr_d, (unsigned __int64) d); + const uint32_t nlz_d = bsr_d ^ 0x3F; + #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) + const uint64_t d_minus_1 = d - 1; + const uint8_t d_is_power_of_2 = (d & d_minus_1) == 0; + unsigned long l_minus_1; + if ((uint32_t) (d_minus_1 >> 32) == 0) { + _BitScanReverse(&l_minus_1, (unsigned long) d_minus_1); + } else { + _BitScanReverse(&l_minus_1, (unsigned long) (uint32_t) (d_minus_1 >> 32)); + l_minus_1 += 32; + } + const uint32_t nlz_d = ((uint8_t) l_minus_1 ^ UINT8_C(0x3F)) - d_is_power_of_2; + #elif defined(__GNUC__) && defined(__x86_64__) && FXDIV_USE_INLINE_ASSEMBLY + uint64_t l_minus_1; + __asm__("BSRQ %[d_minus_1], %[l_minus_1]" + : [l_minus_1] "=r" (l_minus_1) + : [d_minus_1] "r" (d - 1) + : "cc"); + #elif defined(__GNUC__) + const uint32_t l_minus_1 = 63 - __builtin_clzll(d - 1); + const uint32_t nlz_d = __builtin_clzll(d); + #else + /* Based on Algorithm 2 from Hacker's delight */ + const uint64_t d_minus_1 = d - 1; + const uint32_t d_is_power_of_2 = (d & d_minus_1) == 0; + uint32_t l_minus_1 = 0; + uint32_t x = (uint32_t) d_minus_1; + uint32_t y = d_minus_1 >> 32; + if (y != 0) { + l_minus_1 += 32; + x = y; + } + y = x >> 16; + if (y != 0) { + l_minus_1 += 16; + x = y; + } + y = x >> 8; + if (y != 0) { + l_minus_1 += 8; + x = y; + } + y = x >> 4; + if (y != 0) { + l_minus_1 += 4; + x = y; + } + y = x >> 2; + if (y != 0) { + l_minus_1 += 2; + x = y; + } + if ((x & 2) != 0) { + l_minus_1 += 1; + } + const uint32_t nlz_d = (l_minus_1 ^ UINT32_C(0x3F)) - d_is_power_of_2; + #endif + uint64_t u_hi = (UINT64_C(2) << (uint32_t) l_minus_1) - d; + + /* Division of 128-bit number u_hi:UINT64_C(0) by 64-bit number d, 64-bit quotient output q */ + #if defined(__GNUC__) && defined(__x86_64__) && FXDIV_USE_INLINE_ASSEMBLY + uint64_t q; + __asm__("DIVQ %[d]" + : "=a" (q), "+d" (u_hi) + : [d] "r" (d), "a" (UINT64_C(0)) + : "cc"); + #elif 0 && defined(__GNUC__) && defined(__SIZEOF_INT128__) + /* GCC, Clang, and Intel Compiler fail to inline optimized implementation and call into support library for 128-bit division */ + const uint64_t q = (uint64_t) (((unsigned __int128) u_hi << 64) / ((unsigned __int128) d)); + #elif (defined(_MSC_VER) && _MSC_VER >= 1920) && !defined(__clang__) && !defined(__INTEL_COMPILER) && defined(_M_X64) + unsigned __int64 remainder; + const uint64_t q = (uint64_t) _udiv128((unsigned __int64) u_hi, 0, (unsigned __int64) d, &remainder); + #else + /* Implementation based on code from Hacker's delight */ + + /* Normalize divisor and shift divident left */ + d <<= nlz_d; + u_hi <<= nlz_d; + /* Break divisor up into two 32-bit digits */ + const uint64_t d_hi = (uint32_t) (d >> 32); + const uint32_t d_lo = (uint32_t) d; + + /* Compute the first quotient digit, q1 */ + uint64_t q1 = u_hi / d_hi; + uint64_t r1 = u_hi - q1 * d_hi; + + while ((q1 >> 32) != 0 || fxdiv_mulext_uint32_t((uint32_t) q1, d_lo) > (r1 << 32)) { + q1 -= 1; + r1 += d_hi; + if ((r1 >> 32) != 0) { + break; + } + } + + /* Multiply and subtract. */ + u_hi = (u_hi << 32) - q1 * d; + + /* Compute the second quotient digit, q0 */ + uint64_t q0 = u_hi / d_hi; + uint64_t r0 = u_hi - q0 * d_hi; + + while ((q0 >> 32) != 0 || fxdiv_mulext_uint32_t((uint32_t) q0, d_lo) > (r0 << 32)) { + q0 -= 1; + r0 += d_hi; + if ((r0 >> 32) != 0) { + break; + } + } + const uint64_t q = (q1 << 32) | (uint32_t) q0; + #endif + result.m = q + UINT64_C(1); + result.s1 = 1; + result.s2 = (uint8_t) l_minus_1; + } + return result; +} + +static inline struct fxdiv_divisor_size_t fxdiv_init_size_t(size_t d) { +#if SIZE_MAX == UINT32_MAX + const struct fxdiv_divisor_uint32_t uint_result = fxdiv_init_uint32_t((uint32_t) d); +#elif SIZE_MAX == UINT64_MAX + const struct fxdiv_divisor_uint64_t uint_result = fxdiv_init_uint64_t((uint64_t) d); +#else + #error Unsupported platform +#endif + struct fxdiv_divisor_size_t size_result = { + (size_t) uint_result.value, + (size_t) uint_result.m, + uint_result.s1, + uint_result.s2 + }; + return size_result; +} + +static inline uint32_t fxdiv_quotient_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t t = fxdiv_mulhi_uint32_t(n, divisor.m); + return (t + ((n - t) >> divisor.s1)) >> divisor.s2; +} + +static inline uint64_t fxdiv_quotient_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t t = fxdiv_mulhi_uint64_t(n, divisor.m); + return (t + ((n - t) >> divisor.s1)) >> divisor.s2; +} + +static inline size_t fxdiv_quotient_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { +#if SIZE_MAX == UINT32_MAX + const struct fxdiv_divisor_uint32_t uint32_divisor = { + (uint32_t) divisor.value, + (uint32_t) divisor.m, + divisor.s1, + divisor.s2 + }; + return fxdiv_quotient_uint32_t((uint32_t) n, uint32_divisor); +#elif SIZE_MAX == UINT64_MAX + const struct fxdiv_divisor_uint64_t uint64_divisor = { + (uint64_t) divisor.value, + (uint64_t) divisor.m, + divisor.s1, + divisor.s2 + }; + return fxdiv_quotient_uint64_t((uint64_t) n, uint64_divisor); +#else + #error Unsupported platform +#endif +} + +static inline uint32_t fxdiv_remainder_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline uint64_t fxdiv_remainder_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline size_t fxdiv_remainder_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { + const size_t quotient = fxdiv_quotient_size_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline uint32_t fxdiv_round_down_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t granularity) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, granularity); + return quotient * granularity.value; +} + +static inline uint64_t fxdiv_round_down_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t granularity) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, granularity); + return quotient * granularity.value; +} + +static inline size_t fxdiv_round_down_size_t(size_t n, const struct fxdiv_divisor_size_t granularity) { + const size_t quotient = fxdiv_quotient_size_t(n, granularity); + return quotient * granularity.value; +} + +static inline struct fxdiv_result_uint32_t fxdiv_divide_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, divisor); + const uint32_t remainder = n - quotient * divisor.value; + struct fxdiv_result_uint32_t result = { quotient, remainder }; + return result; +} + +static inline struct fxdiv_result_uint64_t fxdiv_divide_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, divisor); + const uint64_t remainder = n - quotient * divisor.value; + struct fxdiv_result_uint64_t result = { quotient, remainder }; + return result; +} + +static inline struct fxdiv_result_size_t fxdiv_divide_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { + const size_t quotient = fxdiv_quotient_size_t(n, divisor); + const size_t remainder = n - quotient * divisor.value; + struct fxdiv_result_size_t result = { quotient, remainder }; + return result; +} + +#endif /* FXDIV_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify-zca.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify-zca.h new file mode 100644 index 0000000000000000000000000000000000000000..36baff866e7d65e1d7d01cdce1eef7bb8f2e4487 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify-zca.h @@ -0,0 +1,86 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +/** + * Zero Cost Annotations (ZCA) + * + * Intel Compiler supports two intrinsics that could be used for code annotations + * without incurring significant run-time costs when the tools are not in use. + * Each annotation is more than a mere mark in the instruction stream. + * It can accept an expression argument like a call to a routine does. + * There are two forms of the intrinsic, with the following signatures: + * + * extern "C" void __notify_intrinsic( const char *annotation, const volatile void *tag); + * extern "C" void __notify_zc_intrinsic(const char *annotation, const volatile void *tag); + * + * The string annotation must be a compile-time constant. It specifies the type of the annotation. + * The pointer tag is computed at run time. It specifies the data associated with the annotation. + * Each intrinsic implies a compiler fence: the compiler must not move any memory + * operation across it. The reason for this restriction is that annotation might denote an + * event that must be precisely placed with respect to memory operations. + * + * The difference between the two intrinsics is that __notify_intrinsic must leave a + * probe-ready instruction sequence in the instruction stream where the instrinsic + * occurs. The __notify_zc_intrinsic does not leave such a sequence, and hence is closer to "zero cost". + **/ + +#pragma once +#include "ittnotify.h" + +#ifndef INTEL_NO_ITTNOTIFY_API +#if (defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER)) && (ITT_PLATFORM == ITT_PLATFORM_WIN || ITT_PLATFORM == ITT_PLATFORM_POSIX) +#define ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#else +#error Zero cost (low overhead) annotations are not supported on this platform +#endif +#endif + +/** + * Zero cost annotations for memory allocation and deallocation + **/ +#ifdef ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#pragma pack(push, 1) +typedef struct ___itt_zca_allocation_info { + size_t size; /*!< Size of allocated memory */ + void** ptr; /*!< Pointer to allocated memory pointer */ + int initialized; /*!< Is allocated memory initialized */ +} __itt_zca_allocation_info; +#pragma pack(pop) + +#define __itt_zca_mem_allocate_begin() __notify_intrinsic((char*)"mem_allocate_begin", 0) +#define __itt_zca_mem_allocate_end(ptr, size, init) { __itt_zca_allocation_info __itt_zca_alloc_info = { size, ptr, init }; __notify_intrinsic((char*)"mem_allocate_end", (void*)&__itt_zca_alloc_info); } +#define __itt_zca_mem_free_begin(ptr) __notify_intrinsic((char*)"mem_free_begin", (void*)ptr) +#define __itt_zca_mem_free_end() __notify_intrinsic((char*)"mem_free_end", 0) +#else +#define __itt_zca_mem_allocate_begin() +#define __itt_zca_mem_allocate_end(ptr, size, init) +#define __itt_zca_mem_free_begin(ptr) +#define __itt_zca_mem_free_end() +#endif + +/** + * Zero cost annotations for threading + **/ +#ifdef ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#define __itt_zca_suppress_push(id) __notify_zc_intrinsic((char*)"__itt_suppress_push", (void*)id); +#define __itt_zca_suppress_pop(id) __notify_zc_intrinsic((char*)"__itt_suppress_pop", (void*)id); +#define __itt_zca_sync_create(id) __notify_zc_intrinsic((char*)"__itt_sync_create", (void*)id) +#define __itt_zca_sync_acquired(id) __notify_zc_intrinsic((char*)"__itt_sync_acquired", (void*)id) +#define __itt_zca_sync_releasing(id) __notify_zc_intrinsic((char*)"__itt_sync_releasing", (void*)id) +#define __itt_zca_sync_destroy(id) __notify_zc_intrinsic((char*)"__itt_sync_destroy", (void*)id) +#else +#define __itt_zca_suppress_push(id) +#define __itt_zca_suppress_pop(id) +#define __itt_zca_sync_create(id) +#define __itt_zca_sync_acquired(id) +#define __itt_zca_sync_releasing(id) +#define __itt_zca_sync_destroy(id) +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify.h new file mode 100644 index 0000000000000000000000000000000000000000..ba87ba3dca8b5d600d7e90f9417b5a11e7f36a44 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/ittnotify.h @@ -0,0 +1,4670 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ +#ifndef _ITTNOTIFY_H_ +#define _ITTNOTIFY_H_ + +/** +@file +@brief Public User API functions and types +@mainpage + +The Instrumentation and Tracing Technology API (ITT API) is used to +annotate a user's program with additional information +that can be used by correctness and performance tools. The user inserts +calls in their program. Those calls generate information that is collected +at runtime, and used by Intel(R) Threading Tools. + +@section API Concepts +The following general concepts are used throughout the API. + +@subsection Unicode Support +Many API functions take character string arguments. On Windows, there +are two versions of each such function. The function name is suffixed +by W if Unicode support is enabled, and by A otherwise. Any API function +that takes a character string argument adheres to this convention. + +@subsection Conditional Compilation +Many users prefer having an option to modify ITT API code when linking it +inside their runtimes. ITT API header file provides a mechanism to replace +ITT API function names inside your code with empty strings. To do this, +define the macros INTEL_NO_ITTNOTIFY_API during compilation and remove the +static library from the linker script. + +@subsection Domains +[see domains] +Domains provide a way to separate notification for different modules or +libraries in a program. Domains are specified by dotted character strings, +e.g. TBB.Internal.Control. + +A mechanism (to be specified) is provided to enable and disable +domains. By default, all domains are enabled. +@subsection Named Entities and Instances +Named entities (frames, regions, tasks, and markers) communicate +information about the program to the analysis tools. A named entity often +refers to a section of program code, or to some set of logical concepts +that the programmer wants to group together. + +Named entities relate to the programmer's static view of the program. When +the program actually executes, many instances of a given named entity +may be created. + +The API annotations denote instances of named entities. The actual +named entities are displayed using the analysis tools. In other words, +the named entities come into existence when instances are created. + +Instances of named entities may have instance identifiers (IDs). Some +API calls use instance identifiers to create relationships between +different instances of named entities. Other API calls associate data +with instances of named entities. + +Some named entities must always have instance IDs. In particular, regions +and frames always have IDs. Task and markers need IDs only if the ID is +needed in another API call (such as adding a relation or metadata). + +The lifetime of instance IDs is distinct from the lifetime of +instances. This allows various relationships to be specified separate +from the actual execution of instances. This flexibility comes at the +expense of extra API calls. + +The same ID may not be reused for different instances, unless a previous +[ref] __itt_id_destroy call for that ID has been issued. +*/ + +/** @cond exclude_from_documentation */ +#ifndef ITT_OS_WIN +# define ITT_OS_WIN 1 +#endif /* ITT_OS_WIN */ + +#ifndef ITT_OS_LINUX +# define ITT_OS_LINUX 2 +#endif /* ITT_OS_LINUX */ + +#ifndef ITT_OS_MAC +# define ITT_OS_MAC 3 +#endif /* ITT_OS_MAC */ + +#ifndef ITT_OS_FREEBSD +# define ITT_OS_FREEBSD 4 +#endif /* ITT_OS_FREEBSD */ + +#ifndef ITT_OS_OPENBSD +# define ITT_OS_OPENBSD 5 +#endif /* ITT_OS_OPENBSD */ + +#ifndef ITT_OS +# if defined WIN32 || defined _WIN32 +# define ITT_OS ITT_OS_WIN +# elif defined( __APPLE__ ) && defined( __MACH__ ) +# define ITT_OS ITT_OS_MAC +# elif defined( __FreeBSD__ ) +# define ITT_OS ITT_OS_FREEBSD +# elif defined( __OpenBSD__) +# define ITT_OS ITT_OS_OPENBSD +# else +# define ITT_OS ITT_OS_LINUX +# endif +#endif /* ITT_OS */ + +#ifndef ITT_PLATFORM_WIN +# define ITT_PLATFORM_WIN 1 +#endif /* ITT_PLATFORM_WIN */ + +#ifndef ITT_PLATFORM_POSIX +# define ITT_PLATFORM_POSIX 2 +#endif /* ITT_PLATFORM_POSIX */ + +#ifndef ITT_PLATFORM_MAC +# define ITT_PLATFORM_MAC 3 +#endif /* ITT_PLATFORM_MAC */ + +#ifndef ITT_PLATFORM_FREEBSD +# define ITT_PLATFORM_FREEBSD 4 +#endif /* ITT_PLATFORM_FREEBSD */ + +#ifndef ITT_PLATFORM_OPENBSD +# define ITT_PLATFORM_OPENBSD 5 +#endif /* ITT_PLATFORM_OPENBSD */ + +#ifndef ITT_PLATFORM +# if ITT_OS==ITT_OS_WIN +# define ITT_PLATFORM ITT_PLATFORM_WIN +# elif ITT_OS==ITT_OS_MAC +# define ITT_PLATFORM ITT_PLATFORM_MAC +# elif ITT_OS==ITT_OS_FREEBSD +# define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# elif ITT_OS==ITT_OS_OPENBSD +# define ITT_PLATFORM ITT_PLATFORM_OPENBSD +# else +# define ITT_PLATFORM ITT_PLATFORM_POSIX +# endif +#endif /* ITT_PLATFORM */ + +#if defined(_UNICODE) && !defined(UNICODE) +#define UNICODE +#endif + +#include +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#include +#if defined(UNICODE) || defined(_UNICODE) +#include +#endif /* UNICODE || _UNICODE */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef ITTAPI_CDECL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define ITTAPI_CDECL __cdecl +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define ITTAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define ITTAPI_CDECL /* actual only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* ITTAPI_CDECL */ + +#ifndef STDCALL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define STDCALL __stdcall +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define STDCALL __attribute__ ((stdcall)) +# else /* _M_IX86 || __i386__ */ +# define STDCALL /* supported only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* STDCALL */ + +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL + +/* TODO: Temporary for compatibility! */ +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +/* use __forceinline (VC++ specific) */ +#if defined(__MINGW32__) && !defined(__cplusplus) +#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__)) +#else +#define ITT_INLINE static __forceinline +#endif /* __MINGW32__ */ + +#define ITT_INLINE_ATTRIBUTE /* nothing */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/* + * Generally, functions are not inlined unless optimization is specified. + * For functions declared inline, this attribute inlines the function even + * if no optimization level was specified. + */ +#ifdef __STRICT_ANSI__ +#define ITT_INLINE static +#define ITT_INLINE_ATTRIBUTE __attribute__((unused)) +#else /* __STRICT_ANSI__ */ +#define ITT_INLINE static inline +#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused)) +#endif /* __STRICT_ANSI__ */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/** @endcond */ + +#ifdef INTEL_ITTNOTIFY_ENABLE_LEGACY +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# pragma message("WARNING!!! Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro") +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# warning "Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro" +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# include "legacy/ittnotify.h" +#endif /* INTEL_ITTNOTIFY_ENABLE_LEGACY */ + +/** @cond exclude_from_documentation */ +/* Helper macro for joining tokens */ +#define ITT_JOIN_AUX(p,n) p##n +#define ITT_JOIN(p,n) ITT_JOIN_AUX(p,n) + +#ifdef ITT_MAJOR +#undef ITT_MAJOR +#endif +#ifdef ITT_MINOR +#undef ITT_MINOR +#endif +#define ITT_MAJOR 3 +#define ITT_MINOR 0 + +/* Standard versioning of a token with major and minor version numbers */ +#define ITT_VERSIONIZE(x) \ + ITT_JOIN(x, \ + ITT_JOIN(_, \ + ITT_JOIN(ITT_MAJOR, \ + ITT_JOIN(_, ITT_MINOR)))) + +#ifndef INTEL_ITTNOTIFY_PREFIX +# define INTEL_ITTNOTIFY_PREFIX __itt_ +#endif /* INTEL_ITTNOTIFY_PREFIX */ +#ifndef INTEL_ITTNOTIFY_POSTFIX +# define INTEL_ITTNOTIFY_POSTFIX _ptr_ +#endif /* INTEL_ITTNOTIFY_POSTFIX */ + +#define ITTNOTIFY_NAME_AUX(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n) +#define ITTNOTIFY_NAME(n) ITT_VERSIONIZE(ITTNOTIFY_NAME_AUX(ITT_JOIN(n,INTEL_ITTNOTIFY_POSTFIX))) + +#define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n) +#define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n) + +#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) + +#ifdef ITT_STUB +#undef ITT_STUB +#endif +#ifdef ITT_STUBV +#undef ITT_STUBV +#endif +#define ITT_STUBV(api,type,name,args) \ + typedef type (api* ITT_JOIN(ITTNOTIFY_NAME(name),_t)) args; \ + extern ITT_JOIN(ITTNOTIFY_NAME(name),_t) ITTNOTIFY_NAME(name); +#define ITT_STUB ITT_STUBV +/** @endcond */ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup public Public API + * @{ + * @} + */ + +/** + * @defgroup control Collection Control + * @ingroup public + * General behavior: application continues to run, but no profiling information is being collected + * + * Pausing occurs not only for the current thread but for all process as well as spawned processes + * - Intel(R) Parallel Inspector and Intel(R) Inspector XE: + * - Does not analyze or report errors that involve memory access. + * - Other errors are reported as usual. Pausing data collection in + * Intel(R) Parallel Inspector and Intel(R) Inspector XE + * only pauses tracing and analyzing memory access. + * It does not pause tracing or analyzing threading APIs. + * . + * - Intel(R) VTune(TM) Profiler: + * - Does continue to record when new threads are started. + * . + * - Other effects: + * - Possible reduction of runtime overhead. + * . + * @{ + */ +/** @brief Pause collection */ +void ITTAPI __itt_pause(void); +/** @brief Resume collection */ +void ITTAPI __itt_resume(void); +/** @brief Detach collection */ +void ITTAPI __itt_detach(void); + +/** + * @enum __itt_collection_scope + * @brief Enumerator for collection scopes + */ +typedef enum { + __itt_collection_scope_host = 1 << 0, + __itt_collection_scope_offload = 1 << 1, + __itt_collection_scope_all = 0x7FFFFFFF +} __itt_collection_scope; + +/** @brief Pause scoped collection */ +void ITTAPI __itt_pause_scoped(__itt_collection_scope); +/** @brief Resume scoped collection */ +void ITTAPI __itt_resume_scoped(__itt_collection_scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, pause, (void)) +ITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, resume, (void)) +ITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, detach, (void)) +#define __itt_pause ITTNOTIFY_VOID(pause) +#define __itt_pause_ptr ITTNOTIFY_NAME(pause) +#define __itt_pause_scoped ITTNOTIFY_VOID(pause_scoped) +#define __itt_pause_scoped_ptr ITTNOTIFY_NAME(pause_scoped) +#define __itt_resume ITTNOTIFY_VOID(resume) +#define __itt_resume_ptr ITTNOTIFY_NAME(resume) +#define __itt_resume_scoped ITTNOTIFY_VOID(resume_scoped) +#define __itt_resume_scoped_ptr ITTNOTIFY_NAME(resume_scoped) +#define __itt_detach ITTNOTIFY_VOID(detach) +#define __itt_detach_ptr ITTNOTIFY_NAME(detach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_pause() +#define __itt_pause_ptr 0 +#define __itt_pause_scoped(scope) +#define __itt_pause_scoped_ptr 0 +#define __itt_resume() +#define __itt_resume_ptr 0 +#define __itt_resume_scoped(scope) +#define __itt_resume_scoped_ptr 0 +#define __itt_detach() +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_pause_ptr 0 +#define __itt_pause_scoped_ptr 0 +#define __itt_resume_ptr 0 +#define __itt_resume_scoped_ptr 0 +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} control group */ +/** @endcond */ + +/** + * @defgroup Intel Processor Trace control + * API from this group provides control over collection and analysis of Intel Processor Trace (Intel PT) data + * Information about Intel Processor Trace technology can be found here (Volume 3 chapter 35): + * https://software.intel.com/sites/default/files/managed/39/c5/325462-sdm-vol-1-2abcd-3abcd.pdf + * Use this API to mark particular code regions for loading detailed performance statistics. + * This mode makes your analysis faster and more accurate. + * @{ +*/ +typedef unsigned char __itt_pt_region; + +/** + * @brief function saves a region name marked with Intel PT API and returns a region id. + * Only 7 names can be registered. Attempts to register more names will be ignored and a region id with auto names will be returned. + * For automatic naming of regions pass NULL as function parameter +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_pt_region ITTAPI __itt_pt_region_createA(const char *name); +__itt_pt_region ITTAPI __itt_pt_region_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_pt_region_create __itt_pt_region_createW +#else /* UNICODE */ +# define __itt_pt_region_create __itt_pt_region_createA +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_pt_region ITTAPI __itt_pt_region_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA ITTNOTIFY_DATA(pt_region_createA) +#define __itt_pt_region_createA_ptr ITTNOTIFY_NAME(pt_region_createA) +#define __itt_pt_region_createW ITTNOTIFY_DATA(pt_region_createW) +#define __itt_pt_region_createW_ptr ITTNOTIFY_NAME(pt_region_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create ITTNOTIFY_DATA(pt_region_create) +#define __itt_pt_region_create_ptr ITTNOTIFY_NAME(pt_region_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA(name) (__itt_pt_region)0 +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW(name) (__itt_pt_region)0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create(name) (__itt_pt_region)0 +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the beginning of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_begin(__itt_pt_region region); +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the end of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_end(__itt_pt_region region); +/** @} Intel PT control group*/ + +/** + * @defgroup threads Threads + * @ingroup public + * Give names to threads + * @{ + */ +/** + * @brief Sets thread name of calling thread + * @param[in] name - name of thread + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_thread_set_nameA(const char *name); +void ITTAPI __itt_thread_set_nameW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_thread_set_name __itt_thread_set_nameW +# define __itt_thread_set_name_ptr __itt_thread_set_nameW_ptr +#else /* UNICODE */ +# define __itt_thread_set_name __itt_thread_set_nameA +# define __itt_thread_set_name_ptr __itt_thread_set_nameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_thread_set_name(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name)) +ITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, thread_set_name, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA ITTNOTIFY_VOID(thread_set_nameA) +#define __itt_thread_set_nameA_ptr ITTNOTIFY_NAME(thread_set_nameA) +#define __itt_thread_set_nameW ITTNOTIFY_VOID(thread_set_nameW) +#define __itt_thread_set_nameW_ptr ITTNOTIFY_NAME(thread_set_nameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name ITTNOTIFY_VOID(thread_set_name) +#define __itt_thread_set_name_ptr ITTNOTIFY_NAME(thread_set_name) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA(name) +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW(name) +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name(name) +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @brief Mark current thread as ignored from this point on, for the duration of its existence. + */ +void ITTAPI __itt_thread_ignore(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, thread_ignore, (void)) +#define __itt_thread_ignore ITTNOTIFY_VOID(thread_ignore) +#define __itt_thread_ignore_ptr ITTNOTIFY_NAME(thread_ignore) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_thread_ignore() +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} threads group */ + +/** + * @defgroup suppress Error suppression + * @ingroup public + * General behavior: application continues to run, but errors are suppressed + * + * @{ + */ + +/*****************************************************************//** + * @name group of functions used for error suppression in correctness tools + *********************************************************************/ +/** @{ */ +/** + * @hideinitializer + * @brief possible value for suppression mask + */ +#define __itt_suppress_all_errors 0x7fffffff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from threading analysis) + */ +#define __itt_suppress_threading_errors 0x000000ff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from memory analysis) + */ +#define __itt_suppress_memory_errors 0x0000ff00 + +/** + * @brief Start suppressing errors identified in mask on this thread + */ +void ITTAPI __itt_suppress_push(unsigned int mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask)) +#define __itt_suppress_push ITTNOTIFY_VOID(suppress_push) +#define __itt_suppress_push_ptr ITTNOTIFY_NAME(suppress_push) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_push(mask) +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effects of the matching call to __itt_suppress_push + */ +void ITTAPI __itt_suppress_pop(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_pop, (void)) +#define __itt_suppress_pop ITTNOTIFY_VOID(suppress_pop) +#define __itt_suppress_pop_ptr ITTNOTIFY_NAME(suppress_pop) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_pop() +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @enum __itt_suppress_mode + * @brief Enumerator for the suppressing modes + */ +typedef enum __itt_suppress_mode { + __itt_unsuppress_range, + __itt_suppress_range +} __itt_suppress_mode_t; + +/** + * @enum __itt_collection_state + * @brief Enumerator for collection state. + */ +typedef enum { + __itt_collection_uninitialized = 0, /* uninitialized */ + __itt_collection_init_fail = 1, /* failed to init */ + __itt_collection_collector_absent = 2, /* non work state collector is absent */ + __itt_collection_collector_exists = 3, /* work state collector exists */ + __itt_collection_init_successful = 4 /* success to init */ +} __itt_collection_state; + +/** + * @brief Mark a range of memory for error suppression or unsuppression for error types included in mask + */ +void ITTAPI __itt_suppress_mark_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_mark_range ITTNOTIFY_VOID(suppress_mark_range) +#define __itt_suppress_mark_range_ptr ITTNOTIFY_NAME(suppress_mark_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_mark_range(mask) +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effect of a matching call to __itt_suppress_mark_range. If not matching + * call is found, nothing is changed. + */ +void ITTAPI __itt_suppress_clear_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_clear_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_clear_range ITTNOTIFY_VOID(suppress_clear_range) +#define __itt_suppress_clear_range_ptr ITTNOTIFY_NAME(suppress_clear_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_clear_range(mask) +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ +/** @} suppress group */ + +/** + * @defgroup sync Synchronization + * @ingroup public + * Indicate user-written synchronization code + * @{ + */ +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_barrier 1 + +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_mutex 2 + +/** +@brief Name a synchronization object +@param[in] addr Handle for the synchronization object. You should +use a real address to uniquely identify the synchronization object. +@param[in] objtype null-terminated object type string. If NULL is +passed, the name will be "User Synchronization". +@param[in] objname null-terminated object name string. If NULL, +no name will be assigned to the object. +@param[in] attribute one of [#__itt_attr_barrier, #__itt_attr_mutex] + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_createA(void *addr, const char *objtype, const char *objname, int attribute); +void ITTAPI __itt_sync_createW(void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_create __itt_sync_createW +# define __itt_sync_create_ptr __itt_sync_createW_ptr +#else /* UNICODE */ +# define __itt_sync_create __itt_sync_createA +# define __itt_sync_create_ptr __itt_sync_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_create (void *addr, const char *objtype, const char *objname, int attribute); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char *objtype, const char *objname, int attribute)) +ITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_create, (void *addr, const char* objtype, const char* objname, int attribute)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA ITTNOTIFY_VOID(sync_createA) +#define __itt_sync_createA_ptr ITTNOTIFY_NAME(sync_createA) +#define __itt_sync_createW ITTNOTIFY_VOID(sync_createW) +#define __itt_sync_createW_ptr ITTNOTIFY_NAME(sync_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create ITTNOTIFY_VOID(sync_create) +#define __itt_sync_create_ptr ITTNOTIFY_NAME(sync_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA(addr, objtype, objname, attribute) +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW(addr, objtype, objname, attribute) +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create(addr, objtype, objname, attribute) +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** +@brief Rename a synchronization object + +You can use the rename call to assign or reassign a name to a given +synchronization object. +@param[in] addr handle for the synchronization object. +@param[in] name null-terminated object name string. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_renameA(void *addr, const char *name); +void ITTAPI __itt_sync_renameW(void *addr, const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_rename __itt_sync_renameW +# define __itt_sync_rename_ptr __itt_sync_renameW_ptr +#else /* UNICODE */ +# define __itt_sync_rename __itt_sync_renameA +# define __itt_sync_rename_ptr __itt_sync_renameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_rename(void *addr, const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char *name)) +ITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_rename, (void *addr, const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA ITTNOTIFY_VOID(sync_renameA) +#define __itt_sync_renameA_ptr ITTNOTIFY_NAME(sync_renameA) +#define __itt_sync_renameW ITTNOTIFY_VOID(sync_renameW) +#define __itt_sync_renameW_ptr ITTNOTIFY_NAME(sync_renameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename ITTNOTIFY_VOID(sync_rename) +#define __itt_sync_rename_ptr ITTNOTIFY_NAME(sync_rename) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA(addr, name) +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW(addr, name) +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename(addr, name) +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + @brief Destroy a synchronization object. + @param addr Handle for the synchronization object. + */ +void ITTAPI __itt_sync_destroy(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_destroy, (void *addr)) +#define __itt_sync_destroy ITTNOTIFY_VOID(sync_destroy) +#define __itt_sync_destroy_ptr ITTNOTIFY_NAME(sync_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_destroy(addr) +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/*****************************************************************//** + * @name group of functions is used for performance measurement tools + *********************************************************************/ +/** @{ */ +/** + * @brief Enter spin loop on user-defined sync object + */ +void ITTAPI __itt_sync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_prepare, (void *addr)) +#define __itt_sync_prepare ITTNOTIFY_VOID(sync_prepare) +#define __itt_sync_prepare_ptr ITTNOTIFY_NAME(sync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_prepare(addr) +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Quit spin loop without acquiring spin object + */ +void ITTAPI __itt_sync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_cancel, (void *addr)) +#define __itt_sync_cancel ITTNOTIFY_VOID(sync_cancel) +#define __itt_sync_cancel_ptr ITTNOTIFY_NAME(sync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_cancel(addr) +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Successful spin loop completion (sync object acquired) + */ +void ITTAPI __itt_sync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_acquired, (void *addr)) +#define __itt_sync_acquired ITTNOTIFY_VOID(sync_acquired) +#define __itt_sync_acquired_ptr ITTNOTIFY_NAME(sync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_acquired(addr) +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Start sync object releasing code. Is called before the lock release call. + */ +void ITTAPI __itt_sync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_releasing, (void *addr)) +#define __itt_sync_releasing ITTNOTIFY_VOID(sync_releasing) +#define __itt_sync_releasing_ptr ITTNOTIFY_NAME(sync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_releasing(addr) +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** @} sync group */ + +/**************************************************************//** + * @name group of functions is used for correctness checking tools + ******************************************************************/ +/** @{ */ +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_prepare(void* addr); + */ +void ITTAPI __itt_fsync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_prepare, (void *addr)) +#define __itt_fsync_prepare ITTNOTIFY_VOID(fsync_prepare) +#define __itt_fsync_prepare_ptr ITTNOTIFY_NAME(fsync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_prepare(addr) +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_cancel(void *addr); + */ +void ITTAPI __itt_fsync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_cancel, (void *addr)) +#define __itt_fsync_cancel ITTNOTIFY_VOID(fsync_cancel) +#define __itt_fsync_cancel_ptr ITTNOTIFY_NAME(fsync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_cancel(addr) +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_acquired(void *addr); + */ +void ITTAPI __itt_fsync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_acquired, (void *addr)) +#define __itt_fsync_acquired ITTNOTIFY_VOID(fsync_acquired) +#define __itt_fsync_acquired_ptr ITTNOTIFY_NAME(fsync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_acquired(addr) +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_releasing(void* addr); + */ +void ITTAPI __itt_fsync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_releasing, (void *addr)) +#define __itt_fsync_releasing ITTNOTIFY_VOID(fsync_releasing) +#define __itt_fsync_releasing_ptr ITTNOTIFY_NAME(fsync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_releasing(addr) +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** + * @defgroup model Modeling by Intel(R) Parallel Advisor + * @ingroup public + * This is the subset of itt used for modeling by Intel(R) Parallel Advisor. + * This API is called ONLY using annotate.h, by "Annotation" macros + * the user places in their sources during the parallelism modeling steps. + * + * site_begin/end and task_begin/end take the address of handle variables, + * which are writeable by the API. Handles must be 0 initialized prior + * to the first call to begin, or may cause a run-time failure. + * The handles are initialized in a multi-thread safe way by the API if + * the handle is 0. The commonly expected idiom is one static handle to + * identify a site or task. If a site or task of the same name has already + * been started during this collection, the same handle MAY be returned, + * but is not required to be - it is unspecified if data merging is done + * based on name. These routines also take an instance variable. Like + * the lexical instance, these must be 0 initialized. Unlike the lexical + * instance, this is used to track a single dynamic instance. + * + * API used by the Intel(R) Parallel Advisor to describe potential concurrency + * and related activities. User-added source annotations expand to calls + * to these procedures to enable modeling of a hypothetical concurrent + * execution serially. + * @{ + */ +#if !defined(_ADVISOR_ANNOTATE_H_) || defined(ANNOTATE_EXPAND_NULL) + +typedef void* __itt_model_site; /*!< @brief handle for lexical site */ +typedef void* __itt_model_site_instance; /*!< @brief handle for dynamic instance */ +typedef void* __itt_model_task; /*!< @brief handle for lexical site */ +typedef void* __itt_model_task_instance; /*!< @brief handle for dynamic instance */ + +/** + * @enum __itt_model_disable + * @brief Enumerator for the disable methods + */ +typedef enum { + __itt_model_disable_observation, + __itt_model_disable_collection +} __itt_model_disable; + +#endif /* !_ADVISOR_ANNOTATE_H_ || ANNOTATE_EXPAND_NULL */ + +/** + * @brief ANNOTATE_SITE_BEGIN/ANNOTATE_SITE_END support. + * + * site_begin/end model a potential concurrency site. + * site instances may be recursively nested with themselves. + * site_end exits the most recently started but unended site for the current + * thread. The handle passed to end may be used to validate structure. + * Instances of a site encountered on different threads concurrently + * are considered completely distinct. If the site name for two different + * lexical sites match, it is unspecified whether they are treated as the + * same or different for data presentation. + */ +void ITTAPI __itt_model_site_begin(__itt_model_site *site, __itt_model_site_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_site_beginW(const wchar_t *name); +#endif +void ITTAPI __itt_model_site_beginA(const char *name); +void ITTAPI __itt_model_site_beginAL(const char *name, size_t siteNameLen); +void ITTAPI __itt_model_site_end (__itt_model_site *site, __itt_model_site_instance *instance); +void ITTAPI __itt_model_site_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_site_begin, (__itt_model_site *site, __itt_model_site_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_site_beginW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_site_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_site_beginAL, (const char *name, size_t siteNameLen)) +ITT_STUBV(ITTAPI, void, model_site_end, (__itt_model_site *site, __itt_model_site_instance *instance)) +ITT_STUBV(ITTAPI, void, model_site_end_2, (void)) +#define __itt_model_site_begin ITTNOTIFY_VOID(model_site_begin) +#define __itt_model_site_begin_ptr ITTNOTIFY_NAME(model_site_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW ITTNOTIFY_VOID(model_site_beginW) +#define __itt_model_site_beginW_ptr ITTNOTIFY_NAME(model_site_beginW) +#endif +#define __itt_model_site_beginA ITTNOTIFY_VOID(model_site_beginA) +#define __itt_model_site_beginA_ptr ITTNOTIFY_NAME(model_site_beginA) +#define __itt_model_site_beginAL ITTNOTIFY_VOID(model_site_beginAL) +#define __itt_model_site_beginAL_ptr ITTNOTIFY_NAME(model_site_beginAL) +#define __itt_model_site_end ITTNOTIFY_VOID(model_site_end) +#define __itt_model_site_end_ptr ITTNOTIFY_NAME(model_site_end) +#define __itt_model_site_end_2 ITTNOTIFY_VOID(model_site_end_2) +#define __itt_model_site_end_2_ptr ITTNOTIFY_NAME(model_site_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_site_begin(site, instance, name) +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW(name) +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA(name) +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL(name, siteNameLen) +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end(site, instance) +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2() +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_TASK_BEGIN/ANNOTATE_TASK_END support + * + * task_begin/end model a potential task, which is contained within the most + * closely enclosing dynamic site. task_end exits the most recently started + * but unended task. The handle passed to end may be used to validate + * structure. It is unspecified if bad dynamic nesting is detected. If it + * is, it should be encoded in the resulting data collection. The collector + * should not fail due to construct nesting issues, nor attempt to directly + * indicate the problem. + */ +void ITTAPI __itt_model_task_begin(__itt_model_task *task, __itt_model_task_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_task_beginW(const wchar_t *name); +void ITTAPI __itt_model_iteration_taskW(const wchar_t *name); +#endif +void ITTAPI __itt_model_task_beginA(const char *name); +void ITTAPI __itt_model_task_beginAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_iteration_taskA(const char *name); +void ITTAPI __itt_model_iteration_taskAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_task_end (__itt_model_task *task, __itt_model_task_instance *instance); +void ITTAPI __itt_model_task_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_task_begin, (__itt_model_task *task, __itt_model_task_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_task_beginW, (const wchar_t *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_task_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_task_beginAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_iteration_taskA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_task_end, (__itt_model_task *task, __itt_model_task_instance *instance)) +ITT_STUBV(ITTAPI, void, model_task_end_2, (void)) +#define __itt_model_task_begin ITTNOTIFY_VOID(model_task_begin) +#define __itt_model_task_begin_ptr ITTNOTIFY_NAME(model_task_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW ITTNOTIFY_VOID(model_task_beginW) +#define __itt_model_task_beginW_ptr ITTNOTIFY_NAME(model_task_beginW) +#define __itt_model_iteration_taskW ITTNOTIFY_VOID(model_iteration_taskW) +#define __itt_model_iteration_taskW_ptr ITTNOTIFY_NAME(model_iteration_taskW) +#endif +#define __itt_model_task_beginA ITTNOTIFY_VOID(model_task_beginA) +#define __itt_model_task_beginA_ptr ITTNOTIFY_NAME(model_task_beginA) +#define __itt_model_task_beginAL ITTNOTIFY_VOID(model_task_beginAL) +#define __itt_model_task_beginAL_ptr ITTNOTIFY_NAME(model_task_beginAL) +#define __itt_model_iteration_taskA ITTNOTIFY_VOID(model_iteration_taskA) +#define __itt_model_iteration_taskA_ptr ITTNOTIFY_NAME(model_iteration_taskA) +#define __itt_model_iteration_taskAL ITTNOTIFY_VOID(model_iteration_taskAL) +#define __itt_model_iteration_taskAL_ptr ITTNOTIFY_NAME(model_iteration_taskAL) +#define __itt_model_task_end ITTNOTIFY_VOID(model_task_end) +#define __itt_model_task_end_ptr ITTNOTIFY_NAME(model_task_end) +#define __itt_model_task_end_2 ITTNOTIFY_VOID(model_task_end_2) +#define __itt_model_task_end_2_ptr ITTNOTIFY_NAME(model_task_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_task_begin(task, instance, name) +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW(name) +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA(name) +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL(name, siteNameLen) +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA(name) +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL(name, siteNameLen) +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end(task, instance) +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2() +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_LOCK_ACQUIRE/ANNOTATE_LOCK_RELEASE support + * + * lock_acquire/release model a potential lock for both lockset and + * performance modeling. Each unique address is modeled as a separate + * lock, with invalid addresses being valid lock IDs. Specifically: + * no storage is accessed by the API at the specified address - it is only + * used for lock identification. Lock acquires may be self-nested and are + * unlocked by a corresponding number of releases. + * (These closely correspond to __itt_sync_acquired/__itt_sync_releasing, + * but may not have identical semantics.) + */ +void ITTAPI __itt_model_lock_acquire(void *lock); +void ITTAPI __itt_model_lock_acquire_2(void *lock); +void ITTAPI __itt_model_lock_release(void *lock); +void ITTAPI __itt_model_lock_release_2(void *lock); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_lock_acquire, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_acquire_2, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release_2, (void *lock)) +#define __itt_model_lock_acquire ITTNOTIFY_VOID(model_lock_acquire) +#define __itt_model_lock_acquire_ptr ITTNOTIFY_NAME(model_lock_acquire) +#define __itt_model_lock_acquire_2 ITTNOTIFY_VOID(model_lock_acquire_2) +#define __itt_model_lock_acquire_2_ptr ITTNOTIFY_NAME(model_lock_acquire_2) +#define __itt_model_lock_release ITTNOTIFY_VOID(model_lock_release) +#define __itt_model_lock_release_ptr ITTNOTIFY_NAME(model_lock_release) +#define __itt_model_lock_release_2 ITTNOTIFY_VOID(model_lock_release_2) +#define __itt_model_lock_release_2_ptr ITTNOTIFY_NAME(model_lock_release_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_lock_acquire(lock) +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2(lock) +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release(lock) +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2(lock) +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_RECORD_ALLOCATION/ANNOTATE_RECORD_DEALLOCATION support + * + * record_allocation/deallocation describe user-defined memory allocator + * behavior, which may be required for correctness modeling to understand + * when storage is not expected to be actually reused across threads. + */ +void ITTAPI __itt_model_record_allocation (void *addr, size_t size); +void ITTAPI __itt_model_record_deallocation(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_record_allocation, (void *addr, size_t size)) +ITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr)) +#define __itt_model_record_allocation ITTNOTIFY_VOID(model_record_allocation) +#define __itt_model_record_allocation_ptr ITTNOTIFY_NAME(model_record_allocation) +#define __itt_model_record_deallocation ITTNOTIFY_VOID(model_record_deallocation) +#define __itt_model_record_deallocation_ptr ITTNOTIFY_NAME(model_record_deallocation) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_record_allocation(addr, size) +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation(addr) +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_INDUCTION_USES support + * + * Note particular storage is inductive through the end of the current site + */ +void ITTAPI __itt_model_induction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_induction_uses, (void *addr, size_t size)) +#define __itt_model_induction_uses ITTNOTIFY_VOID(model_induction_uses) +#define __itt_model_induction_uses_ptr ITTNOTIFY_NAME(model_induction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_induction_uses(addr, size) +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_REDUCTION_USES support + * + * Note particular storage is used for reduction through the end + * of the current site + */ +void ITTAPI __itt_model_reduction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_reduction_uses, (void *addr, size_t size)) +#define __itt_model_reduction_uses ITTNOTIFY_VOID(model_reduction_uses) +#define __itt_model_reduction_uses_ptr ITTNOTIFY_NAME(model_reduction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_reduction_uses(addr, size) +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_OBSERVE_USES support + * + * Have correctness modeling record observations about uses of storage + * through the end of the current site + */ +void ITTAPI __itt_model_observe_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_observe_uses, (void *addr, size_t size)) +#define __itt_model_observe_uses ITTNOTIFY_VOID(model_observe_uses) +#define __itt_model_observe_uses_ptr ITTNOTIFY_NAME(model_observe_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_observe_uses(addr, size) +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_CLEAR_USES support + * + * Clear the special handling of a piece of storage related to induction, + * reduction or observe_uses + */ +void ITTAPI __itt_model_clear_uses(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_clear_uses, (void *addr)) +#define __itt_model_clear_uses ITTNOTIFY_VOID(model_clear_uses) +#define __itt_model_clear_uses_ptr ITTNOTIFY_NAME(model_clear_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_clear_uses(addr) +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_DISABLE_*_PUSH/ANNOTATE_DISABLE_*_POP support + * + * disable_push/disable_pop push and pop disabling based on a parameter. + * Disabling observations stops processing of memory references during + * correctness modeling, and all annotations that occur in the disabled + * region. This allows description of code that is expected to be handled + * specially during conversion to parallelism or that is not recognized + * by tools (e.g. some kinds of synchronization operations.) + * This mechanism causes all annotations in the disabled region, other + * than disable_push and disable_pop, to be ignored. (For example, this + * might validly be used to disable an entire parallel site and the contained + * tasks and locking in it for data collection purposes.) + * The disable for collection is a more expensive operation, but reduces + * collector overhead significantly. This applies to BOTH correctness data + * collection and performance data collection. For example, a site + * containing a task might only enable data collection for the first 10 + * iterations. Both performance and correctness data should reflect this, + * and the program should run as close to full speed as possible when + * collection is disabled. + */ +void ITTAPI __itt_model_disable_push(__itt_model_disable x); +void ITTAPI __itt_model_disable_pop(void); +void ITTAPI __itt_model_aggregate_task(size_t x); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_disable_push, (__itt_model_disable x)) +ITT_STUBV(ITTAPI, void, model_disable_pop, (void)) +ITT_STUBV(ITTAPI, void, model_aggregate_task, (size_t x)) +#define __itt_model_disable_push ITTNOTIFY_VOID(model_disable_push) +#define __itt_model_disable_push_ptr ITTNOTIFY_NAME(model_disable_push) +#define __itt_model_disable_pop ITTNOTIFY_VOID(model_disable_pop) +#define __itt_model_disable_pop_ptr ITTNOTIFY_NAME(model_disable_pop) +#define __itt_model_aggregate_task ITTNOTIFY_VOID(model_aggregate_task) +#define __itt_model_aggregate_task_ptr ITTNOTIFY_NAME(model_aggregate_task) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_disable_push(x) +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop() +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task(x) +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} model group */ + +/** + * @defgroup heap Heap + * @ingroup public + * Heap group + * @{ + */ + +typedef void* __itt_heap_function; + +/** + * @brief Create an identification for heap function + * @return non-zero identifier or NULL + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_heap_function ITTAPI __itt_heap_function_createA(const char* name, const char* domain); +__itt_heap_function ITTAPI __itt_heap_function_createW(const wchar_t* name, const wchar_t* domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_heap_function_create __itt_heap_function_createW +# define __itt_heap_function_create_ptr __itt_heap_function_createW_ptr +#else +# define __itt_heap_function_create __itt_heap_function_createA +# define __itt_heap_function_create_ptr __itt_heap_function_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_heap_function ITTAPI __itt_heap_function_create(const char* name, const char* domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char* name, const char* domain)) +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t* name, const wchar_t* domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_create, (const char* name, const char* domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA ITTNOTIFY_DATA(heap_function_createA) +#define __itt_heap_function_createA_ptr ITTNOTIFY_NAME(heap_function_createA) +#define __itt_heap_function_createW ITTNOTIFY_DATA(heap_function_createW) +#define __itt_heap_function_createW_ptr ITTNOTIFY_NAME(heap_function_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create ITTNOTIFY_DATA(heap_function_create) +#define __itt_heap_function_create_ptr ITTNOTIFY_NAME(heap_function_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation begin occurrence. + */ +void ITTAPI __itt_heap_allocate_begin(__itt_heap_function h, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_begin, (__itt_heap_function h, size_t size, int initialized)) +#define __itt_heap_allocate_begin ITTNOTIFY_VOID(heap_allocate_begin) +#define __itt_heap_allocate_begin_ptr ITTNOTIFY_NAME(heap_allocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_begin(h, size, initialized) +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation end occurrence. + */ +void ITTAPI __itt_heap_allocate_end(__itt_heap_function h, void** addr, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, size_t size, int initialized)) +#define __itt_heap_allocate_end ITTNOTIFY_VOID(heap_allocate_end) +#define __itt_heap_allocate_end_ptr ITTNOTIFY_NAME(heap_allocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_end(h, addr, size, initialized) +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a free begin occurrence. + */ +void ITTAPI __itt_heap_free_begin(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_begin ITTNOTIFY_VOID(heap_free_begin) +#define __itt_heap_free_begin_ptr ITTNOTIFY_NAME(heap_free_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_begin(h, addr) +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a free end occurrence. + */ +void ITTAPI __itt_heap_free_end(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_end ITTNOTIFY_VOID(heap_free_end) +#define __itt_heap_free_end_ptr ITTNOTIFY_NAME(heap_free_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_end(h, addr) +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a reallocation begin occurrence. + */ +void ITTAPI __itt_heap_reallocate_begin(__itt_heap_function h, void* addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_begin ITTNOTIFY_VOID(heap_reallocate_begin) +#define __itt_heap_reallocate_begin_ptr ITTNOTIFY_NAME(heap_reallocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_begin(h, addr, new_size, initialized) +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a reallocation end occurrence. + */ +void ITTAPI __itt_heap_reallocate_end(__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_end, (__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_end ITTNOTIFY_VOID(heap_reallocate_end) +#define __itt_heap_reallocate_end_ptr ITTNOTIFY_NAME(heap_reallocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_end(h, addr, new_addr, new_size, initialized) +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access begin */ +void ITTAPI __itt_heap_internal_access_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void)) +#define __itt_heap_internal_access_begin ITTNOTIFY_VOID(heap_internal_access_begin) +#define __itt_heap_internal_access_begin_ptr ITTNOTIFY_NAME(heap_internal_access_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_begin() +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access end */ +void ITTAPI __itt_heap_internal_access_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_end, (void)) +#define __itt_heap_internal_access_end ITTNOTIFY_VOID(heap_internal_access_end) +#define __itt_heap_internal_access_end_ptr ITTNOTIFY_NAME(heap_internal_access_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_end() +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth begin */ +void ITTAPI __itt_heap_record_memory_growth_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void)) +#define __itt_heap_record_memory_growth_begin ITTNOTIFY_VOID(heap_record_memory_growth_begin) +#define __itt_heap_record_memory_growth_begin_ptr ITTNOTIFY_NAME(heap_record_memory_growth_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_begin() +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth end */ +void ITTAPI __itt_heap_record_memory_growth_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void)) +#define __itt_heap_record_memory_growth_end ITTNOTIFY_VOID(heap_record_memory_growth_end) +#define __itt_heap_record_memory_growth_end_ptr ITTNOTIFY_NAME(heap_record_memory_growth_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_end() +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Specify the type of heap detection/reporting to modify. + */ +/** + * @hideinitializer + * @brief Report on memory leaks. + */ +#define __itt_heap_leaks 0x00000001 + +/** + * @hideinitializer + * @brief Report on memory growth. + */ +#define __itt_heap_growth 0x00000002 + + +/** @brief heap reset detection */ +void ITTAPI __itt_heap_reset_detection(unsigned int reset_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask)) +#define __itt_heap_reset_detection ITTNOTIFY_VOID(heap_reset_detection) +#define __itt_heap_reset_detection_ptr ITTNOTIFY_NAME(heap_reset_detection) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reset_detection() +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief report */ +void ITTAPI __itt_heap_record(unsigned int record_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record, (unsigned int record_mask)) +#define __itt_heap_record ITTNOTIFY_VOID(heap_record) +#define __itt_heap_record_ptr ITTNOTIFY_NAME(heap_record) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record() +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} heap group */ +/** @endcond */ +/* ========================================================================== */ + +/** + * @defgroup domains Domains + * @ingroup public + * Domains group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_domain +{ + volatile int flags; /*!< Zero if disabled, non-zero if enabled. The meaning of different non-zero values is reserved to the runtime */ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_domain* next; +} __itt_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup domains + * @brief Create a domain. + * Create domain using some domain name: the URI naming style is recommended. + * Because the set of domains is expected to be static over the application's + * execution time, there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of + * which thread created the domain. This call is thread-safe. + * @param[in] name name of domain + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_domain* ITTAPI __itt_domain_createA(const char *name); +__itt_domain* ITTAPI __itt_domain_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_domain_create __itt_domain_createW +# define __itt_domain_create_ptr __itt_domain_createW_ptr +#else /* UNICODE */ +# define __itt_domain_create __itt_domain_createA +# define __itt_domain_create_ptr __itt_domain_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_domain* ITTAPI __itt_domain_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA ITTNOTIFY_DATA(domain_createA) +#define __itt_domain_createA_ptr ITTNOTIFY_NAME(domain_createA) +#define __itt_domain_createW ITTNOTIFY_DATA(domain_createW) +#define __itt_domain_createW_ptr ITTNOTIFY_NAME(domain_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create ITTNOTIFY_DATA(domain_create) +#define __itt_domain_create_ptr ITTNOTIFY_NAME(domain_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA(name) (__itt_domain*)0 +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW(name) (__itt_domain*)0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create(name) (__itt_domain*)0 +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} domains group */ + +/** + * @defgroup ids IDs + * @ingroup public + * IDs group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_id +{ + unsigned long long d1, d2, d3; +} __itt_id; + +#pragma pack(pop) +/** @endcond */ + +static const __itt_id __itt_null = { 0, 0, 0 }; + +/** + * @ingroup ids + * @brief A convenience function is provided to create an ID without domain control. + * @brief This is a convenience function to initialize an __itt_id structure. This function + * does not affect the collector runtime in any way. After you make the ID with this + * function, you still must create it with the __itt_id_create function before using the ID + * to identify a named entity. + * @param[in] addr The address of object; high QWORD of the ID value. + * @param[in] extra The extra data to unique identify object; low QWORD of the ID value. + */ + +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) ITT_INLINE_ATTRIBUTE; +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) +{ + __itt_id id = __itt_null; + id.d1 = (unsigned long long)((uintptr_t)addr); + id.d2 = (unsigned long long)extra; + id.d3 = (unsigned long long)0; /* Reserved. Must be zero */ + return id; +} + +/** + * @ingroup ids + * @brief Create an instance of identifier. + * This establishes the beginning of the lifetime of an instance of + * the given ID in the trace. Once this lifetime starts, the ID + * can be used to tag named entity instances in calls such as + * __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * Instance IDs are not domain specific! + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_create(d,x) ITTNOTIFY_VOID_D1(id_create,d,x) +#define __itt_id_create_ptr ITTNOTIFY_NAME(id_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create(domain,id) +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup ids + * @brief Destroy an instance of identifier. + * This ends the lifetime of the current instance of the given ID value in the trace. + * Any relationships that are established after this lifetime ends are invalid. + * This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_destroy(d,x) ITTNOTIFY_VOID_D1(id_destroy,d,x) +#define __itt_id_destroy_ptr ITTNOTIFY_NAME(id_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_destroy(domain,id) +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} ids group */ + +/** + * @defgroup handless String Handles + * @ingroup public + * String Handles group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_string_handle +{ + const char* strA; /*!< Copy of original string in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* strW; /*!< Copy of original string in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* strW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_string_handle* next; +} __itt_string_handle; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup handles + * @brief Create a string handle. + * Create and return handle value that can be associated with a string. + * Consecutive calls to __itt_string_handle_create with the same name + * return the same value. Because the set of string handles is expected to remain + * static during the application's execution time, there is no mechanism to destroy a string handle. + * Any string handle can be accessed by any thread in the process, regardless of which thread created + * the string handle. This call is thread-safe. + * @param[in] name The input string + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_string_handle* ITTAPI __itt_string_handle_createA(const char *name); +__itt_string_handle* ITTAPI __itt_string_handle_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_string_handle_create __itt_string_handle_createW +# define __itt_string_handle_create_ptr __itt_string_handle_createW_ptr +#else /* UNICODE */ +# define __itt_string_handle_create __itt_string_handle_createA +# define __itt_string_handle_create_ptr __itt_string_handle_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_string_handle* ITTAPI __itt_string_handle_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA ITTNOTIFY_DATA(string_handle_createA) +#define __itt_string_handle_createA_ptr ITTNOTIFY_NAME(string_handle_createA) +#define __itt_string_handle_createW ITTNOTIFY_DATA(string_handle_createW) +#define __itt_string_handle_createW_ptr ITTNOTIFY_NAME(string_handle_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create ITTNOTIFY_DATA(string_handle_create) +#define __itt_string_handle_create_ptr ITTNOTIFY_NAME(string_handle_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA(name) (__itt_string_handle*)0 +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW(name) (__itt_string_handle*)0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create(name) (__itt_string_handle*)0 +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} handles group */ + +/** @cond exclude_from_documentation */ +typedef unsigned long long __itt_timestamp; +/** @endcond */ + +#define __itt_timestamp_none ((__itt_timestamp)-1LL) + +/** @cond exclude_from_gpa_documentation */ + +/** + * @ingroup timestamps + * @brief Return timestamp corresponding to the current moment. + * This returns the timestamp in the format that is the most relevant for the current + * host or platform (RDTSC, QPC, and others). You can use the "<" operator to + * compare __itt_timestamp values. + */ +__itt_timestamp ITTAPI __itt_get_timestamp(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void)) +#define __itt_get_timestamp ITTNOTIFY_DATA(get_timestamp) +#define __itt_get_timestamp_ptr ITTNOTIFY_NAME(get_timestamp) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_get_timestamp() +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} timestamps */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @defgroup regions Regions + * @ingroup public + * Regions group + * @{ + */ +/** + * @ingroup regions + * @brief Begin of region instance. + * Successive calls to __itt_region_begin with the same ID are ignored + * until a call to __itt_region_end with the same ID + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance. Must not be __itt_null + * @param[in] parentid The instance ID for the parent of this region instance, or __itt_null + * @param[in] name The name of this region + */ +void ITTAPI __itt_region_begin(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup regions + * @brief End of region instance. + * The first call to __itt_region_end with a given ID ends the + * region. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_region_begin call. + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance + */ +void ITTAPI __itt_region_end(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, region_end, (const __itt_domain *domain, __itt_id id)) +#define __itt_region_begin(d,x,y,z) ITTNOTIFY_VOID_D3(region_begin,d,x,y,z) +#define __itt_region_begin_ptr ITTNOTIFY_NAME(region_begin) +#define __itt_region_end(d,x) ITTNOTIFY_VOID_D1(region_end,d,x) +#define __itt_region_end_ptr ITTNOTIFY_NAME(region_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_region_begin(d,x,y,z) +#define __itt_region_begin_ptr 0 +#define __itt_region_end(d,x) +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_region_begin_ptr 0 +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} regions group */ + +/** + * @defgroup frames Frames + * @ingroup public + * Frames are similar to regions, but are intended to be easier to use and to implement. + * In particular: + * - Frames always represent periods of elapsed time + * - By default, frames have no nesting relationships + * @{ + */ + +/** + * @ingroup frames + * @brief Begin a frame instance. + * Successive calls to __itt_frame_begin with the + * same ID are ignored until a call to __itt_frame_end with the same ID. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + */ +void ITTAPI __itt_frame_begin_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief End a frame instance. + * The first call to __itt_frame_end with a given ID + * ends the frame. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_frame_begin call. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL for current + */ +void ITTAPI __itt_frame_end_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief Submits a frame instance. + * Successive calls to __itt_frame_begin or __itt_frame_submit with the + * same ID are ignored until a call to __itt_frame_end or __itt_frame_submit + * with the same ID. + * Passing special __itt_timestamp_none value as "end" argument means + * take the current timestamp as the end timestamp. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + * @param[in] begin Timestamp of the beginning of the frame + * @param[in] end Timestamp of the end of the frame + */ +void ITTAPI __itt_frame_submit_v3(const __itt_domain *domain, __itt_id *id, + __itt_timestamp begin, __itt_timestamp end); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, frame_begin_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_end_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end)) +#define __itt_frame_begin_v3(d,x) ITTNOTIFY_VOID_D1(frame_begin_v3,d,x) +#define __itt_frame_begin_v3_ptr ITTNOTIFY_NAME(frame_begin_v3) +#define __itt_frame_end_v3(d,x) ITTNOTIFY_VOID_D1(frame_end_v3,d,x) +#define __itt_frame_end_v3_ptr ITTNOTIFY_NAME(frame_end_v3) +#define __itt_frame_submit_v3(d,x,b,e) ITTNOTIFY_VOID_D3(frame_submit_v3,d,x,b,e) +#define __itt_frame_submit_v3_ptr ITTNOTIFY_NAME(frame_submit_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_frame_begin_v3(domain,id) +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3(domain,id) +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3(domain,id,begin,end) +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} frames group */ +/** @endcond */ + +/** + * @defgroup taskgroup Task Group + * @ingroup public + * Task Group + * @{ + */ +/** + * @ingroup task_groups + * @brief Denotes a task_group instance. + * Successive calls to __itt_task_group with the same ID are ignored. + * @param[in] domain The domain for this task_group instance + * @param[in] id The instance ID for this task_group instance. Must not be __itt_null. + * @param[in] parentid The instance ID for the parent of this task_group instance, or __itt_null. + * @param[in] name The name of this task_group + */ +void ITTAPI __itt_task_group(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_group, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +#define __itt_task_group(d,x,y,z) ITTNOTIFY_VOID_D3(task_group,d,x,y,z) +#define __itt_task_group_ptr ITTNOTIFY_NAME(task_group) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_group(d,x,y,z) +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} taskgroup group */ + +/** + * @defgroup tasks Tasks + * @ingroup public + * A task instance represents a piece of work performed by a particular + * thread for a period of time. A call to __itt_task_begin creates a + * task instance. This becomes the current instance for that task on that + * thread. A following call to __itt_task_end on the same thread ends the + * instance. There may be multiple simultaneous instances of tasks with the + * same name on different threads. If an ID is specified, the task instance + * receives that ID. Nested tasks are allowed. + * + * Note: The task is defined by the bracketing of __itt_task_begin and + * __itt_task_end on the same thread. If some scheduling mechanism causes + * task switching (the thread executes a different user task) or task + * switching (the user task switches to a different thread) then this breaks + * the notion of current instance. Additional API calls are required to + * deal with that possibility. + * @{ + */ + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The identifier for this task instance (may be 0) + * @param[in] parentid The parent of this task (may be 0) + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup tasks + * @brief End the current task instance. + * @param[in] domain The domain for this task + */ +void ITTAPI __itt_task_end(const __itt_domain *domain); + +/** + * @ingroup tasks + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped(const __itt_domain* domain, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup tasks + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped(const __itt_domain *domain, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain)) +ITT_STUBV(ITTAPI, void, task_begin_overlapped, (const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id taskid)) +#define __itt_task_begin(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin,d,x,y,z) +#define __itt_task_begin_ptr ITTNOTIFY_NAME(task_begin) +#define __itt_task_begin_fn(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_fn,d,x,y,z) +#define __itt_task_begin_fn_ptr ITTNOTIFY_NAME(task_begin_fn) +#define __itt_task_end(d) ITTNOTIFY_VOID_D0(task_end,d) +#define __itt_task_end_ptr ITTNOTIFY_NAME(task_end) +#define __itt_task_begin_overlapped(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_overlapped,d,x,y,z) +#define __itt_task_begin_overlapped_ptr ITTNOTIFY_NAME(task_begin_overlapped) +#define __itt_task_end_overlapped(d,x) ITTNOTIFY_VOID_D1(task_end_overlapped,d,x) +#define __itt_task_end_overlapped_ptr ITTNOTIFY_NAME(task_end_overlapped) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin(domain,id,parentid,name) +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn(domain,id,parentid,fn) +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end(domain) +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped(domain,taskid,parentid,name) +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped(domain,taskid) +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} tasks group */ + + +/** + * @defgroup markers Markers + * Markers represent a single discreet event in time. Markers have a scope, + * described by an enumerated type __itt_scope. Markers are created by + * the API call __itt_marker. A marker instance can be given an ID for use in + * adding metadata. + * @{ + */ + +/** + * @brief Describes the scope of an event object in the trace. + */ +typedef enum +{ + __itt_scope_unknown = 0, + __itt_scope_global, + __itt_scope_track_group, + __itt_scope_track, + __itt_scope_task, + __itt_scope_marker +} __itt_scope; + +/** @cond exclude_from_documentation */ +#define __itt_marker_scope_unknown __itt_scope_unknown +#define __itt_marker_scope_global __itt_scope_global +#define __itt_marker_scope_process __itt_scope_track_group +#define __itt_marker_scope_thread __itt_scope_track +#define __itt_marker_scope_task __itt_scope_task +/** @endcond */ + +/** + * @ingroup markers + * @brief Create a marker instance + * @param[in] domain The domain for this marker + * @param[in] id The instance ID for this marker or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker(const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker(d,x,y,z) ITTNOTIFY_VOID_D3(marker,d,x,y,z) +#define __itt_marker_ptr ITTNOTIFY_NAME(marker) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker(domain,id,name,scope) +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} markers group */ + +/** + * @defgroup metadata Metadata + * The metadata API is used to attach extra information to named + * entities. Metadata can be attached to an identified named entity by ID, + * or to the current entity (which is always a task). + * + * Conceptually metadata has a type (what kind of metadata), a key (the + * name of the metadata), and a value (the actual data). The encoding of + * the value depends on the type of the metadata. + * + * The type of metadata is specified by an enumerated type __itt_metdata_type. + * @{ + */ + +/** + * @ingroup parameters + * @brief describes the type of metadata + */ +typedef enum { + __itt_metadata_unknown = 0, + __itt_metadata_u64, /**< Unsigned 64-bit integer */ + __itt_metadata_s64, /**< Signed 64-bit integer */ + __itt_metadata_u32, /**< Unsigned 32-bit integer */ + __itt_metadata_s32, /**< Signed 32-bit integer */ + __itt_metadata_u16, /**< Unsigned 16-bit integer */ + __itt_metadata_s16, /**< Signed 16-bit integer */ + __itt_metadata_float, /**< Signed 32-bit floating-point */ + __itt_metadata_double /**< SIgned 64-bit floating-point */ +} __itt_metadata_type; + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add,d,x,y,z,a,b) +#define __itt_metadata_add_ptr ITTNOTIFY_NAME(metadata_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add(d,x,y,z,a,b) +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_addA(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_addW(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add __itt_metadata_str_addW +# define __itt_metadata_str_add_ptr __itt_metadata_str_addW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add __itt_metadata_str_addA +# define __itt_metadata_str_add_ptr __itt_metadata_str_addA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addA,d,x,y,z,a) +#define __itt_metadata_str_addA_ptr ITTNOTIFY_NAME(metadata_str_addA) +#define __itt_metadata_str_addW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addW,d,x,y,z,a) +#define __itt_metadata_str_addW_ptr ITTNOTIFY_NAME(metadata_str_addW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add,d,x,y,z,a) +#define __itt_metadata_str_add_ptr ITTNOTIFY_NAME(metadata_str_add) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW(d,x,y,z,a) +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add_with_scope,d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr ITTNOTIFY_NAME(metadata_add_with_scope) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_add_with_scopeA(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_add_with_scopeW(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeW +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeA +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeA,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeA) +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeW,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scope,d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr ITTNOTIFY_NAME(metadata_str_add_with_scope) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} metadata group */ + +/** + * @defgroup relations Relations + * Instances of named entities can be explicitly associated with other + * instances using instance IDs and the relationship API calls. + * + * @{ + */ + +/** + * @ingroup relations + * @brief The kind of relation between two instances is specified by the enumerated type __itt_relation. + * Relations between instances can be added with an API call. The relation + * API uses instance IDs. Relations can be added before or after the actual + * instances are created and persist independently of the instances. This + * is the motivation for having different lifetimes for instance IDs and + * the actual instances. + */ +typedef enum +{ + __itt_relation_is_unknown = 0, + __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ + __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ + __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ + __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ + __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ + __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ + __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ +} __itt_relation; + +/** + * @ingroup relations + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current(const __itt_domain *domain, __itt_relation relation, __itt_id tail); + +/** + * @ingroup relations + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add(const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add, (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current(d,x,y) ITTNOTIFY_VOID_D2(relation_add_to_current,d,x,y) +#define __itt_relation_add_to_current_ptr ITTNOTIFY_NAME(relation_add_to_current) +#define __itt_relation_add(d,x,y,z) ITTNOTIFY_VOID_D3(relation_add,d,x,y,z) +#define __itt_relation_add_ptr ITTNOTIFY_NAME(relation_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current(d,x,y) +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add(d,x,y,z) +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} relations group */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_info +{ + unsigned long long clock_freq; /*!< Clock domain frequency */ + unsigned long long clock_base; /*!< Clock domain base timestamp */ +} __itt_clock_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef void (ITTAPI *__itt_get_clock_info_fn)(__itt_clock_info* clock_info, void* data); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_domain +{ + __itt_clock_info info; /*!< Most recent clock domain info */ + __itt_get_clock_info_fn fn; /*!< Callback function pointer */ + void* fn_data; /*!< Input argument for the callback function */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_clock_domain* next; +} __itt_clock_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Create a clock domain. + * Certain applications require the capability to trace their application using + * a clock domain different than the CPU, for instance the instrumentation of events + * that occur on a GPU. + * Because the set of domains is expected to be static over the application's execution time, + * there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of which thread created + * the domain. This call is thread-safe. + * @param[in] fn A pointer to a callback function which retrieves alternative CPU timestamps + * @param[in] fn_data Argument for a callback function; may be NULL + */ +__itt_clock_domain* ITTAPI __itt_clock_domain_create(__itt_get_clock_info_fn fn, void* fn_data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data)) +#define __itt_clock_domain_create ITTNOTIFY_DATA(clock_domain_create) +#define __itt_clock_domain_create_ptr ITTNOTIFY_NAME(clock_domain_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_create(fn,fn_data) (__itt_clock_domain*)0 +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Recalculate clock domains frequencies and clock base timestamps. + */ +void ITTAPI __itt_clock_domain_reset(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, clock_domain_reset, (void)) +#define __itt_clock_domain_reset ITTNOTIFY_VOID(clock_domain_reset) +#define __itt_clock_domain_reset_ptr ITTNOTIFY_NAME(clock_domain_reset) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_reset() +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Create an instance of identifier. This establishes the beginning of the lifetime of + * an instance of the given ID in the trace. Once this lifetime starts, the ID can be used to + * tag named entity instances in calls such as __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** + * @ingroup clockdomain + * @brief Destroy an instance of identifier. This ends the lifetime of the current instance of the + * given ID value in the trace. Any relationships that are established after this lifetime ends are + * invalid. This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +ITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +#define __itt_id_create_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_create_ex,d,x,y,z) +#define __itt_id_create_ex_ptr ITTNOTIFY_NAME(id_create_ex) +#define __itt_id_destroy_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_destroy_ex,d,x,y,z) +#define __itt_id_destroy_ex_ptr ITTNOTIFY_NAME(id_destroy_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create_ex(domain,clock_domain,timestamp,id) +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex(domain,clock_domain,timestamp,id) +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, or __itt_null + * @param[in] parentid The parent of this task, or __itt_null + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup clockdomain + * @brief End the current task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + */ +void ITTAPI __itt_task_end_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp)) +#define __itt_task_begin_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_ex,d,x,y,z,a,b) +#define __itt_task_begin_ex_ptr ITTNOTIFY_NAME(task_begin_ex) +#define __itt_task_begin_fn_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_fn_ex,d,x,y,z,a,b) +#define __itt_task_begin_fn_ex_ptr ITTNOTIFY_NAME(task_begin_fn_ex) +#define __itt_task_end_ex(d,x,y) ITTNOTIFY_VOID_D2(task_end_ex,d,x,y) +#define __itt_task_end_ex_ptr ITTNOTIFY_NAME(task_end_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_ex(domain,clock_domain,timestamp,id,parentid,name) +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex(domain,clock_domain,timestamp,id,parentid,fn) +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex(domain,clock_domain,timestamp) +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup counters Counters + * @ingroup public + * Counters are user-defined objects with a monotonically increasing + * value. Counter values are 64-bit unsigned integers. + * Counters have names that can be displayed in + * the tools. + * @{ + */ + +/** + * @brief opaque structure for counter identification + */ +/** @cond exclude_from_documentation */ + +typedef struct ___itt_counter* __itt_counter; + +/** + * @brief Create an unsigned 64 bits integer counter with given name/domain + * + * After __itt_counter_create() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter, where value_ptr is a pointer to an unsigned 64 bits integer + * + * The call is equal to __itt_counter_create_typed(name, domain, __itt_metadata_u64) + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain); +__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create __itt_counter_createW +# define __itt_counter_create_ptr __itt_counter_createW_ptr +#else /* UNICODE */ +# define __itt_counter_create __itt_counter_createA +# define __itt_counter_create_ptr __itt_counter_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA) +#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA) +#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW) +#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create ITTNOTIFY_DATA(counter_create) +#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA(name, domain) +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW(name, domain) +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create(name, domain) +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Increment the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id)) +#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc) +#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc(id) +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Increment the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta) +#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_delta(id, value) +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Decrement the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id)) +#define __itt_counter_dec ITTNOTIFY_VOID(counter_dec) +#define __itt_counter_dec_ptr ITTNOTIFY_NAME(counter_dec) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec(id) +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Decrement the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_dec_delta ITTNOTIFY_VOID(counter_dec_delta) +#define __itt_counter_dec_delta_ptr ITTNOTIFY_NAME(counter_dec_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_delta(id, value) +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup counters + * @brief Increment a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls increment the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Increment a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to increment the counter + */ +void ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x) +#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3) +#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y) +#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_v3(domain,name) +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3(domain,name,delta) +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + + +/** + * @ingroup counters + * @brief Decrement a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls decrement the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_dec_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Decrement a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to decrement the counter + */ +void ITTAPI __itt_counter_dec_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_dec_v3(d,x) ITTNOTIFY_VOID_D1(counter_dec_v3,d,x) +#define __itt_counter_dec_v3_ptr ITTNOTIFY_NAME(counter_dec_v3) +#define __itt_counter_dec_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_dec_delta_v3,d,x,y) +#define __itt_counter_dec_delta_v3_ptr ITTNOTIFY_NAME(counter_dec_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_v3(domain,name) +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3(domain,name,delta) +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} counters group */ + + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value(__itt_counter id, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr)) +#define __itt_counter_set_value ITTNOTIFY_VOID(counter_set_value) +#define __itt_counter_set_value_ptr ITTNOTIFY_NAME(counter_set_value) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value(id, value_ptr) +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value_ex(__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr)) +#define __itt_counter_set_value_ex ITTNOTIFY_VOID(counter_set_value_ex) +#define __itt_counter_set_value_ex_ptr ITTNOTIFY_NAME(counter_set_value_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create a typed counter with given name/domain + * + * After __itt_counter_create_typed() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_create_typedA(const char *name, const char *domain, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_create_typedW(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_typed __itt_counter_create_typedW +# define __itt_counter_create_typed_ptr __itt_counter_create_typedW_ptr +#else /* UNICODE */ +# define __itt_counter_create_typed __itt_counter_create_typedA +# define __itt_counter_create_typed_ptr __itt_counter_create_typedA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_typed(const char *name, const char *domain, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA ITTNOTIFY_DATA(counter_create_typedA) +#define __itt_counter_create_typedA_ptr ITTNOTIFY_NAME(counter_create_typedA) +#define __itt_counter_create_typedW ITTNOTIFY_DATA(counter_create_typedW) +#define __itt_counter_create_typedW_ptr ITTNOTIFY_NAME(counter_create_typedW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed ITTNOTIFY_DATA(counter_create_typed) +#define __itt_counter_create_typed_ptr ITTNOTIFY_NAME(counter_create_typed) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA(name, domain, type) +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW(name, domain, type) +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed(name, domain, type) +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() or + * __itt_counter_create_typed() + */ +void ITTAPI __itt_counter_destroy(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id)) +#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy) +#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_destroy(id) +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} counters group */ + +/** + * @ingroup markers + * @brief Create a marker instance. + * @param[in] domain The domain for this marker + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The instance ID for this marker, or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b) +#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope) +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail); + +/** + * @ingroup clockdomain + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a) +#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex) +#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b) +#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail) +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail) +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum ___itt_track_group_type +{ + __itt_track_group_type_normal = 0 +} __itt_track_group_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track_group +{ + __itt_string_handle* name; /*!< Name of the track group */ + struct ___itt_track* track; /*!< List of child tracks */ + __itt_track_group_type tgtype; /*!< Type of the track group */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track_group* next; +} __itt_track_group; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Placeholder for custom track types. Currently, "normal" custom track + * is the only available track type. + */ +typedef enum ___itt_track_type +{ + __itt_track_type_normal = 0 +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + , __itt_track_type_queue +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ +} __itt_track_type; + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track +{ + __itt_string_handle* name; /*!< Name of the track group */ + __itt_track_group* group; /*!< Parent group to a track */ + __itt_track_type ttype; /*!< Type of the track */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track* next; +} __itt_track; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create logical track group. + */ +__itt_track_group* ITTAPI __itt_track_group_create(__itt_string_handle* name, __itt_track_group_type track_group_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type)) +#define __itt_track_group_create ITTNOTIFY_DATA(track_group_create) +#define __itt_track_group_create_ptr ITTNOTIFY_NAME(track_group_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_group_create(name) (__itt_track_group*)0 +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create logical track. + */ +__itt_track* ITTAPI __itt_track_create(__itt_track_group* track_group, __itt_string_handle* name, __itt_track_type track_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track*, track_create, (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type)) +#define __itt_track_create ITTNOTIFY_DATA(track_create) +#define __itt_track_create_ptr ITTNOTIFY_NAME(track_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_create(track_group,name,track_type) (__itt_track*)0 +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the logical track. + */ +void ITTAPI __itt_set_track(__itt_track* track); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, set_track, (__itt_track *track)) +#define __itt_set_track ITTNOTIFY_VOID(set_track) +#define __itt_set_track_ptr ITTNOTIFY_NAME(set_track) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_set_track(track) +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/* ========================================================================== */ +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup events Events + * @ingroup public + * Events group + * @{ + */ +/** @brief user event type */ +typedef int __itt_event; + +/** + * @brief Create an event notification + * @note name or namelen being null/name and namelen not matching, user event feature not enabled + * @return non-zero event identifier upon success and __itt_err otherwise + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_event LIBITTAPI __itt_event_createA(const char *name, int namelen); +__itt_event LIBITTAPI __itt_event_createW(const wchar_t *name, int namelen); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_event_create __itt_event_createW +# define __itt_event_create_ptr __itt_event_createW_ptr +#else +# define __itt_event_create __itt_event_createA +# define __itt_event_create_ptr __itt_event_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_event LIBITTAPI __itt_event_create(const char *name, int namelen); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen)) +ITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA ITTNOTIFY_DATA(event_createA) +#define __itt_event_createA_ptr ITTNOTIFY_NAME(event_createA) +#define __itt_event_createW ITTNOTIFY_DATA(event_createW) +#define __itt_event_createW_ptr ITTNOTIFY_NAME(event_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create ITTNOTIFY_DATA(event_create) +#define __itt_event_create_ptr ITTNOTIFY_NAME(event_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA(name, namelen) (__itt_event)0 +#define __itt_event_createA_ptr 0 +#define __itt_event_createW(name, namelen) (__itt_event)0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create(name, namelen) (__itt_event)0 +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA_ptr 0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event occurrence. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_start(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_start, (__itt_event event)) +#define __itt_event_start ITTNOTIFY_DATA(event_start) +#define __itt_event_start_ptr ITTNOTIFY_NAME(event_start) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_start(event) (int)0 +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event end occurrence. + * @note It is optional if events do not have durations. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_end(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event)) +#define __itt_event_end ITTNOTIFY_DATA(event_end) +#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_end(event) (int)0 +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} events group */ + + +/** + * @defgroup arrays Arrays Visualizer + * @ingroup public + * Visualize arrays + * @{ + */ + +/** + * @enum __itt_av_data_type + * @brief Defines types of arrays data (for C/C++ intrinsic types) + */ +typedef enum +{ + __itt_e_first = 0, + __itt_e_char = 0, /* 1-byte integer */ + __itt_e_uchar, /* 1-byte unsigned integer */ + __itt_e_int16, /* 2-byte integer */ + __itt_e_uint16, /* 2-byte unsigned integer */ + __itt_e_int32, /* 4-byte integer */ + __itt_e_uint32, /* 4-byte unsigned integer */ + __itt_e_int64, /* 8-byte integer */ + __itt_e_uint64, /* 8-byte unsigned integer */ + __itt_e_float, /* 4-byte floating */ + __itt_e_double, /* 8-byte floating */ + __itt_e_last = __itt_e_double +} __itt_av_data_type; + +/** + * @brief Save an array data to a file. + * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only). + * @param[in] data - pointer to the array data + * @param[in] rank - the rank of the array + * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. + * The size of dimensions must be equal to the rank + * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types) + * @param[in] filePath - the file path; the output format is defined by the file extension + * @param[in] columnOrder - defines how the array is stored in the linear memory. + * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C). + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +int ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_av_save __itt_av_saveW +# define __itt_av_save_ptr __itt_av_saveW_ptr +#else /* UNICODE */ +# define __itt_av_save __itt_av_saveA +# define __itt_av_save_ptr __itt_av_saveA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA) +#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA) +#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW) +#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save ITTNOTIFY_DATA(av_save) +#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA(name) +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW(name) +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save(name) +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +void ITTAPI __itt_enable_attach(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, enable_attach, (void)) +#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach) +#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_enable_attach() +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** @} arrays group */ + +/** @endcond */ + +/** + * @brief Module load notification + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done immidiatelly after this module is loaded to process memory. + * @param[in] start_addr - module start address + * @param[in] end_addr - module end address + * @param[in] path - file system full path to the module + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_module_loadA(void *start_addr, void *end_addr, const char *path); +void ITTAPI __itt_module_loadW(void *start_addr, void *end_addr, const wchar_t *path); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_module_load __itt_module_loadW +# define __itt_module_load_ptr __itt_module_loadW_ptr +#else /* UNICODE */ +# define __itt_module_load __itt_module_loadA +# define __itt_module_load_ptr __itt_module_loadA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_module_load(void *start_addr, void *end_addr, const char *path); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, void, module_loadA, (void *start_addr, void *end_addr, const char *path)) +ITT_STUB(ITTAPI, void, module_loadW, (void *start_addr, void *end_addr, const wchar_t *path)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA ITTNOTIFY_VOID(module_loadA) +#define __itt_module_loadA_ptr ITTNOTIFY_NAME(module_loadA) +#define __itt_module_loadW ITTNOTIFY_VOID(module_loadW) +#define __itt_module_loadW_ptr ITTNOTIFY_NAME(module_loadW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load ITTNOTIFY_VOID(module_load) +#define __itt_module_load_ptr ITTNOTIFY_NAME(module_load) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA(start_addr, end_addr, path) +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW(start_addr, end_addr, path) +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load(start_addr, end_addr, path) +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Report module unload + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done just before the module is unloaded from process memory. + * @param[in] addr - base address of loaded module + */ +void ITTAPI __itt_module_unload(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload, (void *addr)) +#define __itt_module_unload ITTNOTIFY_VOID(module_unload) +#define __itt_module_unload_ptr ITTNOTIFY_NAME(module_unload) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload(addr) +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + __itt_module_type_unknown = 0, + __itt_module_type_elf, + __itt_module_type_coff +} __itt_module_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + itt_section_type_unknown, + itt_section_type_bss, /* notifies that the section contains uninitialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_NOBITS section type + * COFF module: IMAGE_SCN_CNT_UNINITIALIZED_DATA section type + */ + itt_section_type_data, /* notifies that section contains initialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_INITIALIZED_DATA section type + */ + itt_section_type_text /* notifies that the section contains executable code. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_CODE section type + */ +} __itt_section_type; +/** @endcond */ + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be executed as code: + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_X section attribute + * COFF module: IMAGE_SCN_MEM_EXECUTE attribute + */ +#define __itt_section_exec 0x20000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be read. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_R attribute + * COFF module: IMAGE_SCN_MEM_READ attribute + */ +#define __itt_section_read 0x40000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be written to. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_W attribute + * COFF module: IMAGE_SCN_MEM_WRITE attribute + */ +#define __itt_section_write 0x80000000 + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_section_info +{ + const char* name; /*!< Section name in UTF8 */ + __itt_section_type type; /*!< Section content and semantics description */ + size_t flags; /*!< Section bit flags that describe attributes using bit mask + * Zero if disabled, non-zero if enabled + */ + void* start_addr; /*!< Section load(relocated) start address */ + size_t size; /*!< Section file offset */ + size_t file_offset; /*!< Section size */ +} __itt_section_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_module_object +{ + unsigned int version; /*!< API version*/ + __itt_id module_id; /*!< Unique identifier. This is unchanged for sections that belong to the same module */ + __itt_module_type module_type; /*!< Binary module format */ + const char* module_name; /*!< Unique module name or path to module in UTF8 + * Contains module name when module_bufer and module_size exist + * Contains module path when module_bufer and module_size absent + * module_name remains the same for the certain module_id + */ + void* module_buffer; /*!< Module buffer content */ + size_t module_size; /*!< Module buffer size */ + /*!< If module_buffer and module_size exist, the binary module is dumped onto the system. + * If module_buffer and module_size do not exist, + * the binary module exists on the system already. + * The module_name parameter contains the path to the module. + */ + __itt_section_info* section_array; /*!< Reference to section information */ + size_t section_number; +} __itt_module_object; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Load module content and its loaded(relocated) sections. + * This API is useful to save a module, or specify its location on the system and report information about loaded sections. + * The target module is saved on the system if module buffer content and size are available. + * If module buffer content and size are unavailable, the module name contains the path to the existing binary module. + * @param[in] module_obj - provides module and section information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_load_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_load_with_sections ITTNOTIFY_VOID(module_load_with_sections) +#define __itt_module_load_with_sections_ptr ITTNOTIFY_NAME(module_load_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_load_with_sections(module_obj) +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Unload a module and its loaded(relocated) sections. + * This API notifies that the module and its sections were unloaded. + * @param[in] module_obj - provides module and sections information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_unload_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_unload_with_sections ITTNOTIFY_VOID(module_unload_with_sections) +#define __itt_module_unload_with_sections_ptr ITTNOTIFY_NAME(module_unload_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload_with_sections(module_obj) +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_histogram +{ + const __itt_domain* domain; /*!< Domain of the histogram*/ + const char* nameA; /*!< Name of the histogram */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + __itt_metadata_type x_type; /*!< Type of the histogram X axis */ + __itt_metadata_type y_type; /*!< Type of the histogram Y axis */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_histogram* next; +} __itt_histogram; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create a typed histogram instance with given name/domain. + * @param[in] domain The domain controlling the call. + * @param[in] name The name of the histogram. + * @param[in] x_type The type of the X axis in histogram (may be 0 to calculate batch statistics). + * @param[in] y_type The type of the Y axis in histogram. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_histogram* ITTAPI __itt_histogram_createA(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +__itt_histogram* ITTAPI __itt_histogram_createW(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_histogram_create __itt_histogram_createW +# define __itt_histogram_create_ptr __itt_histogram_createW_ptr +#else /* UNICODE */ +# define __itt_histogram_create __itt_histogram_createA +# define __itt_histogram_create_ptr __itt_histogram_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_histogram* ITTAPI __itt_histogram_create(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA ITTNOTIFY_DATA(histogram_createA) +#define __itt_histogram_createA_ptr ITTNOTIFY_NAME(histogram_createA) +#define __itt_histogram_createW ITTNOTIFY_DATA(histogram_createW) +#define __itt_histogram_createW_ptr ITTNOTIFY_NAME(histogram_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create ITTNOTIFY_DATA(histogram_create) +#define __itt_histogram_create_ptr ITTNOTIFY_NAME(histogram_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Submit statistics for a histogram instance. + * @param[in] hist Pointer to the histogram instance to which the histogram statistic is to be dumped. + * @param[in] length The number of elements in dumped axis data array. + * @param[in] x_data The X axis dumped data itself (may be NULL to calculate batch statistics). + * @param[in] y_data The Y axis dumped data itself. +*/ +void ITTAPI __itt_histogram_submit(__itt_histogram* hist, size_t length, void* x_data, void* y_data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* hist, size_t length, void* x_data, void* y_data)) +#define __itt_histogram_submit ITTNOTIFY_VOID(histogram_submit) +#define __itt_histogram_submit_ptr ITTNOTIFY_NAME(histogram_submit) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_histogram_submit(hist, length, x_data, y_data) +#define __itt_histogram_submit_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_histogram_submit_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ + +/** +* @brief function allows to obtain the current collection state at the moment +* @return collection state as a enum __itt_collection_state +*/ +__itt_collection_state __itt_get_collection_state(void); + +/** +* @brief function releases resources allocated by ITT API static part +* this API should be called from the library destructor +* @return void +*/ +void __itt_release_resources(void); +/** @endcond */ + +/** + * @brief Create a typed counter with given domain pointer, string name and counter type +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_createW_v3(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_v3 __itt_counter_createW_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createW_v3_ptr +#else /* UNICODE */ +# define __itt_counter_create_v3 __itt_counter_createA_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createA_v3_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3 ITTNOTIFY_DATA(counter_createA_v3) +#define __itt_counter_createA_v3_ptr ITTNOTIFY_NAME(counter_createA_v3) +#define __itt_counter_createW_v3 ITTNOTIFY_DATA(counter_createW_v3) +#define __itt_counter_createW_v3_ptr ITTNOTIFY_NAME(counter_createW_v3) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3 ITTNOTIFY_DATA(counter_create_v3) +#define __itt_counter_create_v3_ptr ITTNOTIFY_NAME(counter_create_v3) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_v3_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value api + */ +void ITTAPI __itt_counter_set_value_v3(__itt_counter counter, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr)) +#define __itt_counter_set_value_v3 ITTNOTIFY_VOID(counter_set_value_v3) +#define __itt_counter_set_value_v3_ptr ITTNOTIFY_NAME(counter_set_value_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_v3(counter, value_ptr) +#define __itt_counter_set_value_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief describes the type of context metadata +*/ +typedef enum { + __itt_context_unknown = 0, /*!< Undefined type */ + __itt_context_nameA, /*!< ASCII string char* type */ + __itt_context_nameW, /*!< Unicode string wchar_t* type */ + __itt_context_deviceA, /*!< ASCII string char* type */ + __itt_context_deviceW, /*!< Unicode string wchar_t* type */ + __itt_context_unitsA, /*!< ASCII string char* type */ + __itt_context_unitsW, /*!< Unicode string wchar_t* type */ + __itt_context_pci_addrA, /*!< ASCII string char* type */ + __itt_context_pci_addrW, /*!< Unicode string wchar_t* type */ + __itt_context_tid, /*!< Unsigned 64-bit integer type */ + __itt_context_max_val, /*!< Unsigned 64-bit integer type */ + __itt_context_bandwidth_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_latency_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_occupancy_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_on_thread_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_is_abs_val_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_instructions_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_cycles_flag /*!< Unsigned 64-bit integer type */ +} __itt_context_type; + +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_context_name __itt_context_nameW +# define __itt_context_device __itt_context_deviceW +# define __itt_context_units __itt_context_unitsW +# define __itt_context_pci_addr __itt_context_pci_addrW +#else /* UNICODE || _UNICODE */ +# define __itt_context_name __itt_context_nameA +# define __itt_context_device __itt_context_deviceA +# define __itt_context_units __itt_context_unitsA +# define __itt_context_pci_addr __itt_context_pci_addrA +#endif /* UNICODE || _UNICODE */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_context_metadata +{ + __itt_context_type type; /*!< Type of the context metadata value */ + void* value; /*!< Pointer to context metadata value itself */ +} __itt_context_metadata; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_counter_metadata +{ + __itt_counter counter; /*!< Associated context metadata counter */ + __itt_context_type type; /*!< Type of the context metadata value */ + const char* str_valueA; /*!< String context metadata value */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* str_valueW; +#else /* UNICODE || _UNICODE */ + void* str_valueW; +#endif /* UNICODE || _UNICODE */ + unsigned long long value; /*!< Numeric context metadata value */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_counter_metadata* next; +} __itt_counter_metadata; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Bind context metadata to counter instance + * @param[in] counter Pointer to the counter instance to which the context metadata is to be associated. + * @param[in] length The number of elements in context metadata array. + * @param[in] metadata The context metadata itself. +*/ +void ITTAPI __itt_bind_context_metadata_to_counter(__itt_counter counter, size_t length, __itt_context_metadata* metadata); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata)) +#define __itt_bind_context_metadata_to_counter ITTNOTIFY_VOID(bind_context_metadata_to_counter) +#define __itt_bind_context_metadata_to_counter_ptr ITTNOTIFY_NAME(bind_context_metadata_to_counter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_bind_context_metadata_to_counter(counter, length, metadata) +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_H_ */ + +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + +#ifndef _ITTNOTIFY_PRIVATE_ +#define _ITTNOTIFY_PRIVATE_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @ingroup clockdomain + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid)) +#define __itt_task_begin_overlapped_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_overlapped_ex,d,x,y,z,a,b) +#define __itt_task_begin_overlapped_ex_ptr ITTNOTIFY_NAME(task_begin_overlapped_ex) +#define __itt_task_end_overlapped_ex(d,x,y,z) ITTNOTIFY_VOID_D3(task_end_overlapped_ex,d,x,y,z) +#define __itt_task_end_overlapped_ex_ptr ITTNOTIFY_NAME(task_end_overlapped_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_overlapped_ex(domain,clock_domain,timestamp,taskid,parentid,name) +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ex(domain,clock_domain,timestamp,taskid) +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup makrs_internal Marks + * @ingroup internal + * Marks group + * @warning Internal API: + * - It is not shipped to outside of Intel + * - It is delivered to internal Intel teams using e-mail or SVN access only + * @{ + */ +/** @brief user mark type */ +typedef int __itt_mark_type; + +/** + * @brief Creates a user mark type with the specified name using char or Unicode string. + * @param[in] name - name of mark to create + * @return Returns a handle to the mark type + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_mark_type ITTAPI __itt_mark_createA(const char *name); +__itt_mark_type ITTAPI __itt_mark_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_create __itt_mark_createW +# define __itt_mark_create_ptr __itt_mark_createW_ptr +#else /* UNICODE */ +# define __itt_mark_create __itt_mark_createA +# define __itt_mark_create_ptr __itt_mark_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_mark_type ITTAPI __itt_mark_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_mark_type, mark_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA ITTNOTIFY_DATA(mark_createA) +#define __itt_mark_createA_ptr ITTNOTIFY_NAME(mark_createA) +#define __itt_mark_createW ITTNOTIFY_DATA(mark_createW) +#define __itt_mark_createW_ptr ITTNOTIFY_NAME(mark_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create ITTNOTIFY_DATA(mark_create) +#define __itt_mark_create_ptr ITTNOTIFY_NAME(mark_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA(name) (__itt_mark_type)0 +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW(name) (__itt_mark_type)0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create(name) (__itt_mark_type)0 +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates a "discrete" user mark type of the specified type and an optional parameter using char or Unicode string. + * + * - The mark of "discrete" type is placed to collection results in case of success. It appears in overtime view(s) as a special tick sign. + * - The call is "synchronous" - function returns after mark is actually added to results. + * - This function is useful, for example, to mark different phases of application + * (beginning of the next mark automatically meand end of current region). + * - Can be used together with "continuous" marks (see below) at the same collection session + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @param[in] parameter - string parameter of mark + * @return Returns zero value in case of success, non-zero value otherwise. + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_markA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_markW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark __itt_markW +# define __itt_mark_ptr __itt_markW_ptr +#else /* UNICODE */ +# define __itt_mark __itt_markA +# define __itt_mark_ptr __itt_markA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, markA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, markW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA ITTNOTIFY_DATA(markA) +#define __itt_markA_ptr ITTNOTIFY_NAME(markA) +#define __itt_markW ITTNOTIFY_DATA(markW) +#define __itt_markW_ptr ITTNOTIFY_NAME(markW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark ITTNOTIFY_DATA(mark) +#define __itt_mark_ptr ITTNOTIFY_NAME(mark) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA(mt, parameter) (int)0 +#define __itt_markA_ptr 0 +#define __itt_markW(mt, parameter) (int)0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark(mt, parameter) (int)0 +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA_ptr 0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create a "discrete" user event type (mark) for process + * rather then for one thread + * @see int __itt_mark(__itt_mark_type mt, const char* parameter); + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_mark_globalA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_mark_globalW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_global __itt_mark_globalW +# define __itt_mark_global_ptr __itt_mark_globalW_ptr +#else /* UNICODE */ +# define __itt_mark_global __itt_mark_globalA +# define __itt_mark_global_ptr __itt_mark_globalA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark_global(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, mark_globalA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, mark_globalW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark_global, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA ITTNOTIFY_DATA(mark_globalA) +#define __itt_mark_globalA_ptr ITTNOTIFY_NAME(mark_globalA) +#define __itt_mark_globalW ITTNOTIFY_DATA(mark_globalW) +#define __itt_mark_globalW_ptr ITTNOTIFY_NAME(mark_globalW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global ITTNOTIFY_DATA(mark_global) +#define __itt_mark_global_ptr ITTNOTIFY_NAME(mark_global) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA(mt, parameter) (int)0 +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW(mt, parameter) (int)0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global(mt, parameter) (int)0 +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates an "end" point for "continuous" mark with specified name. + * + * - Returns zero value in case of success, non-zero value otherwise. + * Also returns non-zero value when preceding "begin" point for the + * mark with the same name failed to be created or not created. + * - The mark of "continuous" type is placed to collection results in + * case of success. It appears in overtime view(s) as a special tick + * sign (different from "discrete" mark) together with line from + * corresponding "begin" mark to "end" mark. + * @note Continuous marks can overlap and be nested inside each other. + * Discrete mark can be nested inside marked region + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @return Returns zero value in case of success, non-zero value otherwise. + */ +int ITTAPI __itt_mark_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_off, (__itt_mark_type mt)) +#define __itt_mark_off ITTNOTIFY_DATA(mark_off) +#define __itt_mark_off_ptr ITTNOTIFY_NAME(mark_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_off(mt) (int)0 +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create an "end" point for mark of process + * @see int __itt_mark_off(__itt_mark_type mt); + */ +int ITTAPI __itt_mark_global_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt)) +#define __itt_mark_global_off ITTNOTIFY_DATA(mark_global_off) +#define __itt_mark_global_off_ptr ITTNOTIFY_NAME(mark_global_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_global_off(mt) (int)0 +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} marks group */ + +/** + * @defgroup counters_internal Counters + * @ingroup internal + * Counters group + * @{ + */ + + +/** + * @defgroup stitch Stack Stitching + * @ingroup internal + * Stack Stitching group + * @{ + */ +/** + * @brief opaque structure for counter identification + */ +typedef struct ___itt_caller *__itt_caller; + +/** + * @brief Create the stitch point e.g. a point in call stack where other stacks should be stitched to. + * The function returns a unique identifier which is used to match the cut points with corresponding stitch points. + */ +__itt_caller ITTAPI __itt_stack_caller_create(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void)) +#define __itt_stack_caller_create ITTNOTIFY_DATA(stack_caller_create) +#define __itt_stack_caller_create_ptr ITTNOTIFY_NAME(stack_caller_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_create() (__itt_caller)0 +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the information about stitch point identified by the pointer previously returned by __itt_stack_caller_create() + */ +void ITTAPI __itt_stack_caller_destroy(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id)) +#define __itt_stack_caller_destroy ITTNOTIFY_VOID(stack_caller_destroy) +#define __itt_stack_caller_destroy_ptr ITTNOTIFY_NAME(stack_caller_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_destroy(id) +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Sets the cut point. Stack from each event which occurs after this call will be cut + * at the same stack level the function was called and stitched to the corresponding stitch point. + */ +void ITTAPI __itt_stack_callee_enter(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_enter, (__itt_caller id)) +#define __itt_stack_callee_enter ITTNOTIFY_VOID(stack_callee_enter) +#define __itt_stack_callee_enter_ptr ITTNOTIFY_NAME(stack_callee_enter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_enter(id) +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief This function eliminates the cut point which was set by latest __itt_stack_callee_enter(). + */ +void ITTAPI __itt_stack_callee_leave(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_leave, (__itt_caller id)) +#define __itt_stack_callee_leave ITTNOTIFY_VOID(stack_callee_leave) +#define __itt_stack_callee_leave_ptr ITTNOTIFY_NAME(stack_callee_leave) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_leave(id) +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} stitch group */ + +/* ***************************************************************************************************************************** */ + +#include + +/** @cond exclude_from_documentation */ +typedef enum __itt_error_code +{ + __itt_error_success = 0, /*!< no error */ + __itt_error_no_module = 1, /*!< module can't be loaded */ + /* %1$s -- library name; win: %2$d -- system error code; unx: %2$s -- system error message. */ + __itt_error_no_symbol = 2, /*!< symbol not found */ + /* %1$s -- library name, %2$s -- symbol name. */ + __itt_error_unknown_group = 3, /*!< unknown group specified */ + /* %1$s -- env var name, %2$s -- group name. */ + __itt_error_cant_read_env = 4, /*!< GetEnvironmentVariable() failed */ + /* %1$s -- env var name, %2$d -- system error. */ + __itt_error_env_too_long = 5, /*!< variable value too long */ + /* %1$s -- env var name, %2$d -- actual length of the var, %3$d -- max allowed length. */ + __itt_error_system = 6 /*!< pthread_mutexattr_init or pthread_mutex_init failed */ + /* %1$s -- function name, %2$d -- errno. */ +} __itt_error_code; + +typedef void (__itt_error_handler_t)(__itt_error_code code, va_list); +__itt_error_handler_t* __itt_set_error_handler(__itt_error_handler_t*); + +const char* ITTAPI __itt_api_version(void); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#define __itt_error_handler ITT_JOIN(INTEL_ITTNOTIFY_PREFIX, error_handler) +void __itt_error_handler(__itt_error_code code, va_list args); +extern const int ITTNOTIFY_NAME(err); +#define __itt_err ITTNOTIFY_NAME(err) +ITT_STUB(ITTAPI, const char*, api_version, (void)) +#define __itt_api_version ITTNOTIFY_DATA(api_version) +#define __itt_api_version_ptr ITTNOTIFY_NAME(api_version) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_api_version() (const char*)0 +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_PRIVATE_ */ + +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/jitprofiling.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/jitprofiling.h new file mode 100644 index 0000000000000000000000000000000000000000..e52f30697c1dc62f4655669758a16269447f086e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/jitprofiling.h @@ -0,0 +1,647 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +#ifndef __JITPROFILING_H__ +#define __JITPROFILING_H__ + +/** + * @brief JIT Profiling APIs + * + * The JIT Profiling API is used to report information about just-in-time + * generated code that can be used by performance tools. The user inserts + * calls in the code generator to report information before JIT-compiled + * code goes to execution. This information is collected at runtime and used + * by tools like Intel(R) VTune(TM) Profiler to display performance metrics + * associated with JIT-compiled code. + * + * These APIs can be used to\n + * - **Profile trace-based and method-based JIT-compiled + * code**. Some examples of environments that you can profile with these APIs: + * dynamic JIT compilation of JavaScript code traces, JIT execution in OpenCL(TM) + * software technology, Java/.NET managed execution environments, and custom + * ISV JIT engines. + * @code + * #include + * + * if (iJIT_IsProfilingActive != iJIT_SAMPLING_ON) { + * return; + * } + * + * iJIT_Method_Load jmethod = {0}; + * jmethod.method_id = iJIT_GetNewMethodID(); + * jmethod.method_name = "method_name"; + * jmethod.class_file_name = "class_name"; + * jmethod.source_file_name = "source_file_name"; + * jmethod.method_load_address = code_addr; + * jmethod.method_size = code_size; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_SHUTDOWN, NULL); + * @endcode + * + * * Expected behavior: + * * If any iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and its + * memory region is treated as unloaded. VTune Profiler displays the metrics + * collected by the method until it is overwritten. + * * If supplied line number information contains multiple source lines for + * the same assembly instruction (code location), then VTune Profiler picks up + * the first line number. + * * Dynamically generated code can be associated with a module name. + * Use the iJIT_Method_Load_V2 structure.\n + * Clarification of some cases: + * * If you register a function with the same method ID multiple times, + * specifying different module names, then the VTune Profiler picks up + * the module name registered first. If you want to distinguish the same + * function between different JIT engines, supply different method IDs for + * each function. Other symbolic information (for example, source file) + * can be identical. + * + * - **Analyze split functions** (multiple joint or disjoint code regions + * belonging to the same function) **including re-JIT** + * with potential overlapping of code regions in time, which is common in + * resource-limited environments. + * @code + * #include + * + * unsigned int method_id = iJIT_GetNewMethodID(); + * + * iJIT_Method_Load a = {0}; + * a.method_id = method_id; + * a.method_load_address = 0x100; + * a.method_size = 0x20; + * + * iJIT_Method_Load b = {0}; + * b.method_id = method_id; + * b.method_load_address = 0x200; + * b.method_size = 0x30; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&b); + * @endcode + * + * * Expected behaviour: + * * If a iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and + * its memory region is treated as unloaded. + * * All code regions reported with the same method ID are considered as + * belonging to the same method. Symbolic information (method name, + * source file name) will be taken from the first notification, and all + * subsequent notifications with the same method ID will be processed + * only for line number table information. So, the VTune Profiler will map + * samples to a source line using the line number table from the current + * notification while taking the source file name from the very first one.\n + * Clarification of some cases:\n + * * If you register a second code region with a different source file + * name and the same method ID, then this information will be saved and + * will not be considered as an extension of the first code region, but + * VTune Profiler will use the source file of the first code region and map + * performance metrics incorrectly. + * * If you register a second code region with the same source file as + * for the first region and the same method ID, then the source file will be + * discarded but VTune Profiler will map metrics to the source file correctly. + * * If you register a second code region with a null source file and + * the same method ID, then provided line number info will be associated + * with the source file of the first code region. + * + * - **Explore inline functions** including multi-level hierarchy of + * nested inline methods which shows how performance metrics are distributed through them. + * @code + * #include + * + * // method_id parent_id + * // [-- c --] 3000 2000 + * // [---- d -----] 2001 1000 + * // [---- b ----] 2000 1000 + * // [------------ a ----------------] 1000 n/a + * + * iJIT_Method_Load a = {0}; + * a.method_id = 1000; + * + * iJIT_Method_Inline_Load b = {0}; + * b.method_id = 2000; + * b.parent_method_id = 1000; + * + * iJIT_Method_Inline_Load c = {0}; + * c.method_id = 3000; + * c.parent_method_id = 2000; + * + * iJIT_Method_Inline_Load d = {0}; + * d.method_id = 2001; + * d.parent_method_id = 1000; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&b); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&c); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&d); + * @endcode + * + * * Requirements: + * * Each inline (iJIT_Method_Inline_Load) method should be associated + * with two method IDs: one for itself; one for its immediate parent. + * * Address regions of inline methods of the same parent method cannot + * overlap each other. + * * Execution of the parent method must not be started until it and all + * its inline methods are reported. + * * Expected behaviour: + * * In case of nested inline methods an order of + * iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED events is not important. + * * If any event overwrites either inline method or top parent method, + * then the parent, including inline methods, becomes invalid and its memory + * region is treated as unloaded. + * + * **Life time of allocated data**\n + * The client sends an event notification to the agent with event-specific + * data, which is a structure. The pointers in the structure refer to memory + * allocated by the client, which responsible for releasing it. The pointers are + * used by the iJIT_NotifyEvent method to copy client's data in a trace file, + * and they are not used after the iJIT_NotifyEvent method returns. + */ + +/** + * @defgroup jitapi JIT Profiling + * @ingroup internal + * @{ + */ + +/** + * @brief Enumerator for the types of notifications + */ +typedef enum iJIT_jvm_event +{ + iJVM_EVENT_TYPE_SHUTDOWN = 2, /**<\brief Send this to shutdown the agent. + * Use NULL for event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED = 13, /**<\brief Send when dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load as event + * data. */ +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UNLOAD_START, /**<\brief Send when compiled dynamic + * code is being unloaded from memory. + * Use iJIT_Method_Load as event data.*/ +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_UPDATE, /**<\brief Send to provide new content for + * a previously reported dynamic code. + * The previous content will be invalidated + * starting from the time of the notification. + * Use iJIT_Method_Load as event data but + * required fields are following: + * - method_id identify the code to update. + * - method_load_address specify start address + * within identified code range + * where update should be started. + * - method_size specify length of updated code + * range. */ + + + iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, /**<\brief Send when an inline dynamic + * code is JIT compiled and loaded + * into memory by the JIT engine, + * but before the parent code region + * starts executing. + * Use iJIT_Method_Inline_Load as event data.*/ + +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UPDATE_V2, +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 = 21, /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V2 as event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V3 as event data. */ +} iJIT_JVM_EVENT; + +/** + * @brief Enumerator for the agent's mode + */ +typedef enum _iJIT_IsProfilingActiveFlags +{ + iJIT_NOTHING_RUNNING = 0x0000, /**<\brief The agent is not running; + * iJIT_NotifyEvent calls will + * not be processed. */ + iJIT_SAMPLING_ON = 0x0001, /**<\brief The agent is running and + * ready to process notifications. */ +} iJIT_IsProfilingActiveFlags; + +/** + * @brief Description of a single entry in the line number information of a code region. + * @details A table of line number entries gives information about how the reported code region + * is mapped to source file. + * Intel(R) VTune(TM) Profiler uses line number information to attribute + * the samples (virtual address) to a line number. \n + * It is acceptable to report different code addresses for the same source line: + * @code + * Offset LineNumber + * 1 2 + * 12 4 + * 15 2 + * 18 1 + * 21 30 + * + * VTune Profiler constructs the following table using the client data + * + * Code subrange Line number + * 0-1 2 + * 1-12 4 + * 12-15 2 + * 15-18 1 + * 18-21 30 + * @endcode + */ +typedef struct _LineNumberInfo +{ + unsigned int Offset; /**<\brief Offset from the begining of the code region. */ + unsigned int LineNumber; /**<\brief Matching source line number offset (from beginning of source file). */ + +} *pLineNumberInfo, LineNumberInfo; + +/** + * @brief Enumerator for the code architecture. + */ +typedef enum _iJIT_CodeArchitecture +{ + iJIT_CA_NATIVE = 0, /**<\brief Native to the process architecture that is calling it. */ + + iJIT_CA_32, /**<\brief 32-bit machine code. */ + + iJIT_CA_64 /**<\brief 64-bit machine code. */ + +} iJIT_CodeArchitecture; + +#pragma pack(push, 8) + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, data provided with + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table.0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + unsigned int class_id; /**<\brief This field is obsolete. */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Load, iJIT_Method_Load; + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load_V2 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V2 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + The module name can be useful for distinguishing among + different JIT engines. VTune Profiler will display + reported methods grouped by specific module. */ + +} *piJIT_Method_Load_V2, iJIT_Method_Load_V2; + +/** + * @brief Description of a JIT-compiled method + * @details The iJIT_Method_Load_V3 structure is the same as iJIT_Method_Load_V2 + * with a newly introduced 'arch' field that specifies architecture of the code region. + * When you use the iJIT_Method_Load_V3 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V3 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise they are + * treated as regions of different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Cannot be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + * The module name can be useful for distinguishing among + * different JIT engines. VTune Profiler will display + * reported methods grouped by specific module. */ + + iJIT_CodeArchitecture module_arch; /**<\brief Architecture of the method's code region. + * By default, it is the same as the process + * architecture that is calling it. + * For example, you can use it if your 32-bit JIT + * engine generates 64-bit code. + * + * If JIT engine reports both 32-bit and 64-bit types + * of methods then VTune Profiler splits the methods + * with the same module name but with different + * architectures in two different modules. VTune Profiler + * modifies the original name provided with a 64-bit method + * version by ending it with '(64)' */ + +} *piJIT_Method_Load_V3, iJIT_Method_Load_V3; + +/** + * @brief Description of an inline JIT-compiled method + * @details When you use the_iJIT_Method_Inline_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Inline_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + unsigned int parent_method_id; /**<\brief Unique immediate parent's method ID. + * Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /** <\brief The virtual address on which the method + * is inlined. If NULL, then data provided with + * the event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Inline_Load, iJIT_Method_Inline_Load; + +/** @cond exclude_from_documentation */ +/** + * @brief Description of a segment type + * @details Use the segment type to specify a type of data supplied + * with the iJVM_EVENT_TYPE_METHOD_UPDATE_V2 event to be applied to + * a certain code trace. + */ +typedef enum _iJIT_SegmentType +{ + iJIT_CT_UNKNOWN = 0, + + iJIT_CT_CODE, /**<\brief Executable code. */ + + iJIT_CT_DATA, /**<\brief Data (not executable code). + * VTune Profiler uses the format string + * (see iJIT_Method_Update) to represent + * this data in the VTune Profiler GUI */ + + iJIT_CT_KEEP, /**<\brief Use the previous markup for the trace. + * Can be used for the following + * iJVM_EVENT_TYPE_METHOD_UPDATE_V2 events, + * if the type of the previously reported segment + * type is the same. */ + iJIT_CT_EOF +} iJIT_SegmentType; + +/** + * @brief Description of a dynamic update of the content within JIT-compiled method + * @details The JIT engine may generate the methods that are updated at runtime + * partially by mixed (data + executable code) content. When you use the iJIT_Method_Update + * structure to describe the update of the content within a JIT-compiled method, + * use iJVM_EVENT_TYPE_METHOD_UPDATE_V2 as an event type to report it. + * + * On the first Update event, VTune Profiler copies the original code range reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event, then modifies it with the supplied bytes and + * adds the modified range to the original method. For next update events, VTune Profiler + * does the same but it uses the latest modified version of a code region for update. + * Eventually, VTune Profiler GUI displays multiple code ranges for the method reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event. + * Notes: + * - Multiple update events with different types for the same trace are allowed + * but they must be reported for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [code] Ignored + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + * - The types of previously reported events can be changed but they must be reported + * for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + */ + +typedef struct _iJIT_Method_Update +{ + void* load_address; /**<\brief Start address of the update within a method */ + + unsigned int size; /**<\brief The update size */ + + iJIT_SegmentType type; /**<\brief Type of the update */ + + const char* data_format; /**<\brief C string that contains a format string + * that follows the same specifications as format in printf. + * The format string is used for iJIT_CT_CODE only + * and cannot be NULL. + * Format can be changed on the fly. */ +} *piJIT_Method_Update, iJIT_Method_Update; + +/** @endcond */ + +#pragma pack(pop) + +/** @cond exclude_from_documentation */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef JITAPI_CDECL +# if defined WIN32 || defined _WIN32 +# define JITAPI_CDECL __cdecl +# else /* defined WIN32 || defined _WIN32 */ +# if defined _M_IX86 || defined __i386__ +# define JITAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define JITAPI_CDECL /* actual only on x86_64 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* defined WIN32 || defined _WIN32 */ +#endif /* JITAPI_CDECL */ + +#define JITAPI JITAPI_CDECL +/** @endcond */ + +/** + * @brief Generates a new unique method ID. + * + * You must use this API to obtain unique and valid method IDs for methods or + * traces reported to the agent if you don't have your own mechanism to generate + * unique method IDs. + * + * @return a new unique method ID. When out of unique method IDs, this API + * returns 0, which is not an accepted value. + */ +unsigned int JITAPI iJIT_GetNewMethodID(void); + +/** + * @brief Returns the current mode of the agent. + * + * @return iJIT_SAMPLING_ON, indicating that agent is running, or + * iJIT_NOTHING_RUNNING if no agent is running. + */ +iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void); + +/** + * @brief Reports infomation about JIT-compiled code to the agent. + * + * The reported information is used to attribute samples obtained from any + * Intel(R) VTune(TM) Profiler collector. This API needs to be called + * after JIT compilation and before the first entry into the JIT-compiled + * code. + * + * @param[in] event_type - type of the data sent to the agent + * @param[in] EventSpecificData - pointer to event-specific data + * + * @returns 1 on success, otherwise 0. + */ +int JITAPI iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +/** @endcond */ + +/** @} jitapi group */ + +#endif /* __JITPROFILING_H__ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libittnotify.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libittnotify.h new file mode 100644 index 0000000000000000000000000000000000000000..76994f5644d88711ea49344b5b23ce08b4811927 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libittnotify.h @@ -0,0 +1,24 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +#ifndef _LIBITTNOTIFY_H_ +#define _LIBITTNOTIFY_H_ + +#ifndef __ITT_INTERNAL_INCLUDE +# if defined WIN32 || defined _WIN32 +# pragma message("WARNING!!! Include file libittnotify.h is deprecated and should not be included anymore") +# else /* WIN32 */ +# warning "Include file libittnotify.h is deprecated and should not be included anymore" +# endif /* WIN32 */ +#endif /* __ITT_INTERNAL_INCLUDE */ +#include "legacy/ittnotify.h" + +#endif /* _LIBITTNOTIFY_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libshm.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libshm.h new file mode 100644 index 0000000000000000000000000000000000000000..ace7a5358e06748cb9bfd841dc9cc8a6b75d4a14 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/libshm.h @@ -0,0 +1,41 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once + +#include + +#ifdef __cplusplus + +#ifdef SHM_EXPORTS +#define SHM_API __declspec(dllexport) +#else +#define SHM_API __declspec(dllimport) +#endif + +SHM_API void libshm_init(const char* manager_exec_path); + +class SHM_API THManagedMapAllocator : public at::RefcountedMapAllocator { + public: + THManagedMapAllocator( + const char* manager_handle, + const char* filename, + int flags, + size_t size) + : at::RefcountedMapAllocator(filename, flags, size) {} + + static at::DataPtr makeDataPtr( + const char* manager_handle, + const char* filename, + int flags, + size_t size); + static THManagedMapAllocator* fromDataPtr(const at::DataPtr&); + + const char* manager_handle() const { + return "no_manager"; + } +}; + +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/psimd.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/psimd.h new file mode 100644 index 0000000000000000000000000000000000000000..ac18a5e7a94d936a77e0bd1516896634d0ca4707 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/psimd.h @@ -0,0 +1,1389 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef PSIMD_H +#define PSIMD_H + +#if defined(__CUDA_ARCH__) + /* CUDA compiler */ + #define PSIMD_INTRINSIC __forceinline__ __device__ +#elif defined(__OPENCL_VERSION__) + /* OpenCL compiler */ + #define PSIMD_INTRINSIC inline static +#elif defined(__INTEL_COMPILER) + /* Intel compiler, even on Windows */ + #define PSIMD_INTRINSIC inline static __attribute__((__always_inline__)) +#elif defined(__GNUC__) + /* GCC-compatible compiler (gcc/clang/icc) */ + #define PSIMD_INTRINSIC inline static __attribute__((__always_inline__)) +#elif defined(_MSC_VER) + /* MSVC-compatible compiler (cl/icl/clang-cl) */ + #define PSIMD_INTRINSIC __forceinline static +#elif defined(__cplusplus) + /* Generic C++ compiler */ + #define PSIMD_INTRINSIC inline static +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + /* Generic C99 compiler */ + #define PSIMD_INTRINSIC inline static +#else + /* Generic C compiler */ + #define PSIMD_INTRINSIC static +#endif + +#if defined(__GNUC__) || defined(__clang__) + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + #include + #endif + + #if defined(__SSE2__) + #include + #endif + + #if defined(__SSE3__) + #include + #endif + + #if defined(__SSSE3__) + #include + #endif + + #if defined(__SSE4_1__) + #include + #endif + + #if defined(__SSE4_2__) + #include + #endif + + #if defined(__AVX__) + #include + #endif +#elif defined(_MSC_VER) + #include +#endif + +#if defined(__cplusplus) + #define PSIMD_CXX_SYNTAX +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) + #define PSIMD_C11_SYNTAX +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + #define PSIMD_C99_SYNTAX +#else + #define PSIMD_C89_SYNTAX +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201103L) + #include + #include +#elif !defined(__OPENCL_VERSION__) + #include + #include +#endif + +#if defined(__GNUC__) || defined(__clang__) + #define PSIMD_HAVE_F64 0 + #define PSIMD_HAVE_F32 1 + #define PSIMD_HAVE_U8 1 + #define PSIMD_HAVE_S8 1 + #define PSIMD_HAVE_U16 1 + #define PSIMD_HAVE_S16 1 + #define PSIMD_HAVE_U32 1 + #define PSIMD_HAVE_S32 1 + #define PSIMD_HAVE_U64 0 + #define PSIMD_HAVE_S64 0 + + typedef int8_t psimd_s8 __attribute__((vector_size(16), aligned(1))); + typedef uint8_t psimd_u8 __attribute__((vector_size(16), aligned(1))); + typedef int16_t psimd_s16 __attribute__((vector_size(16), aligned(2))); + typedef uint16_t psimd_u16 __attribute__((vector_size(16), aligned(2))); + typedef int32_t psimd_s32 __attribute__((vector_size(16), aligned(4))); + typedef uint32_t psimd_u32 __attribute__((vector_size(16), aligned(4))); + typedef float psimd_f32 __attribute__((vector_size(16), aligned(4))); + + typedef struct { + psimd_s8 lo; + psimd_s8 hi; + } psimd_s8x2; + + typedef struct { + psimd_u8 lo; + psimd_u8 hi; + } psimd_u8x2; + + typedef struct { + psimd_s16 lo; + psimd_s16 hi; + } psimd_s16x2; + + typedef struct { + psimd_u16 lo; + psimd_u16 hi; + } psimd_u16x2; + + typedef struct { + psimd_s32 lo; + psimd_s32 hi; + } psimd_s32x2; + + typedef struct { + psimd_u32 lo; + psimd_u32 hi; + } psimd_u32x2; + + typedef struct { + psimd_f32 lo; + psimd_f32 hi; + } psimd_f32x2; + + /* Bit casts */ + PSIMD_INTRINSIC psimd_u32x2 psimd_cast_s32x2_u32x2(psimd_s32x2 v) { + return (psimd_u32x2) { .lo = (psimd_u32) v.lo, .hi = (psimd_u32) v.hi }; + } + + PSIMD_INTRINSIC psimd_f32x2 psimd_cast_s32x2_f32x2(psimd_s32x2 v) { + return (psimd_f32x2) { .lo = (psimd_f32) v.lo, .hi = (psimd_f32) v.hi }; + } + + PSIMD_INTRINSIC psimd_s32x2 psimd_cast_u32x2_s32x2(psimd_u32x2 v) { + return (psimd_s32x2) { .lo = (psimd_s32) v.lo, .hi = (psimd_s32) v.hi }; + } + + PSIMD_INTRINSIC psimd_f32x2 psimd_cast_u32x2_f32x2(psimd_u32x2 v) { + return (psimd_f32x2) { .lo = (psimd_f32) v.lo, .hi = (psimd_f32) v.hi }; + } + + PSIMD_INTRINSIC psimd_s32x2 psimd_cast_f32x2_s32x2(psimd_f32x2 v) { + return (psimd_s32x2) { .lo = (psimd_s32) v.lo, .hi = (psimd_s32) v.hi }; + } + + PSIMD_INTRINSIC psimd_u32x2 psimd_cast_f32x2_u32x2(psimd_f32x2 v) { + return (psimd_u32x2) { .lo = (psimd_u32) v.lo, .hi = (psimd_u32) v.hi }; + } + + /* Swap */ + PSIMD_INTRINSIC void psimd_swap_s8(psimd_s8 a[1], psimd_s8 b[1]) { + const psimd_s8 new_a = *b; + const psimd_s8 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u8(psimd_u8 a[1], psimd_u8 b[1]) { + const psimd_u8 new_a = *b; + const psimd_u8 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_s16(psimd_s16 a[1], psimd_s16 b[1]) { + const psimd_s16 new_a = *b; + const psimd_s16 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u16(psimd_u16 a[1], psimd_u16 b[1]) { + const psimd_u16 new_a = *b; + const psimd_u16 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_s32(psimd_s32 a[1], psimd_s32 b[1]) { + const psimd_s32 new_a = *b; + const psimd_s32 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u32(psimd_u32 a[1], psimd_u32 b[1]) { + const psimd_u32 new_a = *b; + const psimd_u32 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_f32(psimd_f32 a[1], psimd_f32 b[1]) { + const psimd_f32 new_a = *b; + const psimd_f32 new_b = *a; + *a = new_a; + *b = new_b; + } + + /* Zero-initialization */ + PSIMD_INTRINSIC psimd_s8 psimd_zero_s8(void) { + return (psimd_s8) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u8 psimd_zero_u8(void) { + return (psimd_u8) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_s16 psimd_zero_s16(void) { + return (psimd_s16) { 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u16 psimd_zero_u16(void) { + return (psimd_u16) { 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_zero_s32(void) { + return (psimd_s32) { 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_zero_u32(void) { + return (psimd_u32) { 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_zero_f32(void) { + return (psimd_f32) { 0.0f, 0.0f, 0.0f, 0.0f }; + } + + /* Initialization to the same constant */ + PSIMD_INTRINSIC psimd_s8 psimd_splat_s8(int8_t c) { + return (psimd_s8) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u8 psimd_splat_u8(uint8_t c) { + return (psimd_u8) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_s16 psimd_splat_s16(int16_t c) { + return (psimd_s16) { c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u16 psimd_splat_u16(uint16_t c) { + return (psimd_u16) { c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_splat_s32(int32_t c) { + return (psimd_s32) { c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_splat_u32(uint32_t c) { + return (psimd_u32) { c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat_f32(float c) { + return (psimd_f32) { c, c, c, c }; + } + + /* Load vector */ + PSIMD_INTRINSIC psimd_s8 psimd_load_s8(const void* address) { + return *((const psimd_s8*) address); + } + + PSIMD_INTRINSIC psimd_u8 psimd_load_u8(const void* address) { + return *((const psimd_u8*) address); + } + + PSIMD_INTRINSIC psimd_s16 psimd_load_s16(const void* address) { + return *((const psimd_s16*) address); + } + + PSIMD_INTRINSIC psimd_u16 psimd_load_u16(const void* address) { + return *((const psimd_u16*) address); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load_s32(const void* address) { + return *((const psimd_s32*) address); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load_u32(const void* address) { + return *((const psimd_u32*) address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_f32(const void* address) { + return *((const psimd_f32*) address); + } + + PSIMD_INTRINSIC psimd_s8 psimd_load_splat_s8(const void* address) { + return psimd_splat_s8(*((const int8_t*) address)); + } + + PSIMD_INTRINSIC psimd_u8 psimd_load_splat_u8(const void* address) { + return psimd_splat_u8(*((const uint8_t*) address)); + } + + PSIMD_INTRINSIC psimd_s16 psimd_load_splat_s16(const void* address) { + return psimd_splat_s16(*((const int16_t*) address)); + } + + PSIMD_INTRINSIC psimd_u16 psimd_load_splat_u16(const void* address) { + return psimd_splat_u16(*((const uint16_t*) address)); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load_splat_s32(const void* address) { + return psimd_splat_s32(*((const int32_t*) address)); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load_splat_u32(const void* address) { + return psimd_splat_u32(*((const uint32_t*) address)); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_splat_f32(const void* address) { + return psimd_splat_f32(*((const float*) address)); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load1_s32(const void* address) { + return (psimd_s32) { *((const int32_t*) address), 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load1_u32(const void* address) { + return (psimd_u32) { *((const uint32_t*) address), 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_f32(const void* address) { + return (psimd_f32) { *((const float*) address), 0.0f, 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load2_s32(const void* address) { + const int32_t* address_s32 = (const int32_t*) address; + return (psimd_s32) { address_s32[0], address_s32[1], 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load2_u32(const void* address) { + const uint32_t* address_u32 = (const uint32_t*) address; + return (psimd_u32) { address_u32[0], address_u32[1], 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[1], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load3_s32(const void* address) { + const int32_t* address_s32 = (const int32_t*) address; + return (psimd_s32) { address_s32[0], address_s32[1], address_s32[2], 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load3_u32(const void* address) { + const uint32_t* address_u32 = (const uint32_t*) address; + return (psimd_u32) { address_u32[0], address_u32[1], address_u32[2], 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[1], address_f32[2], 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load4_s32(const void* address) { + return psimd_load_s32(address); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load4_u32(const void* address) { + return psimd_load_u32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_f32(const void* address) { + return psimd_load_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_stride2_f32(const void* address) { + const psimd_f32 v0x1x = psimd_load_f32(address); + const psimd_f32 vx2x3 = psimd_load_f32((const float*) address + 3); + #if defined(__clang__) + return __builtin_shufflevector(v0x1x, vx2x3, 0, 2, 5, 7); + #else + return __builtin_shuffle(v0x1x, vx2x3, (psimd_s32) { 0, 2, 5, 7 }); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_stride2_f32(const void* address) { + return psimd_load_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_stride2_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[2], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_stride2_f32(const void* address) { + const psimd_f32 v0x1x = psimd_load_f32(address); + const psimd_f32 v2zzz = psimd_load1_f32((const float*) address + 2); + #if defined(__clang__) + return __builtin_shufflevector(v0x1x, v2zzz, 0, 2, 4, 6); + #else + return __builtin_shuffle(v0x1x, v2zzz, (psimd_s32) { 0, 2, 4, 6 }); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_stride2_f32(const void* address) { + return psimd_load_stride2_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_stride_f32(const void* address, size_t stride) { + const float* address0_f32 = (const float*) address; + const float* address1_f32 = address0_f32 + stride; + const float* address2_f32 = address1_f32 + stride; + const float* address3_f32 = address2_f32 + stride; + return (psimd_f32) { *address0_f32, *address1_f32, *address2_f32, *address3_f32 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_stride_f32(const void* address, size_t stride) { + return psimd_load1_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_stride_f32(const void* address, size_t stride) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[stride], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_stride_f32(const void* address, size_t stride) { + const float* address0_f32 = (const float*) address; + const float* address1_f32 = address0_f32 + stride; + const float* address2_f32 = address1_f32 + stride; + return (psimd_f32) { *address0_f32, *address1_f32, *address2_f32, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_stride_f32(const void* address, size_t stride) { + return psimd_load_stride_f32(address, stride); + } + + /* Store vector */ + PSIMD_INTRINSIC void psimd_store_s8(void* address, psimd_s8 value) { + *((psimd_s8*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u8(void* address, psimd_u8 value) { + *((psimd_u8*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_s16(void* address, psimd_s16 value) { + *((psimd_s16*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u16(void* address, psimd_u16 value) { + *((psimd_u16*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_s32(void* address, psimd_s32 value) { + *((psimd_s32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u32(void* address, psimd_u32 value) { + *((psimd_u32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_f32(void* address, psimd_f32 value) { + *((psimd_f32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store1_s32(void* address, psimd_s32 value) { + *((int32_t*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store1_u32(void* address, psimd_u32 value) { + *((uint32_t*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store1_f32(void* address, psimd_f32 value) { + *((float*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store2_s32(void* address, psimd_s32 value) { + int32_t* address_s32 = (int32_t*) address; + address_s32[0] = value[0]; + address_s32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store2_u32(void* address, psimd_u32 value) { + uint32_t* address_u32 = (uint32_t*) address; + address_u32[0] = value[0]; + address_u32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store2_f32(void* address, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store3_s32(void* address, psimd_s32 value) { + int32_t* address_s32 = (int32_t*) address; + address_s32[0] = value[0]; + address_s32[1] = value[1]; + address_s32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store3_u32(void* address, psimd_u32 value) { + uint32_t* address_u32 = (uint32_t*) address; + address_u32[0] = value[0]; + address_u32[1] = value[1]; + address_u32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store3_f32(void* address, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[1] = value[1]; + address_f32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store4_s32(void* address, psimd_s32 value) { + psimd_store_s32(address, value); + } + + PSIMD_INTRINSIC void psimd_store4_u32(void* address, psimd_u32 value) { + psimd_store_u32(address, value); + } + + PSIMD_INTRINSIC void psimd_store4_f32(void* address, psimd_f32 value) { + psimd_store_f32(address, value); + } + + PSIMD_INTRINSIC void psimd_store_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address0_f32 = (float*) address; + float* address1_f32 = address0_f32 + stride; + float* address2_f32 = address1_f32 + stride; + float* address3_f32 = address2_f32 + stride; + *address0_f32 = value[0]; + *address1_f32 = value[1]; + *address2_f32 = value[2]; + *address3_f32 = value[3]; + } + + PSIMD_INTRINSIC void psimd_store1_stride_f32(void* address, size_t stride, psimd_f32 value) { + psimd_store1_f32(address, value); + } + + PSIMD_INTRINSIC void psimd_store2_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[stride] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store3_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address0_f32 = (float*) address; + float* address1_f32 = address0_f32 + stride; + float* address2_f32 = address1_f32 + stride; + *address0_f32 = value[0]; + *address1_f32 = value[1]; + *address2_f32 = value[2]; + } + + /* Vector addition */ + PSIMD_INTRINSIC psimd_s8 psimd_add_s8(psimd_s8 a, psimd_s8 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_add_u8(psimd_u8 a, psimd_u8 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_add_s16(psimd_s16 a, psimd_s16 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_add_u16(psimd_u16 a, psimd_u16 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_add_s32(psimd_s32 a, psimd_s32 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_add_u32(psimd_u32 a, psimd_u32 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_add_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vaddq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a + b; + #endif + } + + /* Vector subtraction */ + PSIMD_INTRINSIC psimd_s8 psimd_sub_s8(psimd_s8 a, psimd_s8 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_sub_u8(psimd_u8 a, psimd_u8 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_sub_s16(psimd_s16 a, psimd_s16 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_sub_u16(psimd_u16 a, psimd_u16 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_sub_s32(psimd_s32 a, psimd_s32 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_sub_u32(psimd_u32 a, psimd_u32 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_sub_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vsubq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a - b; + #endif + } + + /* Vector multiplication */ + PSIMD_INTRINSIC psimd_s8 psimd_mul_s8(psimd_s8 a, psimd_s8 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_mul_u8(psimd_u8 a, psimd_u8 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_mul_s16(psimd_s16 a, psimd_s16 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_mul_u16(psimd_u16 a, psimd_u16 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_mul_s32(psimd_s32 a, psimd_s32 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_mul_u32(psimd_u32 a, psimd_u32 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_mul_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vmulq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a * b; + #endif + } + + /* Quasi-Fused Multiply-Add */ + PSIMD_INTRINSIC psimd_f32 psimd_qfma_f32(psimd_f32 a, psimd_f32 b, psimd_f32 c) { + #if defined(__aarch64__) || defined(__ARM_NEON__) && defined(__ARM_FEATURE_FMA) + return (psimd_f32) vfmaq_f32((float32x4_t) a, (float32x4_t) b, (float32x4_t) c); + #elif (defined(__x86_64__) || defined(__i386__) || defined(__i686__)) && defined(__FMA__) + return (psimd_f32) _mm_fmadd_ps((__m128) b, (__m128) c, (__m128) a); + #elif (defined(__x86_64__) || defined(__i386__) || defined(__i686__)) && defined(__FMA4__) + return (psimd_f32) _mm_macc_ps((__m128) b, (__m128) c, (__m128) a); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) && PSIMD_ENABLE_WASM_QFMA + return (psimd_f32) __builtin_wasm_qfma_f32x4(a, b, c); + #else + return a + b * c; + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_div_f32(psimd_f32 a, psimd_f32 b) { + return a / b; + } + + /* Vector and */ + PSIMD_INTRINSIC psimd_f32 psimd_andmask_f32(psimd_s32 mask, psimd_f32 v) { + return (psimd_f32) (mask & (psimd_s32) v); + } + + /* Vector and-not */ + PSIMD_INTRINSIC psimd_f32 psimd_andnotmask_f32(psimd_s32 mask, psimd_f32 v) { + return (psimd_f32) (~mask & (psimd_s32) v); + } + + /* Vector blend */ + PSIMD_INTRINSIC psimd_s8 psimd_blend_s8(psimd_s8 mask, psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vbslq_s8((uint8x16_t) mask, (int8x16_t) a, (int8x16_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s8) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_blend_u8(psimd_s8 mask, psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vbslq_u8((uint8x16_t) mask, (uint8x16_t) a, (uint8x16_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u8) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u8) ((mask & (psimd_s8) a) | (~mask & (psimd_s8) b)); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_blend_s16(psimd_s16 mask, psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vbslq_s16((uint16x8_t) mask, (int16x8_t) a, (int16x8_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s16) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_blend_u16(psimd_s16 mask, psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vbslq_u16((uint16x8_t) mask, (uint16x8_t) a, (uint16x8_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u16) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u16) ((mask & (psimd_s16) a) | (~mask & (psimd_s16) b)); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_blend_s32(psimd_s32 mask, psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vbslq_s32((uint32x4_t) mask, (int32x4_t) a, (int32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s32) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_blend_u32(psimd_s32 mask, psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vbslq_u32((uint32x4_t) mask, (uint32x4_t) a, (uint32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u32) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u32) ((mask & (psimd_s32) a) | (~mask & (psimd_s32) b)); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_blend_f32(psimd_s32 mask, psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vbslq_f32((uint32x4_t) mask, (float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_f32) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_f32) ((mask & (psimd_s32) a) | (~mask & (psimd_s32) b)); + #endif + } + + /* Vector blend on sign */ + PSIMD_INTRINSIC psimd_s8 psimd_signblend_s8(psimd_s8 x, psimd_s8 a, psimd_s8 b) { + return psimd_blend_s8(x >> psimd_splat_s8(7), a, b); + } + + PSIMD_INTRINSIC psimd_u8 psimd_signblend_u8(psimd_s8 x, psimd_u8 a, psimd_u8 b) { + return psimd_blend_u8((x >> psimd_splat_s8(7)), a, b); + } + + PSIMD_INTRINSIC psimd_s16 psimd_signblend_s16(psimd_s16 x, psimd_s16 a, psimd_s16 b) { + return psimd_blend_s16(x >> psimd_splat_s16(15), a, b); + } + + PSIMD_INTRINSIC psimd_u16 psimd_signblend_u16(psimd_s16 x, psimd_u16 a, psimd_u16 b) { + return psimd_blend_u16((x >> psimd_splat_s16(15)), a, b); + } + + PSIMD_INTRINSIC psimd_s32 psimd_signblend_s32(psimd_s32 x, psimd_s32 a, psimd_s32 b) { + return psimd_blend_s32(x >> psimd_splat_s32(31), a, b); + } + + PSIMD_INTRINSIC psimd_u32 psimd_signblend_u32(psimd_s32 x, psimd_u32 a, psimd_u32 b) { + return psimd_blend_u32((x >> psimd_splat_s32(31)), a, b); + } + + PSIMD_INTRINSIC psimd_f32 psimd_signblend_f32(psimd_f32 x, psimd_f32 a, psimd_f32 b) { + const psimd_s32 mask = (psimd_s32) x >> psimd_splat_s32(31); + return psimd_blend_f32(mask, a, b); + } + + /* Vector absolute value */ + PSIMD_INTRINSIC psimd_f32 psimd_abs_f32(psimd_f32 v) { + const psimd_s32 mask = (psimd_s32) psimd_splat_f32(-0.0f); + return (psimd_f32) ((psimd_s32) v & ~mask); + } + + /* Vector negation */ + PSIMD_INTRINSIC psimd_f32 psimd_neg_f32(psimd_f32 v) { + const psimd_s32 mask = (psimd_s32) psimd_splat_f32(-0.0f); + return (psimd_f32) ((psimd_s32) v ^ mask); + } + + /* Vector maximum */ + PSIMD_INTRINSIC psimd_s8 psimd_max_s8(psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vmaxq_s8((int8x16_t) a, (int8x16_t) b); + #else + return psimd_blend_s8(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_max_u8(psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vmaxq_u8((uint8x16_t) a, (uint8x16_t) b); + #else + return psimd_blend_u8(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_max_s16(psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vmaxq_s16((int16x8_t) a, (int16x8_t) b); + #else + return psimd_blend_s16(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_max_u16(psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vmaxq_u16((uint16x8_t) a, (uint16x8_t) b); + #else + return psimd_blend_u16(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_max_s32(psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vmaxq_s32((int32x4_t) a, (int32x4_t) b); + #else + return psimd_blend_s32(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_max_u32(psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vmaxq_u32((uint32x4_t) a, (uint32x4_t) b); + #else + return psimd_blend_u32(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_max_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vmaxq_f32((float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return __builtin_wasm_max_f32x4(a, b); + #else + return psimd_blend_f32(a > b, a, b); + #endif + } + + /* Vector minimum */ + PSIMD_INTRINSIC psimd_s8 psimd_min_s8(psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vminq_s8((int8x16_t) a, (int8x16_t) b); + #else + return psimd_blend_s8(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_min_u8(psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vminq_u8((uint8x16_t) a, (uint8x16_t) b); + #else + return psimd_blend_u8(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_min_s16(psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vminq_s16((int16x8_t) a, (int16x8_t) b); + #else + return psimd_blend_s16(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_min_u16(psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vminq_u16((uint16x8_t) a, (uint16x8_t) b); + #else + return psimd_blend_u16(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_min_s32(psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vminq_s32((int32x4_t) a, (int32x4_t) b); + #else + return psimd_blend_s32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_min_u32(psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vminq_u32((uint32x4_t) a, (uint32x4_t) b); + #else + return psimd_blend_u32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_min_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vminq_f32((float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return __builtin_wasm_min_f32x4(a, b); + #else + return psimd_blend_f32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_cvt_s32_f32(psimd_s32 v) { + #if defined(__clang__) + return __builtin_convertvector(v, psimd_f32); + #elif defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vcvtq_f32_s32((int32x4_t) v); + #elif defined(__SSE2__) + return (psimd_f32) _mm_cvtepi32_ps((__m128i) v); + #else + return (psimd_f32) { (float) v[0], (float) v[1], (float) v[2], (float) v[3] }; + #endif + } + + /* Broadcast vector element */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_f32 psimd_splat0_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 0, 0, 0, 0); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat1_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 1, 1, 1, 1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat2_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 2, 2, 2, 2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat3_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 3, 3, 3, 3); + } + #else + PSIMD_INTRINSIC psimd_f32 psimd_splat0_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 0, 0, 0, 0 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat1_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 1, 1, 1, 1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat2_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 2, 2, 2, 2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat3_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 3, 3, 3 }); + } + #endif + + /* Reversal of vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s8 psimd_reverse_s8(psimd_s8 v) { + return __builtin_shufflevector(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u8 psimd_reverse_u8(psimd_u8 v) { + return __builtin_shufflevector(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_s16 psimd_reverse_s16(psimd_s16 v) { + return __builtin_shufflevector(v, v, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u16 psimd_reverse_u16(psimd_u16 v) { + return __builtin_shufflevector(v, v, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_s32 psimd_reverse_s32(psimd_s32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u32 psimd_reverse_u32(psimd_u32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_f32 psimd_reverse_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + #else + PSIMD_INTRINSIC psimd_s8 psimd_reverse_s8(psimd_s8 v) { + return __builtin_shuffle(v, (psimd_s8) { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_reverse_u8(psimd_u8 v) { + return __builtin_shuffle(v, (psimd_s8) { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_reverse_s16(psimd_s16 v) { + return __builtin_shuffle(v, (psimd_s16) { 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_reverse_u16(psimd_u16 v) { + return __builtin_shuffle(v, (psimd_s16) { 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_reverse_s32(psimd_s32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_reverse_u32(psimd_u32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_reverse_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + #endif + + /* Interleaving of vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s16 psimd_interleave_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3); + } + + PSIMD_INTRINSIC psimd_s16 psimd_interleave_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s16 psimd_interleave_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_interleave_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + #endif + + /* Concatenation of low/high vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s16 psimd_concat_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s16 psimd_concat_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + #endif + + /* Concatenation of even/odd vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s8 psimd_concat_even_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shufflevector(a, b, + 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14); + } + + PSIMD_INTRINSIC psimd_s8 psimd_concat_odd_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shufflevector(a, b, + 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_even_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shufflevector(a, b, + 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_odd_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shufflevector(a, b, + 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_even_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_odd_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_even_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_odd_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_even_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_odd_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_even_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_odd_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_even_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_odd_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s8 psimd_concat_even_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14 }); + } + + PSIMD_INTRINSIC psimd_s8 psimd_concat_odd_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_even_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_odd_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_even_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_odd_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_even_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_odd_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_even_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_odd_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_even_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_odd_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_even_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_odd_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + #endif + + /* Vector reduce */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shufflevector(v, v, 2, 3, 0, 1); + return temp + __builtin_shufflevector(temp, temp, 1, 0, 3, 2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shufflevector(v, v, 2, 3, 0, 1)); + return psimd_max_f32(temp, __builtin_shufflevector(temp, temp, 1, 0, 3, 2)); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shufflevector(v, v, 2, 3, 0, 1)); + return psimd_min_f32(temp, __builtin_shufflevector(temp, temp, 1, 0, 3, 2)); + } + + PSIMD_INTRINSIC float psimd_reduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shufflevector(v, v, 2, 3, -1, -1); + const psimd_f32 result = temp + __builtin_shufflevector(temp, temp, 1, -1, -1, -1); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shufflevector(v, v, 2, 3, -1, -1)); + const psimd_f32 result = psimd_max_f32(temp, __builtin_shufflevector(temp, temp, 1, -1, -1, -1)); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shufflevector(v, v, 2, 3, -1, -1)); + const psimd_f32 result = psimd_min_f32(temp, __builtin_shufflevector(temp, temp, 1, -1, -1, -1)); + return result[0]; + } + #else + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 }); + return temp + __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 })); + return psimd_max_f32(temp, __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 })); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 })); + return psimd_min_f32(temp, __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 })); + } + + PSIMD_INTRINSIC float psimd_reduce_sum_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_sum_f32(v); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_max_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_max_f32(v); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_min_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_min_f32(v); + return result[0]; + } + #endif +#endif + +#endif /* PSIMD_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/pthreadpool.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/pthreadpool.h new file mode 100644 index 0000000000000000000000000000000000000000..01dac3623786225ea13deeb1f856bf1fc96d8092 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/pthreadpool.h @@ -0,0 +1,2560 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#ifndef PTHREADPOOL_H_ +#define PTHREADPOOL_H_ + +#include +#include + +typedef struct pthreadpool* pthreadpool_t; + +typedef void (*pthreadpool_task_1d_t)(void*, size_t); +typedef void (*pthreadpool_task_1d_with_thread_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_1d_tile_1d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_2d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_2d_with_thread_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_1d_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_2d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_thread_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t); + +typedef void (*pthreadpool_task_1d_with_id_t)(void*, uint32_t, size_t); +typedef void (*pthreadpool_task_2d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t, size_t); + +typedef void (*pthreadpool_task_2d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t); + + +/** + * Disable support for denormalized numbers to the maximum extent possible for + * the duration of the computation. + * + * Handling denormalized floating-point numbers is often implemented in + * microcode, and incurs significant performance degradation. This hint + * instructs the thread pool to disable support for denormalized numbers before + * running the computation by manipulating architecture-specific control + * registers, and restore the initial value of control registers after the + * computation is complete. The thread pool temporary disables denormalized + * numbers on all threads involved in the computation (i.e. the caller threads, + * and potentially worker threads). + * + * Disabling denormalized numbers may have a small negative effect on results' + * accuracy. As various architectures differ in capabilities to control + * processing of denormalized numbers, using this flag may also hurt results' + * reproducibility across different instruction set architectures. + */ +#define PTHREADPOOL_FLAG_DISABLE_DENORMALS 0x00000001 + +/** + * Yield worker threads to the system scheduler after the operation is finished. + * + * Force workers to use kernel wait (instead of active spin-wait by default) for + * new commands after this command is processed. This flag affects only the + * immediate next operation on this thread pool. To make the thread pool always + * use kernel wait, pass this flag to all parallelization functions. + */ +#define PTHREADPOOL_FLAG_YIELD_WORKERS 0x00000002 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Create a thread pool with the specified number of threads. + * + * @param threads_count the number of threads in the thread pool. + * A value of 0 has special interpretation: it creates a thread pool with as + * many threads as there are logical processors in the system. + * + * @returns A pointer to an opaque thread pool object if the call is + * successful, or NULL pointer if the call failed. + */ +pthreadpool_t pthreadpool_create(size_t threads_count); + +/** + * Query the number of threads in a thread pool. + * + * @param threadpool the thread pool to query. + * + * @returns The number of threads in the thread pool. + */ +size_t pthreadpool_get_threads_count(pthreadpool_t threadpool); + +/** + * Process items on a 1D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * function(context, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. The + * specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d( + pthreadpool_t threadpool, + pthreadpool_task_1d_t function, + void* context, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * function(context, thread_index, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. The + * specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_1d_with_thread_t function, + void* context, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range; i++) + * function(context, uarch_index, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range the number of items on the 1D grid to process. + * The specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid with specified maximum tile size. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i += tile) + * function(context, i, min(range - i, tile)); + * + * When the call returns, all items have been processed and the thread pool is + * ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, + * the calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. + * @param tile the maximum number of items on the 1D grid to process in + * one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_1d_tile_1d_t function, + void* context, + size_t range, + size_t tile, + uint32_t flags); + +/** + * Process items on a 2D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * function(context, i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d( + pthreadpool_t threadpool, + pthreadpool_task_2d_t function, + void* context, + size_t range_i, + size_t range_j, + uint32_t flags); + +/** + * Process items on a 2D grid passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * function(context, thread_index, i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_2d_with_thread_t function, + void* context, + size_t range_i, + size_t range_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function and passing + * along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, thread_index, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_with_id_with_thread_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the first dimension of + * the 2D grid to process in one function call. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, + * cpuinfo initialization failed, or index returned + * by cpuinfo_get_current_uarch_index() exceeds + * the max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected + * by the specified function. If the index returned + * by cpuinfo_get_current_uarch_index() exceeds this + * value, default_uarch_index will be used instead. + * default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 2D grid. + * @param range_j the number of items to process along the second + * dimension of the 2D grid. + * @param tile_j the maximum number of items along the first + * dimension of the 2D grid to process in one function call. + * @param tile_j the maximum number of items along the second + * dimension of the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 3D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * function(context, i, j, k); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + pthreadpool_task_3d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension and passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, thread_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_thread_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function and passing + * along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, thread_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_id_with_thread_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 3D grid to process in one function call. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_j the maximum number of items along the second + * dimension of the 3D grid to process in one function call. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 4D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * function(context, i, j, k, l); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + pthreadpool_task_4d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, i, j, k, l, min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 4D grid to process in one function call. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, uarch_index, i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 4D grid. + * @param range_j the number of items to process along the second + * dimension of the 4D grid. + * @param range_k the number of items to process along the third + * dimension of the 4D grid. + * @param range_l the number of items to process along the fourth + * dimension of the 4D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 4D grid to process in one function call. + * @param tile_l the maximum number of items along the fourth + * dimension of the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 5D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * function(context, i, j, k, l, m); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + pthreadpool_task_5d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags); + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * function(context, i, j, k, l, m, min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags); + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * for (size_t m = 0; m < range_m; m += tile_m) + * function(context, i, j, k, l, m, + * min(range_l - l, tile_l), min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 5D grid to process in one function call. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_l, + size_t tile_m, + uint32_t flags); + +/** + * Process items on a 6D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n++) + * function(context, i, j, k, l, m, n); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d( + pthreadpool_t threadpool, + pthreadpool_task_6d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + uint32_t flags); + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n += tile_n) + * function(context, i, j, k, l, m, n, min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_6d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_n, + uint32_t flags); + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * for (size_t n = 0; n < range_n; n += tile_n) + * function(context, i, j, k, l, m, n, + * min(range_m - m, tile_m), min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 6D grid to process in one function call. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_6d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_m, + size_t tile_n, + uint32_t flags); + +/** + * Terminates threads in the thread pool and releases associated resources. + * + * @warning Accessing the thread pool after a call to this function constitutes + * undefined behaviour and may cause data corruption. + * + * @param[in,out] threadpool The thread pool to destroy. + */ +void pthreadpool_destroy(pthreadpool_t threadpool); + +#ifndef PTHREADPOOL_NO_DEPRECATED_API + +/* Legacy API for compatibility with pre-existing users (e.g. NNPACK) */ +#if defined(__GNUC__) + #define PTHREADPOOL_DEPRECATED __attribute__((__deprecated__)) +#else + #define PTHREADPOOL_DEPRECATED +#endif + +typedef void (*pthreadpool_function_1d_t)(void*, size_t); +typedef void (*pthreadpool_function_1d_tiled_t)(void*, size_t, size_t); +typedef void (*pthreadpool_function_2d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_function_2d_tiled_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_function_3d_tiled_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_function_4d_tiled_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t); + +void pthreadpool_compute_1d( + pthreadpool_t threadpool, + pthreadpool_function_1d_t function, + void* argument, + size_t range) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_1d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_1d_tiled_t function, + void* argument, + size_t range, + size_t tile) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_2d( + pthreadpool_t threadpool, + pthreadpool_function_2d_t function, + void* argument, + size_t range_i, + size_t range_j) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_2d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_2d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_3d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_3d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_i, + size_t tile_j, + size_t tile_k) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_4d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_4d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_i, + size_t tile_j, + size_t tile_k, + size_t tile_l) PTHREADPOOL_DEPRECATED; + +#endif /* PTHREADPOOL_NO_DEPRECATED_API */ + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#ifdef __cplusplus + +namespace libpthreadpool { +namespace detail { +namespace { + +template +void call_wrapper_1d(void* arg, size_t i) { + (*static_cast(arg))(i); +} + +template +void call_wrapper_1d_tile_1d(void* arg, size_t range_i, size_t tile_i) { + (*static_cast(arg))(range_i, tile_i); +} + +template +void call_wrapper_2d(void* functor, size_t i, size_t j) { + (*static_cast(functor))(i, j); +} + +template +void call_wrapper_2d_tile_1d(void* functor, + size_t i, size_t range_j, size_t tile_j) +{ + (*static_cast(functor))(i, range_j, tile_j); +} + +template +void call_wrapper_2d_tile_2d(void* functor, + size_t range_i, size_t range_j, + size_t tile_i, size_t tile_j) +{ + (*static_cast(functor))(range_i, range_j, tile_i, tile_j); +} + +template +void call_wrapper_3d(void* functor, size_t i, size_t j, size_t k) { + (*static_cast(functor))(i, j, k); +} + +template +void call_wrapper_3d_tile_1d(void* functor, + size_t i, size_t j, size_t range_k, + size_t tile_k) +{ + (*static_cast(functor))(i, j, range_k, tile_k); +} + +template +void call_wrapper_3d_tile_2d(void* functor, + size_t i, size_t range_j, size_t range_k, + size_t tile_j, size_t tile_k) +{ + (*static_cast(functor))(i, range_j, range_k, tile_j, tile_k); +} + +template +void call_wrapper_4d(void* functor, size_t i, size_t j, size_t k, size_t l) { + (*static_cast(functor))(i, j, k, l); +} + +template +void call_wrapper_4d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t range_l, + size_t tile_l) +{ + (*static_cast(functor))(i, j, k, range_l, tile_l); +} + +template +void call_wrapper_4d_tile_2d(void* functor, + size_t i, size_t j, size_t range_k, size_t range_l, + size_t tile_k, size_t tile_l) +{ + (*static_cast(functor))(i, j, range_k, range_l, tile_k, tile_l); +} + +template +void call_wrapper_5d(void* functor, size_t i, size_t j, size_t k, size_t l, size_t m) { + (*static_cast(functor))(i, j, k, l, m); +} + +template +void call_wrapper_5d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t range_m, + size_t tile_m) +{ + (*static_cast(functor))(i, j, k, l, range_m, tile_m); +} + +template +void call_wrapper_5d_tile_2d(void* functor, + size_t i, size_t j, size_t k, size_t range_l, size_t range_m, + size_t tile_l, size_t tile_m) +{ + (*static_cast(functor))(i, j, k, range_l, range_m, tile_l, tile_m); +} + +template +void call_wrapper_6d(void* functor, size_t i, size_t j, size_t k, size_t l, size_t m, size_t n) { + (*static_cast(functor))(i, j, k, l, m, n); +} + +template +void call_wrapper_6d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t m, size_t range_n, + size_t tile_n) +{ + (*static_cast(functor))(i, j, k, l, m, range_n, tile_n); +} + +template +void call_wrapper_6d_tile_2d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t range_m, size_t range_n, + size_t tile_m, size_t tile_n) +{ + (*static_cast(functor))(i, j, k, l, range_m, range_n, tile_m, tile_n); +} + +} /* namespace */ +} /* namespace detail */ +} /* namespace libpthreadpool */ + +/** + * Process items on a 1D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * functor(i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each item. + * @param range the number of items on the 1D grid to process. The + * specified functor will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range, + uint32_t flags = 0) +{ + pthreadpool_parallelize_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_1d, + const_cast(static_cast(&functor)), + range, + flags); +} + +/** + * Process items on a 1D grid with specified maximum tile size. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i += tile) + * functor(i, min(range - i, tile)); + * + * When the call returns, all items have been processed and the thread pool is + * ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, + * the calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range the number of items on the 1D grid to process. + * @param tile the maximum number of items on the 1D grid to process in + * one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_1d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range, + size_t tile, + uint32_t flags = 0) +{ + pthreadpool_parallelize_1d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_1d_tile_1d, + const_cast(static_cast(&functor)), + range, + tile, + flags); +} + +/** + * Process items on a 2D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * functor(i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each item. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + flags); +} + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * functor(i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + tile_j, + flags); +} + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * functor(i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the first dimension of + * the 2D grid to process in one functor call. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + tile_i, + tile_j, + flags); +} + +/** + * Process items on a 3D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * functor(i, j, k); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + flags); +} + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * functor(i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + tile_k, + flags); +} + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * functor(i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 3D grid to process in one functor call. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + tile_j, + tile_k, + flags); +} + +/** + * Process items on a 4D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * functor(i, j, k, l); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + flags); +} + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * functor(i, j, k, l, min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + tile_l, + flags); +} + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * functor(i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 4D grid to process in one functor call. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + tile_k, + tile_l, + flags); +} + +/** + * Process items on a 5D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * functor(i, j, k, l, m); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + flags); +} + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * functor(i, j, k, l, m, min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + tile_m, + flags); +} + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * for (size_t m = 0; m < range_m; m += tile_m) + * functor(i, j, k, l, m, + * min(range_l - l, tile_l), min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 5D grid to process in one functor call. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_l, + size_t tile_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + tile_l, + tile_m, + flags); +} + +/** + * Process items on a 6D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n++) + * functor(i, j, k, l, m, n); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + flags); +} + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n += tile_n) + * functor(i, j, k, l, m, n, min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + tile_n, + flags); +} + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * for (size_t n = 0; n < range_n; n += tile_n) + * functor(i, j, k, l, m, n, + * min(range_m - m, tile_m), min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 6D grid to process in one functor call. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_m, + size_t tile_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + tile_m, + tile_n, + flags); +} + +#endif /* __cplusplus */ + +#endif /* PTHREADPOOL_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/sleef.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/sleef.h new file mode 100644 index 0000000000000000000000000000000000000000..94b496ea1b492c991bd85b0bb5520d455b74003e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/sleef.h @@ -0,0 +1,4221 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright Naoki Shibata and contributors 2010 - 2024. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#ifndef __SLEEF_H__ +#define __SLEEF_H__ + +#define SLEEF_VERSION_MAJOR 3 +#define SLEEF_VERSION_MINOR 8 +#define SLEEF_VERSION_PATCHLEVEL 0 + +#include +#include + +#if defined (__GNUC__) || defined (__clang__) || defined(__INTEL_COMPILER) +#define SLEEF_CONST __attribute__((const)) +#define SLEEF_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define SLEEF_CONST +#define SLEEF_INLINE __forceinline +#endif + +#if defined(__AVX2__) || defined(__aarch64__) || defined(__arm__) || defined(__powerpc64__) || defined(__zarch__) +#ifndef FP_FAST_FMA +#define FP_FAST_FMA +#endif +#ifndef FP_FAST_FMAF +#define FP_FAST_FMAF +#endif +#endif + +#if defined(_MSC_VER) && !defined(__STDC__) +#define __STDC__ 1 +#endif + +#if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) +#ifdef SLEEF_IMPORT_IS_EXPORT +#define SLEEF_IMPORT __declspec(dllexport) +#else // #ifdef SLEEF_IMPORT_IS_EXPORT +#define SLEEF_IMPORT __declspec(dllimport) +#if (defined(_MSC_VER)) +#pragma comment(lib,"sleef.lib") +#endif // #if (defined(_MSC_VER)) +#endif // #ifdef SLEEF_IMPORT_IS_EXPORT +#else // #if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) +#define SLEEF_IMPORT +#endif // #if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) + +#if (defined(__GNUC__) || defined(__CLANG__)) && (defined(__i386__) || defined(__x86_64__)) +#include +#endif + +#if (defined(_MSC_VER)) +#include +#endif + +#if defined(__ARM_NEON__) || defined(__ARM_NEON) +#include +#endif + +#if defined(__ARM_FEATURE_SVE) +#include +#endif + +#if defined(__VSX__) && defined(__PPC64__) && defined(__LITTLE_ENDIAN__) +#include +typedef __vector double SLEEF_VECTOR_DOUBLE; +typedef __vector float SLEEF_VECTOR_FLOAT; +typedef __vector int SLEEF_VECTOR_INT; +typedef __vector unsigned int SLEEF_VECTOR_UINT; +typedef __vector long long SLEEF_VECTOR_LONGLONG; +typedef __vector unsigned long long SLEEF_VECTOR_ULONGLONG; +#endif + +#if defined(__VX__) && defined(__VEC__) +#ifndef SLEEF_VECINTRIN_H_INCLUDED +#include +#define SLEEF_VECINTRIN_H_INCLUDED +#endif +typedef __vector double SLEEF_VECTOR_DOUBLE; +typedef __vector float SLEEF_VECTOR_FLOAT; +typedef __vector int SLEEF_VECTOR_INT; +typedef __vector unsigned int SLEEF_VECTOR_UINT; +typedef __vector long long SLEEF_VECTOR_LONGLONG; +typedef __vector unsigned long long SLEEF_VECTOR_ULONGLONG; +#endif + +// + +#if defined(SLEEF_ENABLE_OMP_SIMD) && (defined(__GNUC__) || defined(__CLANG__)) && !defined(__INTEL_COMPILER) +#if defined(__aarch64__) +//#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(2) notinbranch") +//#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(4) notinbranch") +//#elif defined(__x86_64__) && defined(__AVX512F__) +//#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(8) notinbranch") +//#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(16) notinbranch") +#elif defined(__x86_64__) && defined(__AVX__) +#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(4) notinbranch") +#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(8) notinbranch") +#elif defined(__x86_64__) && defined(__SSE2__) +#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(2) notinbranch") +#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(4) notinbranch") +#endif +#endif + +#ifndef SLEEF_PRAGMA_OMP_SIMD_DP +#define SLEEF_PRAGMA_OMP_SIMD_DP +#define SLEEF_PRAGMA_OMP_SIMD_SP +#endif + +// + +#ifndef SLEEF_FP_ILOGB0 +#define SLEEF_FP_ILOGB0 ((int)0x80000000) +#endif + +#ifndef SLEEF_FP_ILOGBNAN +#define SLEEF_FP_ILOGBNAN ((int)2147483647) +#endif + +// + +SLEEF_IMPORT void *Sleef_malloc(size_t z); +SLEEF_IMPORT void Sleef_free(void *ptr); +SLEEF_IMPORT uint64_t Sleef_currentTimeMicros(); + +#if defined(__i386__) || defined(__x86_64__) || defined(_MSC_VER) +SLEEF_IMPORT void Sleef_x86CpuID(int32_t out[4], uint32_t eax, uint32_t ecx); +#endif + +// + +#if defined(__riscv_v) +#include +typedef vfloat64m2_t Sleef_vfloat64m1_t_2; +typedef vfloat32m2_t Sleef_vfloat32m1_t_2; +typedef vfloat64m4_t Sleef_vfloat64m2_t_2; +typedef vfloat32m4_t Sleef_vfloat32m2_t_2; +#define Sleef_vfloat64m1_t_2_DEFINED +#define Sleef_vfloat32m1_t_2_DEFINED +#define Sleef_vfloat64m2_t_2_DEFINED +#define Sleef_vfloat32m2_t_2_DEFINED +#endif + +#ifndef Sleef_double2_DEFINED +#define Sleef_double2_DEFINED +typedef struct { + double x, y; +} Sleef_double2; +#endif + +#ifndef Sleef_float2_DEFINED +#define Sleef_float2_DEFINED +typedef struct { + float x, y; +} Sleef_float2; +#endif + +#ifndef Sleef_longdouble2_DEFINED +#define Sleef_longdouble2_DEFINED +typedef struct { + long double x, y; +} Sleef_longdouble2; +#endif + +#if (defined(__SIZEOF_FLOAT128__) && __SIZEOF_FLOAT128__ == 16) || (defined(__linux__) && defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || (defined(__PPC64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 8) +#define SLEEF_FLOAT128_IS_IEEEQP +#endif + +#if !defined(SLEEF_FLOAT128_IS_IEEEQP) && defined(__SIZEOF_LONG_DOUBLE__) && __SIZEOF_LONG_DOUBLE__ == 16 && (defined(__aarch64__) || defined(__zarch__)) +#define SLEEF_LONGDOUBLE_IS_IEEEQP +#endif + +#if !defined(Sleef_quad_DEFINED) +#define Sleef_quad_DEFINED +typedef struct { uint64_t x, y; } Sleef_uint64_2t; +#if defined(SLEEF_FLOAT128_IS_IEEEQP) || defined(ENABLEFLOAT128) +typedef __float128 Sleef_quad; +#define SLEEF_QUAD_C(x) (x ## Q) +#elif defined(SLEEF_LONGDOUBLE_IS_IEEEQP) +typedef long double Sleef_quad; +#define SLEEF_QUAD_C(x) (x ## L) +#else +typedef Sleef_uint64_2t Sleef_quad; +#endif +#endif +#if !defined(Sleef_quad2_DEFINED) +#define Sleef_quad2_DEFINED +typedef union { + struct { + Sleef_quad x, y; + }; + Sleef_quad s[2]; +} Sleef_quad2; +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sin_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cos_u35(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincos_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tan_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asin_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acos_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrt_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sin_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cos_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincos_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tan_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asin_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acos_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrt_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_pow_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asinh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atanh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expm1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log10_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log1p_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincospi_u05(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincospi_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinpi_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cospi_u05(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexp(double, int); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST int Sleef_ilogb(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fma(double, double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt_u35(double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypot_u05(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypot_u35(double, double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fabs(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_copysign(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmax(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmin(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fdim(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_trunc(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_floor(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_ceil(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_round(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_rint(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_nextafter(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexp(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST int Sleef_expfrexp(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmod(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_remainder(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_modf(double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_lgamma_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tgamma_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erf_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfc_u15(double); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf_u35(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincosf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincosf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_powf_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf_u3500(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log10f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincospif_u05(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincospif_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif_u05(float d); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cospif_u05(float d); +SLEEF_IMPORT SLEEF_CONST float Sleef_ldexpf(float, int); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST int Sleef_ilogbf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf(float, float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf_u35(float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf_u05(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf_u35(float, float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fminf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_truncf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_floorf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_roundf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_rintf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST int Sleef_expfrexpf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_modff(float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erff_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf_u15(float); + +SLEEF_IMPORT SLEEF_CONST Sleef_longdouble2 Sleef_sincospil_u05(long double); +SLEEF_IMPORT SLEEF_CONST Sleef_longdouble2 Sleef_sincospil_u35(long double); + +#if defined(Sleef_quad2_DEFINED) +SLEEF_IMPORT SLEEF_CONST Sleef_quad2 Sleef_sincospiq_u05(Sleef_quad); +SLEEF_IMPORT SLEEF_CONST Sleef_quad2 Sleef_sincospiq_u35(Sleef_quad); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_powd2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastsind2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastcosd2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastpowd2_u3500sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acoshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expm1d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log1pd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinpid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_sse2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ldexpd2_sse2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_sse2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmad2_sse2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u05sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fabsd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_copysignd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmaxd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmind2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fdimd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_truncd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_floord2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ceild2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_roundd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_rintd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_nextafterd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_frfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmodd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_remainderd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_modfd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_lgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfcd2_u15sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_sse2(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_powf4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastsinf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastcosf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastpowf4_u3500sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acoshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expm1f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log1pf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinpif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_sse2(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaf4_sse2(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u05sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fabsf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_copysignf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaxf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fminf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fdimf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_truncf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_floorf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_ceilf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_roundf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_rintf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_nextafterf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_frfrexpf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmodf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_remainderf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_modff4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_lgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erff4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erfcf4_u15sse2(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf4_sse2(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_powd2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastsind2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastcosd2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastpowd2_u3500sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acoshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expm1d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log1pd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinpid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_sse4(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ldexpd2_sse4(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_sse4(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmad2_sse4(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u05sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fabsd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_copysignd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmaxd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmind2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fdimd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_truncd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_floord2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ceild2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_roundd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_rintd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_nextafterd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_frfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmodd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_remainderd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_modfd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_lgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfcd2_u15sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_sse4(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_powf4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastsinf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastcosf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastpowf4_u3500sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acoshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expm1f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log1pf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinpif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_sse4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaf4_sse4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u05sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fabsf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_copysignf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaxf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fminf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fdimf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_truncf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_floorf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_ceilf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_roundf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_rintf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_nextafterf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_frfrexpf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmodf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_remainderf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_modff4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_lgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erff4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erfcf4_u15sse4(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf4_sse4(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atan2d4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_logd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cbrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atan2d4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_logd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cbrtd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_expd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_powd4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_coshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_coshd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tanhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastsind4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastcosd4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastpowd4_u3500avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acoshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp10d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_expm1d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log1pd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincospid4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinpid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_avx(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_ldexpd4_avx(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_avx(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmad4_avx(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_hypotd4_u05avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_hypotd4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fabsd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_copysignd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmaxd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmind4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fdimd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_truncd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_floord4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_ceild4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_roundd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_rintd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_nextafterd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_frfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmodd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_remainderd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_modfd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_lgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_erfd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_erfcd4_u15avx(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_avx(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atan2f8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_logf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cbrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atan2f8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_logf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cbrtf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_expf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_powf8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_coshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_coshf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastsinf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastcosf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastpowf8_u3500avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acoshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp10f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_expm1f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log1pf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincospif8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinpif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_avx(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmaf8_avx(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_hypotf8_u05avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_hypotf8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fabsf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_copysignf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmaxf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fminf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fdimf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_truncf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_floorf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_ceilf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_roundf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_rintf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_nextafterf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_frfrexpf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmodf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_remainderf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_modff8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_lgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_erff8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_erfcf8_u15avx(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_avx(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf8_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf8_avx(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_powd4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastsind4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastcosd4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastpowd4_u3500fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acoshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expm1d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log1pd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinpid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_fma4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ldexpd4_fma4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_fma4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmad4_fma4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u05fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fabsd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_copysignd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmaxd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmind4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fdimd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_truncd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_floord4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ceild4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_roundd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_rintd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_nextafterd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_frfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmodd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_remainderd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_modfd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_lgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfcd4_u15fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_fma4(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_powf8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastsinf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastcosf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastpowf8_u3500fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acoshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expm1f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log1pf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinpif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_fma4(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaf8_fma4(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u05fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fabsf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_copysignf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaxf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fminf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fdimf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_truncf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_floorf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_ceilf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_roundf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_rintf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_nextafterf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_frfrexpf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmodf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_remainderf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_modff8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_lgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erff8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erfcf8_u15fma4(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf8_fma4(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_powd4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastsind4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastcosd4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastpowd4_u3500avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acoshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expm1d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log1pd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinpid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_avx2(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ldexpd4_avx2(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_avx2(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmad4_avx2(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u05avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fabsd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_copysignd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmaxd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmind4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fdimd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_truncd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_floord4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ceild4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_roundd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_rintd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_nextafterd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_frfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmodd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_remainderd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_modfd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_lgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfcd4_u15avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_avx2(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_powf8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastsinf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastcosf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastpowf8_u3500avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acoshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expm1f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log1pf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinpif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_avx2(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaf8_avx2(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u05avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fabsf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_copysignf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaxf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fminf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fdimf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_truncf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_floorf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_ceilf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_roundf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_rintf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_nextafterf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_frfrexpf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmodf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_remainderf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_modff8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_lgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erff8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erfcf8_u15avx2(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf8_avx2(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atan2d2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_logd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cbrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atan2d2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_logd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cbrtd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_expd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_powd2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_coshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_coshd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tanhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastsind2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastcosd2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastpowd2_u3500avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acoshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp10d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_expm1d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log1pd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincospid2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinpid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_avx2128(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_ldexpd2_avx2128(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_avx2128(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmad2_avx2128(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_hypotd2_u05avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_hypotd2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fabsd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_copysignd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmaxd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmind2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fdimd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_truncd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_floord2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_ceild2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_roundd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_rintd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_nextafterd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_frfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmodd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_remainderd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_modfd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_lgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_erfd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_erfcd2_u15avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_avx2128(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atan2f4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_logf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cbrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atan2f4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_logf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cbrtf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_expf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_powf4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_coshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_coshf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastsinf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastcosf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastpowf4_u3500avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acoshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp10f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_expm1f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log1pf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincospif4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinpif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_avx2128(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmaf4_avx2128(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_hypotf4_u05avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_hypotf4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fabsf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_copysignf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmaxf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fminf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fdimf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_truncf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_floorf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_ceilf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_roundf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_rintf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_nextafterf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_frfrexpf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmodf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_remainderf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_modff4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_lgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_erff4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_erfcf4_u15avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf4_avx2128(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atan2d8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_logd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cbrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atan2d8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_logd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cbrtd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_expd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_powd8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_coshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_coshd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tanhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastsind8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastcosd8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastpowd8_u3500avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acoshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp10d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_expm1d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log1pd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincospid8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinpid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8_avx512f(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_ldexpd8_avx512f(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_finz_ilogbd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8_avx512f(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmad8_avx512f(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_hypotd8_u05avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_hypotd8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fabsd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_copysignd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmaxd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmind8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fdimd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_truncd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_floord8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_ceild8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_roundd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_rintd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_nextafterd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_frfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_finz_expfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmodd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_remainderd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_modfd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_lgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_erfd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_erfcd8_u15avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8_avx512f(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atan2f16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_logf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cbrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atan2f16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_logf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cbrtf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_expf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_powf16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_coshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_coshf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastsinf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastcosf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastpowf16_u3500avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acoshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp10f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_expm1f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log1pf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincospif16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinpif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16_avx512f(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmaf16_avx512f(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_hypotf16_u05avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_hypotf16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fabsf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_copysignf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmaxf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fminf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fdimf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_truncf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_floorf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_ceilf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_roundf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_rintf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_nextafterf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_frfrexpf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmodf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_remainderf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_modff16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_lgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_erff16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_erfcf16_u15avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf16_avx512f(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atan2d8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_logd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cbrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atan2d8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_logd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cbrtd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_expd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_powd8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_coshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_coshd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tanhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastsind8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastcosd8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastpowd8_u3500avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acoshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp10d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_expm1d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log1pd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincospid8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinpid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8_avx512fnofma(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_ldexpd8_avx512fnofma(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_cinz_ilogbd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8_avx512fnofma(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmad8_avx512fnofma(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_hypotd8_u05avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_hypotd8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fabsd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_copysignd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmaxd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmind8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fdimd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_truncd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_floord8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_ceild8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_roundd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_rintd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_nextafterd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_frfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_cinz_expfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmodd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_remainderd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_modfd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_lgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_erfd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_erfcd8_u15avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8_avx512fnofma(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atan2f16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_logf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cbrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atan2f16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_logf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cbrtf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_expf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_powf16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_coshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_coshf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastsinf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastcosf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastpowf16_u3500avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acoshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp10f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_expm1f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log1pf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincospif16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinpif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16_avx512fnofma(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmaf16_avx512fnofma(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_hypotf16_u05avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_hypotf16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fabsf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_copysignf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmaxf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fminf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fdimf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_truncf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_floorf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_ceilf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_roundf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_rintf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_nextafterf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_frfrexpf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmodf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_remainderf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_modff16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_lgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_erff16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_erfcf16_u15avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf16_avx512fnofma(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atan2d1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_logd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cbrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atan2d1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_logd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cbrtd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_expd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_powd1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_coshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_coshd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tanhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastsind1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastcosd1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastpowd1_u3500purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acoshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp10d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_expm1d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log1pd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincospid1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinpid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1_purec(double, int32_t); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_ldexpd1_purec(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_cinz_ilogbd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1_purec(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmad1_purec(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_hypotd1_u05purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_hypotd1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fabsd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_copysignd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmaxd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmind1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fdimd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_truncd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_floord1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_floord1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_ceild1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_roundd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_rintd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_nextafterd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_frfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_cinz_expfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmodd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_remainderd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1_purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_modfd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_lgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_erfd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_erfcd1_u15purec(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1_purec(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atan2f1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_logf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cbrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atan2f1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_logf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cbrtf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_expf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_powf1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_coshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_coshf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastsinf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastcosf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastpowf1_u3500purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acoshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp10f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_expm1f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log1pf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincospif1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinpif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1_purec(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmaf1_purec(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_hypotf1_u05purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_hypotf1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fabsf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_copysignf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmaxf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fminf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fdimf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_truncf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_floorf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_ceilf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_roundf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_rintf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_nextafterf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_frfrexpf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmodf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_remainderf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1_purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_modff1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_lgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_erff1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_erfcf1_u15purec(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1_purec(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf1_purec(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atan2d1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_logd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cbrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atan2d1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_logd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cbrtd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_expd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_powd1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_coshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_coshd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tanhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastsind1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastcosd1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastpowd1_u3500purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acoshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp10d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_expm1d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log1pd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincospid1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinpid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1_purecfma(double, int32_t); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_ldexpd1_purecfma(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_finz_ilogbd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1_purecfma(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmad1_purecfma(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_hypotd1_u05purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_hypotd1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fabsd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_copysignd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmaxd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmind1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fdimd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_truncd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_floord1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_floord1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_ceild1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_roundd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_rintd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_nextafterd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_frfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_finz_expfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmodd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_remainderd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_modfd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_lgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_erfd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_erfcd1_u15purecfma(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1_purecfma(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atan2f1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_logf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cbrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atan2f1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_logf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cbrtf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_expf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_powf1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_coshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_coshf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastsinf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastcosf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastpowf1_u3500purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acoshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp10f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_expm1f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log1pf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincospif1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinpif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1_purecfma(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmaf1_purecfma(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_hypotf1_u05purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_hypotf1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fabsf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_copysignf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmaxf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fminf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fdimf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_truncf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_floorf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_ceilf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_roundf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_rintf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_nextafterf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_frfrexpf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmodf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_remainderf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_modff1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_lgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_erff1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_erfcf1_u15purecfma(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf1_purecfma(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1(double, double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_floord1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1(float, float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1(int); +#endif + +// + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // #ifndef __SLEEF_H__ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/include/xnnpack.h b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/xnnpack.h new file mode 100644 index 0000000000000000000000000000000000000000..0d7657fb5a3b08d637d48de7a4c4b14982c95258 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/include/xnnpack.h @@ -0,0 +1,4860 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright (c) Facebook, Inc. and its affiliates. +// All rights reserved. +// +// Copyright 2019 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include "pthreadpool.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/// The number of bytes XNNPACK may read beyond array bounds. +/// The caller must allocate at least this many extra bytes after the tensor data passed to XNNPACK. +/// +/// Note: XNNPACK reads, but never writes beyond array bounds. +#if XNN_ARCH_HEXAGON +#define XNN_EXTRA_BYTES 128 +#else +#define XNN_EXTRA_BYTES 16 +#endif // XNN_ARCH_HEXAGON + +/// Maximum number of dimensions in tensor shape. +#define XNN_MAX_TENSOR_DIMS 6 + +/// A value ID that cannot be valid. +#define XNN_INVALID_VALUE_ID UINT32_MAX + +/// Allow sparse inference in a Runtime. +/// +/// Note: this flag is a hint to XNNPACK that it should consider sparse inference, but does not guarantee it. +#define XNN_FLAG_HINT_SPARSE_INFERENCE 0x00000001 + +/// Allow IEEE FP16 inference in a Runtime. +/// +/// Note: this flag hints XNNPACK to consider IEEE FP16 inference, but does not guarantee it. +#define XNN_FLAG_HINT_FP16_INFERENCE 0x00000002 + +/// Force IEEE FP16 inference in a Runtime, and fail if FP16 inference is not possible. +/// +/// Note: this flag guarantees that XNNPACK will use IEEE FP16 inference, or fail to create the Runtime object. +/// Warning: on x86 systems FP16 computations will be emulated at a substantial performance cost. +#define XNN_FLAG_FORCE_FP16_INFERENCE 0x00000004 + +/// Enable timing of each operator's runtime. +#define XNN_FLAG_BASIC_PROFILING 0x00000008 + +/// Enable the just-in-time compiler. +#define XNN_FLAG_JIT 0x00000010 + +/// The convolution operator represents a depthwise convolution, and use HWGo layout for filters. +#define XNN_FLAG_DEPTHWISE_CONVOLUTION 0x00000001 + +/// Assume transposed weights in a fully connected operator. +#define XNN_FLAG_TRANSPOSE_WEIGHTS 0x00000001 + +/// The operator assumes NHWC layout for the input, regardless of the output layout. +#define XNN_FLAG_INPUT_NHWC 0x00000002 + +/// Match "SAME" padding in TensorFlow. Exact padding values are computed dynamically depending on input size. +#define XNN_FLAG_TENSORFLOW_SAME_PADDING 0x00000004 + +/// Assume transposed weights in a batch matrix multiply operator. +#define XNN_FLAG_TRANSPOSE_B XNN_FLAG_TRANSPOSE_WEIGHTS + +/// Assume transposed input in a batch matrix multiply operator. +#define XNN_FLAG_TRANSPOSE_A 0x00000002 + +/// Implicitly flatten and reshape input of a Fully Connected operator into a 2D tensor. +#define XNN_FLAG_TENSORFLOW_RESHAPE_2D 0x00000004 + +/// Match behaviour of TensorFlow 1.x. +#define XNN_FLAG_TENSORFLOW_LEGACY_MODE 0x00000004 + +/// Static weights of the FP16 operator are in FP32 format. +#define XNN_FLAG_FP32_STATIC_WEIGHTS 0x00000008 + +/// Static biases of the FP16 operator are in FP32 format. +#define XNN_FLAG_FP32_STATIC_BIASES 0x00000080 + +/// Align corners of input and output images in resize operations. +#define XNN_FLAG_ALIGN_CORNERS 0x00000008 + +/// Yield worker threads of the thread pool to the system scheduler after the inference. +#define XNN_FLAG_YIELD_WORKERS 0x00000010 + +/// Use transient indirection buffer to reduce memory footprint +#define XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER 0x00000020 + +/// Retain reduced dimensions with length 1. +#define XNN_FLAG_KEEP_DIMS 0x00000040 + +// Next unused flag value: 0x00000100. + +/// The number of entries in an array of xnn_quantization_params that XNNPACK may read beyond array bounds. +/// The caller must allocate at least this many extra xnn_quantization_params before passing the array to XNNPACK. +/// +/// Note: XNNPACK reads, but never writes beyond array bounds. +#define XNN_EXTRA_QUANTIZATION_PARAMS 15 + +/// The minimum blocksize for blockwise quantized operators. +#define XNN_MIN_BLOCKSIZE 32 + +#ifdef __GNUC__ +#define XNN_DEPRECATED __attribute__((deprecated)) +#else +#define XNN_DEPRECATED +#endif + +struct xnn_quantization_params { + int32_t zero_point; + float scale; +}; + +/// Status code for any XNNPACK function call. +enum xnn_status { + /// The call succeeded, and all output arguments now contain valid data. + xnn_status_success = 0, + xnn_status_uninitialized = 1, + xnn_status_invalid_parameter = 2, + xnn_status_invalid_state = 3, + xnn_status_unsupported_parameter = 4, + xnn_status_unsupported_hardware = 5, + xnn_status_out_of_memory = 6, + xnn_status_reallocation_required = 7, + xnn_status_deprecated = 8, +}; + +struct xnn_allocator { + /// User-specified pointer that will be passed as-is to all functions in this structure. + void* context; + /// Pointer to a function to be called for general memory allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param size - The size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the allocated memory block of at least @ref size bytes. + /// If allocation fails, the function must return NULL. + void* (*allocate)(void* context, size_t size); + /// Pointer to a function to be called for general memory re-allocation, i.e. to increase or shrink a previously + /// allocated memory block. The content of the old memory block is copied to the new memory block. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref allocate or @ref reallocate functions. Can be NULL. + /// If the pointer is NULL, the @ref reallocate call is equivalent to an @ref allocate call. + /// @param size - The new size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the newly allocated memory block of at least @ref size bytes with the content of the previous + /// memory block. + /// If allocation fails, the function must return NULL, but must not release the previous memory block. + void* (*reallocate)(void* context, void* pointer, size_t size); + /// Pointer to a function to be called for general memory de-allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref allocate or @ref reallocate functions. Can be NULL. + /// If the pointer is NULL, the @ref deallocate call is a no-op. + void (*deallocate)(void* context, void* pointer); + /// Pointer to a function to be called for aligned memory allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param alignment - The alignment of the memory block to allocate, in bytes. Alignment is always a power-of-2. + /// @param size - The size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the allocated memory block of at least @ref size bytes. + /// If allocation fails, the function must return NULL. + void* (*aligned_allocate)(void* context, size_t alignment, size_t size); + /// Pointer to a function to be called for aligned memory deallocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref aligned_allocate function. Can be NULL. + /// If the pointer is NULL, the @ref aligned_deallocate call is a no-op. + void (*aligned_deallocate)(void* context, void* pointer); +}; + +/// Initialize XNNPACK library. +/// +/// XNNPACK must be successfully initialized before use. During initialization, XNNPACK populates internal structures +/// depending on the host processor. Initialization can be time-consuming. +/// +/// @param[in] allocator - structure with function pointers to be use for memory allocation and de-allocation. +/// If this argument is NULL, system-provided memory management functions (e.g. malloc/free) +/// will be used. +/// +/// @retval xnn_status_success - XNNPACK is successfully initialized and ready to use. +/// @retval xnn_status_out_of_memory - initialization failed due to out-of-memory condition. +/// @retval xnn_status_unsupported_hardware - initialization failed because the host processor does not satisfy the +/// minimum hardware requirements for XNNPACK. E.g. this may happen on x86 +/// processors without SSE2 extension, or on 32-bit ARM processors without +/// the NEON SIMD extension. +enum xnn_status xnn_initialize(const struct xnn_allocator* allocator); + +/// Deinitialize XNNPACK library. +/// +/// To avoid memory and resource leaks, users must call xnn_deinitialize once for each successful xnn_initialize call. +/// +/// @retval xnn_status_success - deinitialization call succeeded. +enum xnn_status xnn_deinitialize(void); + +/// Get the microkernel implementation build identifier's data. +/// +/// That identifier will be unique for the current set of microkernels implementations. +/// +/// @returns A pointer to the current identifier's data. +const void* xnn_experimental_get_build_identifier_data(); + +/// Get the microkernel implementation build identifier's data size. +/// +/// @returns The size in bytes of the identifier's data. +size_t xnn_experimental_get_build_identifier_size(); + +/// Check whether the given data matches this version's identifier. +/// +/// @returns The size in bytes of the identifier's data. +bool xnn_experimental_check_build_identifier(const void* data, size_t size); + +/// Subgraph is an abstract representation of a neural network model. +/// Subgraph objects are used to define Values (tensors) and Nodes (operators) comprising the model. +typedef struct xnn_subgraph* xnn_subgraph_t; + +/// Create a empty Subgraph object. +/// +/// @param external_value_ids - number of Value IDs to reserve for communication with external graph representation. +/// The Subgraph object would avoid creating internal Value IDs in the +/// [0, reserved_value_ids-1] range. +/// @param flags - binary features of the subgraph. No supported flags are currently defined. +/// @param subgraph_out - pointer to the variable that will be initialized with a handle to the Subgraph object upon +/// successful return. +enum xnn_status xnn_create_subgraph( + uint32_t external_value_ids, + uint32_t flags, + xnn_subgraph_t* subgraph_out); + +/// Destroy a Subgraph object, as well as Values, and Nodes associated with the subgraph. +/// +/// @param subgraph - the Subgraph object to destroy. +enum xnn_status xnn_delete_subgraph( + xnn_subgraph_t subgraph); + +#define XNN_VALUE_FLAG_EXTERNAL_INPUT 0x00000001 +#define XNN_VALUE_FLAG_EXTERNAL_OUTPUT 0x00000002 +#define XNN_VALUE_FLAG_PERSISTENT 0x00000004 + +#define XNN_INVALID_VALUE_ID UINT32_MAX + +/// Type of elements in a Value object. +enum xnn_datatype { + /// Invalid data type. Valid Values never have this datatype. + xnn_datatype_invalid = 0, + /// IEEE754 single-precision floating-point. + xnn_datatype_fp32 = 1, + /// IEEE754 half-precision floating-point. + xnn_datatype_fp16 = 2, + /// Quantized 8-bit signed integer with shared per-Value quantization + /// parameters. + xnn_datatype_qint8 = 3, + /// Quantized 8-bit unsigned integer with shared per-Value quantization + /// parameters. + xnn_datatype_quint8 = 4, + /// Quantized 32-bit signed integer with shared per-Value quantization + /// parameters. + xnn_datatype_qint32 = 5, + /// Quantized 8-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint8 = 6, + /// Quantized 32-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint32 = 7, + /// Quantized 4-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint4 = 8, + /// Dynamically quantized 8-bit signed integer with per-batch quantization + /// parameters. + xnn_datatype_qdint8 = 9, + /// Dynamically quantized 8-bit signed integers packed with their per-row + /// quantization parameters. + xnn_datatype_qpint8 = 10, + /// 32-bit signed integers. + xnn_datatype_int32 = 11, + /// Quantized 4-bit signed integer with shared per-channel-block quantization + /// parameters. + xnn_datatype_qbint4 = 12, + /// IEEE754 single-precision packed floating-point. + xnn_datatype_pfp32 = 13, + /// BFloat16, i.e. the upper 16 bits of a float32. + xnn_datatype_bf16 = 14, + /// Dynamically quantized 8-bit unsigned integer with per-batch quantization + /// parameters. + xnn_datatype_qduint8 = 15, +}; + +/// Define a tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + size_t num_dims, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - multiplication factor to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + float scale, + size_t num_dims, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +enum xnn_status xnn_define_channelwise_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Validate the dimensions, channel_dim, zero point, datatype, and scale of a quantized tensor-type. +/// +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - multiplication factor to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_validate_quantized_tensor( + enum xnn_datatype datatype, + int32_t zero_point, + float scale, + size_t num_dims, + const size_t* dims); + +/// Validate the dimensions, channel_dim, zero point, datatype, and scales of a channelwise quantized tensor-type. +/// +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - per-channel multiplication factors to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param channel_dim - index of the channel dimension in the tensor with per-channel quantization parameters. +/// Typically this is the first dimension (dimension #0) of the filter tensors in the Convolution, +/// Deconvolution, and Fully Connected operators and the last dimension of the filter tensors in +/// the Depthwise Convolution operators. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_validate_channelwise_quantized_tensor( + enum xnn_datatype datatype, + int32_t zero_point, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims); + +/// Define a channelwise quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - per-channel multiplication factors to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param channel_dim - index of the channel dimension in the tensor with per-channel quantization parameters. +/// Typically this is the first dimension (dimension #0) of the filter tensors in the Convolution, +/// Deconvolution, and Fully Connected operators and the last dimension of the filter tensors in +/// the Depthwise Convolution operators. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_channelwise_quantized_tensor_value_v2( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a blockwise quantized tensor-type Value and add it to a Subgraph. +/// @param block_size - size of a block in the tensor with blockwise quantization parameters. Block is defined as +/// number of input channel element per output channel. +/// For Fully connected operators with 2d filters of size [output_channels, input_channels], +/// expecting number of scale values to be = output_channels * (input_channels / block_size). +enum xnn_status xnn_define_blockwise_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + const uint16_t* scale, + size_t num_dims, + size_t channel_dim, + size_t block_size, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a dynamically quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param num_dims - number of dimensions in the shape. +/// @param num_non_batch_dims - number of non-batch dimensions in the shape. The leading (num_dims - num_non_batch_dims) +/// dimensions will be flattened and treated as batch size. A set of quantization parameters +/// will be calculated for each batch element. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. No supported flags are currently defined. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_dynamically_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + size_t num_dims, + size_t num_nonbatch_dims, + const size_t* dims, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Type of unary operation +enum xnn_unary_operator { + xnn_unary_invalid = -1, + xnn_unary_convert, + xnn_unary_clamp, + xnn_unary_abs, + xnn_unary_bankers_rounding, + xnn_unary_ceiling, + xnn_unary_elu, + xnn_unary_exp, + xnn_unary_floor, + xnn_unary_gelu, + xnn_unary_hardswish, + xnn_unary_leaky_relu, + xnn_unary_log, + xnn_unary_negate, + xnn_unary_sigmoid, + xnn_unary_square, + xnn_unary_square_root, + xnn_unary_reciprocal_square_root, + xnn_unary_tanh, + // The following operators are experimental and may be removed. + xnn_unary_cube_root, + xnn_unary_cosine, + xnn_unary_sine, + xnn_unary_count_leading_zeros, + xnn_unary_bitwise_not, + xnn_unary_popcount, + xnn_unary_sign, +}; + +/// Parameters for xnn_define_unary +union xnn_unary_params { + struct { + /// lower bound for clipping output values. + float min; + /// upper bound for clipping output values. + float max; + } clamp; + struct { + /// scale factor for negative input elements. + float alpha; + } elu; + struct { + /// scale factor for negative input elements. + float negative_slope; + } leaky_relu; +}; + +/// Define a unary operator Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param operator - type of unary operator to define. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param params - parameters to be interpreted by the specific operator type. +/// @param flags - binary features of the Node. No supported flags are currently defined. +enum xnn_status xnn_define_unary( + xnn_subgraph_t subgraph, + enum xnn_unary_operator type, + const union xnn_unary_params* params, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Convert Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Convert Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_convert( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Convolution Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param subsampling_height - height of subsampling region for convolution output (convolution height stride). +/// @param subsampling_width - width of subsampling region for convolution output (convolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param groups - number of convolution groups. +/// @param group_input_channels - number of input channels per group. +/// @param group_output_channels - number of output channels per group. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, groups * group_input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [groups * group_output_channels, kernel_height, kernel_width, group_input_channels] +/// dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Convolution Node without a bias. If +/// present, the bias tensor must be a 1D tensor defined in the @a subgraph with [groups * +/// group_output_channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, groups * group_output_channels] dimensions. +/// @param flags - binary features of the 2D Convolution Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_convolution_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Deconvolution (Transposed Convolution) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param padding_top - implicit padding above 2D output data. +/// @param padding_right - implicit padding to the right of 2D output data. +/// @param padding_bottom - implicit padding below 2D output data. +/// @param padding_left - implicit padding to the left of 2D output data. +/// @param adjustment_height - additional elements in the bottom of the 2D output data. +/// @param adjustment_width - additional elements to the right of the 2D output data. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param upsampling_height - height of upsampling region for deconvolution input (deconvolution height stride). +/// @param upsampling_width - width of upsampling region for deconvolution input (deconvolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param groups - number of convolution groups. +/// @param group_input_channels - number of input channels per group. +/// @param group_output_channels - number of output channels per group. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, groups * group_input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [groups * group_output_channels, kernel_height, kernel_width, group_input_channels] +/// dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Convolution Node without a bias. If +/// present, the bias tensor must be a 1D tensor defined in the @a subgraph with +/// [groups * group_output_channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, groups * group_output_channels] dimensions. +/// @param flags - binary features of the 2D Deconvolution Node. No supported flags are currently defined. +enum xnn_status xnn_define_deconvolution_2d( + xnn_subgraph_t subgraph, + uint32_t padding_top, + uint32_t padding_right, + uint32_t padding_bottom, + uint32_t padding_left, + uint32_t adjustment_height, + uint32_t adjustment_width, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t upsampling_height, + uint32_t upsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Depthwise Convolution Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param subsampling_height - height of subsampling region for convolution output (convolution height stride). +/// @param subsampling_width - width of subsampling region for convolution output (convolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param depth_multiplier - ratio of output channels to input channels. +/// @param input_channels - number of input channels. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [1, kernel_height, kernel_width, input_channels * depth_multiplier] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Depthwise Convolution Node without +/// a bias. If present, the bias tensor must be a 1D tensor defined in the @a subgraph with +/// [input_channels * depth_multiplier] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, input_channels * depth_multiplier] dimensions. +/// @param flags - binary features of the 2D Depthwise Convolution Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_depthwise_convolution_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t depth_multiplier, + size_t input_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Depth To Space Node 2D and add it to a Subgraph. +/// +/// The Depth To Space 2D Node rearranges data from depth into blocks of spatial data (a reverse transform to +/// Space To Depth). For a given input pixel, an output square of pixels with side @a block_size is formed from values +/// in the corresponding number of its channels. The output depth is therefore @a block_size x @a block_size times +/// smaller than that of the input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param block_size - the size of the spatial block. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, OC * block_size * block_size] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH * block_size, IW * block_size, OC] dimensions. +/// @param flags - binary features of the input_channels Node. No supported flags are currently defined. +enum xnn_status xnn_define_depth_to_space_2d( + xnn_subgraph_t subgraph, + uint32_t block_size, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_status xnn_define_depth_to_space( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t block_size, + uint32_t flags); + +/// Define a 1D Global Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 2 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second-innermost dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 2 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 1D Global Average Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_average_pooling_1d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Global Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 3 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second- and third-innermost +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 3 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 2D Global Average Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_average_pooling_2d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 1D Global Sum Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 2 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second-innermost dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 2 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 1D Global Sum Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_sum_pooling_1d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Global Sum Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 3 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second- and third-innermost +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 3 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 2D Global Sum Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_sum_pooling_2d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param pooling_height - pooling (kernel) height. +/// @param pooling_width - pooling (kernel) width. +/// @param stride_height - displacing of the pooling window in the vertical dimension of the input pixels corresponding +/// to vertically adjacent output pixels. +/// @param stride_width - displacing of the pooling window in the horizontal dimension of the input pixels corresponding +/// to horizontally adjacent output pixels. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D Average Pooling Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_average_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Fully Connected Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the +/// @a subgraph. If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the input tensor must be at least +/// 1D and its last dimension must match the last dimension of the filter tensor. In particular, if +/// input is a 2D tensor, it must have [batch_size, input_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, the number of elements in the input tensor must be +/// divisible by the input_channels. The tensor will be first flattened into a 1D tensor of +/// [num_input_elements] dimensions, then reshaped into a 2D tensor of +/// [num_input_elements / input_channels, input_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param filter_id - Value ID for the filter tensor. The filter tensor must a 2D tensor defined in the @a subgraph. +/// If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is not specified, the filter tensor must have +/// [output_channels, input_channels] dimensions. If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is +/// specified, the filter tensor must have [input_channels, output_channels] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a Fully Connected Node without a bias. +/// If present, the bias tensor must be a 1D tensor defined in the @a subgraph with [output_channels] +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the output tensor must have the same +/// dimensionality as the input tensor, all its dimensions but the last one must match the +/// corresponding dimensions of the input tensor, and the last dimensions of the output tensor must +/// match the first dimension of the filter tensor. In particular, if input is a 2D tensor, output +/// must be a 2D tensor of [batch_size, output_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, output must be a 2D tensor of +/// [num_input_elements / input_channels, output_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param flags - binary features of the Fully Connected Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_RESHAPE_2D and XNN_FLAG_TRANSPOSE_WEIGHTS. +enum xnn_status xnn_define_fully_connected( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Sparse Fully Connected Node and add it to a Subgraph. +/// +/// This operator is experimental, and will be removed in the future. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the +/// @a subgraph. If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the input tensor must be at least +/// 1D and its last dimension must match the last dimension of the filter tensor. In particular, if +/// input is a 2D tensor, it must have [batch_size, input_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, the number of elements in the input tensor must be +/// divisible by the input_channels. The tensor will be first flattened into a 1D tensor of +/// [num_input_elements] dimensions, then reshaped into a 2D tensor of +/// [num_input_elements / input_channels, input_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param filter_id - Value ID for the filter tensor. The filter tensor must a 2D tensor defined in the @a subgraph. +/// If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is not specified, the filter tensor must have +/// [output_channels, input_channels] dimensions. If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is +/// specified, the filter tensor must have [input_channels, output_channels] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a Fully Connected Node without a bias. +/// If present, the bias tensor must be a 1D tensor defined in the @a subgraph with [output_channels] +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the output tensor must have the same +/// dimensionality as the input tensor, all its dimensions but the last one must match the +/// corresponding dimensions of the input tensor, and the last dimensions of the output tensor must +/// match the first dimension of the filter tensor. In particular, if input is a 2D tensor, output +/// must be a 2D tensor of [batch_size, output_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, output must be a 2D tensor of +/// [num_input_elements / input_channels, output_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param flags - binary features of the Fully Connected Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_RESHAPE_2D and XNN_FLAG_TRANSPOSE_WEIGHTS. +enum xnn_status xnn_define_fully_connected_sparse( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Max Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param pooling_height - pooling (kernel) height. +/// @param pooling_width - pooling (kernel) width. +/// @param stride_height - displacing of the pooling window in the vertical dimension of the input pixels corresponding +/// to vertically adjacent output pixels. +/// @param stride_width - displacing of the pooling window in the horizontal dimension of the input pixels corresponding +/// to horizontally adjacent output pixels. +/// @param dilation_height - dilation of pooling elements along the height dimension. +/// @param dilation_width - dilation of pooling elements along the width dimension. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D Max Pooling Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_max_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D ArgMax Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. +/// @param pooling_height - pooling (kernel) height. Vertical stride between pooling regions match this value. +/// @param pooling_width - pooling (kernel) width. Horizontal stride between pooling regions match this value. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_value_id - Value ID for the output tensor with the maximum values in the pools. The output tensor must +/// be a 4D tensor defined in the @a subgraph with [N, OH, OW, channels] dimensions. +/// @param output_index_id - Value ID for the output tensor with the indexes of the maximum values in the pools. The +/// output tensor must be a 4D tensor defined in the @a subgraph with [N, OH, OW, channels] +/// dimensions. +/// @param flags - binary features of the 2D ArgMax Pooling Node. No supported flags are currently defined. +enum xnn_status xnn_define_argmax_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t input_id, + uint32_t output_value_id, + uint32_t output_index_id, + uint32_t flags); + +/// Define a 2D UnPooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param padding_top - implicit padding above 2D output data. +/// @param padding_right - implicit padding to the right of 2D output data. +/// @param padding_bottom - implicit padding below 2D output data. +/// @param padding_left - implicit padding to the left of 2D output data. +/// @param pooling_height - height of the pooling window. +/// @param pooling_width - width of the pooling window. +/// @param input_value_id - Value ID for the input tensor with the max-pooling values to invert. The input value tensor +/// must be a 4D tensor defined in the @a subgraph with [N, IH, IW, channels] dimensions. +/// @param input_index_id - Value ID for the input tensor with the indices of the per-pool maximum values produced by +/// a 2D UnPooling Node. The input tensor must be a 4D tensor defined in the @a subgraph with +/// [N, IH, IW, channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D UnPooling Node. No supported flags are currently defined. +enum xnn_status xnn_define_unpooling_2d( + xnn_subgraph_t subgraph, + uint32_t padding_top, + uint32_t padding_right, + uint32_t padding_bottom, + uint32_t padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t input_value_id, + uint32_t input_index_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_binary_operator { + xnn_binary_invalid = -1, + xnn_binary_add, + xnn_binary_subtract, + xnn_binary_multiply, + xnn_binary_divide, + xnn_binary_maximum, + xnn_binary_minimum, + xnn_binary_copysign, + xnn_binary_squared_difference, + xnn_binary_prelu, + // The following operators are experimental and may be removed. + xnn_binary_modulus, + xnn_binary_atan2, + xnn_binary_pow, + xnn_binary_bitwise_and, + xnn_binary_bitwise_or, + xnn_binary_bitwise_xor, + xnn_binary_shift_left, + xnn_binary_shift_right_logical, + xnn_binary_shift_right_arithmetic, +}; + +struct xnn_binary_params { + /// lower bound for clipping output values. + double output_min; + /// upper bound for clipping output values. + double output_max; +}; + +/// Define a 2-Input binary operator Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param type - Type of operator to apply to the two inputs. +/// @param params - Optional parameters for the operator. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Node. No supported flags are currently defined. +enum xnn_status xnn_define_binary( + xnn_subgraph_t subgraph, + enum xnn_binary_operator type, + const struct xnn_binary_params* params, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Add Node and add it to a Subgraph. +/// +/// The 2-Input Add Node computes elementwise addition of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Add Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_add2( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Multiply Node and add it to a Subgraph. +/// +/// The 2-Input Multiply Node computes elementwise multiplication of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Multiply Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_multiply2( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +// Cap operations applied to logits (Q * K) of attention operator. +enum xnn_attention_logits_cap_type { + // No capping. + xnn_attention_logits_cap_type_none = 0, + // Cap the absolute values of logits by tanh: tanh(logits / cap) * cap + xnn_attention_logits_cap_type_tanh +}; + +// Params when the cap type is xnn_attention_logits_cap_type_tanh. +struct xnn_attention_logits_cap_tanh_params { + float cap; +}; + +/// Define a Scaled Dot-Product Attention Node and add it to a Subgraph. +/// +/// This operator is experimental. +/// +/// The Scaled Dot-Product Attention Node computes a multi-head or multi-query scaled dot attention on the query, key, +/// and value tensors. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param cap_type - type of cap to be applied to the logits. +/// @param cap_params - parameters for the cap. Must be a pointer to xnn_attention_logits_cap_tanh_params if cap_type +/// is xnn_attention_logits_cap_type_tanh. +/// @param query_id - Value ID for the query tensor. The query tensor must be a 3+-dimensional tensor defined in the +/// @a subgraph with the dimensions as [*, H, T, C], where H/T/C are the heads/tokens/channels, and * +/// is the 0 or more dimensions treated as batch size. +/// @param key_id - Value ID for the key tensor. The key tensor must be a 2+--dimensional tensor defined in the +/// @a subgraph. It can have the same number of dimensions as the query, with the dimensions as +/// [*, H, U, C] (multi-head), or have 1 less dimension than the query, with the dimensions as +/// as [*, U, C] (multi-query, number of heads omitted implies single head), where H/U/C are the +/// heads/key_value_tokens/channels, and * is the 0 or more dimensions treated as batch size. These +/// batch size dimensions must be the same as query. +/// @param value_id - Value ID for the value tensor. The value tensor must be a 2+--dimensional tensor defined in the +/// @a subgraph. It can have the same number of dimensions as the query, with the dimensions as +/// [*, H, U, D] (multi-head), or have 1 less dimension than the query, with the dimensions as +/// as [*, U, D] (multi-query, number of heads omitted implies single head), where H/U/D are the +/// heads/key_value_tokens/value_channels, and * is the 0 or more dimensions treated as batch size. +/// These batch size dimensions must be the same as query and key. +/// @param scale_id - Value ID for the scale tensor. The scale tensor must be a 1D tensor defined in the @a subgraph +/// with [C] dimensions. The query tensor is multiplied with this scale tensor before the dot product +/// with the key tensor. +/// @param mask_id - Value ID for the mask tensor. The mask tensor must be a 2D tensor defined in the @a subgraph with +/// [T, U] dimensions. The mask tensor is added to the logits (query dot value). +/// @param output_id - Value ID for the output tensor. The output tensor must be a 3+-dimensional tensor defined in the +/// @a subgraph with the dimensions as [*, H, T, D], where H/T/D are the heads/tokens/value_channels, +/// and * is the 0 or more dimensions treated as batch size. These batch size dimensions must be the +/// same as query, key, and value. +/// @param flags - binary features of the Scaled Dot Product Attention Node. No supported flags are currently defined. +enum xnn_status xnn_define_scaled_dot_product_attention( + xnn_subgraph_t subgraph, + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t query_id, + uint32_t key_id, + uint32_t value_id, + uint32_t scale_id, + uint32_t mask_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Subtract Node and add it to a Subgraph. +/// +/// The Subtract Node computes elementwise subtraction of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Subtract Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_subtract( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Divide Node and add it to a Subgraph. +/// +/// The Divide Node computes elementwise division of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Divide Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_divide( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Maximum Node and add it to a Subgraph. +/// +/// The 2-Input Maximum Node computes elementwise maximum of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Maximum Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_maximum2( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Minimum Node and add it to a Subgraph. +/// +/// The 2-Input Minimum Node computes elementwise minimum of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Minimum Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_minimum2( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Squared Difference Node and add it to a Subgraph. +/// +/// The Squared Difference Node computes elementwise squared difference of two tensor inputs with numpy broadcasting +/// rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Squared Difference Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_squared_difference( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Constant Pad Node with static padding specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param pre_paddings - number of padding elements to insert before input elements for every dimension. This array +/// must have as many elements as the number of dimensions in the input tensor. +/// @param post_paddings - number of padding elements to insert after input elements for every dimension. This array +/// must have as many elements as the number of dimensions in the input tensor. +/// @param padding_value - constant value used to initialize padding elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Constant Pad Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_constant_pad( + xnn_subgraph_t subgraph, + const size_t* pre_paddings, + const size_t* post_paddings, + float padding_value, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Expand Dims Node with and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_new_axes - number of new axes of size 1 to be inserted. +/// @param new_axes - The axis positions of the new axes in the expanded dimensions. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Constant Pad Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_expand_dims( + xnn_subgraph_t subgraph, + size_t num_new_axes, + const size_t* new_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Mean Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which mean is computed. +/// @param reduction_axes - axes along which mean is computed. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with at least +/// @a num_reduction_axes dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor defined in the +/// @a subgraph with @a num_reduction_axes fewer dimensions than the input tensor (if +/// XNN_FLAG_KEEP_DIMS is not specified), or has same dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is specified). +/// @param flags - binary features of the Mean Node. The only currently supported value is XNN_FLAG_KEEP_DIMS +XNN_DEPRECATED enum xnn_status xnn_define_static_mean( + xnn_subgraph_t subgraph, + size_t num_reduction_axes, + const size_t* reduction_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_reduce_operator { + xnn_reduce_invalid = -1, + xnn_reduce_sum, + xnn_reduce_mean, +}; + +/// Define a Reduce Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which reduce is computed. +/// @param reduction_axes - axes along which reduce is computed. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with at least +/// @a num_reduction_axes dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor defined in the +/// @a subgraph with @a num_reduction_axes fewer dimensions than the input tensor (if +/// XNN_FLAG_KEEP_DIMS is not specified), or has same dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is specified). +/// @param flags - binary features of the Reduce Node. The only currently supported value is XNN_FLAG_KEEP_DIMS +enum xnn_status xnn_define_static_reduce( + xnn_subgraph_t subgraph, + enum xnn_reduce_operator reduce_operator_type, + size_t num_reduction_axes, + const size_t* reduction_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Reduce Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which reduce is computed. +/// @param reduction_axes - axes along which reduce is computed. Negative values +/// are interpreted as offsets from @a +/// num_reduction_axes. +/// @param input_id - Value ID for the input tensor. The input tensor must be a +/// dense tensor with at least @a num_reduction_axes +/// dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be +/// a dense tensor defined in the @a subgraph with @a +/// num_reduction_axes fewer dimensions than the input tensor +/// (if XNN_FLAG_KEEP_DIMS is not specified), or has same +/// dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is +/// specified). +/// @param flags - binary features of the Reduce Node. The only currently +/// supported value is XNN_FLAG_KEEP_DIMS +enum xnn_status xnn_define_static_reduce_v2( // + xnn_subgraph_t subgraph, // + enum xnn_reduce_operator reduce_operator_type, // + size_t num_reduction_axes, // + const int64_t* reduction_axes, // + uint32_t input_id, // + uint32_t output_id, // + uint32_t flags); + +/// Define a 2-Input Concatenate Node and add it to a Subgraph. +/// +/// The 2-Input Concatenate Node concatenates two tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// second input. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// first input. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of both inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of both inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate2( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 3-Input Concatenate Node and add it to a Subgraph. +/// +/// The 3-Input Concatenate Node concatenates three tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate3( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 4-Input Concatenate Node and add it to a Subgraph. +/// +/// The 4-Input Concatenate Node concatenates four tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input4_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate4( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t input4_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 5-Input Concatenate Node and add it to a Subgraph. +/// +/// The 5-Input Concatenate Node concatenates four tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input4_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input5_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +enum xnn_status xnn_define_concatenate5( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t input4_id, + uint32_t input5_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Copy Sign Node and add it to a Subgraph. +/// +/// The Copy Sign Node copies the sign of the second input to the first input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be defined in the @a subgraph. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. +/// @param flags - binary features of the Copy Sign Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_copysign( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Copy Node and add it to a Subgraph. +/// +/// The Copy Node copies an input tensor to an output tensor. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the first input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Copy Node. No supported flags are currently defined. +enum xnn_status xnn_define_copy( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Output Split Node and add it to a Subgraph. +/// +/// The 2-Output Split Node splits an input tensor into two output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the second output. The split_dim dimension is half of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the first output. The split_dim dimension is half of the input's split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split2( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t flags); + +/// Define a 3-Output Split Node and add it to a Subgraph. +/// +/// The 3-Output Split Node splits an input tensor into three output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the second and third output. The split_dim dimension is one third of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the first and third output. The split_dim dimension is one third of the input's +/// split_dim. +/// @param output3_id - Value ID for the third output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the second and third output. The split_dim dimension is one third of the input's +/// split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split3( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t output3_id, + uint32_t flags); + +/// Define a 4-Output Split Node and add it to a Subgraph. +/// +/// The 4-Output Split Node splits an input tensor into four output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the other output tensors. The split_dim dimension is one fourth of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param output3_id - Value ID for the third output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param output4_id - Value ID for the fourth output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split4( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t output3_id, + uint32_t output4_id, + uint32_t flags); + +/// Define a Reshape Node with static shape specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_dims - number of shape dimensions in the output tensor. +/// @param new_shape - shape dimensions of the output tensor. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Reshape Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_reshape( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* new_shape, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Resize Bilinear Node with static output height & width specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param new_height - height dimension of the output tensor. +/// @param new_width - width dimension of the output tensor. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, C] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, new_height, new_width, C] dimensions. +/// @param flags - binary features of the 2D Resize Bilinear Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_LEGACY_MODE and XNN_FLAG_ALIGN_CORNERS, which are mutually exclusive. +enum xnn_status xnn_define_static_resize_bilinear_2d( + xnn_subgraph_t subgraph, + size_t new_height, + size_t new_width, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a PReLU (Parametric ReLU) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, channels] dimensions. +/// @param slope_id - Value ID for the slope tensor. The slope tensor must be a 1D tensor defined in the @a subgraph with +/// either [1] or [channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, channels] dimensions. +/// @param flags - binary features of the PReLU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_prelu( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t slope_id, + uint32_t output_id, + uint32_t flags); + +/// Define a RoPE (Rotary Positional Embeddings) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param max_tokens - deprecated. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [batch, tokens, heads, channels] dimensions. +/// @param weights_id - Value ID for the weights tensor. The weights tensor must be a 2D tensor defined in the +/// @a subgraph with [max_tokens, channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [batch, tokens, heads, channels] dimensions. +/// @param flags - binary features of the RoPE Node. No supported flags are currently defined. +enum xnn_status xnn_define_rope( + xnn_subgraph_t subgraph, + size_t max_sequence_size, + uint32_t input_id, + uint32_t weights_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Abs Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Abs Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_abs( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Bankers' Rounding Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Bankers' Rounding Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_bankers_rounding( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Batch Matrix Multiply Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph. It must be at least 3D. The first N-2 dimensions must match the second input +/// tensor. The last 2 dimensions are [M, K]. If XNN_FLAG_TRANSPOSE_B is not specified, the last +/// dimension must match the second last dimension of the second input tensor. If +/// XNN_FLAG_TRANSPOSE_B is specified, the last dimension must match the last dimension of the +/// second input tensor. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined +/// in the @a subgraph. It must be at least 3D. The first N-2 dimensions must match the first input +/// tensor. If XNN_FLAG_TRANSPOSE_B is not specified, the last 2 dimensions are [K, N], and the +/// second last dimension must match the last dimension of the first input tensor. If +/// XNN_FLAG_TRANSPOSE_B is specified, the last 2 dimensions are [N, K], and the last dimension must +/// match the last dimension of the first input tensor. +/// @param output_id - Value ID for the output tensor. The output tensor must be an N-dimensional tensor defined in the +/// @a subgraph. It must be at least 3D. The first N-2 dimensions must match the first and second +/// input tensors . The last 2 dimensions must be [M, N]. +/// @param flags - binary features of the Batch Matrix Multiply Node. The only currently supported value is +/// XNN_FLAG_TRANSPOSE_B. +enum xnn_status xnn_define_batch_matrix_multiply( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Ceiling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Ceiling Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_ceiling( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Clamp Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Clamp Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_clamp( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define an ELU (Exponential Linear Unit) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param alpha - scale factor for negative output elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the ELU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_elu( + xnn_subgraph_t subgraph, + float alpha, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Exp Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Exp Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_exp( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Floor Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Floor Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_floor( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define an GELU (Gaussian Error Linear Unit) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the GELU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_gelu( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a HardSwish Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the HardSwish Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_hardswish( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Leaky ReLU Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param negative_slope - scale factor for negative input elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Leaky ReLU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_leaky_relu( + xnn_subgraph_t subgraph, + float negative_slope, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Log Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Log Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_log( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Negate Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Negate Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_negate( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Sigmoid Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Sigmoid Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_sigmoid( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a SoftMax Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph, and have at +/// least one dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the SoftMax Node. No supported flags are currently defined. +enum xnn_status xnn_define_softmax( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Space To Depth 2D Node and add it to a Subgraph. +/// +/// The Space To Depth 2D Node rearranges blocks of spatial data into blocks (a reverse transform to Depth To Space 2D). +/// For a given input pixel, an output square of pixels with side @a block_size is formed from values in the +/// corresponding number of its channels. The output depth is therefore @a block_size x @a block_size times greater +/// than that of the input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param block_size - the size of the spatial block. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH * block_size, IW * block_size, OC] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, OC * block_size * block_size] dimensions. +/// @param flags - binary features of the input_channels Node. No supported flags are currently defined. +enum xnn_status xnn_define_space_to_depth_2d( + xnn_subgraph_t subgraph, + uint32_t block_size, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Square Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_square( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Square Root Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Root Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_square_root( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Reciprocal Square Root Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be +/// defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be +/// defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Root Node. No supported flags +/// are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_reciprocal_square_root( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_status xnn_define_static_slice( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* offsets, + const size_t* sizes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); +/// Define a Static Slice Node add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_dims - number of shape dimensions in the input and output tensor. +/// @param offsets - offsets in each dimension of the input tensor. This array must have @a num_dims elements. Can be +/// negative meaning that the offset is relative to the end of the dimension. +/// @param sizes - size of each dimension in output tensor. This array must have @a num_dims elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// dimensions must match @a sizes. +/// @param flags - binary features of the Static Slice Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_slice_v2( // + xnn_subgraph_t subgraph, // + size_t num_dims, // + const int64_t* offsets, // + const size_t* sizes, // + uint32_t input_id, // + uint32_t output_id, // + uint32_t flags); + +/// Define a Static Transpose Node and add it to a Subgraph. +/// +/// The Static Transpose Node applies a generalized transpose to the input tensor using the permuation in perm. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to its corresponding permuted input dimension. +/// @param num_dims - the number of permutation dimensions. This must be equal to the number of input dimensions. +/// @param perm - The permutation of the axis of the input tensor. The perm array must must contain 0 to N-1 in the +/// permuted order. +/// @param flags - binary features of the Static Transpose Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_transpose( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* perm, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Tanh Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Tanh Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_tanh( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Code cache is a cache for JIT generated code. +typedef struct xnn_code_cache* xnn_code_cache_t; + +/// Weights cache can be finalized in these ways: +enum xnn_weights_cache_finalization_kind { + /// Weights cache is finalized, no insert operations into the weights cache is allowed, even if the "inserted" + /// weights already exist in thee cache. Weights cache memory will also be trimmed to page boundary and set to + /// read-only (to prevent writes). + xnn_weights_cache_finalization_kind_hard, + /// Weights cache will be finalized with some extra space at the end, this allows for "inserting" into the cache only + /// if the weights are already in the cache, and errors on inserting uncached weights. There is memory overhead. + xnn_weights_cache_finalization_kind_soft, +}; + +/// A combination of multiple factors to uniquely locate the weights cache. +struct xnn_weights_cache_look_up_key { + /// The unique seed for each ukernel. It is guaranteed that each ukernel provides + /// a consistent and identical seed. + uint32_t seed; + /// Pointer to the original kernel. + const void* kernel; + /// Pointer to the original bias, could be NULL. + const void* bias; +}; + +/// A group of function pointers to manage weights cache. All functions may be +/// called on multi threads. +struct xnn_weights_cache_provider { + /// User-specified pointer that will be passed as-is to all functions in this + /// structure. + void* context; + + /// Looks up the tuple of {cache_key, kernel, bias} in the cache. If it is found, + /// returns the offset to the found entry for reuse. Otherwise, returns SIZE_MAX. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param cache_key - The key used to locate the weights cache entry. + size_t (*look_up)(void* context, const struct xnn_weights_cache_look_up_key* cache_key); + + /// Ensures that cache has enough space for `n` bytes. Returns the address to + /// store weight cache. Returns NULL if fails to reserve space. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param n - size to be reserved. + void* (*reserve_space)(void* context, size_t n); + + /// Looks up packed weights at `ptr` in the cache. If it is found, reuse it. + /// Otherwise, it is added to the cache. Returns the offset to the cache. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param cache_key - The key used to locate the weights cache entry. + /// @param ptr - pointer pointing to the packed weight. + /// @param size - size of the packed weight. + size_t (*look_up_or_insert)(void* context, const struct xnn_weights_cache_look_up_key* cache_key, void* ptr, size_t size); + + /// Returns whether the cache is finalized. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + bool (*is_finalized)(void* context); + + /// Returns the absolute pointer corresponding to `offset`, where the offset is returned from + /// `look_up` or `get_or_insert`. This function must be called after finalize. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param offset - offset to the start of internal buffer + void* (*offset_to_addr)(void* context, size_t offset); + + /// Destroy a weights cache object, as well as memory used for the cache. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + enum xnn_status (*delete_cache)(void* context); +}; + +/// Weights cache is a cache for packed weights. It can be reused between runtimes. +typedef struct xnn_weights_cache_provider* xnn_weights_cache_t; + +/// Create a weights cache object specifying the initial size of weights cache (in bytes). +/// +/// @param[in] size - initial capacity of the weights cache (in bytes), i.e. it can hold size bytes without growing. +/// @param weights_cache_out - pointer to the variable that will be initialized to a handle to the weights cache provider +/// upon successful return. Once created, the weights cache provider can be shared between +/// different Runtime objects. +enum xnn_status xnn_create_weights_cache_with_size(size_t size, xnn_weights_cache_t* weights_cache_out); + +enum xnn_status xnn_create_weights_cache(xnn_weights_cache_t* weights_cache_out); + +/// Finalizes the weights cache. The kind of finalization is specified by `finalization_kind`. +/// @param weights_cache - the weights cache object to finalize. +/// @param finalization_kind - the kind of finalization. +enum xnn_status xnn_finalize_weights_cache( + xnn_weights_cache_t weights_cache, + enum xnn_weights_cache_finalization_kind finalization_kind); + +// Wrapper function of the function pointers in `xnn_weights_cache_t`. +bool xnn_weights_cache_is_finalized(xnn_weights_cache_t cache); + +/// Destroy a weights cache object, as well as memory used for the cache. +/// @param weights_cache - the weights cache object to destroy. +enum xnn_status xnn_delete_weights_cache(xnn_weights_cache_t weights_cache); + +typedef struct xnn_workspace* xnn_workspace_t; + +/// Create a workspace object. +/// @param workspace_out - pointer to the variable that will be initialized to a handle to the workspace object upon +/// successful return. Once created, the workspace can be shared between different Runtime +/// objects. +enum xnn_status xnn_create_workspace(xnn_workspace_t* workspace_out); +/// Destroy a workspace object, as well as memory used by the workspace. Object destruction can be deferred until all +/// Runtime objects created with this workspace are destroyed. +/// @param workspace - the workspace object to destroy. +enum xnn_status xnn_release_workspace(xnn_workspace_t workspace); + +/// Runtime is a combination of an execution plan for subgraph Nodes and a memory manager for subgraph Values. +typedef struct xnn_runtime* xnn_runtime_t; + +enum xnn_profile_info { + /// Returns a size_t containing the number of operators. + xnn_profile_info_num_operators, + /// Returns a char[] containing the null character separated names of all operators. + xnn_profile_info_operator_name, + /// Returns a uint64_t[] with the runtimes of all operators in the same order as xnn_profile_info_operator_name. + xnn_profile_info_operator_timing, +}; + +/// Return profile information for all operators. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime, @ref xnn_create_runtime_v2 or +/// @ref xnn_create_runtime_v3. +/// @param param_name - type of profile information required. +/// @param param_value_size - the size in bytes of memory pointed to by param_value. If this is not sufficient then +/// param_value_size_ret will be set to the required size and xnn_status_out_of_memory will be +/// returned. +/// @param param_value - a pointer to memory location where appropriate values for a given param_value will be written. +/// @param param_value_size_ret - returns number of bytes required to write the result if param_value_size is not +/// sufficient. +enum xnn_status xnn_get_runtime_profiling_info(xnn_runtime_t runtime, + enum xnn_profile_info param_name, + size_t param_value_size, + void* param_value, + size_t* param_value_size_ret); + +/// Create a Runtime object from a subgraph. +/// +/// @param subgraph - a Subgraph object with all Values and Nodes that would be handled by the runtime. No Values or +/// Nodes can be added to the runtime once it is constructed. +/// @param weights_cache - a cache for packed weights. The runtime will look up and reuse packed weights in this cache, +/// this will reduce memory allocated for packed weights. +/// @param workspace - a workspace to hold internal tensors. The runtime will allocate space used for internal tensors +/// and track them using workspace. Workspace can be shared and reused across different runtimes. If +/// workspace is NULL, there will be no sharing: each runtime has its own workspace. +/// @param threadpool - the thread pool to be used for parallelisation of computations in the runtime. If the thread +/// pool is NULL, the computation would run on the caller thread without parallelization. +/// @param flags - binary features of the runtime. The only currently supported values are +/// XNN_FLAG_HINT_SPARSE_INFERENCE, XNN_FLAG_HINT_FP16_INFERENCE, XNN_FLAG_FORCE_FP16_INFERENCE, +/// XNN_FLAG_YIELD_WORKERS, and XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER. If XNN_FLAG_YIELD_WORKERS is +/// specified, worker threads would be yielded to the system scheduler after processing the last operator +/// in the Runtime. If XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER is specified, convolution operators will +/// initialize indirection buffers on each inference run using temporary memory in the workspace, instead +/// of initializing persistent indirection buffers once. +/// @param runtime_out - pointer to the variable that will be initialized with a handle to the Runtime object upon +/// successful return. Once constructed, the Runtime object is independent of the Subgraph object +/// used to create it. +enum xnn_status xnn_create_runtime_v4( + xnn_subgraph_t subgraph, + xnn_weights_cache_t weights_cache, + xnn_workspace_t workspace, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime_v3( + xnn_subgraph_t subgraph, + xnn_weights_cache_t weights_cache, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime_v2( + xnn_subgraph_t subgraph, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime( + xnn_subgraph_t subgraph, + xnn_runtime_t* runtime_out); + +struct xnn_external_value { + uint32_t id; + void* data; +}; + +/// Reshape an external value. +/// +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_reshape_external_value( + xnn_runtime_t runtime, + uint32_t external_id, + size_t num_dims, + const size_t* dims); + +/// Get the external value shape. +/// +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. The external ID can not be XNN_INVALID_VALUE_ID. +/// @param num_dims - A valid pointer into which the number of dimensions in the shape will be written. It can not be larger than XNN_MAX_TENSOR_DIMS. +/// @param dims - pointer to an array of @a num_dims shape dimensions. This pointer can't be NULL. It must be large enough to hold +/// at least @a num_dims elements. XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_get_external_value_shape( + xnn_runtime_t runtime, + uint32_t external_id, + size_t* num_dims, + size_t* dims); + +/// Reshape the XNNPACK runtime. +/// +/// Propagates the shapes of input tensors through the graph to determine the shapes of intermediate and output tensors. +/// Memory is allocated if required. Output tensor shapes are returned by xnn_get_external_value_shape. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +enum xnn_status xnn_reshape_runtime( + xnn_runtime_t runtime); + +/// Deprecated. Use xnn_reshape_runtime and xnn_setup_runtime_v2. +/// +/// Setup data pointers for external inputs and outputs in a Runtime object and +/// allocate memory. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +/// @param num_external_values - the number of external inputs and outputs specified in this call. This number must +/// match the number of external inputs and outputs in the runtime, i.e. all external +/// inputs and outputs in the runtime must be specified in one call. +/// @param external_values - array with location information for all external inputs and outputs in the runtime. +enum xnn_status xnn_setup_runtime( + xnn_runtime_t runtime, + size_t num_external_values, + const struct xnn_external_value* external_values); + +/// Setup data pointers for external inputs and outputs in a Runtime object. +/// Should be called after xnn_reshape_runtime. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +/// @param num_external_values - the number of external inputs and outputs specified in this call. This number must +/// match the number of external inputs and outputs in the runtime, i.e. all external +/// inputs and outputs in the runtime must be specified in one call. +/// @param external_values - array with location information for all external inputs and outputs in the runtime. +enum xnn_status xnn_setup_runtime_v2( + xnn_runtime_t runtime, + size_t num_external_values, + const struct xnn_external_value* external_values); + +/// Execute forward pass for all operators in the runtime. +/// +/// @param runtime - the Runtime object with the execution plan to invoke. +enum xnn_status xnn_invoke_runtime( + xnn_runtime_t runtime); + +/// Destroy a Runtime object, as well as operators and memory associated with it. +/// +/// @param runtime - the Runtime object to destroy. +enum xnn_status xnn_delete_runtime( + xnn_runtime_t runtime); + +typedef struct xnn_operator* xnn_operator_t; + +enum xnn_status xnn_run_operator( + xnn_operator_t op, + pthreadpool_t threadpool); + +enum xnn_status xnn_delete_operator( + xnn_operator_t op); + +/// Operator API: +/// - create operator will create and populate a xnn_operator_t +/// - reshape operator will update fields in xnn_operator_t with shape/dimensions and parallelization information +/// - setup operator will update pointers to input and outputs +/// Each supported operator must have a create, reshape, and setup function. (Optionally a run function.) +/// Operators listed below are in alphabetical order by operator name; within each operator, we sort alphabetically by +/// data layout and type. We also group create, reshape, setup (and optionally run) functions of each operator together. + +enum xnn_status xnn_create_binary_elementwise_nd( + enum xnn_binary_operator type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input1_quantization, + const struct xnn_quantization_params* input2_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* binary_op_out); + +enum xnn_status xnn_reshape_binary_elementwise_nd( + xnn_operator_t binary_op, + size_t num_input1_dims, + const size_t* input1_shape, + size_t num_input2_dims, + const size_t* input2_shape, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_binary_elementwise_nd( + xnn_operator_t binary_op, + const void* input1, + const void* input2, + void* output); + +enum xnn_status xnn_run_binary_elementwise_nd( + enum xnn_binary_operator type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input1_quantization, + const struct xnn_quantization_params* input2_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + size_t num_input1_dims, + const size_t* input1_shape, + size_t num_input2_dims, + const size_t* input2_shape, + const void* input1, + const void* input2, + void* output, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_unary_elementwise_nc( + enum xnn_unary_operator op_type, + enum xnn_datatype input_datatype, + enum xnn_datatype output_datatype, + const union xnn_unary_params* params, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* op_out); + +enum xnn_status xnn_reshape_unary_elementwise_nc( + xnn_operator_t op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_unary_elementwise_nc( + xnn_operator_t op, + const void* input, + void* output); + +enum xnn_status xnn_run_unary_elementwise_nc( + // create parameters + enum xnn_unary_operator op_type, + enum xnn_datatype input_datatype, + enum xnn_datatype output_datatype, + const union xnn_unary_params* params, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + // reshape parameters + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool, + // setup parameters + const void* input, + void* output); + +enum xnn_status xnn_create_argmax_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t flags, + xnn_operator_t* argmax_pooling_op_out); + +enum xnn_status xnn_reshape_argmax_pooling2d_nhwc_f32( + xnn_operator_t argmax_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_argmax_pooling2d_nhwc_f32( + xnn_operator_t argmax_pooling_op, + void* workspace, + const float* input, + float* output, + uint32_t* index); + +enum xnn_status xnn_create_average_pooling2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_f16( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_f16( + xnn_operator_t average_pooling_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_average_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_f32( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_f32( + xnn_operator_t average_pooling_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_average_pooling2d_nhwc_qu8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint8_t input_zero_point, + float input_scale, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_qu8( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_qu8( + xnn_operator_t average_pooling_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f16( + uint32_t flags, + xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_f16( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, size_t* workspace_size, size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_f16( + xnn_operator_t batch_matrix_multiply_op, void* workspace, + const void* input_a, const void* input_b, void* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f32( + uint32_t flags, xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f32_const_weights( + size_t batch_size_b, size_t k, size_t n, const float* data_b, + uint32_t flags, xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_f32( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, size_t* workspace_size, size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_f32( + xnn_operator_t batch_matrix_multiply_op, void* workspace, + const float* input_a, const float* input_b, float* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_qd8_f32_qc8w( + size_t batch_size_b, size_t k, size_t n, const int8_t* data_b, + const float* scale_b, uint32_t flags, + xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_qd8_f32_qc8w( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_qd8_f32_qc8w( + xnn_operator_t batch_matrix_multiply_op, const int8_t* input_a, + const struct xnn_quantization_params* quantization_params, + float* output); + +enum xnn_status xnn_create_channel_shuffle_nc_x8( + size_t groups, + size_t group_channels, + size_t input_stride, + size_t output_stride, + uint32_t flags, + xnn_operator_t* channel_shuffle_op_out); + +enum xnn_status xnn_reshape_channel_shuffle_nc_x8( + xnn_operator_t channel_shuffle_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_channel_shuffle_nc_x8( + xnn_operator_t channel_shuffle_op, + const void* input, + void* output); + +enum xnn_status xnn_create_channel_shuffle_nc_x32( + size_t groups, + size_t group_channels, + size_t input_stride, + size_t output_stride, + uint32_t flags, + xnn_operator_t* channel_shuffle_op_out); + +enum xnn_status xnn_reshape_channel_shuffle_nc_x32( + xnn_operator_t channel_shuffle_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_channel_shuffle_nc_x32( + xnn_operator_t channel_shuffle_op, + const void* input, + void* output); + +enum xnn_status xnn_create_constant_pad_nd_x8( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x8( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x8( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x8( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_constant_pad_nd_x16( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x16( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x16( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x16( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_constant_pad_nd_x32( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x32( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x32( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x32( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_convert_nc_f16_qd8( + uint32_t flags, + xnn_operator_t* convert_op_out); + +enum xnn_status xnn_reshape_convert_nc_f16_qd8( + xnn_operator_t convert_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +// quantization_params must be padded with at least XNN_EXTRA_QUANTIZATION_PARAMS entries. +enum xnn_status xnn_setup_convert_nc_f16_qd8( + xnn_operator_t convert_op, + const void* input, + int8_t* output, + struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_convert_nc_f32_qd8( + uint32_t flags, + xnn_operator_t* convert_op_out); + +enum xnn_status xnn_reshape_convert_nc_f32_qd8( + xnn_operator_t convert_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +// quantization_params must be padded with at least XNN_EXTRA_QUANTIZATION_PARAMS entries. +enum xnn_status xnn_setup_convert_nc_f32_qd8( + xnn_operator_t convert_op, + const float* input, + int8_t* output, + struct xnn_quantization_params* quantization_params); + +XNN_DEPRECATED enum xnn_status xnn_run_convert_nc_f32_f16( + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + const float* input, + void* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_convolution2d_nchw_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nchw_f16( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nchw_f16( + xnn_operator_t convolution_op, + const void* input, + void* output); + +enum xnn_status xnn_create_convolution2d_nchw_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nchw_f32( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nchw_f32( + xnn_operator_t convolution_op, + const float* input, + float* output); + +enum xnn_status xnn_create_convolution2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_f16( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_f16( + xnn_operator_t convolution_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_convolution2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_f32_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +// Forward declare. +struct xnn_post_operation; + +/// Deprecated +enum xnn_status xnn_create_fused_convolution2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + size_t num_post_operations, + struct xnn_post_operation* post_operations, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_f32( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_f32( + xnn_operator_t convolution_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qd8_f16_qc8w( + uint32_t input_padding_top, uint32_t input_padding_right, + uint32_t input_padding_bottom, uint32_t input_padding_left, + uint32_t kernel_height, uint32_t kernel_width, uint32_t subsampling_height, + uint32_t subsampling_width, uint32_t dilation_height, + uint32_t dilation_width, uint32_t groups, size_t group_input_channels, + size_t group_output_channels, size_t input_channel_stride, + size_t output_channel_stride, const float* kernel_scale, + const int8_t* kernel, const float* bias, float output_min, float output_max, + uint32_t flags, xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_qd8_f32_qc8w( + uint32_t input_padding_top, uint32_t input_padding_right, + uint32_t input_padding_bottom, uint32_t input_padding_left, + uint32_t kernel_height, uint32_t kernel_width, uint32_t subsampling_height, + uint32_t subsampling_width, uint32_t dilation_height, + uint32_t dilation_width, uint32_t groups, size_t group_input_channels, + size_t group_output_channels, size_t input_channel_stride, + size_t output_channel_stride, const float* kernel_scale, + const int8_t* kernel, const float* bias, float output_min, float output_max, + uint32_t flags, xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_qs8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qd8_f16_qc8w( + xnn_operator_t convolution_op, size_t batch_size, size_t input_height, + size_t input_width, size_t* workspace_size, size_t* workspace_alignment, + size_t* output_height_out, size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t convolution_op, size_t batch_size, size_t input_height, + size_t input_width, size_t* workspace_size, size_t* workspace_alignment, + size_t* output_height_out, size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qs8( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qd8_f16_qc8w( + xnn_operator_t convolution_op, void* workspace, const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_setup_convolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t convolution_op, void* workspace, const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_setup_convolution2d_nhwc_qs8( + xnn_operator_t convolution_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qs8_qc8w( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qs8_qc8w( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qs8_qc8w( + xnn_operator_t convolution_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qu8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qu8( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qu8( + xnn_operator_t convolution_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_copy_nc_x8( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x8( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x8( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_create_copy_nc_x16( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x16( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x16( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_create_copy_nc_x32( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x32( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x32( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_run_copy_nc_x32( + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + const uint32_t* input, + uint32_t* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f16( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_f16( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_f16( + xnn_operator_t deconvolution_op, + const void* input, + void* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f32( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f32_f16( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_f32( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_f32( + xnn_operator_t deconvolution_op, + const float* input, + float* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qd8_f32_qc8w( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t deconvolution_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qs8( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qs8( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qs8( + xnn_operator_t deconvolution_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qs8_qc8w( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qs8_qc8w( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qs8_qc8w( + xnn_operator_t deconvolution_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qu8( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qu8( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qu8( + xnn_operator_t deconvolution_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x16( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x16( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x32( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x32( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x8( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x8( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x8( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x16( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x16( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x32( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x32( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_dynamic_fully_connected_nc_f16( + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* dynamic_fully_connected_op_out); + +enum xnn_status xnn_reshape_dynamic_fully_connected_nc_f16( + xnn_operator_t dynamic_fully_connected_op, + size_t batch_size, + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_dynamic_fully_connected_nc_f16( + xnn_operator_t dynamic_fully_connected_op, + void* workspace, + const void* input, + const void* kernel, + const void* bias, + void* output); + +enum xnn_status xnn_create_dynamic_fully_connected_nc_f32( + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* dynamic_fully_connected_op_out); + +enum xnn_status xnn_reshape_dynamic_fully_connected_nc_f32( + xnn_operator_t dynamic_fully_connected_op, + size_t batch_size, + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_dynamic_fully_connected_nc_f32( + xnn_operator_t dynamic_fully_connected_op, + void* workspace, + const float* input, + const float* kernel, + const float* bias, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f16( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f16( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f16( + xnn_operator_t fully_connected_op, + const void* input, + void* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_f16( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_create_fully_connected_nc_f32( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_f16( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_fully_connected_nc_f32( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_f16( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_setup_fully_connected_nc_f32( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const uint8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_qc4w( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_qc8w( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qc4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qb4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t block_size, + uint8_t kernel_zero_point, + const uint16_t* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qb4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qb4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qc4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qb4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t block_size, + uint8_t kernel_zero_point, + const uint16_t* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qb4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qb4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qs8( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qs8( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qs8( + xnn_operator_t fully_connected_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_fully_connected_nc_qs8_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qs8_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qs8_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_fully_connected_nc_qu8( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qu8( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qu8( + xnn_operator_t fully_connected_op, + const uint8_t* input, + uint8_t* output); + + +enum xnn_status xnn_create_max_pooling2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_f16( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_f16( + xnn_operator_t max_pooling_op, + const void* input, + void* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_f32( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_f32( + xnn_operator_t max_pooling_op, + const float* input, + float* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_s8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_s8( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_s8( + xnn_operator_t max_pooling_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_u8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_u8( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_u8( + xnn_operator_t max_pooling_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_reduce_nd( + enum xnn_reduce_operator reduce_operator_type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* reduce_op_out); + +enum xnn_status xnn_reshape_reduce_nd( // + xnn_operator_t reduce_op, // + size_t num_reduction_axes, // + const int64_t* reduction_axes, // + size_t num_input_dims, // + const size_t* input_shape, // + size_t* workspace_size, // + size_t* workspace_alignment, // + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_reduce_nd( + xnn_operator_t reduce_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nchw_f32( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nchw_f32( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nchw_f32( + xnn_operator_t resize_op, + const float* input, + float* output); + +enum xnn_status xnn_create_resize_bilinear2d_nchw_f16( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nchw_f16( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nchw_f16( + xnn_operator_t resize_op, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_f16( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_f16( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_f16( + xnn_operator_t resize_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_f32( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_f32( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_f32( + xnn_operator_t resize_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_s8( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_s8( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_s8( + xnn_operator_t resize_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_u8( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_u8( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_u8( + xnn_operator_t resize_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_rope_nthc_f16( + uint32_t flags, + xnn_operator_t* rope_op_out); + +enum xnn_status xnn_reshape_rope_nthc_f16( + xnn_operator_t rope_op, + size_t batch_size, + size_t tokens, + size_t heads, + size_t channels, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_rope_nthc_f16( + xnn_operator_t rope_op, + const void* input, + const void* weights, + void* output); + +enum xnn_status xnn_create_rope_nthc_f32( + uint32_t flags, + xnn_operator_t* rope_op_out); + +enum xnn_status xnn_reshape_rope_nthc_f32( + xnn_operator_t rope_op, + size_t batch_size, + size_t tokens, + size_t heads, + size_t channels, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_rope_nthc_f32( + xnn_operator_t rope_op, + const float* input, + const float* weights, + float* output); + +// N: batch size +// H: number of heads +// T: tokens (sequence length) +// C: channels (head dimension) +enum xnn_status xnn_create_scaled_dot_product_attention_nhtc_f16( + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t flags, + xnn_operator_t* attention_op_out); + +enum xnn_status xnn_reshape_scaled_dot_product_attention_nhtc_f16( + xnn_operator_t attention_op, + size_t batch_size, + size_t query_heads, + // Number of tokens in query. + size_t query_tokens, + size_t key_value_heads, + // Number of tokens in key/value. For self-attention, this is same as tokens. + size_t key_value_tokens, + size_t query_key_channels, + size_t value_channels, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +// Query is of dimension [batch_size, query_heads, query_tokens, channels]. +// Key and value are of dimension [batch_size, key_value_heads, key_value_tokens, channels]. +// Scale is of dimension [channels]. +// Mask is of dimension [query_tokens, key_value_tokens]. +enum xnn_status xnn_setup_scaled_dot_product_attention_nhtc_f16( + xnn_operator_t attention_op, + void* workspace, + const void* query, + const void* key, + const void* value, + const void* scale, + const void* mask, + void* output); + +// N: batch size +// H: number of heads +// T: tokens (sequence length) +// C: channels (head dimension) +enum xnn_status xnn_create_scaled_dot_product_attention_nhtc_f32( + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t flags, + xnn_operator_t* attention_op_out); + +enum xnn_status xnn_reshape_scaled_dot_product_attention_nhtc_f32( + xnn_operator_t attention_op, + size_t batch_size, + size_t query_heads, + // Number of tokens in query. + size_t query_tokens, + size_t key_value_heads, + // Number of tokens in key/value. For self-attention, this is same as tokens. + size_t key_value_tokens, + size_t query_key_channels, + size_t value_channels, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +// Query is of dimension [batch_size, query_heads, query_tokens, query_key_channels]. +// Key and value are of dimension [batch_size, key_value_heads, key_value_tokens, query_key_channels]. +// Scale is of dimension [query_key_channels]. +// Mask is of dimension [query_tokens, key_value_tokens]. +// Output is of dimension [batch_size, query_heads, query_tokens, value_channels]. +enum xnn_status xnn_setup_scaled_dot_product_attention_nhtc_f32( + xnn_operator_t attention_op, + void* workspace, + const float* query, + const float* key, + const float* value, + const float* scale, + const float* mask, + float* output); + + +enum xnn_status xnn_create_slice_nd_x16( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x16( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x16( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_create_slice_nd_x32( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x32( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x32( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_run_slice_nd_x32( + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + const void* input, + void* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_softmax_nc_f16( + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_f16( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_f16( + xnn_operator_t softmax_op, + const void* input, + void* output); + +enum xnn_status xnn_create_softmax_nc_f32( + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_f32( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_f32( + xnn_operator_t softmax_op, + const float* input, + float* output); + +enum xnn_status xnn_create_softmax_nc_qu8( + float input_scale, + uint8_t output_zero_point, + float output_scale, + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_qu8( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_qu8( + xnn_operator_t softmax_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x16( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x16( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x32( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x32( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +enum xnn_status xnn_create_transpose_nd_x8( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x8( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x8( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x8( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x16( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x16( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x16( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x16( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x32( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x32( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x32( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x32( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x64( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x64( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x64( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x64( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_unpooling2d_nhwc_x32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + uint32_t flags, + xnn_operator_t* unpooling_op_out); + +enum xnn_status xnn_reshape_unpooling2d_nhwc_x32( + xnn_operator_t unpooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_unpooling2d_nhwc_x32( + xnn_operator_t unpooling_op, + const void* input, + const uint32_t* index, + void* output); + +enum xnn_status xnn_create_slice_nd_x8( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x8( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x8( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x8( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x8( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x8( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +#ifdef __cplusplus +} // extern "C" +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fca5909d8a77fc297fbe9f87d6755fed6276b67a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/__init__.py @@ -0,0 +1,295 @@ +# mypy: allow-untyped-defs +import warnings +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import torch._C + +# These are imported so users can access them from the `torch.jit` module +from torch._jit_internal import ( + _Await, + _drop, + _IgnoreContextManager, + _isinstance, + _overload, + _overload_method, + export, + Final, + Future, + ignore, + is_scripting, + unused, +) +from torch.jit._async import fork, wait +from torch.jit._await import _awaitable, _awaitable_nowait, _awaitable_wait +from torch.jit._decomposition_utils import _register_decomposition +from torch.jit._freeze import freeze, optimize_for_inference, run_frozen_optimizations +from torch.jit._fuser import ( + fuser, + last_executed_optimized_graph, + optimized_execution, + set_fusion_strategy, +) +from torch.jit._ir_utils import _InsertPoint +from torch.jit._script import ( + _ScriptProfile, + _unwrap_optional, + Attribute, + CompilationUnit, + interface, + RecursiveScriptClass, + RecursiveScriptModule, + script, + script_method, + ScriptFunction, + ScriptModule, + ScriptWarning, +) +from torch.jit._serialization import ( + jit_module_from_flatbuffer, + load, + save, + save_jit_module_to_flatbuffer, +) +from torch.jit._trace import ( + _flatten, + _get_trace_graph, + _script_if_tracing, + _unique_state_dict, + is_tracing, + ONNXTracedModule, + TopLevelTracedModule, + trace, + trace_module, + TracedModule, + TracerWarning, + TracingCheckError, +) +from torch.utils import set_module + + +__all__ = [ + "Attribute", + "CompilationUnit", + "Error", + "Future", + "ScriptFunction", + "ScriptModule", + "annotate", + "enable_onednn_fusion", + "export", + "export_opnames", + "fork", + "freeze", + "interface", + "ignore", + "isinstance", + "load", + "onednn_fusion_enabled", + "optimize_for_inference", + "save", + "script", + "script_if_tracing", + "set_fusion_strategy", + "strict_fusion", + "trace", + "trace_module", + "unused", + "wait", +] + +# For backwards compatibility +_fork = fork +_wait = wait +_set_fusion_strategy = set_fusion_strategy + + +def export_opnames(m): + r""" + Generate new bytecode for a Script module. + + Returns what the op list would be for a Script Module based off the current code base. + + If you have a LiteScriptModule and want to get the currently present + list of ops call _export_operator_list instead. + """ + return torch._C._export_opnames(m._c) + + +# torch.jit.Error +Error = torch._C.JITException +set_module(Error, "torch.jit") +# This is not perfect but works in common cases +Error.__name__ = "Error" +Error.__qualname__ = "Error" + + +# for use in python if using annotate +def annotate(the_type, the_value): + """Use to give type of `the_value` in TorchScript compiler. + + This method is a pass-through function that returns `the_value`, used to hint TorchScript + compiler the type of `the_value`. It is a no-op when running outside of TorchScript. + + Though TorchScript can infer correct type for most Python expressions, there are some cases where + type inference can be wrong, including: + + - Empty containers like `[]` and `{}`, which TorchScript assumes to be container of `Tensor` + - Optional types like `Optional[T]` but assigned a valid value of type `T`, TorchScript would assume + it is type `T` rather than `Optional[T]` + + Note that `annotate()` does not help in `__init__` method of `torch.nn.Module` subclasses because it + is executed in eager mode. To annotate types of `torch.nn.Module` attributes, + use :meth:`~torch.jit.Attribute` instead. + + Example: + + .. testcode:: + + import torch + from typing import Dict + + @torch.jit.script + def fn(): + # Telling TorchScript that this empty dictionary is a (str -> int) dictionary + # instead of default dictionary type of (str -> Tensor). + d = torch.jit.annotate(Dict[str, int], {}) + + # Without `torch.jit.annotate` above, following statement would fail because of + # type mismatch. + d["name"] = 20 + + .. testcleanup:: + + del fn + + Args: + the_type: Python type that should be passed to TorchScript compiler as type hint for `the_value` + the_value: Value or expression to hint type for. + + Returns: + `the_value` is passed back as return value. + """ + return the_value + + +def script_if_tracing(fn): + """ + Compiles ``fn`` when it is first called during tracing. + + ``torch.jit.script`` has a non-negligible start up time when it is first called due to + lazy-initializations of many compiler builtins. Therefore you should not use + it in library code. However, you may want to have parts of your library work + in tracing even if they use control flow. In these cases, you should use + ``@torch.jit.script_if_tracing`` to substitute for + ``torch.jit.script``. + + Args: + fn: A function to compile. + + Returns: + If called during tracing, a :class:`ScriptFunction` created by `torch.jit.script` is returned. + Otherwise, the original function `fn` is returned. + """ + return _script_if_tracing(fn) + + +# for torch.jit.isinstance +def isinstance(obj, target_type): + """ + Provide container type refinement in TorchScript. + + It can refine parameterized containers of the List, Dict, Tuple, and Optional types. E.g. ``List[str]``, + ``Dict[str, List[torch.Tensor]]``, ``Optional[Tuple[int,str,int]]``. It can also + refine basic types such as bools and ints that are available in TorchScript. + + Args: + obj: object to refine the type of + target_type: type to try to refine obj to + Returns: + ``bool``: True if obj was successfully refined to the type of target_type, + False otherwise with no new type refinement + + + Example (using ``torch.jit.isinstance`` for type refinement): + .. testcode:: + + import torch + from typing import Any, Dict, List + + class MyModule(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, input: Any): # note the Any type + if torch.jit.isinstance(input, List[torch.Tensor]): + for t in input: + y = t.clamp(0, 0.5) + elif torch.jit.isinstance(input, Dict[str, str]): + for val in input.values(): + print(val) + + m = torch.jit.script(MyModule()) + x = [torch.rand(3,3), torch.rand(4,3)] + m(x) + y = {"key1":"val1","key2":"val2"} + m(y) + """ + return _isinstance(obj, target_type) + + +class strict_fusion: + """ + Give errors if not all nodes have been fused in inference, or symbolically differentiated in training. + + Example: + Forcing fusion of additions. + + .. code-block:: python + + @torch.jit.script + def foo(x): + with torch.jit.strict_fusion(): + return x + x + x + + """ + + def __init__(self) -> None: + if not torch._jit_internal.is_scripting(): + warnings.warn("Only works in script mode", stacklevel=2) + + def __enter__(self): + pass + + def __exit__(self, type: Any, value: Any, tb: Any) -> None: + pass + + +# Context manager for globally hiding source ranges when printing graphs. +# Note that these functions are exposed to Python as static members of the +# Graph class, so mypy checks need to be skipped. +@contextmanager +def _hide_source_ranges() -> Iterator[None]: + old_enable_source_ranges = torch._C.Graph.global_print_source_ranges # type: ignore[attr-defined] + try: + torch._C.Graph.set_global_print_source_ranges(False) # type: ignore[attr-defined] + yield + finally: + torch._C.Graph.set_global_print_source_ranges(old_enable_source_ranges) # type: ignore[attr-defined] + + +def enable_onednn_fusion(enabled: bool) -> None: + """Enable or disables onednn JIT fusion based on the parameter `enabled`.""" + torch._C._jit_set_llga_enabled(enabled) + + +def onednn_fusion_enabled(): + """Return whether onednn JIT fusion is enabled.""" + return torch._C._jit_llga_enabled() + + +del Any + +if not torch._C._jit_init(): + raise RuntimeError("JIT initialization failed") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_async.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_async.py new file mode 100644 index 0000000000000000000000000000000000000000..a755a02725d53b71c6809d7ebfa0056004884f67 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_async.py @@ -0,0 +1,115 @@ +# mypy: allow-untyped-defs +"""Async API. + +This module contains the API for parallelism in TorchScript, notably: + * torch.jit.fork + * torch.jit.wait + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import torch +from torch._jit_internal import Future +from torch.jit._builtins import _register_builtin +from torch.utils import set_module + + +set_module(Future, "torch.jit") + + +def fork(func, *args, **kwargs): + r""" + Create an asynchronous task executing `func` and a reference to the value of the result of this execution. + + `fork` will return immediately, so the return value of `func` may not have been computed yet. To force completion + of the task and access the return value invoke `torch.jit.wait` on the Future. `fork` invoked + with a `func` which returns `T` is typed as `torch.jit.Future[T]`. `fork` calls can be arbitrarily + nested, and may be invoked with positional and keyword arguments. + Asynchronous execution will only occur when run in TorchScript. If run in pure python, + `fork` will not execute in parallel. `fork` will also not execute in parallel when invoked + while tracing, however the `fork` and `wait` calls will be captured in the exported IR Graph. + + .. warning:: + `fork` tasks will execute non-deterministically. We recommend only spawning + parallel fork tasks for pure functions that do not modify their inputs, + module attributes, or global state. + + Args: + func (callable or torch.nn.Module): A Python function or `torch.nn.Module` + that will be invoked. If executed in TorchScript, it will execute asynchronously, + otherwise it will not. Traced invocations of fork will be captured in the IR. + ``*args``, ``**kwargs``: arguments to invoke `func` with. + Returns: + `torch.jit.Future[T]`: a reference to the execution of `func`. The value `T` + can only be accessed by forcing completion of `func` through `torch.jit.wait`. + + Example (fork a free function): + + .. code-block:: python + + import torch + from torch import Tensor + + + def foo(a: Tensor, b: int) -> Tensor: + return a + b + + + def bar(a): + fut: torch.jit.Future[Tensor] = torch.jit.fork(foo, a, b=2) + return torch.jit.wait(fut) + + + script_bar = torch.jit.script(bar) + input = torch.tensor(2) + # only the scripted version executes asynchronously + assert script_bar(input) == bar(input) + # trace is not run asynchronously, but fork is captured in IR + graph = torch.jit.trace(bar, (input,)).graph + assert "fork" in str(graph) + + Example (fork a module method): + + .. code-block:: python + + import torch + from torch import Tensor + + + class AddMod(torch.nn.Module): + def forward(self, a: Tensor, b: int): + return a + b + + + class Mod(torch.nn.Module): + def __init__(self) -> None: + super(self).__init__() + self.mod = AddMod() + + def forward(self, input): + fut = torch.jit.fork(self.mod, a, b=2) + return torch.jit.wait(fut) + + + input = torch.tensor(2) + mod = Mod() + assert mod(input) == torch.jit.script(mod).forward(input) + """ + return torch._C.fork(func, *args, **kwargs) + + +def wait(future): + r""" + Force completion of a `torch.jit.Future[T]` asynchronous task, returning the result of the task. + + See :func:`~fork` for docs and examples. + Args: + future (torch.jit.Future[T]): an asynchronous task reference, created through `torch.jit.fork` + Returns: + `T`: the return value of the completed task + """ + return torch._C.wait(future) + + +_register_builtin(wait, "aten::wait") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_await.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_await.py new file mode 100644 index 0000000000000000000000000000000000000000..aa9bd35c3c2f765beb5530151a2079df07bb2551 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_await.py @@ -0,0 +1,27 @@ +# mypy: allow-untyped-defs +import torch +from torch._jit_internal import _Await +from torch.jit._builtins import _register_builtin +from torch.utils import set_module + + +set_module(_Await, "torch.jit") + + +def _awaitable(func, *args, **kwargs): + r"""Create Await object that will call specified functioni with specified args, when it is requested for the result.""" + return torch._C._awaitable(func, *args, **kwargs) + + +def _awaitable_wait(aw): + r"""Request await the result of execution, if Await is not completed yet, the func will be called immediately.""" + return torch._C._awaitable_wait(aw) + + +def _awaitable_nowait(o): + r"""Create completed Await with specified result.""" + return torch._C._awaitable_nowait(o) + + +_register_builtin(_awaitable_wait, "prim::awaitable_wait") +_register_builtin(_awaitable_nowait, "prim::awaitable_nowait") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_builtins.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_builtins.py new file mode 100644 index 0000000000000000000000000000000000000000..370c8ad131a4b655fe25f262cc60912be7f78708 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_builtins.py @@ -0,0 +1,204 @@ +# mypy: allow-untyped-defs +import cmath +import math +import warnings +from collections import OrderedDict +from typing import Optional + +import torch +import torch.backends.cudnn as cudnn +from torch.nn.modules.utils import ( + _list_with_default, + _pair, + _quadruple, + _single, + _triple, +) + + +_builtin_table: Optional[dict[int, str]] = None + +_modules_containing_builtins = ( + torch, + torch._C._nn, + torch._C._fft, # type: ignore[attr-defined] + torch._C._linalg, # type: ignore[attr-defined] + torch._C._nested, # type: ignore[attr-defined] + torch._C._sparse, # type: ignore[attr-defined] + torch._C._special, # type: ignore[attr-defined] +) + +_builtin_ops = [ + # Pairs of (function, op_name) + (_pair, "aten::_pair"), + (_quadruple, "aten::_quadruple"), + (_single, "aten::_single"), + (_triple, "aten::_triple"), + (_list_with_default, "aten::list_with_default"), + (OrderedDict, "aten::dict"), + (dict, "aten::dict"), + (cudnn.is_acceptable, "aten::cudnn_is_acceptable"), + (math.ceil, "aten::ceil"), + (math.copysign, "aten::copysign"), + (math.erf, "aten::erf"), + (math.erfc, "aten::erfc"), + (math.exp, "aten::exp"), + (math.expm1, "aten::expm1"), + (math.fabs, "aten::fabs"), + (math.floor, "aten::floor"), + (math.gamma, "aten::gamma"), + (math.lgamma, "aten::lgamma"), + (math.log, "aten::log"), + (math.log10, "aten::log10"), + (math.log1p, "aten::log1p"), + (math.pow, "aten::pow"), + (math.sqrt, "aten::sqrt"), + (math.isnan, "aten::isnan"), + (math.asinh, "aten::asinh"), + (math.atanh, "aten::atanh"), + (math.cosh, "aten::cosh"), + (math.sinh, "aten::sinh"), + (math.tanh, "aten::tanh"), + (math.acos, "aten::acos"), + (math.asin, "aten::asin"), + (math.atan, "aten::atan"), + (math.atan2, "aten::atan2"), + (math.cos, "aten::cos"), + (math.sin, "aten::sin"), + (math.tan, "aten::tan"), + (math.asinh, "aten::asinh"), + (math.atanh, "aten::atanh"), + (math.acosh, "aten::acosh"), + (math.fmod, "aten::fmod"), + (math.modf, "aten::modf"), + (math.factorial, "aten::factorial"), + (math.frexp, "aten::frexp"), + (math.isinf, "aten::isinf"), + (math.degrees, "aten::degrees"), + (math.radians, "aten::radians"), + (cmath.isnan, "aten::isnan"), + (cmath.isfinite, "aten::isfinite"), + (cmath.isinf, "aten::isinf"), + (cmath.phase, "aten::angle"), + (cmath.rect, "aten::polar"), + (cmath.log, "aten::log"), + (cmath.log10, "aten::log10"), + (cmath.sqrt, "aten::sqrt"), + (cmath.exp, "aten::exp"), + (cmath.sin, "aten::sin"), + (cmath.tan, "aten::tan"), + (cmath.cos, "aten::cos"), + (cmath.asin, "aten::asin"), + (cmath.acos, "aten::acos"), + (cmath.atan, "aten::atan"), + (cmath.sinh, "aten::sinh"), + (cmath.cosh, "aten::cosh"), + (cmath.tanh, "aten::tanh"), + (cmath.asinh, "aten::asinh"), + (cmath.acosh, "aten::acosh"), + (cmath.atanh, "aten::atanh"), + (math.ldexp, "aten::ldexp"), + (torch._assert, "aten::_assert"), + (torch.autograd.grad, "aten::grad"), + (torch.autograd.backward, "aten::backward"), + (torch._C._infer_size, "aten::_infer_size"), + ( + torch.nn.functional._no_grad_embedding_renorm_, # type: ignore[attr-defined] + "aten::_no_grad_embedding_renorm_", + ), + (torch.nn.functional.assert_int_or_pair, "aten::_assert_int_or_pair"), + (torch.nn.init._no_grad_fill_, "aten::_no_grad_fill_"), + (torch.nn.init._no_grad_normal_, "aten::_no_grad_normal_"), + (torch.nn.init._no_grad_uniform_, "aten::_no_grad_uniform_"), + (torch.nn.init._no_grad_zero_, "aten::_no_grad_zero_"), + (torch._C._get_tracing_state, "aten::_get_tracing_state"), + (torch._C._get_cpu_capability, "aten::_get_cpu_capability"), + (warnings.warn, "aten::warn"), + (torch._VF.stft, "aten::stft"), # type: ignore[attr-defined] + (torch._VF.istft, "aten::istft"), # type: ignore[attr-defined] + (torch._VF.cdist, "aten::cdist"), # type: ignore[attr-defined] + (torch._VF.norm, "aten::norm"), # type: ignore[attr-defined] + (torch._VF.unique_dim, "aten::unique_dim"), + (torch._VF.unique_consecutive, "aten::unique_consecutive"), # type: ignore[attr-defined] + (torch._VF.nuclear_norm, "aten::nuclear_norm"), + (torch._VF.frobenius_norm, "aten::frobenius_norm"), + (torch._VF.tensordot, "aten::tensordot"), # type: ignore[attr-defined] +] + +# ops in torch.functional are bound to torch +# in these cases, we want to resolve the function to their python implementation +# instead looking up a builtin "aten::" schema + + +def _gen_torch_functional_registered_ops(): + # eventually ops should encompass all of torch/functional.py, (torch.functional.__all__) + # but we are currently only able to compile some of the functions. additionally, + # some functions directly map to their aten:: implementations. + # TODO: add support for more ops + ops = [ + "stft", + "istft", + "lu", + "cdist", + "norm", + "unique", + "unique_consecutive", + "tensordot", + ] + return {getattr(torch.functional, name) for name in ops} + + +_functional_registered_ops = _gen_torch_functional_registered_ops() + + +def _is_special_functional_bound_op(fn): + return fn in _functional_registered_ops + + +# lazily built to ensure the correct initialization order +def _get_builtin_table(): + global _builtin_table + if _builtin_table is not None: + return _builtin_table + _builtin_table = {} + + def register_all(mod) -> None: + for name in dir(mod): + v = getattr(mod, name) + if ( + callable(v) + and not _is_special_functional_bound_op(v) + and v is not torch.no_grad + and v is not torch.autocast + ): + # Fixup inconsistency in segment_reduce + if name == "_segment_reduce": + name = name[1:] + _builtin_ops.append((v, "aten::" + name)) + + for mod in _modules_containing_builtins: + register_all(mod) + + _builtin_ops.append((math.gcd, "aten::gcd")) + _builtin_ops.append((math.isfinite, "aten::isfinite")) + _builtin_ops.append((math.remainder, "aten::mathremainder")) # type: ignore[attr-defined] + + import torch.distributed.autograd as dist_autograd + + if dist_autograd.is_available(): + _builtin_ops.append((dist_autograd.get_gradients, "aten::get_gradients")) + _builtin_ops.append((dist_autograd.backward, "aten::dist_backward")) + + # populate the _builtin_table from _builtin_ops + for builtin, aten_op in _builtin_ops: + _builtin_table[id(builtin)] = aten_op + + return _builtin_table + + +def _register_builtin(fn, op) -> None: + _get_builtin_table()[id(fn)] = op + + +def _find_builtin(fn): + return _get_builtin_table().get(id(fn)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_check.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_check.py new file mode 100644 index 0000000000000000000000000000000000000000..2ead04ce9d169d8bdb6d5de84b8ca0a5c0760105 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_check.py @@ -0,0 +1,261 @@ +# mypy: allow-untyped-defs +import ast +import inspect +import sys +import textwrap +import warnings + +import torch + + +class AttributeTypeIsSupportedChecker(ast.NodeVisitor): + """Check the ``__init__`` method of a given ``nn.Module``. + + It ensures that all instance-level attributes can be properly initialized. + + Specifically, we do type inference based on attribute values...even + if the attribute in question has already been typed using + Python3-style annotations or ``torch.jit.annotate``. This means that + setting an instance-level attribute to ``[]`` (for ``List``), + ``{}`` for ``Dict``), or ``None`` (for ``Optional``) isn't enough + information for us to properly initialize that attribute. + + An object of this class can walk a given ``nn.Module``'s AST and + determine if it meets our requirements or not. + + Known limitations + 1. We can only check the AST nodes for certain constructs; we can't + ``eval`` arbitrary expressions. This means that function calls, + class instantiations, and complex expressions that resolve to one of + the "empty" values specified above will NOT be flagged as + problematic. + 2. We match on string literals, so if the user decides to use a + non-standard import (e.g. `from typing import List as foo`), we + won't catch it. + + Example: + .. code-block:: python + + class M(torch.nn.Module): + def fn(self): + return [] + + def __init__(self) -> None: + super().__init__() + self.x: List[int] = [] + + def forward(self, x: List[int]): + self.x = x + return 1 + + The above code will pass the ``AttributeTypeIsSupportedChecker`` + check since we have a function call in ``__init__``. However, + it will still fail later with the ``RuntimeError`` "Tried to set + nonexistent attribute: x. Did you forget to initialize it in + __init__()?". + + Args: + nn_module - The instance of ``torch.nn.Module`` whose + ``__init__`` method we wish to check + """ + + def check(self, nn_module: torch.nn.Module) -> None: + source_lines = inspect.getsource(nn_module.__class__.__init__) + + # Ignore comments no matter the indentation + def is_useless_comment(line): + line = line.strip() + return line.startswith("#") and not line.startswith("# type:") + + source_lines = "\n".join( + [l for l in source_lines.split("\n") if not is_useless_comment(l)] + ) + + # This AST only contains the `__init__` method of the nn.Module + init_ast = ast.parse(textwrap.dedent(source_lines)) + + # Get items annotated in the class body + if sys.version_info >= (3, 14): + import annotationlib + + self.class_level_annotations = list( + annotationlib.get_annotations( + nn_module, format=annotationlib.Format.FORWARDREF + ).keys() + ) + else: + self.class_level_annotations = list(nn_module.__annotations__.keys()) + + # Flag for later + self.visiting_class_level_ann = False + + self.visit(init_ast) + + def _is_empty_container(self, node: ast.AST, ann_type: str) -> bool: + if ann_type == "List": + # Assigning `[]` to a `List` type gives you a Node where + # value=List(elts=[], ctx=Load()) + if not isinstance(node, ast.List): + return False + if node.elts: + return False + elif ann_type == "Dict": + # Assigning `{}` to a `Dict` type gives you a Node where + # value=Dict(keys=[], values=[]) + if not isinstance(node, ast.Dict): + return False + if node.keys: + return False + elif ann_type == "Optional": + # Assigning `None` to an `Optional` type gives you a + # Node where value=Constant(value=None, kind=None) + if not isinstance(node, ast.Constant): + return False + if node.value: # type: ignore[attr-defined] + return False + + return True + + def visit_Assign(self, node) -> None: + """Store assignment state when assigning to a Call Node. + + If we're visiting a Call Node (the right-hand side of an + assignment statement), we won't be able to check the variable + that we're assigning to (the left-hand side of an assignment). + Because of this, we need to store this state in visitAssign. + (Luckily, we only have to do this if we're assigning to a Call + Node, i.e. ``torch.jit.annotate``. If we're using normal Python + annotations, we'll be visiting an AnnAssign Node, which has its + target built in.) + """ + try: + if ( + isinstance(node.value, ast.Call) + and node.targets[0].attr in self.class_level_annotations + ): + self.visiting_class_level_ann = True + except AttributeError: + return + self.generic_visit(node) + self.visiting_class_level_ann = False + + def visit_AnnAssign(self, node) -> None: + """Visit an AnnAssign node in an ``nn.Module``'s ``__init__`` method. + + It checks if it conforms to our attribute annotation rules.""" + # If we have a local variable + try: + if node.target.value.id != "self": + return + except AttributeError: + return + + # If we have an attribute that's already been annotated at the + # class level + if node.target.attr in self.class_level_annotations: + return + + # TODO @ansley: add `Union` once landed + + # NB: Even though `Tuple` is a "container", we don't want to + # check for it here. `Tuple` functions as an type with an + # "infinite" number of subtypes, in the sense that you can have + # `Tuple[())]`, `Tuple[T1]`, `Tuple[T2]`, `Tuple[T1, T2]`, + # `Tuple[T2, T1]` and so on, and none of these subtypes can be + # used in place of the other. Therefore, assigning an empty + # tuple in `__init__` CORRECTLY means that that variable + # cannot be reassigned later to a non-empty tuple. Same + # deal with `NamedTuple` + + containers = {"List", "list", "Dict", "dict", "Optional"} + + # If we're not evaluating one of the specified problem types + try: + if node.annotation.value.id not in containers: + return + except AttributeError: + # To evaluate a base type (`str`, `int`, etc.), we would + # have needed to get the name through `node.annotation.id` + # instead of `node.annotation.value.id`. Seems that we're + # not evaluating one of our "containers" + return + + # Check if the assigned variable is empty + ann_type = node.annotation.value.id + if not self._is_empty_container(node.value, ann_type): + return + + warnings.warn( + "The TorchScript type system doesn't support " + "instance-level annotations on empty non-base " + "types in `__init__`. Instead, either 1) use a " + "type annotation in the class body, or 2) wrap " + "the type in `torch.jit.Attribute`.", + stacklevel=2, + ) + + def visit_Call(self, node) -> None: + """Determine if a Call node is 'torch.jit.annotate' in __init__. + + Visit a Call node in an ``nn.Module``'s ``__init__`` + method and determine if it's ``torch.jit.annotate``. If so, + see if it conforms to our attribute annotation rules. + """ + # If we have an attribute that's already been annotated at the + # class level + if self.visiting_class_level_ann: + return + + # If this isn't a call to `torch.jit.annotate` + try: + if ( + node.func.value.value.id != "torch" + or node.func.value.attr != "jit" + or node.func.attr != "annotate" + ): + self.generic_visit(node) + elif ( + node.func.value.value.id != "jit" or node.func.value.attr != "annotate" + ): + self.generic_visit(node) + except AttributeError: + # Looks like we didn't even have the right node structure + # to check for `torch.jit.annotate` in the first place + self.generic_visit(node) + + # Invariant: we have a `torch.jit.annotate` or a + # `torch.annotate` call + + # A Call Node for `torch.jit.annotate` should have an `args` + # list of length 2 where args[0] represents the annotation and + # args[1] represents the actual value + if len(node.args) != 2: + return + + if not isinstance(node.args[0], ast.Subscript): + return + + # See notes in `visit_AnnAssign` r.e. containers + + containers = {"List", "Dict", "Optional"} + + try: + ann_type = node.args[0].value.id # type: ignore[attr-defined] + except AttributeError: + return + + if ann_type not in containers: + return + + # Check if the assigned variable is empty + if not self._is_empty_container(node.args[1], ann_type): + return + + warnings.warn( + "The TorchScript type system doesn't support " + "instance-level annotations on empty non-base " + "types in `__init__`. Instead, either 1) use a " + "type annotation in the class body, or 2) wrap " + "the type in `torch.jit.Attribute`.", + stacklevel=2, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_dataclass_impls.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_dataclass_impls.py new file mode 100644 index 0000000000000000000000000000000000000000..757e9cf83bb814db9a86e24c2f29fca1aed6797a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_dataclass_impls.py @@ -0,0 +1,190 @@ +# mypy: allow-untyped-defs +# Functions for synthesizing magic methods for JIT-compiled dataclasses +import ast +import dataclasses +import inspect +import os +from collections.abc import Callable +from functools import partial + +from torch._jit_internal import FAKE_FILENAME_PREFIX, is_optional +from torch._sources import ParsedDef, SourceContext + + +def _get_fake_filename(cls, method_name): + return os.path.join(FAKE_FILENAME_PREFIX, cls.__name__, method_name) + + +def compose_fn(cls, name: str, body_lines: list[str], signature: str) -> ParsedDef: + body = "\n".join(f" {b}" for b in body_lines) + decl = f"def {name}{signature}:\n{body}" + + # Parse the function declaration + try: + py_ast = ast.parse(decl) + except SyntaxError as e: + # This should only happen if there's some unforeseeable change + # in the dataclasses module that makes our synthesized code fail + raise RuntimeError( + f"TorchScript failed to synthesize dataclass method '{name}' for class '{cls.__name__}'. " + "Please file a bug report at " + ) from e + fake_filename = _get_fake_filename(cls, name) + # Parse the function + return ParsedDef( + py_ast, + ctx=SourceContext( + source=decl, filename=fake_filename, file_lineno=0, leading_whitespace_len=0 + ), + source=decl, + filename=fake_filename, + file_lineno=0, + ) + + +def synthesize__init__(cls) -> ParsedDef: + # Supporting default factories in the way that people expect would sort of require us to + # allow compiling lambda functions, which is not currently supported. + if any( + field.default_factory is not dataclasses.MISSING + for field in dataclasses.fields(cls) + ): + raise NotImplementedError( + "Default factory initializers are not supported in TorchScript dataclasses" + ) + + # Simply read off the generated __init__ signature from CPython's implementation. It'll be + # almost correct except for InitVar annotations, which we need to handle specially. + signature = inspect.signature(cls.__init__) + + # Handle InitVars if needed (only works on Python 3.8+, when a `type` attribute was added to InitVar); + # see CPython commit here https://github.com/python/cpython/commit/01ee12ba35a333e8a6a25c4153c4a21838e9585c + init_vars: list[str] = [] + params = [] + for name, param in signature.parameters.items(): + ann = param.annotation + + if isinstance(ann, dataclasses.InitVar): + # The TorchScript interpreter can't handle InitVar annotations, so we unwrap the underlying type here + init_vars.append(name) + params.append(param.replace(annotation=ann.type)) # type: ignore[attr-defined] + else: + params.append(param) + + signature = signature.replace(parameters=params) + + body = [ + # Assign all attributes to self + f"self.{field.name} = {field.name}" + for field in dataclasses.fields(cls) + if field.init and field.name not in init_vars + ] + # Call user's impl of __post_init__ if it exists + if hasattr(cls, "__post_init__"): + body.append("self.__post_init__(" + ", ".join(init_vars) + ")") + + return compose_fn(cls, "__init__", body or ["pass"], signature=str(signature)) + + +# This is a placeholder at the moment since the TorchScript interpreter doesn't call __repr__ +def synthesize__repr__(cls) -> ParsedDef: + return compose_fn( + cls, + "__repr__", + [ + f"return '{cls.__name__}(" + + ", ".join( + [ + f"{field.name}=self.{field.name}" + for field in dataclasses.fields(cls) + if field.repr + ] + ) + + ")'" + ], + signature="(self) -> str", + ) + + +def synthesize__hash__(cls) -> ParsedDef: + return compose_fn( + cls, + "__hash__", + [ + # This is just a placeholder to prevent compilation from failing; this won't even get called at + # all right now because the TorchScript interpreter doesn't call custom __hash__ implementations + "raise NotImplementedError('__hash__ is not supported for dataclasses in TorchScript')" + ], + signature="(self) -> int", + ) + + +# Implementation for __eq__ and __ne__ +def synthesize_equality(cls, name: str, converse: str) -> ParsedDef: + return synthesize_comparison( + cls, + name, + allow_eq=True, + raise_on_none=False, + inner=[f"if val1 {converse} val2: return False"], + ) + + +def synthesize_inequality(cls, name: str, op: str, allow_eq: bool) -> ParsedDef: + return synthesize_comparison( + cls, + name, + allow_eq, + raise_on_none=True, + inner=[ + f"if val1 {op} val2: return True", + f"elif val2 {op} val1: return False", + ], + ) + + +def synthesize_comparison( + cls, name: str, allow_eq: bool, raise_on_none: bool, inner: list[str] +) -> ParsedDef: + body = [] + for field in dataclasses.fields(cls): + if not field.compare: + continue + + body.extend( + [ + f"val1 = self.{field.name}", + f"val2 = other.{field.name}", + ] + ) + body.extend( + inner + if not is_optional(field.type) + else [ + # Type refinement for optional fields; we need this to avoid type errors from the interpreter + "if val1 is not None and val2 is not None:", + *[" " + line for line in inner], + "elif (val1 is None) != (val2 is None):", + f" raise TypeError('Cannot compare {cls.__name__} with None')" + if raise_on_none + else " return False", + ] + ) + + body.append(f"return {allow_eq}") + return compose_fn( + cls, name, body, signature=f"(self, other: {cls.__name__}) -> bool" + ) + + +DATACLASS_MAGIC_METHODS: dict[str, Callable] = { + "__init__": synthesize__init__, + "__repr__": synthesize__repr__, + "__hash__": synthesize__hash__, + "__eq__": partial(synthesize_equality, name="__eq__", converse="!="), + "__ne__": partial(synthesize_equality, name="__ne__", converse="=="), + "__lt__": partial(synthesize_inequality, name="__lt__", op="<", allow_eq=False), + "__le__": partial(synthesize_inequality, name="__le__", op="<", allow_eq=True), + "__gt__": partial(synthesize_inequality, name="__gt__", op=">", allow_eq=False), + "__ge__": partial(synthesize_inequality, name="__ge__", op=">", allow_eq=True), +} diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decomposition_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decomposition_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4ef33f8999352536a924718c219486ecd2e7abe2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decomposition_utils.py @@ -0,0 +1,12 @@ +# mypy: allow-untyped-defs +import torch +from torch._ops import OpOverload, OpOverloadPacket + + +def _register_decomposition(op: OpOverload, graph: torch._C.Graph) -> None: + assert not isinstance(op, OpOverloadPacket), ( + f"Must pass specific op overload, not overload packet, found {op}" + ) + assert isinstance(op, OpOverload) + + torch._C._jit_register_decomposition_for_schema(op._schema, graph) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decompositions.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decompositions.py new file mode 100644 index 0000000000000000000000000000000000000000..21afb5c44ceaee602fd782294c9010e5b3313c9d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_decompositions.py @@ -0,0 +1,141 @@ +# mypy: allow-untyped-defs +import torch +from torch import Tensor + + +aten = torch.ops.aten +import inspect +import warnings +from collections.abc import Callable +from typing import Optional, TypeVar +from typing_extensions import ParamSpec + +from torch.types import Number + + +decomposition_table: dict[str, torch.jit.ScriptFunction] = {} +function_name_set: set[str] = set() + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def check_decomposition_has_type_annotations(f) -> None: + inspect_empty = inspect._empty # type: ignore[attr-defined] + sig = inspect.signature(f) + for param in sig.parameters.values(): + assert param.annotation != inspect_empty, ( + f"No signature on param {param.name} for function {f.name}" + ) + + assert sig.return_annotation != inspect_empty, ( + f"No return annotation for function {f.name}" + ) + + +def signatures_match(decomposition_sig, torch_op_sig): + decomp_params = decomposition_sig.parameters + op_params = torch_op_sig.parameters + + if len(decomp_params) != len(op_params): + return False + + for decomp_param, op_param in zip(decomp_params.values(), op_params.values()): + # can't check full equality yet because not all fields are correctly deduced + # in the torch_op_sig - like default value + # can't check 'kind' bc + # kwarg-only values with defaults not yet supported in TS + inspect_empty = inspect._empty # type: ignore[attr-defined] + for field in ["name", "annotation"]: + if field == "name" and decomp_param.name == "self": + warnings.warn( + "PyTorch uses 'input' instead of 'self' on public api", stacklevel=2 + ) + + if getattr(decomp_param, field) != getattr(op_param, field): + return False + + decomp_default = decomp_param.default + op_default = op_param.default + # default value not always correctly inferred as being present on torch schema, + # but if specified on both they should be equal + if decomp_default != inspect_empty and op_default != inspect_empty: + if decomp_default != op_default: + return False + + return decomposition_sig.return_annotation == torch_op_sig.return_annotation + + +def register_decomposition( + aten_op: torch._ops.OpOverload, + registry: Optional[dict[str, torch.jit.ScriptFunction]] = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + def decomposition_decorator(f: Callable[_P, _T]) -> Callable[_P, _T]: + nonlocal registry + if registry is None: + registry = decomposition_table + + assert isinstance(aten_op, torch._ops.OpOverload) + + # Need unique name for jit function serialization + assert f.__name__ not in function_name_set, ( + f"Duplicated function name {f.__name__}" + ) + function_name_set.add(f.__name__) + + scripted_func = torch.jit.script(f) + torch._C._jit_pass_inline(scripted_func.graph) + + for _ in range(2): + torch._C._jit_pass_peephole(scripted_func.graph) + torch._C._jit_pass_constant_propagation(scripted_func.graph) + + registry[str(aten_op._schema)] = scripted_func + return f + + return decomposition_decorator + + +# TODO: replace torch.sigmoid -> aten.sigmoid + + +@register_decomposition(aten.var.correction) +def var_decomposition( + input: Tensor, + dim: Optional[list[int]] = None, + correction: Optional[Number] = None, + keepdim: bool = False, +) -> Tensor: + if dim is None: + dim_i: list[int] = [] + dim = dim_i + + if isinstance(dim, (tuple, list)) and len(dim) == 0: + n = input.numel() + else: + n = 1 + for dim_i in dim: # type: ignore[assignment] + n *= input.shape[dim_i] # type: ignore[call-overload] + + mean = aten.mean(input, dim, True) + sub = input - mean + sq = sub * sub + sum = aten.sum(sq, dim, keepdim) + + if correction is None: + denom = float(n - 1) + else: + if isinstance(correction, int): + denom = float(n - correction) + elif isinstance(correction, float): + denom = float(n) - correction + else: + raise RuntimeError("correction must be int or float") + + # pyrefly: ignore [no-matching-overload] + return sum / max(0, denom) + + +@register_decomposition(aten.var.default) +def var(input: Tensor, unbiased: bool = True) -> Tensor: + return var_decomposition(input, correction=(1 if unbiased else 0)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_freeze.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_freeze.py new file mode 100644 index 0000000000000000000000000000000000000000..efffacd9a9fef0c84e7de41bac9ce094e1bb43e9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_freeze.py @@ -0,0 +1,234 @@ +# mypy: allow-untyped-defs +"""Freezing. + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +from typing import Optional + +import torch +from torch.jit._script import RecursiveScriptModule, ScriptModule + + +def freeze( + mod, preserved_attrs: Optional[list[str]] = None, optimize_numerics: bool = True +): + r"""Freeze ScriptModule, inline submodules, and attributes as constants. + + Freezing a :class:`ScriptModule` will clone it and attempt to inline the cloned + module's submodules, parameters, and attributes as constants in the TorchScript IR Graph. + By default, `forward` will be preserved, as well as attributes & methods specified in + `preserved_attrs`. Additionally, any attribute that is modified within a preserved + method will be preserved. + + Freezing currently only accepts ScriptModules that are in eval mode. + + Freezing applies generic optimization that will speed up your model regardless of machine. + To further optimize using server-specific settings, run `optimize_for_inference` after + freezing. + + Args: + mod (:class:`ScriptModule`): a module to be frozen + preserved_attrs (Optional[List[str]]): a list of attributes to preserve in addition to the forward method. + Attributes modified in preserved methods will also be preserved. + optimize_numerics (bool): If ``True``, a set of optimization passes will be run that does not strictly + preserve numerics. Full details of optimization can be found at `torch.jit.run_frozen_optimizations`. + + Returns: + Frozen :class:`ScriptModule`. + + Example (Freezing a simple module with a Parameter): + + .. testcode:: + import torch + class MyModule(torch.nn.Module): + def __init__(self, N, M): + super().__init__() + self.weight = torch.nn.Parameter(torch.rand(N, M)) + self.linear = torch.nn.Linear(N, M) + + def forward(self, input): + output = self.weight.mm(input) + output = self.linear(output) + return output + + scripted_module = torch.jit.script(MyModule(2, 3).eval()) + frozen_module = torch.jit.freeze(scripted_module) + # parameters have been removed and inlined into the Graph as constants + assert len(list(frozen_module.named_parameters())) == 0 + # See the compiled graph as Python code + print(frozen_module.code) + + Example (Freezing a module with preserved attributes) + + .. testcode:: + import torch + class MyModule2(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.modified_tensor = torch.tensor(10.) + self.version = 1 + + def forward(self, input): + self.modified_tensor += 1 + return input + self.modified_tensor + + scripted_module = torch.jit.script(MyModule2().eval()) + frozen_module = torch.jit.freeze(scripted_module, preserved_attrs=["version"]) + # we've manually preserved `version`, so it still exists on the frozen module and can be modified + assert frozen_module.version == 1 + frozen_module.version = 2 + # `modified_tensor` is detected as being mutated in the forward, so freezing preserves + # it to retain model semantics + assert frozen_module(torch.tensor(1)) == torch.tensor(12) + # now that we've run it once, the next result will be incremented by one + assert frozen_module(torch.tensor(1)) == torch.tensor(13) + + Note: + Freezing submodule attributes is also supported: + frozen_module = torch.jit.freeze(scripted_module, preserved_attrs=["submodule.version"]) + + Note: + If you're not sure why an attribute is not being inlined as a constant, you can run + `dump_alias_db` on frozen_module.forward.graph to see if freezing has detected the + attribute is being modified. + + Note: + Because freezing makes weights constants and removes module hierarchy, `to` and other + nn.Module methods to manipulate device or dtype no longer work. As a workaround, + You can remap devices by specifying `map_location` in `torch.jit.load`, however + device-specific logic may have been baked into the model. + """ + if not isinstance(mod, ScriptModule): + raise RuntimeError( + "Freezing expects a ScriptModule as input. " + "Please use torch.jit.script or torch.jit.trace to script your 'nn.Module'." + ) + + if mod.training: + raise RuntimeError( + "Freezing is currently only implemented for modules in eval mode. " + "Please call .eval() on your module before freezing." + ) + + preserved_attrs = preserved_attrs if preserved_attrs is not None else [] + + out = RecursiveScriptModule(torch._C._freeze_module(mod._c, preserved_attrs)) + RecursiveScriptModule._finalize_scriptmodule(out) + + preserved_methods = [x for x in preserved_attrs if mod._c._has_method(x)] + run_frozen_optimizations(out, optimize_numerics, preserved_methods) + + return out + + +def run_frozen_optimizations( + mod, optimize_numerics: bool = True, preserved_methods: Optional[list[str]] = None +) -> None: + r""" + Run a series of optimizations looking for patterns that occur in frozen graphs. + + The current set of optimizations includes: + - Dropout Removal + - Pretranspose Linear Layers + - Concat Linear Layers with same input Tensor + - Conv -> Batchnorm folding + - Conv -> Add/Sub folding + - Conv -> Mul/Div folding + + Args: + mod (:class:`ScriptModule`): a frozen module to be optimized + + optimize_numerics (bool): If ``True``, a set of optimization passes will be run that does not strictly + preserve numerics. These optimizations preserve default rtol and atol of `torch.testing.assert_close` + when applied on a single transformation, however in a module where many transformations are applied + the rtol or atol may no longer fall within the default `assert_close` tolerance. Conv -> Batchnorm folding, + Conv-Add/Sub, and Conv -> Mul/Div folding all may alter numerics. + + Returns: + None + + Note: + In rare occasions, this can result in slower execution. + + Example (Freezing a module with Conv->Batchnorm) + .. code-block:: python + import torch + + in_channels, out_channels = 3, 32 + conv = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=True + ) + bn = torch.nn.BatchNorm2d(out_channels, eps=0.001) + mod = torch.nn.Sequential(conv, bn) + # set optimize to False here, by default freezing runs run_frozen_optimizations + frozen_mod = torch.jit.freeze(torch.jit.script(mod.eval()), optimize=False) + # inspect frozen mod + assert "batch_norm" in str(frozen_mod.graph) + torch.jit.run_frozen_optimizations(frozen_mod) + assert "batch_norm" not in str(frozen_mod.graph) + + """ + if mod._c._has_method("forward"): + torch._C._jit_pass_optimize_frozen_graph(mod.graph, optimize_numerics) + + if preserved_methods is None: + preserved_methods = [] + + for method in preserved_methods: + torch._C._jit_pass_optimize_frozen_graph( + mod.__getattr__(method).graph, optimize_numerics + ) + + +def optimize_for_inference( + mod: ScriptModule, other_methods: Optional[list[str]] = None +) -> ScriptModule: + """ + Perform a set of optimization passes to optimize a model for the purposes of inference. + + If the model is not already frozen, optimize_for_inference + will invoke `torch.jit.freeze` automatically. + + In addition to generic optimizations that should speed up your model regardless + of environment, prepare for inference will also bake in build specific settings + such as the presence of CUDNN or MKLDNN, and may in the future make transformations + which speed things up on one machine but slow things down on another. Accordingly, + serialization is not implemented following invoking `optimize_for_inference` and + is not guaranteed. + + This is still in prototype, and may have the potential to slow down your model. + Primary use cases that have been targeted so far have been vision models on cpu + and gpu to a lesser extent. + + Example (optimizing a module with Conv->Batchnorm):: + + import torch + + in_channels, out_channels = 3, 32 + conv = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=True + ) + bn = torch.nn.BatchNorm2d(out_channels, eps=0.001) + mod = torch.nn.Sequential(conv, bn) + frozen_mod = torch.jit.optimize_for_inference(torch.jit.script(mod.eval())) + assert "batch_norm" not in str(frozen_mod.graph) + # if built with MKLDNN, convolution will be run with MKLDNN weights + assert "MKLDNN" in frozen_mod.graph + """ + if not isinstance(mod, ScriptModule): + raise RuntimeError( + "optimize_for_inference expects a ScriptModule as input. " + "Please use torch.jit.script or torch.jit.trace to script your 'nn.Module'." + ) + + if other_methods is None: + other_methods = [] + + if hasattr(mod, "training"): + mod = freeze(mod.eval(), preserved_attrs=other_methods) + + torch._C._jit_pass_optimize_for_inference(mod._c, other_methods) + + return mod diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_fuser.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_fuser.py new file mode 100644 index 0000000000000000000000000000000000000000..f7c3db8846699d71a3556ad4f8a512a28250998d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_fuser.py @@ -0,0 +1,160 @@ +# mypy: allow-untyped-defs +import contextlib + +import torch + + +@contextlib.contextmanager +def optimized_execution(should_optimize): + """Context manager that controls whether the JIT's executor will run optimizations before executing a function.""" + stored_flag = torch._C._get_graph_executor_optimize() + torch._C._set_graph_executor_optimize(should_optimize) + try: + yield + finally: + torch._C._set_graph_executor_optimize(stored_flag) + + +@contextlib.contextmanager +def fuser(name): + """Context manager that facilitates switching between backend fusers. + + Valid names: + * ``fuser0`` - enables only legacy fuser + * ``fuser1`` - enables only NNC + * ``fuser2`` - enables only nvFuser + * ``fuser3`` - enables oneDNN Graph + """ + old_cpu_fuse = torch._C._jit_can_fuse_on_cpu() + old_gpu_fuse = torch._C._jit_can_fuse_on_gpu() + old_texpr_fuser_state = torch._C._jit_texpr_fuser_enabled() + old_nvfuser_state = torch._C._jit_nvfuser_enabled() + old_llga_state = torch._C._jit_llga_enabled() + if name == "fuser0": # legacy fuser + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser1": # NNC + old_profiling_executor = torch._C._jit_set_profiling_executor(True) + old_profiling_mode = torch._C._get_graph_executor_optimize(True) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + torch._C._jit_set_texpr_fuser_enabled(True) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser2": # nvFuser + torch._C._jit_override_can_fuse_on_cpu(False) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(True) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser3": # oneDNN Graph + old_profiling_executor = torch._C._jit_set_profiling_executor(True) + old_profiling_mode = torch._C._get_graph_executor_optimize(True) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(True) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(True) + elif name == "none": # Turn Pytorch fuser off + torch._C._jit_override_can_fuse_on_cpu(False) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + else: + raise Exception(f"unrecognized fuser option (name: {name})") # noqa: TRY002 + try: + yield + finally: + if name in ["fuser1", "fuser3"]: # NNC or oneDNN Graph + torch._C._jit_set_profiling_executor(old_profiling_executor) # type: ignore[possibly-undefined] + torch._C._get_graph_executor_optimize(old_profiling_mode) # type: ignore[possibly-undefined] + # recover the previous values + torch._C._jit_override_can_fuse_on_cpu(old_cpu_fuse) + torch._C._jit_override_can_fuse_on_gpu(old_gpu_fuse) + torch._C._jit_set_texpr_fuser_enabled(old_texpr_fuser_state) + torch._C._jit_set_nvfuser_enabled(old_nvfuser_state) + torch._C._jit_set_llga_enabled(old_llga_state) + + +last_executed_optimized_graph = torch._C._last_executed_optimized_graph + + +def _get_differentiable_graph_node(node, diff_node) -> None: + if node.kind() == "prim::DifferentiableGraph": + diff_node.append(node) + else: + for block in node.blocks(): + for n in block.nodes(): + _get_differentiable_graph_node(n, diff_node) + + +def _graph_for(self, *args, **kwargs): + return _script_method_graph_for(self, self, *args, **kwargs) + + +def _script_method_graph_for(self, parent, *args, **kwargs): + try: + dbs = parent.get_debug_state() + eps = list(dbs.execution_plans.values()) + assert len(eps) == 1 + graph = eps[0].graph.copy() + + # graph_executor_states for differentiable node + fw_states = eps[0].code.differentiable_op_executor_states() + diff_nodes: list[torch._C.Node] = [] + for n in graph.nodes(): + _get_differentiable_graph_node(n, diff_nodes) + + assert len(fw_states) == len(diff_nodes) + # swap each differentiable graph with optimized graph in their execution plan + for n, state in zip(diff_nodes, fw_states): + fw_execution_plans = list(state.execution_plans.values()) + # we can only update the subgraph when there's a unique execution + # plan. Avoid assert here so we would skip the ones that can't be + # updated while try the best effort to update other nodes. + if len(fw_execution_plans) == 1: + n.g_("Subgraph", fw_execution_plans[0].graph) + + return graph + except Exception: + # fallback approach, we just ran the graph and return the recorded optimized + # graph + self(*args, **kwargs) + return last_executed_optimized_graph() + + +def set_fusion_strategy(strategy: list[tuple[str, int]]): + """Set the type and number of specializations that can occur during fusion. + + Usage: provide a list of pairs (type, depth) where type is one of "STATIC" or "DYNAMIC" + and depth is an integer. + + Behavior - static vs dynamic: + In STATIC fusion, fused ops are compiled to have fixed input shapes. The shape is determined + based on some initial profiling runs. + In DYNAMIC fusion, fused ops are compiled to have variable input shapes, so that multiple + shapes are possible. + + In both cases, we also recompile on new striding behavior, device, or dtype. + + Behavior - fallback functions & depth: + When an input doesn't match the format required by the specialized compiled op, it will run + a fallback function. Fallback functions are recursively be compiled and specialized based + on the observed tensor shapes. Since compilation can be slow, the "depth" parameter is provided to + limit the number of specializations that can be compiled, before giving up on recompiling and + falling back to a completely un-fused, un-specialized implementation. + + The list of (type, depth) pairs controls the type of specializations and the number of + specializations. For example: [("STATIC", 2), ("DYNAMIC", 2)] indicates that the first + two specializations will use static fusions, the following two specializations will use + dynamic fusion, and any inputs that satisfy none of the 4 options will run an + unfused implementation. + + NB: in the future, if more as more fusion backends are added there may be more granular + apis for specific fusers. + """ + return torch._C._jit_set_fusion_strategy(strategy) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_ir_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_ir_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6e91cc8ad04825b6fff925d2119289d5e10350db --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_ir_utils.py @@ -0,0 +1,33 @@ +from types import TracebackType +from typing import Optional, Union + +import torch + + +class _InsertPoint: + def __init__( + self, + insert_point_graph: torch._C.Graph, + insert_point: Union[torch._C.Node, torch._C.Block], + ) -> None: + self.insert_point = insert_point + self.g = insert_point_graph + self.guard = None + + def __enter__(self) -> None: + self.prev_insert_point = self.g.insertPoint() + self.g.setInsertPoint(self.insert_point) + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.g.setInsertPoint(self.prev_insert_point) + + +def insert_point_guard( + self: torch._C.Graph, insert_point: Union[torch._C.Node, torch._C.Block] +) -> _InsertPoint: + return _InsertPoint(self, insert_point) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_logging.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..7f82f79ca5eec9cec4da02574d397239cec5c9bc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_logging.py @@ -0,0 +1,11 @@ +import torch + + +add_stat_value = torch.ops.prim.AddStatValue + +set_logger = torch._C._logging_set_logger +LockingLogger = torch._C.LockingLogger +AggregationType = torch._C.AggregationType +NoopLogger = torch._C.NoopLogger + +time_point = torch.ops.prim.TimePoint diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_monkeytype_config.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_monkeytype_config.py new file mode 100644 index 0000000000000000000000000000000000000000..f4098daa523dc0b86ee5d15f1c8c42c927fb7071 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_monkeytype_config.py @@ -0,0 +1,196 @@ +# mypy: allow-untyped-defs +import inspect +import sys +import typing +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from types import CodeType +from typing import Optional + +import torch + + +_IS_MONKEYTYPE_INSTALLED = True +try: + import monkeytype # type: ignore[import] + + # pyrefly: ignore [import-error] + from monkeytype import trace as monkeytype_trace + from monkeytype.config import _startswith, LIB_PATHS # type: ignore[import] + from monkeytype.db.base import ( # type: ignore[import] + CallTraceStore, + CallTraceStoreLogger, + CallTraceThunk, + ) + from monkeytype.tracing import CallTrace, CodeFilter # type: ignore[import] +except ImportError: + _IS_MONKEYTYPE_INSTALLED = False + + +# Checks whether a class is defined in `torch.*` modules +def is_torch_native_class(cls): + if not hasattr(cls, "__module__"): + return False + + parent_modules = cls.__module__.split(".") + if not parent_modules: + return False + + root_module = sys.modules.get(parent_modules[0]) + return root_module is torch + + +def get_type(type): + """Convert the given type to a torchScript acceptable format.""" + if isinstance(type, str): + return type + elif inspect.getmodule(type) == typing: + # If the type is a type imported from typing + # like Tuple, List, Dict then replace `typing.` + # with a null string. This needs to be done since + # typing.List is not accepted by TorchScript. + type_to_string = str(type) + return type_to_string.replace(type.__module__ + ".", "") + elif is_torch_native_class(type): + # If the type is a subtype of torch module, then TorchScript expects a fully qualified name + # for the type which is obtained by combining the module name and type name. + return type.__module__ + "." + type.__name__ + else: + # For all other types use the name for the type. + return type.__name__ + + +def get_optional_of_element_type(types): + """Extract element type, return as `Optional[element type]` from consolidated types. + + Helper function to extracts the type of the element to be annotated to Optional + from the list of consolidated types and returns `Optional[element type]`. + TODO: To remove this check once Union support lands. + """ + elem_type = types[1] if type(None) is types[0] else types[0] + elem_type = get_type(elem_type) + + # Optional type is internally converted to Union[type, NoneType], which + # is not supported yet in TorchScript. Hence, representing the optional type as string. + return "Optional[" + elem_type + "]" + + +def get_qualified_name(func): + return func.__qualname__ + + +if _IS_MONKEYTYPE_INSTALLED: + + class JitTypeTraceStoreLogger(CallTraceStoreLogger): + """A JitTypeCallTraceLogger that stores logged traces in a CallTraceStore.""" + + def log(self, trace: CallTrace) -> None: + # pyrefly: ignore [missing-attribute] + self.traces.append(trace) + + class JitTypeTraceStore(CallTraceStore): + def __init__(self) -> None: + super().__init__() + # A dictionary keeping all collected CallTrace + # key is fully qualified name of called function + # value is list of all CallTrace + self.trace_records: dict[str, list] = defaultdict(list) + + def add(self, traces: Iterable[CallTrace]) -> None: + for t in traces: + qualified_name = get_qualified_name(t.func) + self.trace_records[qualified_name].append(t) + + def filter( + self, + qualified_name: str, + qualname_prefix: Optional[str] = None, + limit: int = 2000, + ) -> list[CallTraceThunk]: + return self.trace_records[qualified_name] + + def analyze(self, qualified_name: str) -> dict: + # Analyze the types for the given module + # and create a dictionary of all the types + # for arguments. + records = self.trace_records[qualified_name] + all_args = defaultdict(set) + for record in records: + for arg, arg_type in record.arg_types.items(): + all_args[arg].add(arg_type) + return all_args + + def consolidate_types(self, qualified_name: str) -> dict: + all_args = self.analyze(qualified_name) + # If there are more types for an argument, + # then consolidate the type to `Any` and replace the entry + # by type `Any`. + for arg, types in all_args.items(): + types = list(types) + type_length = len(types) + if type_length == 2 and type(None) in types: + # TODO: To remove this check once Union support in TorchScript lands. + all_args[arg] = get_optional_of_element_type(types) + elif type_length > 1: + all_args[arg] = "Any" + elif type_length == 1: + all_args[arg] = get_type(types[0]) + return all_args + + def get_args_types(self, qualified_name: str) -> dict: + return self.consolidate_types(qualified_name) + + class JitTypeTraceConfig(monkeytype.config.Config): + def __init__(self, s: JitTypeTraceStore) -> None: + super().__init__() + self.s = s + + def trace_logger(self) -> JitTypeTraceStoreLogger: + """Return a JitCallTraceStoreLogger that logs to the configured trace store.""" + # pyrefly: ignore [bad-argument-count] + return JitTypeTraceStoreLogger(self.trace_store()) + + def trace_store(self) -> CallTraceStore: + return self.s + + def code_filter(self) -> Optional[CodeFilter]: + return jit_code_filter + +else: + # When MonkeyType is not installed, we provide dummy class definitions + # for the below classes. + class JitTypeTraceStoreLogger: # type: ignore[no-redef] + def __init__(self) -> None: + pass + + class JitTypeTraceStore: # type: ignore[no-redef] + def __init__(self) -> None: + self.trace_records = None + + class JitTypeTraceConfig: # type: ignore[no-redef] + def __init__(self) -> None: + pass + + monkeytype_trace = None # type: ignore[assignment] # noqa: F811 + + +def jit_code_filter(code: CodeType) -> bool: + """Codefilter for Torchscript to trace forward calls. + + The custom CodeFilter is required while scripting a FX Traced forward calls. + FX Traced forward calls have `code.co_filename` start with '<' which is used + to exclude tracing of stdlib and site-packages in the default code filter. + Since we need all forward calls to be traced, this custom code filter + checks for code.co_name to be 'forward' and enables tracing for all such calls. + The code filter is similar to default code filter for monkeytype and + excludes tracing of stdlib and site-packages. + """ + # Filter code without a source file and exclude this check for 'forward' calls. + if code.co_name != "forward" and ( + not code.co_filename or code.co_filename[0] == "<" + ): + return False + + filename = Path(code.co_filename).resolve() + return not any(_startswith(filename, lib_path) for lib_path in LIB_PATHS) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_pickle.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..549c74c900dc5b6bc57e1b3ce24824deb1bcc9bb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_pickle.py @@ -0,0 +1,40 @@ +# These functions are referenced from the pickle archives produced by +# ScriptModule.save() + + +# These (`build_*`) functions used to be used by `pickler.cpp` to specify +# the type of the list for certain special types, but now all lists get +# a type attached and restored via `restore_type_tag` below. The legacy +# functions should stick around for backwards-compatibility. + +from typing import Union + + +def build_intlist(data: list[int]) -> list[int]: + return data + + +def build_tensorlist(data: list[object]) -> list[object]: + return data + + +def build_doublelist(data: list[float]) -> list[float]: + return data + + +def build_boollist(data: list[bool]) -> list[bool]: + return data + + +def build_tensor_from_id(data: Union[int, object]) -> Union[int, None]: + if isinstance(data, int): + # just the id, can't really do anything + return data + return None + + +def restore_type_tag(value: object, type_str: str) -> object: + # The type_ptr is used by the jit unpickler to restore the full static type + # to container types like list when they are re-loaded, but this doesn't + # matter for Python, so just return the plain value + return value diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_recursive.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_recursive.py new file mode 100644 index 0000000000000000000000000000000000000000..85bfed94a5382c526d7cbd3584b0eecd3963151c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_recursive.py @@ -0,0 +1,1062 @@ +# mypy: allow-untyped-defs +import collections +import functools +import inspect +import textwrap +import types +import warnings + +import torch +import torch._jit_internal as _jit_internal +from torch._sources import fake_range +from torch.jit._builtins import _find_builtin +from torch.jit._check import AttributeTypeIsSupportedChecker +from torch.jit._state import _add_script_class, _get_script_class, _python_cu +from torch.jit.frontend import ( + get_class_properties, + get_default_args, + get_jit_class_def, + get_jit_def, +) +from torch.nn import Module + + +ScriptMethodStub = collections.namedtuple( + "ScriptMethodStub", ("resolution_callback", "def_", "original_method") +) +PropertyStub = collections.namedtuple("PropertyStub", ("resolution_callback", "def_")) + + +# TODO: there should be a more principled way of doing this. +ignored_attributes = [ + "_version", + "_parameters", + "_buffers", + "_non_persistent_buffers_set", + "_backward_hooks", + "_backward_pre_hooks", + "_forward_hooks", + "_forward_hooks_with_kwargs", + "_forward_pre_hooks", + "_forward_pre_hooks_with_kwargs", + "_forward_hooks_always_called", + "_state_dict_hooks", + "_state_dict_pre_hooks", + "_load_state_dict_pre_hooks", + "_load_state_dict_post_hooks", + "_modules", + "_initializing", + "dump_patches", +] + + +def _compile_and_register_class(obj, rcb, qualified_name): + script_class = _get_script_class(obj) + + if not script_class: + ast = get_jit_class_def(obj, obj.__name__) + defaults = torch.jit.frontend.get_default_args_for_class(obj) + script_class = torch._C._jit_script_class_compile( + qualified_name, ast, defaults, rcb + ) + _add_script_class(obj, script_class) + + return script_class + + +def make_stub(func, name): + rcb = _jit_internal.createResolutionCallbackFromClosure(func) + ast = get_jit_def(func, name, self_name="RecursiveScriptModule") + return ScriptMethodStub(rcb, ast, func) + + +def make_stub_from_method(nn_module, method_name): + func = getattr(nn_module, method_name) + if isinstance(func, ScriptMethodStub): + return func + # Make sure the name present in the resulting AST will match the name + # requested here. The only time they don't match is if you do something + # like: + # def _forward(self): + # pass + # forward = _forward + # In this case, the actual function object will have the name `_forward`, + # even though we requested a stub for `forward`. + return make_stub(func, method_name) + + +def make_stubs_from_exported_methods(mod): + stubs = [] + for name in dir(mod): + item = getattr(mod, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.EXPORT + ): + stubs.append(make_stub_from_method(mod, name)) + + return stubs + + +def jit_ignored_properties(module): + user_annotated_ignored_attributes = getattr( + module, "__jit_ignored_attributes__", [] + ) + + def get_properties_names(module): + return {k for k, v in vars(module).items() if isinstance(v, property)} + + properties = get_properties_names(type(module)) + user_annoted_ignored_properties = set() + + for ignored_attr in user_annotated_ignored_attributes: + if ignored_attr in properties: + user_annoted_ignored_properties.add(ignored_attr) + return user_annoted_ignored_properties + + +# base types that can be constants +# in addition, tuples and lists of these base types are also considered constants +# If you edit this list, then you also need to edit the handlers in +# ConstantValue in jit/script/init.cpp +_constant_types = ( + bool, + float, + int, + str, + type(None), + torch.device, + torch.layout, + torch.dtype, + torch.qscheme, +) + + +def _get_valid_constant(attr, v, owner_type): + if isinstance(v, _constant_types): + return v + elif isinstance(v, (tuple, list)): + return tuple(_get_valid_constant(attr, x, owner_type) for x in v) + constants = ", ".join(torch.typename(typ) for typ in _constant_types) + raise TypeError( + textwrap.dedent( + f""" + '{torch.typename(type(v))}' object in attribute '{owner_type}.{attr}' is not a valid constant. + Valid constants are: + 1. a nn.ModuleList + 2. a value of type {{{constants}}} + 3. a list or tuple of (2) + """ + ) + ) + + +class SourceContext(torch._C._jit_tree_views.SourceRangeFactory): + pass + + +def get_annotations(obj): + # In Python-3.10+ it is recommended to use inspect.get_annotations + # See https://docs.python.org/3.10/howto/annotations.html + # But also, in 3.10 annotations from base class are not inherited + # by unannotated derived one, so they must be manually extracted + annotations = inspect.get_annotations(obj) + if annotations: + return annotations + + def get_cls_annotations(cls): + cls_annotations = inspect.get_annotations(cls) + if cls_annotations: + return cls_annotations + for base in cls.__bases__: + cls_annotations = get_cls_annotations(base) + if cls_annotations: + return cls_annotations + return {} + + cls = obj if isinstance(obj, type) else type(obj) + return get_cls_annotations(cls) + + +def infer_concrete_type_builder(nn_module, share_types=True): + """ + Build a ConcreteModuleTypeBuilder from an nn.Module. + + This ConcreteModuleType doesn't have a JIT type associated with it yet, it + must be filled in by the caller. + """ + concrete_type_builder = torch._C.ConcreteModuleTypeBuilder(type(nn_module)) + if isinstance(nn_module, (torch.nn.ModuleDict)): + concrete_type_builder.set_module_dict() + if isinstance(nn_module, (torch.nn.ModuleList, torch.nn.Sequential)): + concrete_type_builder.set_module_list() + if isinstance(nn_module, (torch.nn.ParameterList)): + concrete_type_builder.set_parameter_list() + if isinstance(nn_module, (torch.nn.ParameterDict)): + concrete_type_builder.set_parameter_dict() + + class_annotations = get_annotations(nn_module) + if isinstance(nn_module, (torch.ao.quantization.QuantWrapper)): + class_annotations = {} + + # Get user-annotated ignored attributes. + user_annotated_ignored_attributes = getattr( + nn_module, "__jit_ignored_attributes__", [] + ) + concrete_type_builder.add_ignored_attributes(user_annotated_ignored_attributes) + ignored_properties = jit_ignored_properties(nn_module) + + # try to infer the type from type annotation or from the object itself + def infer_type(name, item): + # The forward function from Module is special; never use this annotations; we + # need to infer type directly using JIT. I originally wanted to write + # this test as isinstance(class_annotations[name], Callable) but + # isinstance on typing things doesn't seem to work: isinstance(list, Callable) + # is also true! + inferred = False + try: + if ( + name in class_annotations + and class_annotations[name] + != torch.nn.Module.__annotations__["forward"] + ): + ann_to_type = torch.jit.annotations.ann_to_type( + class_annotations[name], fake_range() + ) + attr_type = torch._C.InferredType(ann_to_type) + elif isinstance(item, torch.jit.Attribute): + ann_to_type = torch.jit.annotations.ann_to_type(item.type, fake_range()) + attr_type = torch._C.InferredType(ann_to_type) + else: + attr_type = torch._C._jit_try_infer_type(item) + inferred = True + except RuntimeError as re: + raise RuntimeError(f"Error inferring type for {name}: {item}: {re}") from re + + return attr_type, inferred + + added_names = set() + + for name, item in nn_module._parameters.items(): + if name in user_annotated_ignored_attributes: + continue + + assert item is None or isinstance(item, torch.Tensor) + attr_type, _ = infer_type(name, item) + # We currently have the invariant in various places in our code + # that parameters must be Tensors. However, the nn.Module API also + # allows NoneType parameters. These parameters are not returned as + # part of `parameters()` and its variants, but are available + # through direct attribute access. + concrete_type_builder.add_attribute(name, attr_type.type(), True, False) + added_names.add(name) + + for name, item in nn_module._buffers.items(): + if name in user_annotated_ignored_attributes: + continue + + assert item is None or isinstance(item, torch.Tensor) + attr_type, _ = infer_type(name, item) + concrete_type_builder.add_attribute(name, attr_type.type(), False, True) + added_names.add(name) + + for name, item in nn_module._modules.items(): + if name in user_annotated_ignored_attributes: + continue + + attr_type, _ = infer_type(name, item) + if item is None: + # Modules can be None. We don't have direct support for optional + # Modules, so the register it as an NoneType attribute instead. + concrete_type_builder.add_attribute(name, attr_type.type(), False, False) + continue + if attr_type.success(): + assert attr_type.type().is_interface_type() + # if the type can be inferred, it should be a module interface type + sub_concrete_type = torch._C.ConcreteModuleType.from_jit_type( + attr_type.type() + ) + else: + # otherwise we get the concrete module type for item and add it to concrete_type + sub_concrete_type = get_module_concrete_type(item, share_types) + concrete_type_builder.add_module(name, sub_concrete_type) + + added_names.add(name) + + # populate constants_set + constants_set = set(getattr(nn_module, "__constants__", ())) + + # Constants annotated via `Final[T]` rather than being added to `__constants__` + for name, ann in class_annotations.items(): + if torch._jit_internal.is_final(ann): + constants_set.add(name) + + for name in constants_set: + if name in added_names: + # TODO: We should really error in this case, but its bc-breaking so + # we need to warn for at least one release + if name in nn_module._modules: + hint = "submodule" + elif name in nn_module._buffers: + hint = "buffer" + elif name in nn_module._parameters: + hint = "parameter" + else: + raise AssertionError( + "added_names must be submodule, parameter, or buffer" + ) + + warnings.warn( + f"'{name}' was found in ScriptModule constants, " + f" but it is a non-constant {hint}. Consider removing it.", + stacklevel=2, + ) + continue + if not hasattr(nn_module, name): + # TODO: We should really error in this case, but its bc-breaking so + # we need to warn for at least one release + warnings.warn( + f"'{name}' was found in ScriptModule constants, " + "but was not actually set in __init__. " + "Consider removing it.", + stacklevel=2, + ) + continue + value = getattr(nn_module, name) + concrete_type_builder.add_constant( + name, _get_valid_constant(name, value, type(nn_module).__name__) + ) + added_names.add(name) + + # populate overloads + overloads = getattr(nn_module, "__overloads__", {}) + # update with any annotated overloads + overloads.update( + get_overload_name_mapping( + get_overload_annotations(nn_module, ignored_properties) + ) + ) + for name, overloaded_names in overloads.items(): + concrete_type_builder.add_overload(name, overloaded_names) + + for name, value in nn_module.__dict__.items(): + if name in ignored_attributes or name.startswith("__"): + # Python objects have lots of random attributes attached to them; + # PyTorch adds a few more. Prevent these from getting compiled. + continue + + if name in user_annotated_ignored_attributes: + continue + + if name in added_names: + # Don't re-add anything we already added + continue + + isoverloadpacket = isinstance(value, torch._ops.OpOverloadPacket) + if isoverloadpacket: + value = value.op + # Handle Python function attributes + if inspect.isfunction(value): + try: + scripted_fn = torch.jit.script(value) + concrete_type_builder.add_function_attribute( + name, torch._C._jit_try_infer_type(scripted_fn).type(), value + ) + except Exception as e: + # If we fail to script the function, it isn't a hard error. + # Instead, we will add it to the list of attributes we failed + # to convert, with the compilation error. + hint = ( + "(This function exists as an attribute on the Python module, " + "but we failed to compile it to a TorchScript function. " + f"\nThe error stack is reproduced here:\n{e})" + ) + concrete_type_builder.add_failed_attribute(name, hint) + + continue + + # Handle calls to builtin functions (either bespoke builtins from torch.jit._builtins or + # a call to an aten function like torch.add) + builtin_symbol_name = _find_builtin(value) + if builtin_symbol_name: + concrete_type_builder.add_builtin_function(name, builtin_symbol_name) + continue + + # Handle Script function attributes + if isinstance(value, torch.jit.ScriptFunction): + concrete_type_builder.add_function_attribute( + name, torch._C._jit_try_infer_type(value).type(), value + ) + continue + + # If we got here, this is a regular "data" attribute, add it to the concrete type + attr_type, inferred = infer_type(name, value) + if attr_type.success(): + concrete_type_builder.add_attribute(name, attr_type.type(), False, False) + else: + # TODO: could add more detail here. For example, what the user should do + # when the pytype is `list` or `NoneType` + inferred_msg = ( + "Its type was inferred; try adding a type annotation for the attribute." + if inferred + else "" + ) + additional_info = f"{attr_type.reason()}. {inferred_msg}" + hint = ( + "(This attribute exists on the Python module, " + f"but we failed to convert Python type: '{torch.typename(type(value))}' " + f"to a TorchScript type. {additional_info})" + ) + concrete_type_builder.add_failed_attribute(name, hint) + + # add hooks to concrete type + for hook in nn_module._forward_hooks.values(): + concrete_type_builder.add_forward_hook(hook) + for pre_hook in nn_module._forward_pre_hooks.values(): + concrete_type_builder.add_forward_pre_hook(pre_hook) + + return concrete_type_builder + + +class ConcreteTypeStore: + type_store: dict[type[Module], list[torch._C.ConcreteModuleType]] + methods_compiled: set[torch._C.ConcreteModuleType] + + def __init__(self) -> None: + # Python module type => List[ConcreteModuleType)] + self.type_store = {} + # ConcreteTypes that have had their methods already compiled + self.methods_compiled = set() + + def get_or_create_concrete_type(self, nn_module): + """Infer a ConcreteType from this `nn.Module` instance. Underlying JIT types are reused if possible.""" + concrete_type_builder = infer_concrete_type_builder(nn_module) + + nn_module_type = type(nn_module) + if nn_module_type not in self.type_store: + self.type_store[nn_module_type] = [] + + # Search the type store for an already-available JIT type + known_types = self.type_store[nn_module_type] + for known_type in known_types: + if known_type.equals(concrete_type_builder): + return known_type + + # We didn't find anything; generate a new JIT type from this concrete type + concrete_type = concrete_type_builder.build() + self.type_store[nn_module_type].append(concrete_type) + return concrete_type + + +concrete_type_store = ConcreteTypeStore() + + +def create_methods_and_properties_from_stubs( + concrete_type, method_stubs, property_stubs +) -> None: + method_defs = [m.def_ for m in method_stubs] + method_rcbs = [m.resolution_callback for m in method_stubs] + method_defaults = [get_default_args(m.original_method) for m in method_stubs] + + property_defs = [p.def_ for p in property_stubs] + property_rcbs = [p.resolution_callback for p in property_stubs] + + concrete_type._create_methods_and_properties( + property_defs, property_rcbs, method_defs, method_rcbs, method_defaults + ) + + +def create_hooks_from_stubs(concrete_type, hook_stubs, pre_hook_stubs) -> None: + hook_defs = [h.def_ for h in hook_stubs] + hook_rcbs = [h.resolution_callback for h in hook_stubs] + + pre_hook_defs = [h.def_ for h in pre_hook_stubs] + pre_hook_rcbs = [h.resolution_callback for h in pre_hook_stubs] + + concrete_type._create_hooks(hook_defs, hook_rcbs, pre_hook_defs, pre_hook_rcbs) + + +def get_module_concrete_type(nn_module, share_types=True): + """ + Get a concrete type for nn_modules. + + If share_types is True, the concrete type is fetched from concrete_type_store. + If it is False, a new concrete type is created without first searching concrete_type_store. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + share_types = Whether to share underlying JIT types between modules (if possible). + + Returns: + A concrete type for nn_module. + """ + assert isinstance(nn_module, Module) + if isinstance(nn_module, torch.jit.ScriptModule) and hasattr( + nn_module, "_concrete_type" + ): + return nn_module._concrete_type + + if share_types: + # Look into the store of cached JIT types + concrete_type = concrete_type_store.get_or_create_concrete_type(nn_module) + else: + # Get a concrete type directly, without trying to reuse an existing JIT + # type from the type store. + concrete_type_builder = infer_concrete_type_builder(nn_module, share_types) + concrete_type_builder.set_poisoned() + concrete_type = concrete_type_builder.build() + + return concrete_type + + +def create_script_class(obj): + """ + Create and return a RecursiveScriptClass instance from a Python object. + + Arguments: + obj: A Python object. + """ + qualified_class_name = _jit_internal._qualified_name(type(obj)) + rcb = _jit_internal.createResolutionCallbackForClassMethods(type(obj)) + # Script the type of obj if it hasn't already been scripted. + _compile_and_register_class(type(obj), rcb, qualified_class_name) + class_ty = _python_cu.get_class(qualified_class_name) + # Create an empty torch._C.ScriptObject with the scripted type. + cpp_object = torch._C._create_object_with_type(class_ty) + # Copy all of the attributes over to the torch._C.ScriptObject. + for name, value in obj.__dict__.items(): + cpp_object.setattr(name, value) + + # Wrap the torch._C.ScriptObject in a RecursiveScriptClass instance. + return wrap_cpp_class(cpp_object) + + +def create_script_module(nn_module, stubs_fn, share_types=True, is_tracing=False): + """ + Create a new ScriptModule from an nn.Module. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + stubs_fn: Lambda that takes an nn.Module and generates a list of ScriptMethodStubs to compile. + share_types: Whether to share underlying JIT types between modules (if possible). + NOTE: Only set to False this when we cannot guarantee type sharing will work + correctly. This only happens today for traced modules, where the same + module can produce different traced methods depending on the inputs. + is_tracing: Whether this function is called during tracing or scripting. If tracing, + we don't need to do AttributeTypeIsSupportedChecker because all the unsupported + attributes will be baked as constant in the tracing graph. In addition, + this check significantly slows down the traced modules when the module size is big. + """ + assert not isinstance(nn_module, torch.jit.RecursiveScriptModule) + check_module_initialized(nn_module) + concrete_type = get_module_concrete_type(nn_module, share_types) + if not is_tracing: + AttributeTypeIsSupportedChecker().check(nn_module) + return create_script_module_impl(nn_module, concrete_type, stubs_fn) + + +def create_script_module_impl(nn_module, concrete_type, stubs_fn): + """ + Convert an nn.Module to a RecursiveScriptModule. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + concrete_type: The fully initialized ConcreteType of the module. + stubs_fn: Lambda that takes an nn.Module and generates a list of ScriptMethodStubs to compile. + """ + cpp_module = torch._C._create_module_with_type(concrete_type.jit_type) + method_stubs = stubs_fn(nn_module) + property_stubs = get_property_stubs(nn_module) + hook_stubs, pre_hook_stubs = get_hook_stubs(nn_module) + ignored_properties = jit_ignored_properties(nn_module) + + def init_fn(script_module) -> None: + # Initialize the ScriptModule: + # 1. Copy the attributes/parameters/buffers from the original `nn_module` to the new ScriptModule. + for name in concrete_type.get_attributes(): + orig_value = getattr(nn_module, name) + orig_value = ( + orig_value.value + if isinstance(orig_value, torch.jit.Attribute) + else orig_value + ) + cpp_module.setattr(name, orig_value) + + # 2. Copy the submodules from the original `nn_module` to the new ScriptModule, + # recursively scripting them. + for name, sub_concrete_type in concrete_type.get_modules(): + orig_value = getattr(nn_module, name) + assert isinstance(orig_value, Module), ( + f"Expected Module but got {type(orig_value)}" + ) + module_type = sub_concrete_type.jit_type + if isinstance(module_type, torch._C.InterfaceType): + # use the interface inference rule to compile the module + scripted = interface_script(module_type, orig_value) + elif isinstance(orig_value, torch.jit.ScriptModule): + scripted = orig_value + else: + # always reuse the provided stubs_fn to infer the methods to compile + scripted = create_script_module_impl( + orig_value, sub_concrete_type, stubs_fn + ) + + cpp_module.setattr(name, scripted) + script_module._modules[name] = scripted + + # 3. Copy @ignored/@unused methods and attrs from the original `nn_module` to the new ScriptModule. + # This ensures we can access these Python methods on the ScriptModule. + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if inspect.ismethod(item) and _jit_internal.is_ignored_fn(item): + unbound_function = getattr(nn_module, name).__func__ + bound_method = unbound_function.__get__(script_module) + setattr(script_module, name, bound_method) + elif concrete_type.is_ignored_attribute(name): + setattr(script_module, name, item) + + # For convenience, attach the concrete type to the new ScriptModule + script_module._concrete_type = concrete_type + + # Actually create the ScriptModule, initializing it with the function we just defined + script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + + # Compile methods if necessary + if concrete_type not in concrete_type_store.methods_compiled: + create_methods_and_properties_from_stubs( + concrete_type, method_stubs, property_stubs + ) + # Create hooks after methods to ensure no name collisions between hooks and methods. + # If done before, hooks can overshadow methods that aren't exported. + create_hooks_from_stubs(concrete_type, hook_stubs, pre_hook_stubs) + torch._C._run_emit_module_hook(cpp_module) + concrete_type_store.methods_compiled.add(concrete_type) + + # Copy the forward hooks and pre-hooks to the new ScriptModule + # to allow the hooks to be run from eager as ScriptFunctions + for idx, fn in enumerate(script_module._c._get_forward_pre_hooks()): + script_module._forward_pre_hooks[idx] = fn + for idx, fn in enumerate(script_module._c._get_forward_hooks()): + script_module._forward_hooks[idx] = fn + + # Special handling so methods like __len__ work in script methods on classes derived from containers + if ( + isinstance( + nn_module, (torch.nn.ModuleList, torch.nn.Sequential, torch.nn.ModuleDict) + ) + and "__len__" not in cpp_module._method_names() + ): + script_module.define(f"def __len__(self):\n return {len(nn_module)}\n") + if ( + isinstance(nn_module, torch.nn.ModuleDict) + and "__contains__" not in cpp_module._method_names() + ): + if len(nn_module.keys()): + keys = repr(list(nn_module.keys())) + script_module.define( + f"def __contains__(self, key: str):\n return key in {keys}\n" + ) + else: + script_module.define("def __contains__(self, key: str):\n return False\n") + + # Make the compiled methods available to the Python ScriptModule class. + for method_stub in method_stubs: + if method_stub.original_method is None: + # define()'d methods don't have an Python original_method, so we + # don't need to do any Python re-wrapping stuff + continue + + name = method_stub.original_method.__name__ + if name != method_stub.def_.name().name: + # TODO: Why skip this? Because @torch.jit._overload_method will + # mangle the name of the function. + continue + script_method = cpp_module._get_method(name) + + # Wrap the original to propagate docstrings and such. + # TODO: we don't currently do this functions that are recursively + # compiled, we should. + wrapped_script_method = functools.wraps(method_stub.original_method)( + script_method + ) + + # Add the methods to the script_module directly. This ensures they will + # be found first when `name` is looked up (as opposed to the stubs or + # nn.Module.forward) + script_module.__dict__[name] = wrapped_script_method + + # Make module properties available on the Python ScriptModule class. + for property_stub in property_stubs: + property_name = property_stub.def_.name().name + fget = cpp_module._get_method(property_stub.def_.getter_name().name) + # Setter is optional, so it may not exist. + setter_name = property_stub.def_.setter_name() + fset = cpp_module._get_method(setter_name.name) if setter_name else None + script_module.__dict__[property_name] = property(property_name, fget, fset) # type: ignore[arg-type] + + # copy over python methods to script module if they aren't defined on the script module + # this is currently an internal api used only on module containers + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.COPY_TO_SCRIPT_WRAPPER + ): + add_python_attr_to_scripted_model(script_module, nn_module, name) + + return script_module + + +# We define shims of certain attributes on the RecursiveScriptModule to support +# magic methods. To check if a script model defines an attribute we need +# to also check that the attribute is not the shim +def script_model_defines_attr(script_model, attr): + script_attr = getattr(script_model, attr, None) + if script_attr is None: + return False + default_attr = getattr(torch.jit.RecursiveScriptModule, attr, None) + if default_attr is None: + return False + return script_attr != default_attr + + +def add_python_attr_to_scripted_model(script_model, orig, attr) -> None: + if hasattr(orig, attr) and script_model_defines_attr(script_model, attr): + setattr(script_model, attr, getattr(orig, attr)) + + +def get_overload_annotations(mod, jit_ignored_properties): + # original function => [(mangled overload name, overload function)] + overloads = {} + + for name in dir(type(mod)): + if name in jit_ignored_properties: + continue + item = getattr(mod, name, None) + if not callable(item): + continue + + # builtin functions like repr() in python 2 do not have __module__ defined + if hasattr(item, "__module__") and item.__module__ is not None: + method_overloads = _jit_internal._get_overloaded_methods( + item, mod.__class__ + ) + if method_overloads is None: + continue + + # pyrefly: ignore [missing-attribute] + if item.__func__ in method_overloads: + raise RuntimeError( + _jit_internal.get_overload_no_implementation_error_message( + "method", item.__func__ + ) + ) + + names = [name + "__" + str(i) for i in range(len(method_overloads))] + overloads[item] = list(zip(names, method_overloads)) + + return overloads + + +def get_overload_name_mapping(overload_info): + # Same format as __overloads__ + # original function => [overload names] + overload_name_mappings: dict[str, list[str]] = {} + for orig_fn, overloads in overload_info.items(): + original_name = orig_fn.__name__ + if original_name not in overload_name_mappings: + overload_name_mappings[original_name] = [] + + for overload_name, _ in overloads: + overload_name_mappings[original_name].append(overload_name) + return overload_name_mappings + + +def _check_no_signature(func) -> None: + signature = torch.jit.annotations.get_signature( + func, None, fake_range(), inspect.ismethod(func) + ) + if signature is None: + qual_name = _jit_internal._qualified_name(func) + raise RuntimeError( + f"Must explicitly add type annotations to overloaded functions: {qual_name}" + ) + + +def make_stubs_for_overloads(overload_info): + overload_stubs = [] + for orig_fn, overloads in overload_info.items(): + orig_ast = get_jit_def( + orig_fn, orig_fn.__name__, self_name="RecursiveScriptModule" + ) + for overload_name, overload_fn in overloads: + _check_no_signature(overload_fn) + over_ast = get_jit_def( + overload_fn, overload_fn.__name__, self_name="RecursiveScriptModule" + ) + new_ast = torch._C._replace_overloaded_method_decl( + over_ast.decl(), orig_ast, overload_name + ) + _rcb = _jit_internal.createResolutionCallbackFromClosure(orig_fn) + overload_stubs.append(ScriptMethodStub(_rcb, new_ast, overload_fn)) + return overload_stubs + + +def check_module_initialized(mod) -> None: + assert isinstance(mod, torch.nn.Module) + if not hasattr(mod, "_parameters"): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has not been initialized, did you forget to call 'super()'?" + ) + + # This is to avoid importing torch.distributed.nn + if not hasattr(mod, "remote_parameters"): + for name, param in mod._parameters.items(): + if param is not None and torch.nn.parameter.is_lazy(param): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has uninitialized parameters {name}. Did you forget to run a forward pass?" + ) + for name, buf in mod._buffers.items(): + if buf is not None and torch.nn.parameter.is_lazy(buf): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has uninitialized buffers {name}. Did you forget to run a forward pass?" + ) + + +def infer_methods_to_compile(nn_module): + """Implement the default rules for which methods should act as starting points for compilation. + + (TODO add a link when the rules are published). + """ + check_module_initialized(nn_module) + ignored_properties = jit_ignored_properties(nn_module) + + methods: list[str] = [] + if hasattr(nn_module, "forward") and not _jit_internal.is_ignored_fn( + nn_module.forward + ): + forward_func = getattr(nn_module.forward, "__func__", None) + module_forward = getattr(torch.nn.Module, "forward", None) + if forward_func != module_forward: + methods = ["forward"] + + exported = [] + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.EXPORT + ): + exported.append(name) + + methods = methods + exported + + overload_name_mappings = dict(getattr(nn_module, "__overloads__", {})) + overload_info = get_overload_annotations(nn_module, ignored_properties) + overload_name_mappings.update(get_overload_name_mapping(overload_info)) + overload_stubs = make_stubs_for_overloads(overload_info) + + nn_module.__overloads__ = overload_name_mappings + + # we shouldn't directly compile overloaded methods, just its overloads + def ignore_overloaded(method_name): + return method_name not in overload_name_mappings + + filtered_methods = filter(ignore_overloaded, methods) + + # Unique the methods. We don't want to use a set to store the methods because it + # introduces non-determinism to compile order. + uniquer: set[str] = set() + uniqued_methods = [] + for name in filtered_methods: + if name in uniquer: + continue + uniqued_methods.append(name) + uniquer.add(name) + + stubs = [make_stub_from_method(nn_module, method) for method in uniqued_methods] + return overload_stubs + stubs + + +def get_hook_stubs(nn_module): + """Return forward hook and pre_hook ScriptModuleStubs.""" + check_module_initialized(nn_module) + hook_map: dict = {} + + hook_stubs = [] + for hook in nn_module._forward_hooks.values(): + if hook.__name__ in hook_map: + if id(hook) != id(hook_map[hook.__name__]): + raise RuntimeError( + f"Hook '{hook.__name__}' on {type(nn_module).__name__} " + "has at least two different python definitions." + " Please use unique names for all hooks." + ) + else: + hook_map[hook.__name__] = hook + hook_stubs.append(make_stub(hook, hook.__name__)) + + pre_hook_stubs = [] + for pre_hook in nn_module._forward_pre_hooks.values(): + if pre_hook.__name__ in hook_map: + if id(pre_hook) != id(hook_map[pre_hook.__name__]): + raise RuntimeError( + f"Pre-hook '{pre_hook.__name__}' on {type(nn_module).__name__} " + "has at least two different python definitions." + " Please use unique names for all hooks." + ) + else: + hook_map[pre_hook.__name__] = pre_hook + pre_hook_stubs.append(make_stub(pre_hook, pre_hook.__name__)) + + return hook_stubs, pre_hook_stubs + + +def get_property_stubs(nn_module): + """Create property stubs for the properties of the module by creating method stubs for the getter and setter.""" + module_ty = type(nn_module) + properties_asts = get_class_properties(module_ty, self_name="RecursiveScriptModule") + rcbs = {} + + for name in dir(module_ty): + item = getattr(module_ty, name, None) + if isinstance(item, property): + if not item.fget: + raise RuntimeError( + f"Property {name} of {nn_module.__name__} must have a getter" + ) + + rcbs[name] = _jit_internal.createResolutionCallbackFromClosure(item.fget) + + stubs = [PropertyStub(rcbs[ast.name().name], ast) for ast in properties_asts] + return stubs + + +def interface_script(mod_interface, nn_module): + """ + Make a ScriptModule from an nn.Module, using the interface methods rule for determining which methods to compile. + + Args: + mod_interface: the interface type that the module have + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + """ + if isinstance(nn_module, torch.jit.ScriptModule): + return nn_module + + check_module_initialized(nn_module) + + def infer_interface_methods_to_compile(nn_module): + """Rule to infer the methods from the interface type. + + It is used to know which methods need to act as starting points for compilation. + """ + stubs = [ + make_stub_from_method(nn_module, method) + for method in mod_interface.getMethodNames() + ] + return stubs + + return create_script_module(nn_module, infer_interface_methods_to_compile) + + +def try_compile_fn(fn, loc): + if _jit_internal.is_ignored_fn(fn): + # Don't do anything for @ignore'd functions + return None + + if isinstance(fn, torch.nn.Module): + # Since modules are callable pybind recognizes them as functions, but + # don't do anything for them + return None + + if not inspect.isfunction(fn) and not inspect.ismethod(fn): + raise RuntimeError( + f"`{fn}` is not a function. Recursive scripting only supports " + "Python functions or methods currently.\n" + f"Consider manually annotating `{fn}` with @torch.jit.script." + ) + + # The object returned by __prepare_scriptable__ might have a different closure. + # Resolve it here to get the right resolution callback. + fn = fn.__prepare_scriptable__() if hasattr(fn, "__prepare_scriptable__") else fn # type: ignore[operator] + + # We don't have the actual scope where the function was defined, but we can + # extract the necessary info from the closed over variables on the function + # object + rcb = _jit_internal.createResolutionCallbackFromClosure(fn) + return torch.jit.script(fn, _rcb=rcb) + + +def wrap_cpp_class(cpp_class): + """Wrap this torch._C.Object in a Python RecursiveScriptClass.""" + return torch.jit.RecursiveScriptClass(cpp_class) + + +def wrap_cpp_module(cpp_module): + """Wrap this torch._C.ScriptModule in a Python ScriptModule, recursively for all submodules.""" + + def init_fn(script_module) -> None: + for name, cpp_module in torch._C.ModuleDict(script_module._c).items(): + setattr(script_module, name, wrap_cpp_module(cpp_module)) + script_module._concrete_type = torch._C.ConcreteModuleType.from_jit_type( + script_module._c._type() + ) + + for idx, fn in enumerate(script_module._c._get_forward_pre_hooks()): + script_module._forward_pre_hooks[idx] = fn + for idx, fn in enumerate(script_module._c._get_forward_hooks()): + script_module._forward_hooks[idx] = fn + + return torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + + +def compile_unbound_method(concrete_type, fn): + if _jit_internal.is_ignored_fn(fn): + return None + stub = make_stub(fn, fn.__name__) + with torch._jit_internal._disable_emit_hooks(): + # We don't want to call the hooks here since the graph that is calling + # this function is not yet complete + create_methods_and_properties_from_stubs(concrete_type, (stub,), ()) + return stub + + +def lazy_bind(concrete_type, unbound_method): + """ + Return a function that lazily binds `unbound_method` to a provided Module IValue, then invokes the method. + + We do this so that any Python shenanigans that + will poison type sharing are impossible at compile time. + """ + + def lazy_binding_method(cpp_module, *args): + def init_fn(script_module) -> None: + orig_class = concrete_type.py_class + + # Copy @ignored/@unused methods from the original module to the new one. + # This ensures they are available during execution. + for name in dir(orig_class): + item = getattr(orig_class, name, None) + if _jit_internal.is_ignored_fn(item): + setattr(script_module, name, item) + + # Copy constants over so they are available during execution. + for name, value in concrete_type.get_constants().items(): + setattr(script_module, name, value) + + script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + method = types.MethodType(unbound_method, script_module) + return method(*args) + + # make the lazy binding method "look like" the original method + lazy_binding_method.original_fn = unbound_method # type: ignore[attr-defined] + lazy_binding_method.__name__ = unbound_method.__name__ + torch._jit_internal.copy_torchscript_modifier(unbound_method, lazy_binding_method) + + return lazy_binding_method diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.py new file mode 100644 index 0000000000000000000000000000000000000000..3870d43285a9c4990b7604dd67664b06dc96c03d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.py @@ -0,0 +1,1787 @@ +"""TorchScript. + +This module contains functionality to support the JIT's scripting frontend, notably: + - torch.jit.script + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import collections +import copy +import enum +import functools +import inspect +import pickle +import sys +import warnings +from collections.abc import Callable +from typing import Any, Union +from typing_extensions import deprecated + +import torch +import torch._jit_internal as _jit_internal +from torch._classes import classes +from torch._jit_internal import _get_model_id, _qualified_name +from torch._utils_internal import log_torchscript_usage +from torch.jit._builtins import _register_builtin +from torch.jit._fuser import _graph_for, _script_method_graph_for +from torch.jit._monkeytype_config import ( + JitTypeTraceConfig, + JitTypeTraceStore, + monkeytype_trace, +) +from torch.jit._recursive import ( + _compile_and_register_class, + infer_methods_to_compile, + ScriptMethodStub, + wrap_cpp_module, +) +from torch.jit._state import ( + _enabled, + _set_jit_function_cache, + _set_jit_overload_cache, + _try_get_jit_cached_function, + _try_get_jit_cached_overloads, +) +from torch.jit.frontend import get_default_args, get_jit_class_def, get_jit_def +from torch.nn import Module +from torch.overrides import ( + has_torch_function, + has_torch_function_unary, + has_torch_function_variadic, +) +from torch.package import PackageExporter, PackageImporter +from torch.utils import set_module + +from ._serialization import validate_map_location + + +type_trace_db = JitTypeTraceStore() # DB to hold all call traces from MonkeyType + +torch._C.ScriptMethod.graph_for = _script_method_graph_for # type: ignore[attr-defined] +torch._C.ScriptFunction.graph_for = _graph_for # type: ignore[attr-defined] +ScriptFunction = torch._C.ScriptFunction +ScriptFunction.__doc__ = """ +Functionally equivalent to a :class:`ScriptModule`, but represents a single +function and does not have any attributes or Parameters. +""" +ScriptFunction.__name__ = "ScriptFunction" +ScriptFunction.__qualname__ = "torch.jit.ScriptFunction" +set_module(ScriptFunction, "torch.jit") + + +# Throws an error if a jit function is pickled. +# Helps to avoid Python crashes for Python versions 3.9.5 + when protocol 0 or 1 is given as an argument. +def _reduce(cls): + raise pickle.PickleError("ScriptFunction cannot be pickled") + + +ScriptFunction.__reduce__ = _reduce # type: ignore[assignment] + + +if _enabled: + Attribute = collections.namedtuple("Attribute", ["value", "type"]) +else: + + def Attribute(value, type): # type: ignore[no-redef] + return value + + +Attribute.__doc__ = """ + This method is a pass-through function that returns `value`, mostly + used to indicate to the TorchScript compiler that the left-hand side + expression is a class instance attribute with type of `type`. Note that + `torch.jit.Attribute` should only be used in `__init__` method of `jit.ScriptModule` + subclasses. + + Though TorchScript can infer correct type for most Python expressions, there are some cases where + type inference can be wrong, including: + + - Empty containers like `[]` and `{}`, which TorchScript assumes to be container of `Tensor` + - Optional types like `Optional[T]` but assigned a valid value of type `T`, TorchScript would assume + it is type `T` rather than `Optional[T]` + + In eager mode, it is simply a pass-through function that returns `value` + without other implications. + + Example: + + .. testcode:: + + import torch + from typing import Dict + + class AttributeModule(torch.jit.ScriptModule): + def __init__(self) -> None: + super().__init__() + self.foo = torch.jit.Attribute(0.1, float) + + # we should be able to use self.foo as a float here + assert 0.0 < self.foo + + self.names_ages = torch.jit.Attribute({}, Dict[str, int]) + self.names_ages["someone"] = 20 + assert isinstance(self.names_ages["someone"], int) + + m = AttributeModule() + # m will contain two attributes + # 1. foo of type float + # 2. names_ages of type Dict[str, int] + + .. testcleanup:: + + del AttributeModule + del m + + Note: it's now preferred to instead use type annotations instead of `torch.jit.Attribute`: + + .. testcode:: + + import torch + from typing import Dict + + class AttributeModule(torch.nn.Module): + names: Dict[str, int] + + def __init__(self) -> None: + super().__init__() + self.names = {} + + m = AttributeModule() + + .. testcleanup:: + + del AttributeModule + del m + + Args: + value: An initial value to be assigned to attribute. + type: A Python type + + Returns: + Returns `value` +""" + + +def _get_type_trace_db(): + # This is a private API. Use of this for external purposes is discouraged. + return type_trace_db + + +# Gets a function from the name of a method on a type +def _get_function_from_type(cls, name): + return getattr(cls, name, None) + + +# ScriptClasses must be new-style classes because we construct them using their +# __new__ method. +def _is_new_style_class(cls): + if hasattr(cls, "__class__"): + return "__dict__" in dir(cls) or hasattr(cls, "__slots__") + + +# These OrderedDictWrapper classes replace the actual OrderedDicts in +# module with versions that get/set properties inside of Module. +# This allows us to reuse most of nn.Module while still storing the +# data in C++. +# Each OrderedDict needs to support: +# x not in view +# x in view +# view[name] = ... +# view.values() +# del view[name] +# view.items() +# view.keys() +# len(view) + + +class OrderedDictWrapper: + def __init__(self, _c): + self._c = _c + + def keys(self): + return [k for k, v in self.items()] + + def values(self): + return [v for k, v in self.items()] + + def __len__(self): + return len(self.values()) + + def __delitem__(self, k): + raise RuntimeError("cannot delete methods or parameters of a script module") + + def items(self): + return self._c.items() + + def __setitem__(self, k, v): + if k not in self: + raise RuntimeError( + f"Can't add a new parameter after ScriptModule construction. Tried to add '{k}" + ) + self._c.setattr(k, v) + + def __contains__(self, k): + return self._c.contains(k) + + def __getitem__(self, k): + if k not in self: + raise KeyError(k) + return self._c.getattr(k) + + +class OrderedModuleDict(OrderedDictWrapper): + def __init__(self, module, python_dict): + super().__init__(torch._C.ModuleDict(module)) + # contains _both_ script modules and non-script python-only modules + + # because script modules are subclassed in python and the + # C++ Module class will not hold references to them, + # to ensure that you always get the same python value here + # we store it in the python dict as well + self._python_modules = python_dict + + def items(self): + r = self._python_modules.items() + return r + + def __contains__(self, k): + return k in self._python_modules + + def __setitem__(self, k, v): + # Cases where sub-module can be re-assigned after ScriptModule construction + # 1. If the attr is an module interface type, it's guaranteed that the module is + # not inlined in the graph, so it's safe to swap a new ScriptModule in. + # 2. if the new value if a ScriptModule with the same JIT type, IR won't change + # and it's legit to swap a new module in. + # In these two cases we allow swapping a new scripted module and update the + # corresponding python module dict to keep sync. + # Note: the value to be swapped in has to be ScriptModule instead of nn.Module, + # otherwise it's illegal and we throw error. + if isinstance(v, ScriptModule): + self._c.setattr(k, v) + self._python_modules[k] = v + else: + raise RuntimeError( + "Cannot re-assign modules in a ScriptModule with non-scripted " + f"module, tried to replace existing module '{k}': {v}" + ) + + def __getitem__(self, k): + return self._python_modules[k] + + +# For each user-defined class that subclasses ScriptModule, this meta-class: +# (1) finds all the methods annotated with @script_method in a ScriptModule and +# removes them from the class attributes +# (2) puts a wrapper around the class's __init__ method to recursively compile +# all of the script_methods with the module after the original __init__ has +# run. This has to occur after the user-defined __init__ so that submodules and +# parameters are initialized _before_ the script compiler resolve references to +# `self.param` or `self.module`. +class ScriptMeta(type): + def __init__(cls, name, bases, attrs): # noqa: B902 + # Aggregate all the ScriptMethods and constants from superclasses + cls._methods: dict[str, Any] = {} + cls._constants_set = set(getattr(cls, "__constants__", ())) + for base in reversed(bases): + for k, v in getattr(base, "_methods", {}).items(): + cls._methods[k] = v + base_constants: set = getattr(base, "_constants_set", set()) + cls._constants_set = cls._constants_set.union(base_constants) + + # find all the script methods of the current class + for k, v in sorted(attrs.items()): + if isinstance(v, ScriptMethodStub): + delattr(cls, k) + cls._methods[v.original_method.__name__] = v + + if getattr(cls, "_disable_script_meta", False): + # We leave built-in ScriptModule types alone, since this metaclass + # is only for compiling user classes that inherit from + # ScriptModule. + super().__init__(name, bases, attrs) + return + + original_init = getattr(cls, "__init__", lambda self: None) + + @functools.wraps(original_init) + def init_then_script(self, *args, **kwargs): + num_methods = len(cls._methods) + original_init(self, *args, **kwargs) + added_methods_in_init = len(cls._methods) > num_methods + + if type(self) is cls: + + def make_stubs(module): + cls = type(module) + if hasattr(cls, "_methods"): + return [v for k, v in sorted(cls._methods.items())] + else: + return infer_methods_to_compile(module) + + self.__dict__["_actual_script_module"] = ( + torch.jit._recursive.create_script_module( + self, make_stubs, share_types=not added_methods_in_init + ) + ) + + # Delete the Python attributes that now shadow the ScriptModule + # ones, so that __getattr__ and __setattr__ will properly find + # the scripted versions. + concrete_type = self._actual_script_module._concrete_type + for name in concrete_type.get_attributes(): + delattr(self, name) + for name, _ in concrete_type.get_modules(): + delattr(self, name) + for name in ("_parameters", "_buffers", "_modules"): + delattr(self, name) + + cls.__init__ = init_then_script # type: ignore[misc] + super().__init__(name, bases, attrs) + + +class _CachedForward: + def __get__(self, obj, cls): + return self.__getattr__("forward") # type: ignore[attr-defined] + + +class ScriptWarning(Warning): + pass + + +def script_method(fn): + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.script_method` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return fn + # NOTE: we need to traverse two frames here because the meta-class frame + # for ScriptModule will be present, as opposed to invoking @script on a + # a function or invoking define() on a CompilationUnit. + # The stack will look like: + # + # 0. createResolutionCallback() + # 1. script_method() + # 2. ScriptModule metaclass frame + # 3. Surrounding scope + # + # createResolutionCallback internally adds 1 to get us to the scope of this + # function (the calling function). Adding 2 gets us to the proper surrounding scope. + _rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=2) + ast = get_jit_def(fn, fn.__name__, self_name="ScriptModule") + return ScriptMethodStub(_rcb, ast, fn) + + +class ConstMap: + def __init__(self, const_mapping): + self.const_mapping = const_mapping + + def __getattr__(self, attr): + return self.const_mapping[attr] + + +def unpackage_script_module( + importer: PackageImporter, script_module_id: str +) -> torch.nn.Module: + """ + Call by ``torch.package.PackageImporter``'s Pickler's ``persistent_load`` function. + + Performs work of loading and returning a ScriptModule from a ``torch.package`` archive. + """ + if not isinstance(importer.zip_reader, torch._C.PyTorchFileReader): + raise RuntimeError( + "Loading ScriptObjects from a PackageImporter created from a " + "directory is not supported. Use a package archive file instead." + ) + cu = torch._C.CompilationUnit() + cpp_module = torch._C._import_ir_module_from_package( + cu, + importer.zip_reader, + importer.storage_context, + validate_map_location(importer.last_map_location), + script_module_id, + ) + return wrap_cpp_module(cpp_module) + + +if _enabled: + _magic_methods = [ + "__iter__", + "__len__", + "__neg__", + "__mul__", + "__contains__", + "__add__", + "__sub__", + "__pow__", + "__truediv__", + "__mod__", + "__ne__", + "__eq__", + "__lt__", + "__gt__", + "__le__", + "__ge__", + "__and__", + "__or__", + "__xor__", + "__getitem__", + "__setitem__", + "__call__", + "__int__", + "__float__", + "__bool__", + "__str__", + "__enter__", + "__exit__", + ] + + class RecursiveScriptClass: + """Wrapper for a TorchScript class instance for use in Python. + + An analogue of RecursiveScriptModule for regular objects that are not modules. + This class is a wrapper around a torch._C.ScriptObject that represents an instance + of a TorchScript class and allows it to be used in Python. + + Attributes: + _c [torch._C.ScriptObject]: The C++ object to which attribute lookups and method + calls are forwarded. + _props [Dict[str, property]]: A dictionary of properties fetched from self._c and + exposed on this wrppaer. + """ + + def __init__(self, cpp_class): + super().__init__() + self.__dict__["_initializing"] = True + self._c = cpp_class + + # Add wrapped object's properties to this class instance. + self._props = { + prop.name: property(prop.getter, prop.setter) + for prop in self._c._properties() + } + + self.__dict__["_initializing"] = False + + def __getattr__(self, attr): + if self.__dict__.get("_initializing"): + return super().__getattr__(attr) # type: ignore[misc] + + if attr in self._props: + return self._props[attr].fget() # type: ignore[call-arg, misc] + + return getattr(self._c, attr) + + def __setattr__(self, attr, value): + if self.__dict__.get("_initializing"): + return super().__setattr__(attr, value) + + if attr in self._props: + return self._props[attr].fset(value) # type: ignore[call-arg, misc] + + setattr(self._c, attr, value) + + # Delegate calls to magic methods like __len__ to the C++ module backing the + # RecursiveScriptClass. + def forward_magic_method(self, method_name, *args, **kwargs): + if not self._c._has_method(method_name): + raise TypeError + + self_method = self.__getattr__(method_name) + return self_method(*args, **kwargs) + + def __getstate__(self): + raise pickle.PickleError("ScriptClasses cannot be pickled") + + def __iadd__(self, other): + if self._c._has_method("__iadd__"): + return self.forward_magic_method("__iadd__", other) + else: + return self.forward_magic_method("__add__", other) + + for method_name in _magic_methods: + + def method_template(self, *args, **kwargs): + return self.forward_magic_method(method_name, *args, **kwargs) + + setattr(RecursiveScriptClass, method_name, method_template) + + # this is a Python 'non-data descriptor' that causes the first access + # to ScriptModule's forward to look up the forward method and stash + # it in the objects dict. Due to the standard rules for attribute lookup, + # subsequent lookups will just directly return the previously looked up method. + # This is necessary because nn.Module defines forward as a method. If we + # did nothing, __getattr__ would not be called. Instead we'd get nn.Module.forward + # which always throws an exception. + + class ScriptModule(Module, metaclass=ScriptMeta): + r"""Wrapper for C++ torch::jit::Module with methods, attributes, and parameters. + + A wrapper around C++ ``torch::jit::Module``. ``ScriptModule``\s + contain methods, attributes, parameters, and + constants. These can be accessed the same way as on a normal ``nn.Module``. + """ + + __jit_unused_properties__ = [ + "code", + "code_with_constants", + "graph", + "inlined_graph", + "original_name", + ] + + def __init__(self) -> None: + super().__init__() + + forward: Callable[..., Any] = _CachedForward() # type: ignore[assignment] + + def __getattr__(self, attr): + if "_actual_script_module" not in self.__dict__: + return super().__getattr__(attr) + return getattr(self._actual_script_module, attr) + + def __setattr__(self, attr, value): + if "_actual_script_module" not in self.__dict__: + # Unwrap torch.jit.Attribute into a regular setattr + record + # the provided type in __annotations__. + # + # This ensures that if we use the attr again in `__init__`, it + # will look like the actual value, not an instance of Attribute. + # pyrefly: ignore [invalid-argument] + if isinstance(value, Attribute): + # NB: Ensure that we set __annotations__ on the specific + # class in question, and not on a superclass (which would + # be wrong wrong wrong!). + # See also https://github.com/pytorch/pytorch/issues/39463 + if "__annotations__" not in self.__class__.__dict__: + self.__class__.__annotations__ = {} + self.__annotations__[attr] = value.type + value = value.value + return super().__setattr__(attr, value) + + setattr(self._actual_script_module, attr, value) + + def define(self, src): + if "_actual_script_module" in self.__dict__: + # If we have completed initialization, just defer to the + # backing RecursiveScriptModule to eagerly compile the provided + # source. + return self._actual_script_module.define(src) + + # Otherwise, we are still in the object's __init__. + # In that case, add `src` as a stub to be compiled. + # + # We use frames_up=1 to get to the proper surrounding scope. The stack + # will look like: + # 0. createResolutionCallback + # 1. define() + # 2. surrounding scope. + # + # createResolutionCallback internally adds 1 to get us to our frame, then + # we add 1 to get to the proper surrounding scope. + rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=1) + ast = torch._C._parse_source_def(src) + self._methods[ast.name().name] = ScriptMethodStub(rcb, ast, None) + + def _replicate_for_data_parallel(self): + return self._actual_script_module._replicate_for_data_parallel() + + def __reduce_package__(self, exporter: PackageExporter): + """Save a ScriptModule inside of a ``torch.package`` archive. + + Called by ``torch.package.PackageExporter``'s Pickler's ``persistent_id`` when + saving TorchScript objects. Performs act of saving a ScriptModule inside of + a ``torch.package`` archive. + + Returns method to load the ScriptModule from a ``torch.package.PackageImporter``'s + Pickler's ``persistent_load`` function. + """ + script_module_id = exporter.get_unique_id() + exporter.script_module_serializer.serialize(self._c, int(script_module_id)) + return (unpackage_script_module, (script_module_id,)) + + class RecursiveScriptModule(ScriptModule): + # XXX: RecursiveScriptModule inherits from ScriptModule for the sole + # reason that it retains the existing isinstance(ScriptModule) + # behavior. + r"""Retain the existing isinstance(ScriptModule) behavior. + + The core data structure in TorchScript is the ``ScriptModule``. It is an + analogue of torch's ``nn.Module`` and represents an entire model as a tree of + submodules. Like normal modules, each individual module in a ``ScriptModule`` can + have submodules, parameters, and methods. In ``nn.Module``\s methods are implemented + as Python functions, but in ``ScriptModule``\s methods are implemented as + TorchScript functions, a statically-typed subset of Python that contains all + of PyTorch's built-in Tensor operations. This difference allows your + ``ScriptModule``\s code to run without the need for a Python interpreter. + + ``ScriptModule``\s should not be created manually, instead use + either :func:`tracing ` or :func:`scripting `. + Tracing and scripting can be applied incrementally and :ref:`composed as necessary `. + + * Tracing records the tensor operations as executed with a set of example inputs and uses these + operations to construct a computation graph. You can use the full dynamic behavior of Python with tracing, + but values other than Tensors and control flow aren't captured in the graph. + + * Scripting inspects the Python code of the model + and compiles it to TorchScript. Scripting allows the use of many `types`_ of values and supports dynamic control flow. + Many, but not all features of Python are supported by the compiler, so changes to the source code may be necessary. + """ + + _disable_script_meta = True + + def __init__(self, cpp_module): + self.__dict__["_initializing"] = True + self._c = cpp_module + super().__init__() + # Delete the 'training' attribute set up by `Module.__init__`. It + # will get set on the underlying cpp module, so we delete it here + # to avoid this version shadowing the cpp module version. + delattr(self, "training") + + @staticmethod + def _construct(cpp_module, init_fn): + """ + Construct a RecursiveScriptModule that's ready for use. + + PyTorch code should use this to construct a RecursiveScriptModule instead + of instead of calling `__init__` directly, as it makes sure the + object is properly finalized (and in the future, we may take + control of how the RecursiveScriptModule instance is created). + + Args: + cpp_module: The C++ Module that will hold the actual state of + this RecursiveScriptModule instance. + init_fn: Lambda that initializes the RecursiveScriptModule passed to it. + """ + script_module = RecursiveScriptModule(cpp_module) + init_fn(script_module) + + # Finalize the ScriptModule: replace the nn.Module state with our + # custom implementations and flip the _initializing bit. + # pyrefly: ignore [missing-attribute] + RecursiveScriptModule._finalize_scriptmodule(script_module) + return script_module + + @staticmethod + def _finalize_scriptmodule(script_module): + script_module._parameters = OrderedDictWrapper( + torch._C.ParameterDict(script_module._c) + ) + script_module._buffers = OrderedDictWrapper( + torch._C.BufferDict(script_module._c) + ) + script_module._modules = OrderedModuleDict( + script_module._c, script_module._modules + ) + script_module._initializing = False + + def _reconstruct(self, cpp_module): + """ + Re-construct an instance of RecursiveScriptModule using an instance of a C++ module. + + Args: + cpp_module: The C++ module that this RecursiveScriptModule will be rebuilt around. + """ + self.__init__(cpp_module) # type: ignore[misc] + + # Copy the concrete type from the C++ module to this ScriptModule. + self._concrete_type = torch._C.ConcreteModuleType.from_jit_type( + self._c._type() + ) + + # Copy submodules from the C++ module to this ScriptModule. + modules = {} + for name, cpp_module in torch._C.ModuleDict(self._c).items(): + modules[name] = wrap_cpp_module(cpp_module) + self._modules = OrderedModuleDict(self._c, modules) # type: ignore[assignment] + + # Copy parameters and buffers. + self._parameters = OrderedDictWrapper(torch._C.ParameterDict(self._c)) # type: ignore[assignment] + self._buffers = OrderedDictWrapper(torch._C.BufferDict(self._c)) # type: ignore[assignment] + + # Get rid of the functions from the old C++ module. + self.__dict__ = { + k: v + for k, v in self.__dict__.items() + if not isinstance(v, torch._C.ScriptMethod) + } + self.__dict__["_initializing"] = False + + @property + def graph(self): + r"""Return a string representation of the internal graph for the ``forward`` method.""" + return self._c._get_method("forward").graph + + @property + def inlined_graph(self): + r""" + Return a string representation of the internal graph for the ``forward`` method. + + This graph will be preprocessed to inline all function and method calls. + """ + return self.forward.inlined_graph # type: ignore[attr-defined] + + @property + def code(self): + r""" + Return a pretty-printed representation (as valid Python syntax) of the internal graph for the ``forward`` method. + + """ + return self.forward.code # type: ignore[attr-defined] + + @property + def code_with_constants(self): + r"""Return a tuple. + + Returns a tuple of: + + [0] a pretty-printed representation (as valid Python syntax) of + the internal graph for the ``forward`` method. See `code`. + [1] a ConstMap following the CONSTANT.cN format of the output in [0]. + The indices in the [0] output are keys to the underlying constant's values. + + """ + r = self.forward.code_with_constants # type: ignore[attr-defined] + return (r[0], ConstMap(r[1])) + + def save(self, f, **kwargs): + r"""Save with a file-like object. + + save(f, _extra_files={}) + + See :func:`torch.jit.save ` which accepts a file-like object. + This function, torch.save(), converts the object to a string, treating it as a path. + DO NOT confuse these two functions when it comes to the 'f' parameter functionality. + """ + return self._c.save(str(f), **kwargs) + + @deprecated( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html" + ) + def _save_for_lite_interpreter(self, *args, **kwargs): + r"""Add (or update) the bytecode session to the script model. + + _save_for_lite_interpreter(f) + + The updated model is used + in lite interpreter for mobile applications. + + Args: + f: a string containing a file name. + _extra_files: Map from filename to contents which will be stored as part of 'f'. + + """ + warnings.warn( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html", + DeprecationWarning, + stacklevel=2, + ) + return self._c._save_for_mobile(*args, **kwargs) + + @deprecated( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html" + ) + def _save_to_buffer_for_lite_interpreter(self, *args, **kwargs): + warnings.warn( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html", + DeprecationWarning, + stacklevel=2, + ) + return self._c._save_to_buffer_for_mobile(*args, **kwargs) + + def save_to_buffer(self, *args, **kwargs): + return self._c.save_to_buffer(*args, **kwargs) + + def get_debug_state(self, *args, **kwargs): + return self._c.get_debug_state() + + def extra_repr(self): + return f"original_name={self.original_name}" + + def graph_for(self, *args, **kwargs): + return self.forward.graph_for(self, *args, **kwargs) # type: ignore[attr-defined] + + @property + def original_name(self): + if type(self) is str(self._c._type().name()): + return "" + return str(self._c._type().name()) + + def define(self, src): + # We use frames_up=1 to get to the proper surrounding scope. The stack + # will look like: + # 0. createResolutionCallback + # 1. define() + # 2. surrounding scope. + # + # createResolutionCallback internally adds 1 to get us to our frame, then + # we add 1 to get to the proper surrounding scope. + rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=1) + self._c._define(self._concrete_type, src, rcb) + + def __getattr__(self, attr): + if "_initializing" not in self.__dict__: + raise RuntimeError( + "ScriptModule has not been initialized, did you forget to call super's init?" + ) + + if self._initializing: + return super().__getattr__(attr) + + # _modules check is before hasattr since modules are included as attributes in _c, + # but we want to get the python wrapper from _modules instead of the raw _c object. + if attr in self._modules: + return self._modules[attr] + elif self._c.hasattr(attr): + return self._c.getattr(attr) + elif self._c._has_method(attr): + script_method = self._c._get_method(attr) + # cache method so future calls do not go through __getattr__ + # to improve invocation performance + self.__dict__[attr] = script_method + return script_method + + return super().__getattr__(attr) + + def __setattr__(self, attr, value): + if self._initializing: + return super().__setattr__(attr, value) + + if attr in self._modules: + self._modules[attr] = value + elif self._c.hasattr(attr): + self._c.setattr(attr, value) + elif ( + hasattr(self, "_concrete_type") + and attr in self._concrete_type.get_constants() + ): + # TODO: we don't have _concrete_type set after load(), and in general we lose constant information. + # We should encode constants as class type attributes (or something) so it persists across save/load. + raise AttributeError( + f"Cannot mutate TorchScript constant value: '{attr}'. Value: '{value}'" + ) + else: + # We allow setting Python attributes on the ScriptModule, for + # when people want to stash some convenience info on it. + # TODO: it's possible that the following is confusing: + # s = torch.jit.script(...) + # s.python_attr = ... + # s.save() <--- this doesn't have `python_attr` + # It's fairly trivial to save enough info to warn in this case. + return super().__setattr__(attr, value) + + def __copy__(self): + return torch.jit._recursive.wrap_cpp_module(copy.copy(self._c)) + + def __deepcopy__(self, memo): + return torch.jit._recursive.wrap_cpp_module(copy.deepcopy(self._c, memo)) + + # Python magic methods do method lookups on an object's class type, instead of looking up + # the method defines on the class instance. In order to continue to expose the magic methods + # of builtin-containers (ModuleList, Sequential, ModuleDict) to Python, we + # define magic methods here as a shim to the correct attribute. + def forward_magic_method(self, method_name, *args, **kwargs): + self_method = getattr(self, method_name) + if getattr(self_method, "__func__", None) == getattr( + RecursiveScriptModule, method_name + ): + raise NotImplementedError + return self_method(*args, **kwargs) + + def __iter__(self): + return self.forward_magic_method("__iter__") + + def __getitem__(self, idx): + return self.forward_magic_method("__getitem__", idx) + + def __len__(self): + return self.forward_magic_method("__len__") + + def __contains__(self, key): + return self.forward_magic_method("__contains__", key) + + # dir is defined by the base nn.Module, so instead of throwing if + # it is not overridden, we call into the nn.Module __dir__ method + def __dir__(self): + self_method = self.__dir__ + if ( + self_method.__func__ # type: ignore[attr-defined] + is _get_function_from_type(RecursiveScriptModule, "__dir__") + ): + return super().__dir__() + return self_method() + + # to resolve bool(value), Python looks if __bool__ is defined then __iter__ + # is defined then returns true for classes. Since __iter__() on this + # class throws if it isn't overridden, we define __bool__ to preserve default behavior + def __bool__(self): + self_method = self.__bool__ + if ( + self_method.__func__ # type: ignore[attr-defined] + is _get_function_from_type(RecursiveScriptModule, "__bool__") + ): + return True + return self_method() + + def _replicate_for_data_parallel(self): + # we have to initialize ScriptModule properly so that + # it works with pybind11 + def init_fn(script_module): + # Don't do anything here, we'll initialize the ScriptModule below + return + + # pyrefly: ignore [missing-attribute] + return RecursiveScriptModule._construct( + self._c._replicate_for_data_parallel(), init_fn + ) + + # Need to copy all RecursiveScriptModule methods to ScriptModule. + # + # This is because `super().foo()` does not use + # `__getattr__` to look up `foo`. So we need to make each method available on + # the ScriptModule manually. + # pyrefly: ignore [missing-attribute] + for name, item in RecursiveScriptModule.__dict__.items(): + if not callable(item) and not isinstance(item, property): + continue + if name.startswith("__") or hasattr(ScriptModule, name): + continue + # We can copy over the implementation wholesale because besides the + # `super()` thing above, ScriptModule behaves exactly like + # RecursiveScriptModule + setattr(ScriptModule, name, item) + + def _get_methods(cls): + import inspect + + # In Python 3 unbound methods are functions, but in Python 2 they are methods + return inspect.getmembers( + cls, predicate=lambda x: inspect.isfunction(x) or inspect.ismethod(x) + ) + + _compiled_methods_allowlist = { + "forward", + "register_buffer", + "register_parameter", + "register_module", + "add_module", + "_apply", + "apply", + "cuda", + "cpu", + "to", + "type", + "float", + "double", + "half", + "state_dict", + "_save_to_state_dict", + "load_state_dict", + "_load_from_state_dict", + "_named_members", + "parameters", + "named_parameters", + "buffers", + "named_buffers", + "children", + "named_children", + "modules", + "named_modules", + "zero_grad", + "share_memory", + "_get_name", + "extra_repr", + "_slow_forward", + "_tracing_name", + "eval", + "train", + "get_extra_state", + "set_extra_state", + } + + def _make_fail(name): + def fail(self, *args, **kwargs): + raise RuntimeError(name + " is not supported on ScriptModules") + + return fail + + for name, method in _get_methods(torch.nn.Module): + if name.startswith("__") or name.endswith("_call_impl"): + continue + if ( + # pyrefly: ignore [missing-attribute] + name not in RecursiveScriptModule.__dict__ + and name not in _compiled_methods_allowlist + ): + setattr(RecursiveScriptModule, method.__name__, _make_fail(name)) + + +else: + # TODO MAKE SURE THAT DISABLING WORKS + class RecursiveScriptClass: # type: ignore[no-redef] + pass + + class ScriptModule(torch.nn.Module): # type: ignore[no-redef] + def __init__(self, arg=None): + super().__init__() + + class RecursiveScriptModule(ScriptModule): # type: ignore[no-redef] + def __init__(self, arg=None): + super().__init__() + + +def call_prepare_scriptable_func_impl(obj, memo): + if not isinstance(obj, torch.nn.Module): + return obj + + obj_id = id(obj) + + # If obj_id is in memo, obj has already been prepared or is being + # prepared in another call up the stack. + if obj_id in memo: + return memo[id(obj)] + + obj = ( + # pyrefly: ignore [not-callable] + obj.__prepare_scriptable__() if hasattr(obj, "__prepare_scriptable__") else obj + ) # type: ignore[operator] + # Record obj in memo to avoid infinite recursion in the case of cycles in the module + # hierarchy when recursing below. + memo[obj_id] = obj + + new_obj_dict = {} + + for name, sub_module in obj.__dict__.items(): + if name == "_modules": + for k, v in sub_module.items(): + sub_module[k] = call_prepare_scriptable_func_impl(v, memo) + new_obj_dict[name] = sub_module + elif isinstance(sub_module, torch.nn.Module) and not isinstance( + sub_module, ScriptModule + ): + new_obj_dict[name] = call_prepare_scriptable_func_impl(sub_module, memo) + else: + new_obj_dict[name] = sub_module + + for v in new_obj_dict.values(): + obj.__dict__[name] = v + + return obj + + +def call_prepare_scriptable_func(obj): + memo: dict[int, torch.nn.Module] = {} + return call_prepare_scriptable_func_impl(obj, memo) + + +def create_script_dict(obj): + """ + Create a ``torch._C.ScriptDict`` instance with the data from ``obj``. + + Args: + obj (dict): The Python dictionary that is used to initialize the ``ScriptDict`` + returned by this function. + + Returns: + An instance of ``torch._C.ScriptDict`` that has the same data as ``obj`` + and can be passed between Python and TorchScript with reference semantics and + zero copy overhead. + """ + return torch._C.ScriptDict(obj) # type: ignore[attr-defined] + + +def create_script_list(obj, type_hint=None): + """ + Create a ``torch._C.ScriptList`` instance with the data from ``obj``. + + Args: + obj (dict): The Python list that is used to initialize the ``ScriptList`` + returned by this function. + Returns: + An instance of ``torch._C.ScriptList`` that has the same data as ``obj`` + and can be passed between Python and TorchScript with reference semantics and + zero copy overhead. + """ + return torch._C.ScriptList(obj) # type: ignore[attr-defined] + + +_TOPLEVEL: bool = True + + +def _script_impl( + obj, + optimize=None, + _frames_up=0, + _rcb=None, + example_inputs: Union[list[tuple], dict[Callable, list[tuple]], None] = None, +): + global type_trace_db + + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=3, + ) + + # No-op for modules, functions, class instances that are already scripted + if isinstance(obj, RecursiveScriptClass): + return obj + if isinstance(obj, ScriptModule): + return obj + if isinstance(obj, ScriptFunction): + return obj + + if example_inputs: + # If MonkeyType is installed, enable profile directed type annotation + # Check if example_inputs are defined and generate call traces + # for the method by running eager mode version of the method with + # the provide example inputs. This logs all the traces in type_trace_db + type_trace_db = JitTypeTraceStore() + if monkeytype_trace: + # pyrefly: ignore [bad-argument-count] + monkeytype_config = JitTypeTraceConfig(type_trace_db) + with monkeytype_trace(monkeytype_config): + if isinstance(example_inputs, dict): + # If the obj is an nn.Module or a class, then each method is + # executed with the arguments provided in the example inputs. + # example inputs here will be of type Dict(class.method, (arguments)) + # This is used to infer type annotations for those methods + # which are not called directly under the hood of monkeytype. + for module, example_input in example_inputs.items(): + for example in example_input: + module(*example) + elif isinstance(example_inputs, list): + for examples in example_inputs: + obj(*examples) + else: + raise ValueError( + "Error: Unable to infer types. Please format the inputs to type `List[Tuple]`" + " or `Dict[Callable, List[Tuple]]` to be run with MonkeyType." + ) + else: + warnings.warn( + "Warning: monkeytype is not installed. Please install https://github.com/Instagram/MonkeyType " + "to enable Profile-Directed Typing in TorchScript. Refer to " + "https://github.com/Instagram/MonkeyType/blob/master/README.rst to install MonkeyType. ", + stacklevel=2, + ) + + if isinstance(obj, torch.nn.Module): + obj = call_prepare_scriptable_func(obj) + return torch.jit._recursive.create_script_module( + obj, torch.jit._recursive.infer_methods_to_compile + ) + else: + obj = ( + obj.__prepare_scriptable__() + if hasattr(obj, "__prepare_scriptable__") + else obj + ) # type: ignore[operator] + + if isinstance(obj, dict): + return create_script_dict(obj) + if isinstance(obj, list): + return create_script_list(obj) + + if inspect.isclass(obj): + qualified_name = _qualified_name(obj) + # If this type is a `nn.Module` subclass, they probably meant to pass + # an instance instead of a Module + if issubclass(obj, torch.nn.Module): + raise RuntimeError( + f"Type '{obj}' cannot be compiled since it inherits from nn.Module, pass an instance instead" + ) + + # Enums are automatically usable in TorchScript, explicitly scripting + # is not necessary, but not harmful either. + if issubclass(obj, enum.Enum): + return obj + + if not _is_new_style_class(obj): + raise RuntimeError( + "TorchScript classes must be new-style classes. " + "Please inherit from 'object'." + ) + if len(obj.mro()) > 2: + raise RuntimeError( + "TorchScript classes does not support inheritance yet. " + "Please directly inherit from 'object'." + ) + if _rcb is None: + _rcb = _jit_internal.createResolutionCallbackFromFrame(_frames_up + 1) + _compile_and_register_class(obj, _rcb, qualified_name) + return obj + elif inspect.isfunction(obj) or inspect.ismethod(obj): + qualified_name = _qualified_name(obj) + # this is a decorated fn, and we need to the underlying fn and its rcb + if hasattr(obj, "__script_if_tracing_wrapper"): + obj = obj.__original_fn # type: ignore[union-attr] + _rcb = _jit_internal.createResolutionCallbackFromClosure(obj) + + # some functions are explicitly marked as not supported in script mode + if hasattr(obj, "__script_unsupported"): + raise RuntimeError("TorchScript error: " + obj.__script_unsupported) + + _check_directly_compile_overloaded(obj) + maybe_already_compiled_fn = _try_get_jit_cached_function(obj) + if maybe_already_compiled_fn: + maybe_already_compiled_fn._torchdynamo_inline = obj # type: ignore[attr-defined] + return maybe_already_compiled_fn + ast = get_jit_def(obj, obj.__name__) + if _rcb is None: + _rcb = _jit_internal.createResolutionCallbackFromClosure(obj) + fn = torch._C._jit_script_compile( + qualified_name, ast, _rcb, get_default_args(obj) + ) + # Forward docstrings + fn.__doc__ = obj.__doc__ + fn.__name__ = "ScriptFunction" + fn.__qualname__ = "torch.jit.ScriptFunction" + # Allow torch.compile() to inline + fn._torchdynamo_inline = obj # type: ignore[attr-defined] + _set_jit_function_cache(obj, fn) + return fn + else: + return torch.jit._recursive.create_script_class(obj) + + +def script( + obj, + optimize=None, + _frames_up=0, + _rcb=None, + example_inputs: Union[list[tuple], dict[Callable, list[tuple]], None] = None, +): + r"""Script the function. + + Scripting a function or ``nn.Module`` will inspect the source code, compile + it as TorchScript code using the TorchScript compiler, and return a :class:`ScriptModule` or + :class:`ScriptFunction`. TorchScript itself is a subset of the Python language, so not all + features in Python work, but we provide enough functionality to compute on + tensors and do control-dependent operations. For a complete guide, see the + :ref:`language-reference`. + + Scripting a dictionary or list copies the data inside it into a TorchScript instance than can be + subsequently passed by reference between Python and TorchScript with zero copy overhead. + + ``torch.jit.script`` can be used as a function for modules, functions, dictionaries and lists + and as a decorator ``@torch.jit.script`` for torchscript-classes and functions. + + Args: + obj (Callable, class, or nn.Module): The ``nn.Module``, function, class type, + dictionary, or list to compile. + example_inputs (Union[List[Tuple], Dict[Callable, List[Tuple]], None]): Provide example inputs + to annotate the arguments for a function or ``nn.Module``. + + Returns: + If ``obj`` is ``nn.Module``, ``script`` returns + a :class:`ScriptModule` object. The returned :class:`ScriptModule` will + have the same set of sub-modules and parameters as the + original ``nn.Module``. If ``obj`` is a standalone function, + a :class:`ScriptFunction` will be returned. If ``obj`` is a ``dict``, then + ``script`` returns an instance of `torch._C.ScriptDict`. If ``obj`` is a ``list``, + then ``script`` returns an instance of `torch._C.ScriptList`. + + **Scripting a function** + The ``@torch.jit.script`` decorator will construct a :class:`ScriptFunction` + by compiling the body of the function. + + Example (scripting a function): + + .. testcode:: + + import torch + + @torch.jit.script + def foo(x, y): + if x.max() > y.max(): + r = x + else: + r = y + return r + + print(type(foo)) # torch.jit.ScriptFunction + + # See the compiled graph as Python code + print(foo.code) + + # Call the function using the TorchScript interpreter + foo(torch.ones(2, 2), torch.ones(2, 2)) + + .. testoutput:: + :hide: + + ... + + ****Scripting a function using example_inputs** + Example inputs can be used to annotate a function arguments. + + Example (annotating a function before scripting): + + .. testcode:: + + import torch + + def test_sum(a, b): + return a + b + + # Annotate the arguments to be int + scripted_fn = torch.jit.script(test_sum, example_inputs=[(3, 4)]) + + print(type(scripted_fn)) # torch.jit.ScriptFunction + + # See the compiled graph as Python code + print(scripted_fn.code) + + # Call the function using the TorchScript interpreter + scripted_fn(20, 100) + + .. testoutput:: + :hide: + + ... + + **Scripting an nn.Module** + Scripting an ``nn.Module`` by default will compile the ``forward`` method and recursively + compile any methods, submodules, and functions called by ``forward``. If a ``nn.Module`` only uses + features supported in TorchScript, no changes to the original module code should be necessary. ``script`` + will construct :class:`ScriptModule` that has copies of the attributes, parameters, and methods of + the original module. + + Example (scripting a simple module with a Parameter): + + .. testcode:: + + import torch + + class MyModule(torch.nn.Module): + def __init__(self, N, M): + super().__init__() + # This parameter will be copied to the new ScriptModule + self.weight = torch.nn.Parameter(torch.rand(N, M)) + + # When this submodule is used, it will be compiled + self.linear = torch.nn.Linear(N, M) + + def forward(self, input): + output = self.weight.mv(input) + + # This calls the `forward` method of the `nn.Linear` module, which will + # cause the `self.linear` submodule to be compiled to a `ScriptModule` here + output = self.linear(output) + return output + + scripted_module = torch.jit.script(MyModule(2, 3)) + + Example (scripting a module with traced submodules): + + .. testcode:: + + import torch + import torch.nn as nn + import torch.nn.functional as F + + class MyModule(nn.Module): + def __init__(self) -> None: + super().__init__() + # torch.jit.trace produces a ScriptModule's conv1 and conv2 + self.conv1 = torch.jit.trace(nn.Conv2d(1, 20, 5), torch.rand(1, 1, 16, 16)) + self.conv2 = torch.jit.trace(nn.Conv2d(20, 20, 5), torch.rand(1, 20, 16, 16)) + + def forward(self, input): + input = F.relu(self.conv1(input)) + input = F.relu(self.conv2(input)) + return input + + scripted_module = torch.jit.script(MyModule()) + + To compile a method other than ``forward`` (and recursively compile anything it calls), add + the :func:`@torch.jit.export ` decorator to the method. To opt out of compilation + use :func:`@torch.jit.ignore ` or :func:`@torch.jit.unused `. + + Example (an exported and ignored method in a module):: + + import torch + import torch.nn as nn + + + class MyModule(nn.Module): + def __init__(self) -> None: + super().__init__() + + @torch.jit.export + def some_entry_point(self, input): + return input + 10 + + @torch.jit.ignore + def python_only_fn(self, input): + # This function won't be compiled, so any + # Python APIs can be used + import pdb + + pdb.set_trace() + + def forward(self, input): + if self.training: + self.python_only_fn(input) + return input * 99 + + + scripted_module = torch.jit.script(MyModule()) + print(scripted_module.some_entry_point(torch.randn(2, 2))) + print(scripted_module(torch.randn(2, 2))) + + Example ( Annotating forward of nn.Module using example_inputs):: + + import torch + import torch.nn as nn + from typing import NamedTuple + + class MyModule(NamedTuple): + result: List[int] + + class TestNNModule(torch.nn.Module): + def forward(self, a) -> MyModule: + result = MyModule(result=a) + return result + + pdt_model = TestNNModule() + + # Runs the pdt_model in eager model with the inputs provided and annotates the arguments of forward + scripted_model = torch.jit.script(pdt_model, example_inputs={pdt_model: [([10, 20, ], ), ], }) + + # Run the scripted_model with actual inputs + print(scripted_model([20])) + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.script` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return obj + try: + global _TOPLEVEL + prev = _TOPLEVEL + _TOPLEVEL = False + ret = _script_impl( + obj=obj, + optimize=optimize, + _frames_up=_frames_up + 1, + _rcb=_rcb, + example_inputs=example_inputs, + ) + + if prev: + log_torchscript_usage("script", model_id=_get_model_id(ret)) + + return ret + finally: + _TOPLEVEL = prev + + +# overloads are registered in _jit_internal and compiled here so that _overload +# can be used in nn/functional.py without an import cycle + + +def _check_overload_defaults(impl_defaults, overload_defaults, loc): + for name, overload_value in overload_defaults.items(): + if name not in impl_defaults or impl_defaults[name] != overload_value: + raise torch.jit.frontend.FrontendError( + loc, + "Default parameters on overloads do not affect the runtime so they " + "must equal to the default parameter on the implementation function. Found on " + f"parameter {name}", + ) + + +def _compile_function_with_overload(overload_fn, qual_name, impl_fn): + overload_decl = get_jit_def(overload_fn, overload_fn.__name__).decl() + overload_signature = torch.jit.annotations.get_signature( + overload_fn, None, None, inspect.ismethod(overload_fn) + ) + impl_ast = get_jit_def(impl_fn, impl_fn.__name__) + overload_defaults = get_default_args(overload_fn) + implementation_defaults = get_default_args(impl_fn) + _rcb = _jit_internal.createResolutionCallbackFromClosure(impl_fn) + _check_overload_defaults( + implementation_defaults, overload_defaults, overload_decl.range() + ) + fn = torch._C._jit_script_compile_overload( + qual_name, + overload_decl, + impl_ast, + _rcb, + implementation_defaults, + overload_signature, + ) + return fn + + +def _get_overloads(obj): + # check for cached compiled fns + existing_compiled_fns = _try_get_jit_cached_overloads(obj) + qual_name = _qualified_name(obj) + uncompiled_overloads = _jit_internal._get_fn_overloads(qual_name) + if uncompiled_overloads is None: + return existing_compiled_fns + + if obj in uncompiled_overloads: + raise RuntimeError( + _jit_internal.get_overload_no_implementation_error_message("function", obj) + ) + + compiled_fns = [ + _compile_function_with_overload(overload_fn, qual_name, obj) + for overload_fn in uncompiled_overloads + ] + + if existing_compiled_fns: + compiled_fns = existing_compiled_fns + compiled_fns + + # cache compilation, remove information stored to do compilation + _set_jit_overload_cache(obj, compiled_fns) + _jit_internal._clear_fn_overloads(qual_name) + return compiled_fns + + +def _check_directly_compile_overloaded(obj): + qual_name = _qualified_name(obj) + if _jit_internal._get_fn_overloads(qual_name) or _try_get_jit_cached_overloads(obj): + raise RuntimeError( + f"Function {qual_name} cannot be directly compiled because it" + " is overloaded. It must be used in a context of a function" + " where its inputs can determine which overload to call." + ) + + +def interface(obj): + r"""Decorate to annotate classes or modules of different types. + + This decorator can be used to define an interface that can be used to annotate + classes or modules of different types. This can be used for to annotate a submodule + or attribute class that could have different types that implement the same + interface, or which could be swapped at runtime; or to store a list of modules or + classes of varying types. + + It is sometimes used to implement "Callables" - functions or modules that implement + an interface but whose implementations differ and which can be swapped out. + + Example: + .. testcode:: + + import torch + from typing import List + + @torch.jit.interface + class InterfaceType: + def run(self, x: torch.Tensor) -> torch.Tensor: + pass + + # implements InterfaceType + @torch.jit.script + class Impl1: + def run(self, x: torch.Tensor) -> torch.Tensor: + return x.relu() + + class Impl2(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.val = torch.rand(()) + + @torch.jit.export + def run(self, x: torch.Tensor) -> torch.Tensor: + return x + self.val + + def user_fn(impls: List[InterfaceType], idx: int, val: torch.Tensor) -> torch.Tensor: + return impls[idx].run(val) + + user_fn_jit = torch.jit.script(user_fn) + + impls = [Impl1(), torch.jit.script(Impl2())] + val = torch.rand(4, 4) + user_fn_jit(impls, 0, val) + user_fn_jit(impls, 1, val) + """ + if not inspect.isclass(obj): + raise RuntimeError("interface must be applied to a class") + if not _is_new_style_class(obj): + raise RuntimeError("TorchScript interfaces must inherit from 'object'") + + # Expected MRO is: + # User module + # torch.nn.modules.module.Module + # object + is_module_interface = issubclass(obj, torch.nn.Module) and len(obj.mro()) == 3 + + if not is_module_interface and len(obj.mro()) > 2: + raise RuntimeError( + "TorchScript interface does not support inheritance yet. " + "Please directly inherit from 'object' or 'nn.Module'." + ) + + qualified_name = _qualified_name(obj) + rcb = _jit_internal.createResolutionCallbackFromFrame(1) + # if this type is a `nn.Module` subclass, generate a module interface type + # instead of a class interface type; a module interface type only compiles + # the user provided methods as part of the interface + ast = get_jit_class_def(obj, obj.__name__) + mangled_classname = torch._C._jit_script_interface_compile( + qualified_name, ast, rcb, is_module_interface + ) + obj.__torch_script_interface__ = mangled_classname + return obj + + +def _recursive_compile_class(obj, loc): + _qual_name = _qualified_name(obj) + # We're starting a new compilation, so update the error call stack in + # case it fails + error_stack = torch._C.CallStack(_qual_name, loc) # noqa: F841 + rcb = _jit_internal.createResolutionCallbackForClassMethods(obj) + return _compile_and_register_class(obj, rcb, _qual_name) + + +CompilationUnit = torch._C.CompilationUnit +set_module(CompilationUnit, "torch.jit") + + +def pad(s: str, padding: int, offset: int = 0, char: str = " "): + if padding >= len(s): + padding -= len(s) + return "".join([char for _ in range(padding + offset)]) + s + + +class _ScriptProfileColumn: + def __init__(self, header: str, alignment: int = 4, offset: int = 0): + self.header = header + self.alignment = alignment + self.offset = offset + self.rows: dict[int, Any] = {} + + def add_row(self, lineno: int, value: Any): + self.rows[lineno] = value + + def materialize(self): + max_length = len(self.header) + rows: list[tuple[int, str]] = [] + for key, value in self.rows.items(): + cell = str(value) + rows.append((key, cell)) + max_length = max(len(cell), max_length) + + if self.alignment > 0: + padding = max_length + self.alignment + padding -= padding % self.alignment + else: + padding = 0 + + rows = [(key, pad(cell, padding, self.offset)) for key, cell in rows] + return pad(self.header, padding, self.offset), rows + + +class _ScriptProfileTable: + def __init__(self, cols: list[_ScriptProfileColumn], source_range: list[int]): + self.cols = cols + self.source_range = source_range + + def dump_string(self): + outputs: list[str] = [] + cells: list[tuple[str, dict[int, str]]] = [] + header_buffer = "" + for col in self.cols: + header, rows = col.materialize() + header_buffer += header + cells.append((header, dict(rows))) + + outputs.append(header_buffer) + outputs.append(pad("", len(header_buffer), 0, "=")) + for line in self.source_range: + row_buffer = "" + for header, rows in cells: + cell = rows.get(line) + if cell is None: + row_buffer += pad("", len(header)) + else: + row_buffer += cell + outputs.append(row_buffer) + return "\n".join(outputs) + + +class _ScriptProfile: + def __init__(self) -> None: + self.profile = classes.profiling._ScriptProfile() + + def enable(self): + self.profile.enable() + + def disable(self): + self.profile.disable() + + def dump_string(self) -> str: + outputs: list[str] = [] + for source_stats in self.profile._dump_stats(): + source_ref = source_stats.source() + source_lines = source_ref.text().splitlines() + dedent = min(len(line) - len(line.lstrip(" ")) for line in source_lines) + source_lines = [line[dedent:] for line in source_lines] + + start_line = source_ref.starting_lineno() + end_line = start_line + len(source_lines) + source_range = range(start_line, end_line) + lineno = _ScriptProfileColumn("Line #") + hits = _ScriptProfileColumn("Hits") + time_ns = _ScriptProfileColumn("Time (ns)") + line_contents = _ScriptProfileColumn("Line Contents", 0, 1) + stats = source_stats.line_map() + for line in source_range: + lineno.add_row(line, line) + line_contents.add_row(line, source_lines[line - start_line]) + stat = stats.get(line) + if stat is not None: + hits.add_row(line, stat.count()) + time_ns.add_row(line, stat.duration_ns()) + + table = _ScriptProfileTable( + [lineno, hits, time_ns, line_contents], list(source_range) + ) + outputs.append(table.dump_string()) + return "\n\n".join(outputs) + + def dump(self): + print(self.dump_string()) + + +def _unwrap_optional(x): + assert x is not None, "Unwrapping null optional" + return x + + +_register_builtin(_unwrap_optional, "aten::_unwrap_optional") +_register_builtin(_jit_internal.is_scripting, "aten::is_scripting") +_register_builtin(has_torch_function, "aten::has_torch_function") +_register_builtin(has_torch_function_unary, "aten::has_torch_function") +_register_builtin(has_torch_function_variadic, "aten::has_torch_function") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.pyi b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.pyi new file mode 100644 index 0000000000000000000000000000000000000000..594837e5e7dce17c81114e8b71b9a22ee799120f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_script.pyi @@ -0,0 +1,297 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code="type-arg" +from collections.abc import Callable +from typing import Any, NamedTuple, overload, TypeAlias, TypeVar +from typing_extensions import Never + +from _typeshed import Incomplete + +import torch +from torch._classes import classes as classes +from torch._jit_internal import _qualified_name as _qualified_name +from torch.jit._builtins import _register_builtin as _register_builtin +from torch.jit._fuser import ( + _graph_for as _graph_for, + _script_method_graph_for as _script_method_graph_for, +) +from torch.jit._monkeytype_config import ( + JitTypeTraceConfig as JitTypeTraceConfig, + JitTypeTraceStore as JitTypeTraceStore, + monkeytype_trace as monkeytype_trace, +) +from torch.jit._recursive import ( + _compile_and_register_class as _compile_and_register_class, + infer_methods_to_compile as infer_methods_to_compile, + ScriptMethodStub as ScriptMethodStub, + wrap_cpp_module as wrap_cpp_module, +) +from torch.jit._serialization import validate_map_location as validate_map_location +from torch.jit._state import ( + _enabled as _enabled, + _set_jit_function_cache as _set_jit_function_cache, + _set_jit_overload_cache as _set_jit_overload_cache, + _try_get_jit_cached_function as _try_get_jit_cached_function, + _try_get_jit_cached_overloads as _try_get_jit_cached_overloads, +) +from torch.jit.frontend import ( + get_default_args as get_default_args, + get_jit_class_def as get_jit_class_def, + get_jit_def as get_jit_def, +) +from torch.nn import Module as Module +from torch.overrides import ( + has_torch_function as has_torch_function, + has_torch_function_unary as has_torch_function_unary, + has_torch_function_variadic as has_torch_function_variadic, +) +from torch.package import ( + PackageExporter as PackageExporter, + PackageImporter as PackageImporter, +) +from torch.utils import set_module as set_module + +ScriptFunction = torch._C.ScriptFunction + +type_trace_db: JitTypeTraceStore + +# Defined in torch/csrc/jit/python/script_init.cpp +ResolutionCallback: TypeAlias = Callable[[str], Callable[..., Any]] +_ClassVar = TypeVar("_ClassVar", bound=type) + +def _reduce(cls) -> None: ... + +class Attribute(NamedTuple): + value: Incomplete + type: Incomplete + +def _get_type_trace_db(): ... +def _get_function_from_type(cls, name): ... +def _is_new_style_class(cls): ... + +class OrderedDictWrapper: + _c: Incomplete + def __init__(self, _c) -> None: ... + def keys(self): ... + def values(self): ... + def __len__(self) -> int: ... + def __delitem__(self, k) -> None: ... + def items(self): ... + def __setitem__(self, k, v) -> None: ... + def __contains__(self, k) -> bool: ... + def __getitem__(self, k): ... + +class OrderedModuleDict(OrderedDictWrapper): + _python_modules: Incomplete + def __init__(self, module, python_dict) -> None: ... + def items(self): ... + def __contains__(self, k) -> bool: ... + def __setitem__(self, k, v) -> None: ... + def __getitem__(self, k): ... + +class ScriptMeta(type): + def __init__(cls, name, bases, attrs) -> None: ... + +class _CachedForward: + def __get__(self, obj, cls): ... + +class ScriptWarning(Warning): ... + +def script_method(fn): ... + +class ConstMap: + const_mapping: Incomplete + def __init__(self, const_mapping) -> None: ... + def __getattr__(self, attr): ... + +def unpackage_script_module( + importer: PackageImporter, + script_module_id: str, +) -> torch.nn.Module: ... + +_magic_methods: Incomplete + +class RecursiveScriptClass: + _c: Incomplete + _props: Incomplete + def __init__(self, cpp_class) -> None: ... + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def forward_magic_method(self, method_name, *args, **kwargs): ... + def __getstate__(self) -> None: ... + def __iadd__(self, other): ... + +def method_template(self, *args, **kwargs): ... + +class ScriptModule(Module, metaclass=ScriptMeta): + __jit_unused_properties__: Incomplete + def __init__(self) -> None: ... + forward: Callable[..., Any] + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def define(self, src): ... + def _replicate_for_data_parallel(self): ... + def __reduce_package__(self, exporter: PackageExporter): ... + # add __jit_unused_properties__ + @property + def code(self) -> str: ... + @property + def code_with_constants(self) -> tuple[str, ConstMap]: ... + @property + def graph(self) -> torch.Graph: ... + @property + def inlined_graph(self) -> torch.Graph: ... + @property + def original_name(self) -> str: ... + +class RecursiveScriptModule(ScriptModule): + _disable_script_meta: bool + _c: Incomplete + def __init__(self, cpp_module) -> None: ... + @staticmethod + def _construct(cpp_module, init_fn): ... + @staticmethod + def _finalize_scriptmodule(script_module) -> None: ... + _concrete_type: Incomplete + _modules: Incomplete + _parameters: Incomplete + _buffers: Incomplete + __dict__: Incomplete + def _reconstruct(self, cpp_module) -> None: ... + def save(self, f, **kwargs): ... + def _save_for_lite_interpreter(self, *args, **kwargs): ... + def _save_to_buffer_for_lite_interpreter(self, *args, **kwargs): ... + def save_to_buffer(self, *args, **kwargs): ... + def get_debug_state(self, *args, **kwargs): ... + def extra_repr(self): ... + def graph_for(self, *args, **kwargs): ... + def define(self, src) -> None: ... + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def __copy__(self): ... + def __deepcopy__(self, memo): ... + def forward_magic_method(self, method_name, *args, **kwargs): ... + def __iter__(self): ... + def __getitem__(self, idx): ... + def __len__(self) -> int: ... + def __contains__(self, key) -> bool: ... + def __dir__(self): ... + def __bool__(self) -> bool: ... + def _replicate_for_data_parallel(self): ... + +def _get_methods(cls): ... + +_compiled_methods_allowlist: Incomplete + +def _make_fail(name): ... +def call_prepare_scriptable_func_impl(obj, memo): ... +def call_prepare_scriptable_func(obj): ... +def create_script_dict(obj): ... +def create_script_list(obj, type_hint: Incomplete | None = ...): ... +@overload +def script( + obj: type[Module], + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> Never: ... +@overload +def script( + obj: dict, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> torch.ScriptDict: ... +@overload +def script( + obj: list, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> torch.ScriptList: ... +@overload +def script( # type: ignore[overload-overlap] + obj: Module, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> RecursiveScriptModule: ... +@overload +def script( # type: ignore[overload-overlap] + obj: _ClassVar, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> _ClassVar: ... +@overload +def script( + obj: Callable, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> ScriptFunction: ... +@overload +def script( + obj: Any, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> RecursiveScriptClass: ... +@overload +def script( + obj, + optimize: Incomplete | None = ..., + _frames_up: int = ..., + _rcb: Incomplete | None = ..., + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = ..., +): ... +def _check_overload_defaults(impl_defaults, overload_defaults, loc) -> None: ... +def _compile_function_with_overload(overload_fn, qual_name, impl_fn): ... +def _get_overloads(obj): ... +def _check_directly_compile_overloaded(obj) -> None: ... +def interface(obj): ... +def _recursive_compile_class(obj, loc): ... + +CompilationUnit: Incomplete + +def pad(s: str, padding: int, offset: int = ..., char: str = ...): ... + +class _ScriptProfileColumn: + header: Incomplete + alignment: Incomplete + offset: Incomplete + rows: Incomplete + def __init__( + self, + header: str, + alignment: int = ..., + offset: int = ..., + ) -> None: ... + def add_row(self, lineno: int, value: Any): ... + def materialize(self): ... + +class _ScriptProfileTable: + cols: Incomplete + source_range: Incomplete + def __init__( + self, + cols: list[_ScriptProfileColumn], + source_range: list[int], + ) -> None: ... + def dump_string(self): ... + +class _ScriptProfile: + profile: Incomplete + def __init__(self) -> None: ... + def enable(self) -> None: ... + def disable(self) -> None: ... + def dump_string(self) -> str: ... + def dump(self) -> None: ... + +def _unwrap_optional(x): ... diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_serialization.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_serialization.py new file mode 100644 index 0000000000000000000000000000000000000000..dd1e0f71f0a11e42f0f5b9e0e4eb2f98835d7d12 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_serialization.py @@ -0,0 +1,315 @@ +# mypy: allow-untyped-defs +"""Serialization. + +This module contains functionality for serializing TorchScript modules, notably: + * torch.jit.save + * torch.jit.load + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import os +import sys +import warnings + +import torch +from torch._jit_internal import _get_model_id +from torch._utils_internal import log_torchscript_usage +from torch.jit._recursive import wrap_cpp_module +from torch.serialization import validate_cuda_device + + +def save(m, f, _extra_files=None) -> None: + r""" + Save an offline version of this module for use in a separate process. + + The saved module serializes all of the methods, submodules, parameters, and + attributes of this module. It can be loaded into the C++ API using + ``torch::jit::load(filename)`` or into the Python API with + :func:`torch.jit.load `. + + To be able to save a module, it must not make any calls to native Python + functions. This means that all submodules must be subclasses of + :class:`ScriptModule` as well. + + .. DANGER:: + All modules, no matter their device, are always loaded onto the CPU + during loading. This is different from :func:`torch.load`'s semantics + and may change in the future. + + Args: + m: A :class:`ScriptModule` to save. + f: A file-like object (has to implement write and flush) or a string + containing a file name. + _extra_files: Map from filename to contents which will be stored as part of `f`. + + .. note:: + torch.jit.save attempts to preserve the behavior of some operators + across versions. For example, dividing two integer tensors in + PyTorch 1.5 performed floor division, and if the module + containing that code is saved in PyTorch 1.5 and loaded in PyTorch 1.6 + its division behavior will be preserved. The same module saved in + PyTorch 1.6 will fail to load in PyTorch 1.5, however, since the + behavior of division changed in 1.6, and 1.5 does not know how to + replicate the 1.6 behavior. + + Example: + .. testcode:: + + import torch + import io + + class MyModule(torch.nn.Module): + def forward(self, x): + return x + 10 + + m = torch.jit.script(MyModule()) + + # Save to file + torch.jit.save(m, 'scriptmodule.pt') + # This line is equivalent to the previous + m.save("scriptmodule.pt") + + # Save to io.BytesIO buffer + buffer = io.BytesIO() + torch.jit.save(m, buffer) + + # Save with extra files + extra_files = {'foo.txt': b'bar'} + torch.jit.save(m, 'scriptmodule.pt', _extra_files=extra_files) + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.save` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.save` is deprecated. Please switch to `torch.export`.", + DeprecationWarning, + ) + log_torchscript_usage("save", model_id=_get_model_id(m)) + if _extra_files is None: + _extra_files = {} + if isinstance(f, (str, os.PathLike)): + m.save(f, _extra_files=_extra_files) + else: + ret = m.save_to_buffer(_extra_files=_extra_files) + f.write(ret) + + +def load(f, map_location=None, _extra_files=None, _restore_shapes=False): + r""" + Load a :class:`ScriptModule` or :class:`ScriptFunction` previously saved with :func:`torch.jit.save `. + + All previously saved modules, no matter their device, are first loaded onto CPU, + and then are moved to the devices they were saved from. If this fails (e.g. + because the run time system doesn't have certain devices), an exception is + raised. + + Args: + f: a file-like object (has to implement read, readline, tell, and seek), + or a string containing a file name + map_location (string or torch.device): A simplified version of + ``map_location`` in `torch.jit.save` used to dynamically remap + storages to an alternative set of devices. + _extra_files (dictionary of filename to content): The extra + filenames given in the map would be loaded and their content + would be stored in the provided map. + _restore_shapes (bool): Whether or not to retrace the module on load using stored inputs + + Returns: + A :class:`ScriptModule` object. + + .. warning:: + It is possible to construct malicious pickle data which will execute arbitrary code + during func:`torch.jit.load`. Never load data that could have come from an untrusted + source, or that could have been tampered with. **Only load data you trust**. + + Example: + .. testcode:: + + import torch + import io + + torch.jit.load('scriptmodule.pt') + + # Load ScriptModule from io.BytesIO object + with open('scriptmodule.pt', 'rb') as f: + buffer = io.BytesIO(f.read()) + + # Load all tensors to the original device + torch.jit.load(buffer) + + # Load all tensors onto CPU, using a device + buffer.seek(0) + torch.jit.load(buffer, map_location=torch.device('cpu')) + + # Load all tensors onto CPU, using a string + buffer.seek(0) + torch.jit.load(buffer, map_location='cpu') + + # Load with extra files. + extra_files = {'foo.txt': ''} # values will be replaced with data + torch.jit.load('scriptmodule.pt', _extra_files=extra_files) + print(extra_files['foo.txt']) + + .. testoutput:: + :hide: + + ... + + .. testcleanup:: + + import os + os.remove("scriptmodule.pt") + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.load` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.load` is deprecated. Please switch to `torch.export`.", + DeprecationWarning, + ) + if isinstance(f, (str, os.PathLike)): + if not os.path.exists(f): + raise ValueError(f"The provided filename {f} does not exist") + if os.path.isdir(f): + raise ValueError(f"The provided filename {f} is a directory") + + map_location = validate_map_location(map_location) + if _extra_files is None: + _extra_files = {} + + cu = torch._C.CompilationUnit() + if isinstance(f, (str, os.PathLike)): + cpp_module = torch._C.import_ir_module( + cu, + os.fspath(f), + map_location, + _extra_files, + # pyrefly: ignore [bad-argument-count] + _restore_shapes, + ) # type: ignore[call-arg] + else: + cpp_module = torch._C.import_ir_module_from_buffer( + cu, + # pyrefly: ignore [missing-attribute] + f.read(), + map_location, + _extra_files, + # pyrefly: ignore [bad-argument-count] + _restore_shapes, + ) # type: ignore[call-arg] + + # TODO: Pretty sure this approach loses ConstSequential status and such + ret = wrap_cpp_module(cpp_module) + log_torchscript_usage("load", model_id=_get_model_id(ret)) + return ret + + +def validate_map_location(map_location=None): + if isinstance(map_location, str): + map_location = torch.device(map_location) + elif not (map_location is None or isinstance(map_location, torch.device)): + raise ValueError( + "map_location should be either None, string or torch.device, " + "but got type: " + str(type(map_location)) + ) + + if str(map_location).startswith("cuda"): + validate_cuda_device(map_location) + + return map_location + + +def jit_module_from_flatbuffer(f): + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + return wrap_cpp_module(torch._C._load_jit_module_from_file(f)) + else: + return wrap_cpp_module(torch._C._load_jit_module_from_bytes(f.read())) + + +def save_jit_module_to_flatbuffer(m, f, _extra_files=None) -> None: + r""" + Save an offline version of this module for use in a separate process. + + The saved module serializes all of the methods, submodules, parameters, and + attributes of this module. It can be loaded into the C++ API using + ``torch::jit::load_jit_module_from_file(filename)`` or into the Python API with + :func:`torch.jit.jit_module_from_flatbuffer`. + + To be able to save a module, it must not make any calls to native Python + functions. This means that all submodules must be subclasses of + :class:`ScriptModule` as well. + + .. DANGER:: + All modules, no matter their device, are always loaded onto the CPU + during loading. This is different from :func:`torch.load`'s semantics + and may change in the future. + + Args: + m: A :class:`ScriptModule` to save. + f: A string for file path + + + Example: + .. testcode:: + + import torch + import io + + class MyModule(torch.nn.Module): + def forward(self, x): + return x + 10 + + m = torch.jit.script(MyModule()) + + # Save to file + torch.jit.save_jit_module_to_flatbuffer(m, 'scriptmodule.ff') + """ + extra_files = _extra_files + if extra_files is None: + extra_files = {} + + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + torch._C._save_jit_module(m._c, f, extra_files) + else: + s = torch._C._save_jit_module_to_bytes(m._c, extra_files) + f.write(s) + + +def get_flatbuffer_module_info(path_or_file): + r"""Get some information regarding a model file in flatbuffer format. + + Args: + path_or_file: Either str, Path or file like object (BytesIO OK). + If it's str or Path, we will read the file referenced by that + path as Bytes. + + Returns: + A dict with metadata on what that file contains, currently looks like + this: + { + 'bytecode_version': 4, # int + 'operator_version': 4, # int + 'function_names': { + '__torch__.___torch_mangle_0.Foo.forward'}, # set + 'type_names': set(), # set + 'opname_to_num_args': {'aten::linear': 3} # Dict[str, int] + } + """ + if isinstance(path_or_file, (str, os.PathLike)): + with open(path_or_file, "rb") as f: + all_bytes = f.read() + else: + all_bytes = path_or_file.read() + return torch._C._get_module_info_from_flatbuffer(all_bytes) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_shape_functions.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_shape_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..22f4f7ffa15782a7b12361dd78e17a999c71bccb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_shape_functions.py @@ -0,0 +1,1475 @@ +# mypy: allow-untyped-defs +import math +from collections.abc import Callable +from typing import Any, Optional, Union + + +number = Union[int, float] +# flake8: noqa + +### +# There are generated files that depend on this file +# To re-generate, please run from the root of the repo: +# python torchgen/shape_functions/gen_jit_shape_functions.py + +# How to test: +# After regenerating files, compile PyTorch. +# Then run: ./build/bin/test_jit --gtest_filter=TestShapeGraphLinting.Basic +# If you have enabled opinfo testing for the op, also run: +# python test/test_ops_jit.py TestJitCPU.test_variant_consistency_jit_[FAILING_OP]_cpu_float32 +# to reproduce errors from opinfo tests. + +# Example PR: https://github.com/pytorch/pytorch/pull/80860/files +#### + +import torch + + +def broadcast(a: list[int], b: list[int]): + dimsA = len(a) + dimsB = len(b) + ndim = max(dimsA, dimsB) + expandedSizes: list[int] = [] + + for i in range(ndim): + offset = ndim - 1 - i + dimA = dimsA - 1 - offset + dimB = dimsB - 1 - offset + sizeA = a[dimA] if (dimA >= 0) else 1 + sizeB = b[dimB] if (dimB >= 0) else 1 + + if sizeA != sizeB and sizeA != 1 and sizeB != 1: + # TODO: only assertion error is bound in C++ compilation right now + raise AssertionError( + f"The size of tensor a {sizeA} must match the size of tensor b ({sizeB}) at non-singleton dimension {i}" + ) + + expandedSizes.append(sizeB if sizeA == 1 else sizeA) + + return expandedSizes + + +def broadcast_three(a: list[int], b: list[int], c: list[int]): + return broadcast(broadcast(a, b), c) + + +def broadcast_one_three(a: list[int], b: Any, c: list[int]): + return broadcast(a, c) + + +def adaptive_avg_pool2d(self: list[int], out: list[int]): + assert len(out) == 2 + assert len(self) == 3 or len(self) == 4 + for i in range(1, len(self)): + assert self[i] != 0 + + shape: list[int] = [] + for i in range(0, len(self) - 2): + shape.append(self[i]) + for elem in out: + shape.append(elem) + return shape + + +def _copy(self: list[int]): + out: list[int] = [] + for elem in self: + out.append(elem) + return out + + +def unary(self: list[int]): + return _copy(self) + + +def broadcast_inplace(a: list[int], b: list[int]): + dimsA = len(a) + dimsB = len(b) + if dimsB > dimsA: + raise AssertionError( + f"The dims of tensor b ({dimsB}) must be less than or equal to the dims of tensor a ({dimsA}) " + ) + for dimA in range(dimsA): + dimB = dimsB - dimsA + dimA + sizeA = a[dimA] + sizeB = b[dimB] if (dimB >= 0) else 1 + if sizeA != sizeB and sizeB != 1: + # TODO: only assertion error is bound in C++ compilation right now + raise AssertionError( + "The size of tensor a {} must match the size of tensor b (" + "{}) at non-singleton dimension {}".format(sizeA, sizeB, dimA) + ) + return _copy(a) + + +def expand(self: list[int], sizes: list[int]): + assert len(sizes) >= len(self) + ndim = len(sizes) + tensor_dim = len(self) + if ndim == 0: + return _copy(sizes) + out: list[int] = [] + for i in range(ndim): + offset = ndim - 1 - i + dim = tensor_dim - 1 - offset + size = self[dim] if dim >= 0 else 1 + targetSize = sizes[i] + if targetSize == -1: + assert dim >= 0 + targetSize = size + if size != targetSize: + assert size == 1 + size = targetSize + out.append(size) + return out + + +def expand_one_unused(self: list[int], sizes: list[int], inp0: Any): + return expand(self, sizes) + + +def infer_size_impl(shape: list[int], numel: int) -> list[int]: + newsize = 1 + infer_dim: Optional[int] = None + for dim in range(len(shape)): + if shape[dim] == -1: + if infer_dim is not None: + raise AssertionError("only one dimension can be inferred") + infer_dim = dim + elif shape[dim] >= 0: + newsize *= shape[dim] + else: + raise AssertionError("invalid shape dimensions") + if not ( + numel == newsize + or (infer_dim is not None and newsize > 0 and numel % newsize == 0) + ): + raise AssertionError("invalid shape") + out = _copy(shape) + if infer_dim is not None: + out[infer_dim] = numel // newsize + return out + + +def numel(sizes: list[int]): + numel = 1 + for elem in sizes: + numel *= elem + return numel + + +def view(self: list[int], sizes: list[int]): + return infer_size_impl(sizes, numel(self)) + + +def view_one_unused(self: list[int], sizes: list[int], *, implicit: bool = False): + return view(self, sizes) + + +def sum_mean_dim( + self: list[int], opt_dims: Optional[list[int]], keep_dim: bool, dt: Any +): + out: list[int] = [] + if opt_dims is None or len(opt_dims) == 0: + dims: list[int] = list(range(len(self))) + else: + dims = opt_dims + + for idx in range(len(self)): + is_mean_dim: bool = False + for reduce_dim in dims: + if idx == maybe_wrap_dim(reduce_dim, len(self)): + is_mean_dim = True + if is_mean_dim: + if keep_dim: + out.append(1) + else: + out.append(self[idx]) + return out + + +def max_dim(self: list[int], dim: int, keep_dim: bool): + out = sum_mean_dim(self, [dim], keep_dim, None) + return out, out + + +# note: python already rounds down towards negative infinity on integer division, special arithmetic not needed +def div_rtn(x: int, y: int): + return x // y + + +def pooling_output_shape_pad_lr( + inputSize: int, + kernelSize: int, + pad_l: int, + pad_r: int, + stride: int, + dilation: int, + ceil_mode: bool, +): + outputSize = ( + div_rtn( + inputSize + + pad_l + + pad_r + - dilation * (kernelSize - 1) + - 1 + + (stride - 1 if ceil_mode else 0), + stride, + ) + + 1 + ) + if ceil_mode: + if (outputSize - 1) * stride >= inputSize + pad_l: + outputSize = outputSize - 1 + return outputSize + + +def pooling_output_shape( + inputSize: int, + kernelSize: int, + pad_l: int, + stride: int, + dilation: int, + ceil_mode: bool, +): + assert stride != 0, "stride should not be zeero" + return pooling_output_shape_pad_lr( + inputSize, kernelSize, pad_l, pad_l, stride, dilation, ceil_mode + ) + + +def pool2d_shape_check( + input: list[int], + kH: int, + kW: int, + dH: int, + dW: int, + padH: int, + padW: int, + dilationH: int, + dilationW: int, + nInputPlane: int, + inputHeight: int, + inputWidth: int, + outputHeight: int, + outputWidth: int, +): + ndim = len(input) + + assert kW > 0 and kH > 0 + assert dW > 0 and dH > 0 + assert dilationH > 0 and dilationW > 0 + + valid_dims = input[1] != 0 and input[2] != 0 + assert ( + ndim == 3 + and input[0] != 0 + and valid_dims + or (ndim == 4 and valid_dims and input[3] != 0) + ) + + assert kW // 2 >= padW and kH // 2 >= padH + assert outputWidth >= 1 and outputHeight >= 1 + + +def max_pool2d( + input: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], + ceil_mode: bool, +): + assert len(kernel_size) == 1 or len(kernel_size) == 2, ( + "max_pool2d: kernel_size must either be a single int, or a tuple of two ints" + ) + kH = kernel_size[0] + kW = kH if len(kernel_size) == 1 else kernel_size[1] + + assert len(stride) == 0 or len(stride) == 1 or len(stride) == 2, ( + "max_pool2d: stride must either be omitted, a single int, or a tuple of two ints" + ) + dH = kH if len(stride) == 0 else stride[0] + if len(stride) == 0: + dW = kW + elif len(stride) == 1: + dW = dH + else: + dW = stride[1] + + assert len(padding) == 1 or len(padding) == 2, ( + "max_pool2d: padding must either be a single int, or a tuple of two ints" + ) + padH = padding[0] + padW = padH if len(padding) == 1 else padding[1] + + assert len(dilation) == 1 or len(dilation) == 2, ( + "max_pool2d: dilation must be either a single int, or a tuple of two ints" + ) + dilationH = dilation[0] + dilationW = dilationH if len(dilation) == 1 else dilation[1] + + assert len(input) == 3 or len(input) == 4 + + nbatch = input[-4] if len(input) == 4 else 1 + nInputPlane = input[-3] + inputHeight = input[-2] + inputWidth = input[-1] + + outputHeight = pooling_output_shape(inputHeight, kH, padH, dH, dilationH, ceil_mode) + outputWidth = pooling_output_shape(inputWidth, kW, padW, dW, dilationW, ceil_mode) + + pool2d_shape_check( + input, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + nInputPlane, + inputHeight, + inputWidth, + outputHeight, + outputWidth, + ) + + if len(input) == 3: + return [nInputPlane, outputHeight, outputWidth] + else: + return [nbatch, nInputPlane, outputHeight, outputWidth] + + +def max_pool2d_with_indices( + input: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], + ceil_mode: bool, +): + out = max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode) + return (out, out) + + +def upsample_nearest2d( + input: list[int], + output_size: Optional[list[int]], + scale_factors: Optional[list[float]], +): + out: list[int] = [] + out.append(input[0]) + out.append(input[1]) + + if scale_factors is None and output_size is None: + assert 0, "Either output_size or scale_factors must be presented" + + if output_size is not None: + assert scale_factors is None, ( + "Must specify exactly one of output_size and scale_factors" + ) + assert len(output_size) == 2 + out.append(output_size[0]) + out.append(output_size[1]) + + if scale_factors is not None: + assert output_size is None, ( + "Must specify exactly one of output_size and scale_factors" + ) + assert len(scale_factors) == 2 + out.append(int(input[2] * scale_factors[0])) + out.append(int(input[3] * scale_factors[1])) + + return out + + +def mm(self: list[int], mat2: list[int]): + assert len(self) == 2, "self must be a matrix" + assert len(mat2) == 2, "mat2 must be a matrix" + + assert self[1] == mat2[0] + return [self[0], mat2[1]] + + +def dot(self: list[int], tensor: list[int]): + assert len(self) == 1 and len(tensor) == 1 + assert self[0] == tensor[0] + out: list[int] = [] + return out + + +def mv(self: list[int], vec: list[int]): + assert len(self) == 2 and len(vec) == 1 + assert self[1] == vec[0] + # TODO: return self + return [self[0]] + + +def unsqueeze(li: list[int], dim: int): + dim = maybe_wrap_dim(dim, len(li) + 1) + out = _copy(li) + out.insert(dim, 1) + return out + + +def squeeze_nodim(li: list[int]): + out: list[int] = [] + for i in range(len(li)): + if li[i] != 1: + out.append(li[i]) + return out + + +def squeeze(li: list[int], dim: int): + out: list[int] = [] + wrapped_dim = maybe_wrap_dim(dim, len(li)) + for i in range(len(li)): + if i == wrapped_dim: + if li[i] != 1: + out.append(li[i]) + else: + out.append(li[i]) + return out + + +def squeeze_dims(li: list[int], dims: list[int]): + if len(dims) == 0: + return li + wrapped_dims = _copy(dims) + for i in range(len(dims)): + wrapped_dims[i] = maybe_wrap_dim(wrapped_dims[i], len(li)) + result: list[int] = [] + for i in range(len(li)): + if li[i] == 1: + if i not in wrapped_dims: + result.append(li[i]) + else: + result.append(li[i]) + return result + + +def index_select(self: list[int], dim: int, index: list[int]): + dim = maybe_wrap_dim(dim, len(self)) + numel = multiply_integers(index) + assert len(index) <= 1 + assert dim == 0 or dim < len(self) + result_size: list[int] = [] + for i in range(len(self)): + if dim == i: + result_size.append(numel) + else: + result_size.append(self[i]) + return result_size + + +def embedding( + weight: list[int], + indices: list[int], + padding_idx: int = -1, + scale_grad_by_freq: bool = False, + sparse: bool = False, +): + assert len(weight) == 2 + if len(indices) == 1: + return index_select(weight, 0, indices) + size = _copy(indices) + size.append(weight[1]) + return size + + +def max_int(): + return 9223372036854775807 + + +def slice( + self: list[int], dim: int, start: Optional[int], end: Optional[int], step: int +): + ndim = len(self) + assert ndim != 0 + dim = maybe_wrap_dim(dim, ndim) + start_val = start if start is not None else 0 + end_val = end if end is not None else max_int() + assert step > 0 + if start_val == max_int(): + start_val = 0 + if start_val < 0: + start_val += self[dim] + if end_val < 0: + end_val += self[dim] + if start_val < 0: + start_val = 0 + elif start_val > self[dim]: + start_val = self[dim] + if end_val < start_val: + end_val = start_val + elif end_val >= self[dim]: + end_val = self[dim] + slice_len = end_val - start_val + out = _copy(self) + out[dim] = (slice_len + step - 1) // step + return out + + +def check_cat_no_zero_dim(tensors: list[list[int]]): + for tensor in tensors: + assert len(tensor) > 0 + + +def legacy_cat_wrap_dim(dim: int, tensor_sizes: list[list[int]]): + out_dim: Optional[int] = None + for size in tensor_sizes: + if not (len(size) == 1 and size[0] == 0): + if out_dim is None: + out_dim = maybe_wrap_dim(dim, len(size)) + if out_dim is None: + out_dim = dim + return out_dim + + +def should_skip(tensor: list[int]): + return numel(tensor) == 0 and len(tensor) == 1 + + +def check_cat_shape_except_dim( + first: list[int], second: list[int], dimension: int, index: int +): + first_dims = len(first) + second_dims = len(second) + assert first_dims == second_dims, "Tensors must have same number of dimensions" + for dim in range(0, first_dims): + if dim != dimension: + assert first[dim] == second[dim], ( + "Sizes of tensors must match except in dimension" + ) + + +def cat(tensors: list[list[int]], dim: int): + check_cat_no_zero_dim(tensors) + dim = legacy_cat_wrap_dim(dim, tensors) + assert len(tensors) > 0 + not_skipped_tensor: Optional[list[int]] = None + for tensor in tensors: + if not should_skip(tensor): + not_skipped_tensor = tensor + if not_skipped_tensor is None: + return [0] + + cat_dim_size = 0 + + for i in range(len(tensors)): + tensor = tensors[i] + if not should_skip(tensor): + check_cat_shape_except_dim(not_skipped_tensor, tensor, dim, i) + cat_dim_size = cat_dim_size + tensor[dim] + + result_size = _copy(not_skipped_tensor) + result_size[dim] = cat_dim_size + return result_size + + +def stack(tensors: list[list[int]], dim: int): + unsqueezed_tensors: list[list[int]] = [] + for tensor in tensors: + unsqueezed = unsqueeze(tensor, dim) + unsqueezed_tensors.append(unsqueezed) + return cat(unsqueezed_tensors, dim) + + +def select(self: list[int], dim: int, index: int): + ndim = len(self) + assert ndim != 0 + dim = maybe_wrap_dim(dim, ndim) + size = self[dim] + assert not (index < -size or index >= size) + if index < 0: + index += size + out: list[int] = [] + for i in range(ndim): + if i != dim: + out.append(self[i]) + return out + + +def matmul(tensor1: list[int], tensor2: list[int]): + dim_tensor1 = len(tensor1) + dim_tensor2 = len(tensor2) + if dim_tensor1 == 1 and dim_tensor2 == 1: + return dot(tensor1, tensor2) + elif dim_tensor1 == 2 and dim_tensor2 == 1: + return mv(tensor1, tensor2) + elif dim_tensor1 == 1 and dim_tensor2 == 2: + return squeeze(mm(unsqueeze(tensor1, 0), tensor2), 0) + elif dim_tensor1 == 2 and dim_tensor2 == 2: + return mm(tensor1, tensor2) + elif dim_tensor1 >= 1 and dim_tensor2 >= 1: + # We are multiplying b1 x n x m1 by x2 x m2 x p (where b1 can be a list); + # we track m1 vs m2 separately even though they must match for nicer error messages + n = tensor1[-2] if dim_tensor1 > 1 else 1 + batch_tensor1: list[int] = [] + # TODO: handling of slice + for i in range(dim_tensor1 - 2): + batch_tensor1.append(tensor1[i]) + p = tensor2[-1] + batch_tensor2: list[int] = [] + # TODO: handling of slice + for i in range(dim_tensor2 - 2): + batch_tensor2.append(tensor2[i]) + + # expand the batch portion (i.e. cut off matrix dimensions and expand rest) + expand_batch_portion = broadcast(batch_tensor1, batch_tensor2) + + # todo: copy ? + output_shape = expand_batch_portion + if dim_tensor1 > 1: + output_shape.append(n) + + if dim_tensor2 > 1: + output_shape.append(p) + + return output_shape + else: + assert False, "both arguments to matmul need to be at least 1D" + + +def t(self: list[int]): + assert len(self) <= 2 + self_len = len(self) + if self_len == 0: + out: list[int] = [] + return out + elif self_len == 1: + return [self[0]] + else: + return [self[1], self[0]] + + +def transpose(self: list[int], dim0: int, dim1: int): + ndims = len(self) + dim0 = maybe_wrap_dim(dim0, ndims) + dim1 = maybe_wrap_dim(dim1, ndims) + if dim0 == dim1: + return _copy(self) + out: list[int] = [] + for i in range(ndims): + if i == dim0: + out.append(self[dim1]) + elif i == dim1: + out.append(self[dim0]) + else: + out.append(self[i]) + return out + + +def linear(input: list[int], weight: list[int], bias: Optional[list[int]]): + out = matmul(input, t(weight)) + if bias is not None: + assert broadcast(bias, out) == out + return out + + +def addmm(self: list[int], mat1: list[int], mat2: list[int], beta: Any, alpha: Any): + return broadcast(self, mm(mat1, mat2)) + + +def check_non_negative(array: list[int]) -> bool: + # TODO: look into rewriting with early return and getting loop unrolling to fire + non_negative = False + for val in array: + if val < 0: + non_negative = True + return non_negative + + +def check_shape_forward( + input: list[int], + weight_sizes: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + k = len(input) + weight_dim = len(weight_sizes) + + # TODO: assertions could be expanded with the error messages + assert not check_non_negative(padding) + assert not check_non_negative(stride) + + assert weight_dim == k + assert weight_sizes[0] >= groups + assert (weight_sizes[0] % groups) == 0 + # only handling not transposed + assert input[1] == weight_sizes[1] * groups + assert bias is None or (len(bias) == 1 and bias[0] == weight_sizes[0]) + + for i in range(2, k): + assert (input[i] + 2 * padding[i - 2]) >= ( + dilation[i - 2] * (weight_sizes[i] - 1) + 1 + ) + + # this is not handling transposed convolution yet + + +def conv_output_size( + input_size: list[int], + weight_size: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + check_shape_forward( + input_size, weight_size, bias, stride, padding, dilation, groups + ) + + has_dilation = len(dilation) > 0 + dim = len(input_size) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 0 + output_size.append(input_size[input_batch_size_dim]) + output_size.append(weight_size[weight_output_channels_dim]) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + kernel = dilation_ * (weight_size[d] - 1) + 1 + output_size.append( + (input_size[d] + (2 * padding[d - 2]) - kernel) // stride[d - 2] + 1 + ) + return output_size + + +def conv1d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 3 + assert len(input) == 3 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def conv2d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 4 + assert len(input) == 4 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def conv_backwards( + grad_output: list[int], + input: list[int], + weight: list[int], + biases: Optional[list[int]], +): + # Bias gradient is always generated regardess of if biases is supplied + return _copy(input), _copy(weight), [grad_output[1]] + + +def conv_transpose2d_input( + input: list[int], + weight: list[int], + bias: Optional[list[int]] = None, + stride: Optional[list[int]] = None, + padding: Optional[list[int]] = None, + output_padding: Optional[list[int]] = None, + groups: int = 1, + dilation: Optional[list[int]] = None, +) -> list[int]: + if stride is None: + stride = [1, 1] + if padding is None: + padding = [0, 0] + if output_padding is None: + output_padding = [0, 0] + if dilation is None: + dilation = [1, 1] + has_dilation = len(dilation) > 0 + dim = len(input) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 1 + output_size.append(input[input_batch_size_dim]) + output_size.append(weight[weight_output_channels_dim] * groups) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + kernel = dilation_ * (weight[d] - 1) + output_size.append( + (input[d] - 1) * stride[d - 2] + - 2 * padding[d - 2] + + kernel + + output_padding[d - 2] + + 1 + ) + return output_size + + +def conv_forwards( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + transposed: bool, + output_padding: list[int], + groups: int, +) -> list[int]: + has_dilation = len(dilation) > 0 + has_output_padding = len(output_padding) > 0 + dim = len(input) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 1 if transposed else 0 + output_size.append(input[input_batch_size_dim]) + if transposed: + output_size.append(weight[weight_output_channels_dim] * groups) + else: + output_size.append(weight[weight_output_channels_dim]) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + output_padding_ = output_padding[d - 2] if has_output_padding else 0 + if transposed: + kernel = dilation_ * (weight[d] - 1) + output_size.append( + (input[d] - 1) * stride[d - 2] + - 2 * padding[d - 2] + + kernel + + output_padding_ + + 1 + ) + else: + kernel = dilation_ * (weight[d] - 1) + 1 + output_size.append( + (input[d] + (2 * padding[d - 2]) - kernel) // stride[d - 2] + 1 + ) + return output_size + + +def _conv_forwards( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + transposed: bool, + output_padding: list[int], + groups: int, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, +) -> list[int]: + return conv_forwards( + input, + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + + +def batch_norm( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], + training: bool, + momentum: float, + eps: float, + cudnn_enabled: bool, +): + out: list[int] = [] + for elem in input: + out.append(elem) + return out + + +def conv3d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 5 + assert len(input) == 5 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def maybe_wrap_dim(dim: int, dim_post_expr: int, wrap_scalar: bool = True): + if dim_post_expr <= 0: + assert wrap_scalar + dim_post_expr = 1 + min = -dim_post_expr + max = dim_post_expr - 1 + assert not (dim < min or dim > max) + if dim < 0: + dim += dim_post_expr + return dim + + +def zero_dim_tensor(input: Any): + out: list[int] = [] + return out + + +def multiply_integers(li: list[int]): + out = 1 + for elem in li: + out = out * elem + return out + + +def arange_end(end: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any): + assert end >= 0 + return [int(math.ceil(end))] + + +def arange_start( + start: number, end: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any +): + assert end >= 0 + assert end >= start + return [int(math.ceil(end - start))] + + +def arange_start_step( + start: number, end: number, step: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any +): + assert step != 0 + if step < 0: + assert start >= end + else: + assert end >= start + return [int(math.ceil((end - start) / step))] + + +def permute(input: list[int], dims: list[int]): + assert len(input) == len(dims) + ndim = len(dims) + seen_dims: list[int] = [] + newSizes: list[int] = [] + for i in range(ndim): + dim = maybe_wrap_dim(dims[i], ndim) + seen_dims.append(dim) + newSizes.append(input[dim]) + for i in range(1, ndim): + for j in range(i): + assert seen_dims[i] != seen_dims[j] + return newSizes + + +def movedim(self: list[int], source: list[int], destination: list[int]) -> list[int]: + self_dim = len(self) + if self_dim <= 1: + return self + normalized_src: list[int] = [] + normalized_dst: list[int] = [] + for i in range(len(source)): + normalized_src.append(maybe_wrap_dim(source[i], self_dim)) + normalized_dst.append(maybe_wrap_dim(destination[i], self_dim)) + order = [-1 for i in range(self_dim)] + src_dims = [i for i in range(self_dim)] + dst_dims = [i for i in range(self_dim)] + + for i in range(len(source)): + order[normalized_dst[i]] = normalized_src[i] + src_dims[normalized_src[i]] = -1 + dst_dims[normalized_dst[i]] = -1 + + source_dims: list[int] = [] + destination_dims: list[int] = [] + for ele in src_dims: + if ele != -1: + source_dims.append(ele) + for ele in dst_dims: + if ele != -1: + destination_dims.append(ele) + + rest_dim = self_dim - len(source) + for i in range(rest_dim): + order[destination_dims[i]] = source_dims[i] + return permute(self, order) + + +def flatten(input: list[int], start_dim: int, end_dim: int): + start_dim = maybe_wrap_dim(start_dim, len(input)) + end_dim = maybe_wrap_dim(end_dim, len(input)) + assert start_dim <= end_dim + if len(input) == 0: + return [1] + if start_dim == end_dim: + # TODO: return self + out: list[int] = [] + for elem in input: + out.append(elem) + return out + slice_numel = 1 + for i in range(start_dim, end_dim + 1): + slice_numel *= input[i] + # TODO: use slicing when slice optimization has landed + # slice_numel = multiply_integers(input[start_dim:end_dim - start_dim + 1]) + shape: list[int] = [] + for i in range(start_dim): + shape.append(input[i]) + shape.append(slice_numel) + for i in range(end_dim + 1, len(input)): + shape.append(input[i]) + return shape + + +def nonzero_lower_bound(input: list[int]): + return [0, len(input)] + + +def nonzero_upper_bound(input: list[int]): + return [numel(input), len(input)] + + +def _reduce_along_dim(self: list[int], dim: int, keepdim: bool): + dim = maybe_wrap_dim(dim, len(self)) + out: list[int] = [] + for i, self_dim in enumerate(self): + if i == dim: + if keepdim: + out.append(1) + else: + out.append(self_dim) + return out + + +def argmax( + self: list[int], dim: Optional[int] = None, keepdim: bool = False +) -> list[int]: + if dim is None: + return [] + return _reduce_along_dim(self, dim, keepdim) + + +def bmm(self: list[int], mat2: list[int]) -> list[int]: + assert len(self) == 3, "bmm only supports 3D tensors" + assert len(mat2) == 3, "bmm only supports 3D tensors" + assert self[0] == mat2[0], "mismatching batch dimension" + assert self[2] == mat2[1], "mismatching contracting dimension" + return [self[0], self[1], mat2[2]] + + +def _shape_as_tensor(self: list[int]) -> list[int]: + return [len(self)] + + +def topk(self: list[int], k: int, dim: int = -1) -> tuple[list[int], list[int]]: + if len(self) == 0: + result: list[int] = [] + else: + assert k <= self[dim], ( + f"k ({k}) is too big for dimension {dim} of size {self[dim]}" + ) + result = _copy(self) + result[dim] = k + return result, result + + +def nll_loss_forward( + self: list[int], target: list[int], weight: Optional[list[int]], reduction: int +) -> tuple[list[int], list[int]]: + # This is taken shamelessly from the meta function in LossNLL.cpp + self_dim = len(self) + target_dim = len(target) + assert 0 < self_dim <= 2 + assert target_dim <= 1 + no_batch_dim = self_dim == 1 and target_dim == 0 + assert no_batch_dim or (self[0] == target[0]) + n_classes = self[-1] + scalar_shape: list[int] = [] + assert weight is None or (len(weight) == 1 and weight[0] == n_classes) + if reduction == 0 and self_dim == 2: + reduction_shape = [self[0]] + else: + reduction_shape = scalar_shape + return reduction_shape, scalar_shape + + +def native_layer_norm( + input: list[int], normalized_shape: list[int] +) -> tuple[list[int], list[int], list[int]]: + reduction_shape: list[int] = [] + num_unreduced_dimensions = len(input) - len(normalized_shape) + assert num_unreduced_dimensions >= 0 + for i in range(num_unreduced_dimensions): + reduction_shape.append(input[i]) + for i in range(num_unreduced_dimensions, len(input)): + reduction_shape.append(1) + return _copy(input), reduction_shape, reduction_shape + + +def native_batch_norm( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], + training: bool, +) -> tuple[list[int], list[int], list[int]]: + if training: + _size = [input[1]] + else: + _size = [0] + return _copy(input), _size, _size + + +def _batch_norm_with_update( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], +) -> tuple[list[int], list[int], list[int], list[int]]: + _size = [input[1]] + return _copy(input), _size, _size, [0] + + +def cross_entropy_loss( + self: list[int], + target: list[int], + weight: Optional[list[int]] = None, + reduction: int = 1, + ignore_index: int = -100, + label_smoothing: float = 0.0, +) -> list[int]: + result_shape = nll_loss_forward(self, target, weight, reduction)[0] + return result_shape + + +""" +Currently deferring the enabling of this, as part of the propoasal to suspend +adding ops. +There are currently cases in the test case where this is being called +in the SSA opinfo tests with with unexpected values (eg list of two ints, see the first +opinfo test). The behavior of index is significantly dependent on the inputs. + +This could be an error with how we are matching up shape functions, or that this +function needs to just implement everything. + +def index_Tensor(self: List[int], indices: List[Optional[List[int]]]) -> List[int]: + assert len(indices) <= len(self), "More indices than dimensions to index" + broadcasted_shape: List[int] = [] + for index_tensor_shape in indices: + if index_tensor_shape is not None: + broadcasted_shape = broadcast(broadcasted_shape, index_tensor_shape) + return broadcasted_shape +""" + +ScriptFn = torch._C.ScriptFunction +shape_compute_graph_mapping: dict[str, ScriptFn] = {} +bounded_compute_graph_mapping: dict[str, tuple[ScriptFn, ScriptFn]] = {} +script_func_map: dict[Callable, ScriptFn] = {} + + +def process_func(func: Callable): + if func not in script_func_map: + scripted_func = torch.jit.script(func) + + torch._C._jit_pass_inline(scripted_func.graph) + + for _ in range(2): + torch._C._jit_pass_peephole(scripted_func.graph) + torch._C._jit_pass_constant_propagation(scripted_func.graph) + + script_func_map[func] = scripted_func + return script_func_map[func] + + +def add_shape_compute_mapping(operator_schema: str, func: Callable): + global shape_compute_graph_mapping + + shape_compute_graph_mapping[operator_schema] = process_func(func) + + +def add_bounded_compute_mapping( + operator_schema: str, lower_bound_func: Callable, upper_bound_func: Callable +): + # Adds a shape compute function for both upper and lower bounds + fns = (process_func(lower_bound_func), process_func(upper_bound_func)) + bounded_compute_graph_mapping[operator_schema] = fns + + +add_shape_compute_mapping( + "aten::contiguous(Tensor(a) self, *, MemoryFormat memory_format=contiguous_format) -> Tensor(a)", + unary, +) +add_shape_compute_mapping( + "aten::rsub.Tensor(Tensor self, Scalar other, Scalar alpha=1) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::dropout(Tensor input, float p, bool train) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::adaptive_avg_pool2d(Tensor self, int[2] output_size) -> Tensor", + adaptive_avg_pool2d, +) +add_shape_compute_mapping( + "prim::NumToTensor.Scalar(Scalar a) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping("prim::NumToTensor.bool(bool a) -> Tensor", zero_dim_tensor) +add_shape_compute_mapping( + "aten::zeros(int[] size, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None) -> (Tensor)", + unary, +) +add_shape_compute_mapping( + "aten::to.dtype(Tensor(a) self, int dtype, bool non_blocking=False, bool copy=False, int? memory_format=None) -> (Tensor(a))", + unary, +) +add_shape_compute_mapping( + "aten::arange(Scalar end, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None) -> (Tensor)", + arange_end, +) +add_shape_compute_mapping( + "aten::arange.start(Scalar start, Scalar end, *, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor", + arange_start, +) +add_shape_compute_mapping( + "aten::arange.start_step(Scalar start, Scalar end, Scalar step, *, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor", + arange_start_step, +) +add_shape_compute_mapping("aten::squeeze(Tensor(a) self) -> Tensor(a)", squeeze_nodim) +add_shape_compute_mapping( + "aten::squeeze.dim(Tensor(a) self, int dim) -> Tensor(a)", squeeze +) +add_shape_compute_mapping( + "aten::squeeze.dims(Tensor(a) self, int[] dim) -> Tensor(a)", squeeze_dims +) +add_shape_compute_mapping( + "aten::unsqueeze(Tensor(a) self, int dim) -> Tensor(a)", unsqueeze +) +add_shape_compute_mapping( + "aten::slice.Tensor(Tensor(a) self, int dim=0, int? start=None, int? end=None, int step=1) -> Tensor(a)", + slice, +) +add_shape_compute_mapping( + "aten::select.int(Tensor(a) self, int dim, int index) -> Tensor(a)", select +) +add_shape_compute_mapping( + "aten::index_select(Tensor self, int dim, Tensor index) -> Tensor", index_select +) +add_shape_compute_mapping( + "aten::layer_norm(Tensor input, int[] normalized_shape, Tensor? weight=None, Tensor? bias=None, " + "float eps=1e-05, bool cudnn_enable=True) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::softmax.int(Tensor self, int dim, ScalarType? dtype=None) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::_no_grad_embedding_renorm_(Tensor weight, Tensor input, float max_norm, float norm_type) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::embedding_renorm_(Tensor(a!) self, Tensor indices, float max_norm, float norm_type) -> Tensor(a!)", + unary, +) +add_shape_compute_mapping( + "aten::embedding(Tensor weight, Tensor indices, int padding_idx=-1, bool scale_grad_by_freq=False, bool sparse=False) -> Tensor", + embedding, +) +add_shape_compute_mapping("aten::mm(Tensor self, Tensor mat2) -> Tensor", mm) +add_shape_compute_mapping("aten::dot(Tensor self, Tensor tensor) -> Tensor", dot) +add_shape_compute_mapping("aten::mv(Tensor self, Tensor vec) -> Tensor", mv) +add_shape_compute_mapping("aten::matmul(Tensor self, Tensor other) -> Tensor", matmul) +add_shape_compute_mapping( + "aten::linear(Tensor input, Tensor weight, Tensor? bias=None) -> Tensor", linear +) +add_shape_compute_mapping( + "aten::max_pool2d(Tensor self, int[2] kernel_size, int[2] stride=[], int[2] padding=0, int[2] dilation=1, bool ceil_mode=False) -> Tensor", + max_pool2d, +) +add_shape_compute_mapping( + "aten::max_pool2d_with_indices(Tensor self, int[2] kernel_size, int[2] stride=[], int[2] padding=0, int[2] dilation=1, bool ceil_mode=False) -> (Tensor, Tensor)", + max_pool2d_with_indices, +) +add_shape_compute_mapping("aten::t(Tensor(a) self) -> Tensor(a)", t) +add_shape_compute_mapping( + "aten::transpose.int(Tensor(a) self, int dim0, int dim1) -> Tensor(a)", transpose +) +add_shape_compute_mapping( + "aten::conv1d(Tensor input, Tensor weight, Tensor? bias=None, int[1] stride=1, int[1] padding=0, int[1] dilation=1, int groups=1) -> Tensor", + conv1d, +) +add_shape_compute_mapping( + "aten::conv2d(Tensor input, Tensor weight, Tensor? bias=None, int[2] stride=1, int[2] padding=0, int[2] dilation=1, int groups=1) -> Tensor", + conv2d, +) +add_shape_compute_mapping( + "aten::batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool training, float momentum, float eps, bool cudnn_enabled) -> Tensor", + batch_norm, +) +add_shape_compute_mapping( + "aten::conv3d(Tensor input, Tensor weight, Tensor? bias=None, int[3] stride=1, int[3] padding=0, int[3] dilation=1, int groups=1) -> Tensor", + conv3d, +) +add_shape_compute_mapping( + "aten::convolution_backward(Tensor grad_output, Tensor input, Tensor weight, int[]? bias_sizes, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups, bool[3] output_mask) -> (Tensor, Tensor, Tensor)", + conv_backwards, +) +add_shape_compute_mapping( + "aten::convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups) -> Tensor", + conv_forwards, +) +add_shape_compute_mapping( + "aten::_convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups, bool benchmark, bool deterministic, bool cudnn_enabled, bool allow_tf32) -> Tensor", + _conv_forwards, +) +add_shape_compute_mapping( + "aten::conv_transpose2d.input(Tensor input, Tensor weight, Tensor? bias=None, int[2] stride=1, int[2] padding=0, int[2] output_padding=0, int groups=1, int[2] dilation=1) -> Tensor", + conv_transpose2d_input, +) +add_shape_compute_mapping( + "aten::flatten.using_ints(Tensor(a) self, int start_dim=0, int end_dim=-1) -> Tensor(a)", + flatten, +) +add_shape_compute_mapping("aten::cat(Tensor[] tensors, int dim=0) -> Tensor", cat) +add_shape_compute_mapping("aten::stack(Tensor[] tensors, int dim=0) -> Tensor", stack) +add_shape_compute_mapping( + "aten::permute(Tensor(a) self, int[] dims) -> Tensor(a)", permute +) +add_shape_compute_mapping( + "aten::movedim.intlist(Tensor(a) self, int[] source, int[] destination) -> Tensor(a)", + movedim, +) +add_shape_compute_mapping("aten::view(Tensor(a) self, int[] size) -> Tensor(a)", view) +add_shape_compute_mapping( + "aten::expand_as(Tensor(a) self, Tensor other) -> Tensor(a)", expand +) +add_shape_compute_mapping( + "aten::expand(Tensor(a) self, int[] size, *, bool implicit=False) -> Tensor(a)", + expand_one_unused, +) +add_shape_compute_mapping( + "aten::mean.dim(Tensor self, int[1]? dim, bool keepdim=False, *, ScalarType? dtype=None) -> Tensor", + sum_mean_dim, +) +add_shape_compute_mapping( + "aten::sum.dim_IntList(Tensor self, int[1]? dim, bool keepdim=False, *, ScalarType? dtype=None) -> Tensor", + sum_mean_dim, +) +add_shape_compute_mapping( + "aten::max.dim(Tensor self, int dim, bool keepdim=False) -> (Tensor values, Tensor indices)", + max_dim, +) +add_shape_compute_mapping( + "aten::mean(Tensor self, *, ScalarType? dtype=None) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping( + "aten::sum(Tensor self, *, ScalarType? dtype=None) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping( + "aten::addmm(Tensor self, Tensor mat1, Tensor mat2, *, Scalar beta=1, Scalar alpha=1) -> Tensor", + addmm, +) +add_shape_compute_mapping( + "aten::upsample_nearest2d.vec(Tensor input, int[]? output_size, float[]? scale_factors) -> (Tensor)", + upsample_nearest2d, +) +add_shape_compute_mapping( + "aten::quantize_per_tensor(Tensor self, float scale, int zero_point, ScalarType dtype) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::quantize_per_tensor.tensor_qparams(Tensor self, Tensor scale, Tensor zero_point, ScalarType dtype) -> Tensor", + unary, +) +add_shape_compute_mapping("aten::dequantize(Tensor self) -> Tensor", unary) +add_shape_compute_mapping( + "quantized::add(Tensor qa, Tensor qb, float scale, int zero_point) -> Tensor qc", + broadcast, +) +add_shape_compute_mapping( + "aten::argmax(Tensor self, int? dim=None, bool keepdim=False) -> Tensor", argmax +) +add_shape_compute_mapping("aten::bmm(Tensor self, Tensor mat2) -> Tensor", bmm) +add_shape_compute_mapping( + "aten::_shape_as_tensor(Tensor self) -> Tensor", _shape_as_tensor +) +add_shape_compute_mapping( + "aten::topk(Tensor self, int k, int dim=-1, bool largest=True, bool sorted=True) -> (Tensor values, Tensor indices)", + topk, +) +add_shape_compute_mapping( + "aten::nll_loss_forward(Tensor self, Tensor target, Tensor? weight, int reduction, int ignore_index) -> (Tensor output, Tensor total_weight)", + nll_loss_forward, +) +add_shape_compute_mapping( + "aten::native_layer_norm(Tensor input, int[] normalized_shape, Tensor? weight, Tensor? bias, float eps) -> (Tensor, Tensor, Tensor)", + native_layer_norm, +) +add_shape_compute_mapping( + "aten::native_batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "aten::_native_batch_norm_legit(Tensor input, Tensor? weight, Tensor? bias, Tensor running_mean, Tensor running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "aten::_native_batch_norm_legit.no_stats(Tensor input, Tensor? weight, Tensor? bias, Tensor running_mean, Tensor running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "_batch_norm_with_update(Tensor input, Tensor? weight, Tensor? bias, Tensor(a!) running_mean, Tensor(b!) running_var, float momentum, float eps) -> (Tensor, Tensor, Tensor, Tensor)", + _batch_norm_with_update, +) + +add_shape_compute_mapping( + "aten::cross_entropy_loss(Tensor self, Tensor target, Tensor? weight=None, int reduction=Mean, SymInt ignore_index=-100, float label_smoothing=0.0) -> Tensor", + cross_entropy_loss, +) +# add_shape_compute_mapping("aten::index.Tensor(Tensor self, Tensor?[] indices) -> Tensor", index_Tensor) + +# TODO: migrate over all of symbolic_shape_registry_util.cpp +# These are duplicated here so that the functions will be serialized +add_shape_compute_mapping( + "aten::lerp.Tensor(Tensor self, Tensor end, Tensor weight) -> Tensor", + broadcast_three, +) +add_shape_compute_mapping( + "aten::where.ScalarSelf(Tensor condition, Scalar self, Tensor other) -> Tensor", + broadcast_one_three, +) +add_shape_compute_mapping( + "aten::add_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> Tensor(a!)", + broadcast_inplace, +) + +# quantized_conv_prepack TODO + +# Shape Compute Fn with upper and lower bounds +add_bounded_compute_mapping( + "aten::nonzero(Tensor self) -> (Tensor)", nonzero_lower_bound, nonzero_upper_bound +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_state.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_state.py new file mode 100644 index 0000000000000000000000000000000000000000..f72b7a517e1dcbf5309fa2e08ab37d808cd83da7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_state.py @@ -0,0 +1,128 @@ +# mypy: allow-untyped-defs +"""JIT-related state. + +This module stores various pieces of Python-global state relating to the JIT. + +This is not intended to be imported directly; please the exposed +functionalities in `torch.jit`. +""" + +import os +import weakref +from typing import Any + +import torch + + +class EnabledProxy: + """Stores whether the JIT is enabled or not. + + This is just a wrapper for a bool, so that we get reference semantics + """ + + def __init__(self) -> None: + self.enabled = self.parse_env( + "PYTORCH_JIT", True, "> Using PyTorch JIT", "> PyTorch JIT DISABLED" + ) + + def parse_env(self, name, default, true_message, false_message): + value = os.environ.get(name) + if value is None: + return default + if value.lower() in {"1", "true", "yes"}: + return True + elif value.lower() in {"0", "false", "no"}: + return False + if value == "1v": + print(true_message) + return True + elif value == "0v": + print(false_message) + return False + raise ValueError(f"Unknown setting of {name}. Try using 0 or 1.") + + def __bool__(self) -> bool: + return self.enabled + + +_enabled = EnabledProxy() + + +def disable() -> None: + _enabled.enabled = False + + +def enable() -> None: + _enabled.enabled = True + + +# The Python CompilationUnit. All functions and modules defined in Python will +# live in here. It's defined in Python because doing in cpp creates static +# destruction order issues. +_python_cu = torch._C.CompilationUnit() + + +# python class => ScriptClass mapping +_script_classes: dict[type[Any], type[Any]] = {} +_name_to_pyclass: dict[str, type[Any]] = {} + + +def _add_script_class(python_class, script_class) -> None: + _script_classes[python_class] = script_class + _name_to_pyclass[script_class.qualified_name()] = python_class + + +def _get_script_class(python_class): + override = getattr(python_class, "_jit_override_qualname", None) + if override is not None: + python_class = _get_python_class(override) + return _script_classes.get(python_class) + + +def _get_python_class(qualified_name): + return _name_to_pyclass.get(qualified_name) + + +def _clear_class_state() -> None: + _script_classes.clear() + _name_to_pyclass.clear() + + +# Caching: we currently cache compilation of free functions and overloaded functions. +# To cache free functions we hold a weak ref to the function object and +# map to the compiled fn's qualified name. +# To cache overloaded functions we hold a weak ref to the function obj and +# map to all of its overloaded compiled fns. +# In the future we could consider caching more types of objects so that +# aliasing is preserved across separate compilations of the same object. + +_jit_caching_layer: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() +_jit_function_overload_caching: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + + +def _try_get_jit_cached_overloads(key): + qual_names = _jit_function_overload_caching.get(key, None) + if qual_names: + return [_python_cu.find_function(qual_name) for qual_name in qual_names] + else: + return None + + +def _set_jit_overload_cache(key, compiled_fns) -> None: + _jit_function_overload_caching[key] = [fn.qualified_name for fn in compiled_fns] + + +def _try_get_jit_cached_function(key): + if getattr(key, "__disable_jit_function_caching__", False) is True: + return None + qual_name = _jit_caching_layer.get(key, None) + if qual_name: + return _python_cu.find_function(qual_name) + else: + return None + + +def _set_jit_function_cache(key, value) -> None: + # only free functions currently supported + assert isinstance(value, torch.jit.ScriptFunction) + _jit_caching_layer[key] = value.qualified_name diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_trace.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..4533602b1f5294124ccb43f351f6e1692cfb5abb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/_trace.py @@ -0,0 +1,1435 @@ +# mypy: allow-untyped-defs +"""Tracing. + +This module contains functionality to support the JIT's tracing frontend, notably: + * torch.jit.trace + * torch.jit.trace_module + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import contextlib +import copy +import functools +import inspect +import os +import re +import sys +import warnings +from collections.abc import Callable +from enum import Enum +from typing import Any, Optional, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch._jit_internal import ( + _get_model_id, + _qualified_name, + get_callable_argument_names, + is_scripting, +) +from torch.autograd import function +from torch.jit._script import _CachedForward, script, ScriptModule +from torch.jit._state import _enabled, _python_cu +from torch.nn import Module +from torch.testing._comparison import default_tolerances + + +_flatten = torch._C._jit_flatten +_unflatten = torch._C._jit_unflatten + +R = TypeVar("R", covariant=True) # return type (always covariant) +P = ParamSpec("P") + + +def _create_interpreter_name_lookup_fn(frames_up=1): + def _get_interpreter_name_for_var(var): + frame = inspect.currentframe() + if not frame: + raise RuntimeError("failed to inspect frame") + + i = 0 + while i < frames_up + 1: + frame = frame.f_back + if not frame: + raise RuntimeError("failed to get frame") + i += 1 + + f_locals = frame.f_locals + + for k, v in f_locals.items(): + if isinstance(v, torch.Tensor) and var is v: + return k if k != "self" else "" + return "" + + return _get_interpreter_name_for_var + + +def _unique_state_dict(module, keep_vars=False): + # since Parameter.detach() always creates a new torch.Tensor instance, + # id(v) doesn't work with it. So we always get the Parameter or Buffer + # as values, and deduplicate the params using Parameters and Buffers + state_dict = module.state_dict(keep_vars=True) + filtered_dict = type(state_dict)() + seen_ids: set[int] = set() + for k, v in state_dict.items(): + if id(v) in seen_ids: + continue + seen_ids.add(id(v)) + if keep_vars: + filtered_dict[k] = v + else: + filtered_dict[k] = v.detach() + return filtered_dict + + +class ONNXTracedModule(torch.nn.Module): + def __init__( + self, + inner, + strict=True, + force_outplace=False, + return_inputs=False, + return_inputs_states=False, + ): + super().__init__() + # inner may be a Module, or it may be an arbitrary callable + # If it's a Module, we get its parameters automatically, which lets + # us avoid a special casing functions versus modules. + self.inner = inner + self.strict = strict + self._force_outplace = force_outplace + self._return_inputs = return_inputs + self._return_inputs_states = return_inputs_states + + def forward(self, *args: torch.Tensor): + in_vars, in_desc = _flatten(args) + # NOTE: use full state, because we need it for BatchNorm export + # This differs from the compiler path, which doesn't support it at the moment. + module_state = list(_unique_state_dict(self, keep_vars=True).values()) + + ret_inputs = [] + inputs_states = [] + outs = [] + + def wrapper(*args): + in_args: list[torch.Tensor] = [] + for i in range(len(in_vars)): + if not isinstance(args[i], torch.Tensor): + raise RuntimeError("Expected Tensor argument") + in_args.append(args[i]) + + trace_inputs = _unflatten(in_args, in_desc) + + if self._return_inputs: + ret_inputs.append( + tuple(x.clone(memory_format=torch.preserve_format) for x in args) + ) + if self._return_inputs_states: + inputs_states.append(_unflatten(in_args, in_desc)) + outs.append(self.inner(*trace_inputs)) + if self._return_inputs_states: + inputs_states[0] = (inputs_states[0], trace_inputs) + out_vars, _ = _flatten(outs) + if len(out_vars) == 1: + return out_vars[0] + else: + return tuple(out_vars) + + graph, _out = torch._C._create_graph_by_tracing( + wrapper, + in_vars + module_state, + _create_interpreter_name_lookup_fn(), + self.strict, + self._force_outplace, + ) + + if self._return_inputs: + return graph, outs[0], ret_inputs[0] + if self._return_inputs_states: + return graph, outs[0], inputs_states[0] + else: + return graph, outs[0] + + +def _clone_inputs(args): + def clone_input(a): + if a is None: + return None + elif isinstance(a, torch.Tensor): + # TODO: figure out one liner to .clone() and set requires_grad + v = ( + a.detach() + .clone(memory_format=None if a.is_mkldnn else torch.preserve_format) + .requires_grad_(a.requires_grad) + ) + if a.grad is not None: + v.grad = clone_input(v.grad) + return v + else: + return a.clone(memory_format=torch.preserve_format) + + # pyrefly: ignore [missing-attribute] + return function._nested_map( + lambda x: isinstance(x, torch.Tensor), clone_input, condition_msg="tensors" + )(args) + + +# This is purely for developer debugging. We are not going to advertise it. +_JIT_TIME = os.environ.get("PYTORCH_JIT_TIME", False) # CUDA-only timing +_JIT_DISABLE = os.environ.get("PYTORCH_JIT_DISABLE", False) +_JIT_STATS = os.environ.get("PYTORCH_JIT_STATS", False) + + +@contextlib.contextmanager +def _time(trace_name, name, time=True): + if (not _JIT_TIME and not time) or not torch.cuda.is_available(): + yield + return + stream = torch.cuda.current_stream() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + stream.record_event(start) + try: + yield + finally: + stream.record_event(end) + end.synchronize() + print(f"{trace_name} {name} time: {start.elapsed_time(end)} ms") + + +def verify(model, args, loss_fn=torch.sum, devices=None): + """ + Verify that a JIT compiled model has the same behavior as its uncompiled version along with its backwards pass. + + If your model returns multiple outputs, + you must also specify a `loss_fn` to produce a loss for which + the backwards will be computed. + + This function has side-effects (e.g., it executes your model / saves and loads + parameters), so don't expect the model to come out exactly the same as what + you passed in. + + Args: + model (compiled torch.nn.Module or function): the module/function to be + verified. The module/function definition MUST have been decorated with + `@torch.jit.compile`. + args (tuple or Tensor): the positional arguments to pass to the + compiled function/module to be verified. A non-tuple is assumed to + be a single positional argument to be passed to the model. + loss_fn (function, optional): the loss function to be applied to + the output of the model, before backwards is invoked. By default, + we assume that a model returns a single result, and we :func:`torch.sum` + before calling backwards; if this is inappropriate, you can pass your + own loss function. Note that if a model returns a tuple of results, + these are passed as separate positional arguments to `loss_fn`. + devices (iterable of device IDs, optional): the GPU devices which the + compiled module will be run on. This determines the RNG state we + must save when running both compiled and uncompiled versions of the model. + """ + # TODO: In principle, we track device information in our trace, so it + # should be possible to check if our execution actually obeyed the 'devices' + # the user provided. + + # TODO: Consider adding a utility function to torch.jit to test + # for this case + if not isinstance(model, torch._C.CompiledFunction): # type: ignore[attr-defined] + raise TypeError( + "Cannot verify an uncompiled module. Add @torch.jit.compile to compile it" + ) + is_module = isinstance(model, Module) + + if not isinstance(args, tuple): + args = (args,) + + if is_module: + saved_state = copy.deepcopy(model.state_dict()) + + def run_fwd_bwd(args, force_trace=False, assert_compiled=False): + params = list(model.parameters()) if is_module else [] + in_vars, _ = _flatten((args, params)) + # We use a special API to reset the trace and compile it from scratch. + compiled_fn = model + if force_trace: + compiled_fn.clear_cache() + if assert_compiled: + hits = compiled_fn.hits + out = model(*args) + if assert_compiled and compiled_fn.hits == hits: # type: ignore[possibly-undefined] + raise RuntimeError("failed to use the compiled function") + if not isinstance(out, tuple): + out = (out,) + if loss_fn == torch.sum and len(out) != 1: + raise ValueError( + f"Model returns {len(out)} outputs, but default loss function " + "(torch.sum) can only handle a single output" + ) + out_vars, _ = _flatten(out) + saved_outs = [ + v.detach().clone(memory_format=torch.preserve_format) for v in out_vars + ] + loss = loss_fn(*out) + grads = torch.autograd.grad([loss], in_vars) + # TODO: I'm not sure if the clone here is necessary but it is safer + saved_grads = [ + v.detach().clone(memory_format=torch.preserve_format) for v in grads + ] + return (saved_outs, saved_grads) + + with torch.random.fork_rng(devices, _caller="torch.jit.verify"): + uncompiled_outs, uncompiled_grads = run_fwd_bwd(args, force_trace=True) + assert model.has_trace_for(*args) + + if is_module: + model.load_state_dict(saved_state) # type: ignore[possibly-undefined] + compiled_outs, compiled_grads = run_fwd_bwd(args, assert_compiled=True) + + _verify_equal(uncompiled_outs, compiled_outs) + _verify_equal(uncompiled_grads, compiled_grads) + + +def _verify_equal(xs, ys): + for x, y in zip(xs, ys): + if x.sub(y).abs().max() > 1e-6: + raise RuntimeError("JIT and real computation mismatch") + + +def indent(s): + return "\n".join(["\t" + line for line in s.splitlines()]) + + +class TracingCheckError(Exception): + def __init__(self, graph_diff_error, tensor_compare_error, extra_msg=None): + self.message = "Tracing failed sanity checks!\n" + if extra_msg is not None: + self.message += extra_msg + "\n" + if graph_diff_error is not None: + self.message += "ERROR: Graphs differed across invocations!\n" + self.message += indent(graph_diff_error) + "\n" + if tensor_compare_error is not None: + self.message += ( + "ERROR: Tensor-valued Constant nodes differed in value " + "across invocations. This often indicates that the tracer has" + " encountered untraceable code.\n" + ) + self.message += indent(tensor_compare_error) + "\n" + super().__init__(self.message) + + +# Check the traced module against a set of user-provided validation inputs +@torch.no_grad() +def _check_trace( + check_inputs, + func, + traced_func, + check_tolerance, + strict, + force_outplace, + is_trace_module, + _module_class, + example_inputs_is_kwarg=False, +): + # Note: tracing is independent of optimizations, which consume the trace + for inputs in check_inputs: + if isinstance(inputs, torch.Tensor): + inputs = (inputs,) + + if is_trace_module: + copied_dict = {} + # pyrefly: ignore [missing-attribute] + for name, data in inputs.items(): + copied_dict[name] = _clone_inputs(data) + check_mod = torch.jit.trace_module( + getattr(func, "__self__", func), + copied_dict, + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + _compilation_unit=torch._C.CompilationUnit(), + example_inputs_is_kwarg=example_inputs_is_kwarg, + _store_inputs=False, + ) + check_mod_func = check_mod._c._get_method(traced_func.name) + inputs = inputs[traced_func.name] + if ( + isinstance(inputs, (torch.Tensor)) + or isinstance(inputs, dict) + and not example_inputs_is_kwarg + ): + inputs = (inputs,) + else: + if example_inputs_is_kwarg: + check_mod = torch.jit.trace( + func, + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + example_kwarg_inputs=_clone_inputs(inputs), + _store_inputs=False, + ) + else: + check_mod = torch.jit.trace( + func, + _clone_inputs(inputs), + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + _store_inputs=False, + ) + check_mod_func = check_mod + + def graph_diagnostic_info(): + mod_canonicalized = torch._C._jit_pass_canonicalize(traced_func.graph) + torch._C._jit_pass_inline(mod_canonicalized) + torch._C._jit_pass_erase_shape_information(mod_canonicalized) + mod_str = str(mod_canonicalized) + mod_str = re.sub(r"___torch_mangle_[0-9]+\.", "", mod_str) + check_canonicalized = torch._C._jit_pass_canonicalize(check_mod_func.graph) + torch._C._jit_pass_inline(check_canonicalized) + torch._C._jit_pass_erase_shape_information(check_canonicalized) + check_str = str(check_canonicalized) + check_str = re.sub(r"___torch_mangle_[0-9]+\.", "", check_str) + + graph_diff_errors = None + if mod_str != check_str: + import difflib + + graph_diff = difflib.ndiff( + mod_str.splitlines(True), check_str.splitlines(True) + ) + graph_diff_errors = "Graph diff:\n" + indent("".join(graph_diff)) + "\n" + + for n_mod, n_check in zip( + mod_canonicalized.nodes(), check_canonicalized.nodes() + ): + if str(n_mod) != str(n_check): + graph_diff_errors += "First diverging operator:\n" + node_diff = difflib.ndiff( + str(n_mod).splitlines(True), str(n_check).splitlines(True) + ) + source_printout = ( + "Node diff:\n" + indent("".join(node_diff)) + "\n" + ) + mod_stack = n_mod.sourceRange() + if mod_stack: + source_printout += ( + "Trace source location:\n" + indent(mod_stack) + "\n" + ) + check_stack = n_check.sourceRange() + if check_stack: + source_printout += ( + "Check source location:\n" + indent(check_stack) + "\n" + ) + graph_diff_errors += source_printout + + break # For now, only print out the first pair of nodes that diverges + + tensor_compare_errors = None + # Check Tensor-valued constant nodes + for n_mod, n_check in zip( + mod_canonicalized.nodes(), check_canonicalized.nodes() + ): + if n_mod.kind() != n_check.kind(): + break # Graphs have already diverged + + if n_mod.kind() == "prim::Constant" and not ( + n_mod.mustBeNone() or n_check.mustBeNone() + ): + if not n_mod.hasAttribute("value"): + continue + if n_mod.kindOf("value") != "t" or n_check.kindOf("value") != "t": + continue + + mod_tensor_val = n_mod.t("value") + check_tensor_val = n_check.t("value") + + try: + torch.testing.assert_close( + mod_tensor_val, check_tensor_val, equal_nan=True + ) + except (RuntimeError, AssertionError) as e: + if tensor_compare_errors is None: + tensor_compare_errors = "" + tensor_compare_errors += "Node:\n" + indent(str(n_mod)) + "\n" + compare_stack = n_mod.sourceRange() + if compare_stack: + tensor_compare_errors += ( + "Source Location:\n" + indent(compare_stack) + "\n" + ) + tensor_compare_errors += "Comparison exception: " + indent( + str(e) + ) + + break # For now, only print the first diverging pair + + return graph_diff_errors, tensor_compare_errors + + def wrap_retval(x): + return x if isinstance(x, tuple) else (x,) + + def run_mod_and_filter_tensor_outputs(mod, inputs, running_what): + try: + if isinstance(inputs, dict) and example_inputs_is_kwarg: + outs = wrap_retval(mod(**inputs)) + else: + outs = wrap_retval(mod(*_clone_inputs(inputs))) + outs = [out for out in outs if isinstance(out, torch.Tensor)] + return outs + except Exception as e: + graph_diff_errors, tensor_compare_errors = graph_diagnostic_info() + msg = f"encountered an exception while running the {running_what} with test inputs.\nException:\n{indent(str(e))}" + raise TracingCheckError( + graph_diff_errors, + tensor_compare_errors, + extra_msg=msg, + ) from e + + has_warned = [False] + + def maybe_warn_nondeterministic(): + if has_warned[0]: + return + has_warned[0] = True + nondeterm_ops = [ + op for op in traced_func.graph.nodes() if op.isNondeterministic() + ] + if len(nondeterm_ops) > 0: + nondeterministic_ops_warning = "Trace had nondeterministic nodes. " + nondeterministic_ops_warning += ( + "Did you forget call .eval() on your model? Nodes:\n" + ) + nondeterministic_ops_warning += "\n".join( + [indent(str(op)) for op in nondeterm_ops][:20] + ) + nondeterministic_ops_warning += ( + "\nThis may cause errors in trace checking. To disable trace checking," + " pass check_trace=False to torch.jit.trace()" + ) + warnings.warn( + nondeterministic_ops_warning, category=TracerWarning, stacklevel=5 + ) + + def compare_outputs(original, reference, match_what): + all_ok = True + for i, (orig, ref) in enumerate(zip(original, reference)): + try: + if orig.is_quantized: + orig = orig.dequantize() + if ref.is_quantized: + ref = ref.dequantize() + if orig.is_mkldnn: + orig = orig.to_dense() + if ref.is_mkldnn: + ref = ref.to_dense() + if ref.is_complex() or orig.is_complex(): + torch.testing.assert_close( + orig.to(torch.cdouble), + ref.to(torch.cdouble), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + else: + if orig.is_mps or ref.is_mps: + torch.testing.assert_close( + orig.float(), + ref.float(), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + elif getattr(orig, "is_nested", None) or getattr( + ref, "is_nested", None + ): + assert getattr(orig, "is_nested", None) == getattr( + ref, "is_nested", None + ) + for t_orig, t_ref in zip(orig.unbind(), ref.unbind()): + torch.testing.assert_close( + t_orig.double(), + t_ref.double(), + rtol=check_tolerance, + atol=default_tolerances(t_orig, t_ref)[1], + equal_nan=True, + ) + else: + torch.testing.assert_close( + orig.double(), + ref.double(), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + except AssertionError as e: + maybe_warn_nondeterministic() + warnings.warn( + "Output nr " + + str(i + 1) + + ". of the traced function does not match " + "the corresponding output of the " + + match_what + + ". Detailed error:\n" + + str(e), + category=TracerWarning, + stacklevel=4, + ) + all_ok = False + + return all_ok + + traced_outs = run_mod_and_filter_tensor_outputs(traced_func, inputs, "trace") + fn_outs = run_mod_and_filter_tensor_outputs(func, inputs, "Python function") + if compare_outputs(traced_outs, fn_outs, "Python function"): + check_outs = run_mod_and_filter_tensor_outputs( + check_mod_func, inputs, "repeated trace" + ) + compare_outputs(traced_outs, check_outs, "repeated trace") + + diag_info = graph_diagnostic_info() + if any(info is not None for info in diag_info): + raise TracingCheckError(*diag_info) + + +class TracerWarning(Warning): + @staticmethod + def ignore_lib_warnings(): + # We ignore warnings from all submodules excluding the JIT, because we need them e.g. for _check_trace + warnings.filterwarnings( + "ignore", category=TracerWarning, module="torch.(?!jit)" + ) + warnings.filterwarnings("ignore", "torch::jit::fuser::cuda") + + +# We ignore the tracer warnings coming form inside the library, because all our shape +# checks in nn will trigger them. +TracerWarning.ignore_lib_warnings() +torch._C._tracer_warn_use_python() + + +def make_tuple(example_inputs): + if isinstance(example_inputs, (torch.Tensor, dict)): + return (example_inputs,) + # done primarily so that weird iterables fail here and not pybind11 code + if not isinstance(example_inputs, tuple): + return tuple(example_inputs) + return example_inputs + + +def make_module(mod, _module_class, _compilation_unit): + if isinstance(mod, ScriptModule): + return mod + elif torch._jit_internal.module_has_exports(mod): + infer_methods_stubs_fn = torch.jit._recursive.make_stubs_from_exported_methods + return torch.jit._recursive.create_script_module( + mod, infer_methods_stubs_fn, share_types=False, is_tracing=True + ) + else: + if _module_class is None: + _module_class = TopLevelTracedModule + return _module_class(mod, _compilation_unit=_compilation_unit) + + +def wrap_check_inputs(check_inputs): + if check_inputs is None: + return None + + return [{"forward": c} for c in check_inputs] + + +def analyze_ts_result_with_export_result(export, trace): + import torch.utils._pytree as pytree + + flat_export = pytree.tree_leaves(export) + flat_trace = pytree.tree_leaves(trace) + + for orig, loaded in zip(flat_export, flat_trace): + if orig.layout != loaded.layout: + return False + # mkldnn is not supported for torch.allclose + if orig.layout == torch._mkldnn: # type: ignore[attr-defined] + return True + if type(orig) is not type(loaded): + return False + + if isinstance(orig, torch._subclasses.FakeTensor): + # Skip for FakeTensor. + return True + elif isinstance(orig, torch.Tensor): + if orig.dtype != loaded.dtype: + return False + if not torch.allclose(orig, loaded): + return False + else: + if orig != loaded: + return False + return True + + +def _trace_impl( + func, + example_inputs=None, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_kwarg_inputs=None, + _store_inputs=True, +): + if isinstance(func, torch.jit.ScriptModule): + # it is hard to trace it because the forward method on ScriptModule is already defined, so it + # would result in an error. + warnings.warn( + "The input to trace is already a ScriptModule, tracing it is a no-op. Returning the object as is.", + stacklevel=2, + ) + return func + + if isinstance(func, torch.nn.Module): + if example_inputs is None: + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + else: + raise RuntimeError("example_kwarg_inputs should be a dict") + return trace_module( + func, + {"forward": example_inputs}, + None, + check_trace, + wrap_check_inputs(check_inputs), + check_tolerance, + strict, + _force_outplace, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + _store_inputs=_store_inputs, + ) + if ( + hasattr(func, "__self__") + and isinstance(func.__self__, torch.nn.Module) + and func.__name__ == "forward" + ): + if example_inputs is None: + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + else: + raise RuntimeError("example_kwarg_inputs should be a dict") + return trace_module( + func.__self__, + {"forward": example_inputs}, + None, + check_trace, + wrap_check_inputs(check_inputs), + check_tolerance, + strict, + _force_outplace, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + _store_inputs=_store_inputs, + ) + + # Special case for common case of passing a single Tensor + if ( + isinstance(example_inputs, (torch.Tensor, dict)) + and example_kwarg_inputs is None + ): + example_inputs = (example_inputs,) + # done primarily so that weird iterables fail here and not pybind11 code + elif example_kwarg_inputs is None and not isinstance(example_inputs, tuple): + # pyrefly: ignore [bad-argument-type] + example_inputs = tuple(example_inputs) + + var_lookup_fn = _create_interpreter_name_lookup_fn(0) + + if hasattr(func, "__self__") and isinstance(func.__self__, torch.nn.Module): + raise AttributeError( + "trace doesn't support compiling individual module's functions.\n" + "Please use trace_module" + ) + + name = _qualified_name(func) + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + traced = torch._C._create_function_from_trace_with_dict( + name, + func, + example_kwarg_inputs, + var_lookup_fn, + strict, + _force_outplace, + get_callable_argument_names(func), + ) + else: + traced = torch._C._create_function_from_trace( + name, + func, + # pyrefly: ignore [bad-argument-type] + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + get_callable_argument_names(func), + ) + + # Check the trace against new traces created from user-specified inputs + if check_trace: + if check_inputs is not None: + _check_trace( + check_inputs, + func, + traced, + check_tolerance, + strict, + _force_outplace, + False, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + ) + else: + _check_trace( + [example_inputs], + func, + traced, + check_tolerance, + strict, + _force_outplace, + False, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + ) + + # Allow torch.compile() to inline + traced._torchdynamo_inline = func # type: ignore[attr-defined] + return traced + + +class _ExportType(str, Enum): + DIRECT_EXPORT = "DIRECT_EXPORT" + TRACE_AND_EXPORT = "TRACE_AND_EXPORT" + SOURCE_TO_SOURCE = "SOURCE_TO_SOURCE" + + def __str__(self) -> str: + return self.value + + +class _ExportOutcome(str, Enum): + SUCCESS = "SUCCESS" + FAILED_TO_EXPORT = "FAILED_TO_EXPORT" + FAILED_TO_RUN = "FAILED_TO_RUN" + ACCURACY_ERROR = "ACCURACY_ERROR" + + def __str__(self) -> str: + return self.value + + +def trace( + func, + example_inputs=None, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_kwarg_inputs=None, + _store_inputs=True, +): + r""" + Trace a function and return an executable or :class:`ScriptFunction` that will be optimized using just-in-time compilation. + + Tracing is ideal for code that operates only on + ``Tensor``\\s and lists, dictionaries, and + tuples of ``Tensor``\\s. + + Using `torch.jit.trace` and `torch.jit.trace_module`, you can turn an + existing module or Python function into a TorchScript + :class:`ScriptFunction` or :class:`ScriptModule`. You must provide example + inputs, and we run the function, recording the operations performed on all + the tensors. + + * The resulting recording of a standalone function produces `ScriptFunction`. + * The resulting recording of `nn.Module.forward` or `nn.Module` produces + `ScriptModule`. + + This module also contains any parameters that the original + module had as well. + + Warning: + Tracing only correctly records functions and modules which are not data + dependent (e.g., do not have conditionals on data in tensors) and do not have + any untracked external dependencies (e.g., perform input/output or + access global variables). Tracing only records operations done when the given + function is run on the given tensors. Therefore, the returned + `ScriptModule` will always run the same traced graph on any input. This + has some important implications when your module is expected to run + different sets of operations, depending on the input and/or the module + state. For example, + + * Tracing will not record any control-flow like if-statements or loops. + When this control-flow is constant across your module, this is fine + and it often inlines the control-flow decisions. But sometimes the + control-flow is actually part of the model itself. For instance, a + recurrent network is a loop over the (possibly dynamic) length of an + input sequence. + * In the returned :class:`ScriptModule`, operations that have different + behaviors in ``training`` and ``eval`` modes will always behave as if + it is in the mode it was in during tracing, no matter which mode the + `ScriptModule` is in. + + In cases like these, tracing would not be appropriate and + :func:`scripting ` is a better choice. If you trace + such models, you may silently get incorrect results on subsequent + invocations of the model. The tracer will try to emit warnings when + doing something that may cause an incorrect trace to be produced. + + Args: + func (callable or torch.nn.Module): A Python function or `torch.nn.Module` + that will be run with `example_inputs`. `func` arguments and return + values must be tensors or (possibly nested) tuples that contain + tensors. When a module is passed `torch.jit.trace`, only the + ``forward`` method is run and traced (see :func:`torch.jit.trace + ` for details). + + Keyword arguments: + example_inputs (tuple or torch.Tensor or None, optional): A tuple of example + inputs that will be passed to the function while tracing. + Default: ``None``. Either this argument or ``example_kwarg_inputs`` + should be specified. The resulting trace can be run with inputs of + different types and shapes assuming the traced operations support those + types and shapes. `example_inputs` may also be a single Tensor in which + case it is automatically wrapped in a tuple. When the value is None, + ``example_kwarg_inputs`` should be specified. + + check_trace (``bool``, optional): Check if the same inputs run through + traced code produce the same outputs. Default: ``True``. You might want + to disable this if, for example, your network contains non- + deterministic ops or if you are sure that the network is correct despite + a checker failure. + + check_inputs (list of tuples, optional): A list of tuples of input + arguments that should be used to check the trace against what is + expected. Each tuple is equivalent to a set of input arguments that + would be specified in ``example_inputs``. For best results, pass in + a set of checking inputs representative of the space of shapes and + types of inputs you expect the network to see. If not specified, + the original ``example_inputs`` are used for checking + check_tolerance (float, optional): Floating-point comparison tolerance + to use in the checker procedure. This can be used to relax the + checker strictness in the event that results diverge numerically + for a known reason, such as operator fusion. + strict (``bool``, optional): run the tracer in a strict mode or not + (default: ``True``). Only turn this off when you want the tracer to + record your mutable container types (currently ``list``/``dict``) + and you are sure that the container you are using in your + problem is a ``constant`` structure and does not get used as + control flow (if, for) conditions. + example_kwarg_inputs (dict, optional): This parameter is a pack of keyword + arguments of example inputs that will be passed to the function while + tracing. Default: ``None``. Either this argument or ``example_inputs`` + should be specified. The dict will be unpacking by the arguments name + of the traced function. If the keys of the dict don't not match with + the traced function's arguments name, a runtime exception will be raised. + + Returns: + If `func` is `nn.Module` or ``forward`` of `nn.Module`, `trace` returns + a :class:`ScriptModule` object with a single ``forward`` method + containing the traced code. The returned `ScriptModule` will + have the same set of sub-modules and parameters as the original + ``nn.Module``. If ``func`` is a standalone function, ``trace`` + returns `ScriptFunction`. + + Example (tracing a function): + + .. testcode:: + + import torch + + def foo(x, y): + return 2 * x + y + + # Run `foo` with the provided inputs and record the tensor operations + traced_foo = torch.jit.trace(foo, (torch.rand(3), torch.rand(3))) + + # `traced_foo` can now be run with the TorchScript interpreter or saved + # and loaded in a Python-free environment + + Example (tracing an existing module):: + + import torch + import torch.nn as nn + + + class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = nn.Conv2d(1, 1, 3) + + def forward(self, x): + return self.conv(x) + + + n = Net() + example_weight = torch.rand(1, 1, 3, 3) + example_forward_input = torch.rand(1, 1, 3, 3) + + # Trace a specific method and construct `ScriptModule` with + # a single `forward` method + module = torch.jit.trace(n.forward, example_forward_input) + + # Trace a module (implicitly traces `forward`) and construct a + # `ScriptModule` with a single `forward` method + module = torch.jit.trace(n, example_forward_input) + + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.trace` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.trace` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return func + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=2, + ) + + from torch._utils_internal import log_torchscript_usage + + traced_func = _trace_impl( + func, + example_inputs, + optimize, + check_trace, + check_inputs, + check_tolerance, + strict, + _force_outplace, + _module_class, + _compilation_unit, + example_kwarg_inputs, + _store_inputs, + ) + log_torchscript_usage("trace", model_id=_get_model_id(traced_func)) + return traced_func + + +_trace_module_map: Optional[dict[Any, Any]] = None + + +def trace_module( + mod, + inputs, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_inputs_is_kwarg=False, + _store_inputs=True, +): + """ + Trace a module and return an executable :class:`ScriptModule` that will be optimized using just-in-time compilation. + + When a module is passed to :func:`torch.jit.trace `, only + the ``forward`` method is run and traced. With ``trace_module``, you can specify a dictionary of + method names to example inputs to trace (see the ``inputs``) argument below. + + See :func:`torch.jit.trace ` for more information on tracing. + + Args: + mod (torch.nn.Module): A ``torch.nn.Module`` containing methods whose names are + specified in ``inputs``. The given methods will be compiled + as a part of a single `ScriptModule`. + inputs (dict): A dict containing sample inputs indexed by method names in ``mod``. + The inputs will be passed to methods whose names correspond to inputs' + keys while tracing. + ``{ 'forward' : example_forward_input, 'method2': example_method2_input}`` + Keyword arguments: + check_trace (``bool``, optional): Check if the same inputs run through + traced code produce the same outputs. Default: ``True``. You might want + to disable this if, for example, your network contains non- + deterministic ops or if you are sure that the network is correct despite + a checker failure. + + check_inputs (list of dicts, optional): A list of dicts of input arguments that should be used + to check the trace against what is expected. Each tuple + is equivalent to a set of input arguments that would + be specified in ``inputs``. For best results, pass in a + set of checking inputs representative of the space of + shapes and types of inputs you expect the network to see. + If not specified, the original ``inputs`` are used for checking + check_tolerance (float, optional): Floating-point comparison tolerance to use in the checker procedure. + This can be used to relax the checker strictness in the event that + results diverge numerically for a known reason, such as operator fusion. + example_inputs_is_kwarg (``bool``, optional): This parameter indicate whether the example inputs is a pack + pack of keyword arguments. Default: ``False``. + + Returns: + A :class:`ScriptModule` object with a single ``forward`` method containing the traced code. + When ``func`` is a ``torch.nn.Module``, the returned :class:`ScriptModule` will have the same set of + sub-modules and parameters as ``func``. + + Example (tracing a module with multiple methods):: + + import torch + import torch.nn as nn + + + class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = nn.Conv2d(1, 1, 3) + + def forward(self, x): + return self.conv(x) + + def weighted_kernel_sum(self, weight): + return weight * self.conv.weight + + + n = Net() + example_weight = torch.rand(1, 1, 3, 3) + example_forward_input = torch.rand(1, 1, 3, 3) + + # Trace a specific method and construct `ScriptModule` with + # a single `forward` method + module = torch.jit.trace(n.forward, example_forward_input) + + # Trace a module (implicitly traces `forward`) and construct a + # `ScriptModule` with a single `forward` method + module = torch.jit.trace(n, example_forward_input) + + # Trace specific methods on a module (specified in `inputs`), constructs + # a `ScriptModule` with `forward` and `weighted_kernel_sum` methods + inputs = { + "forward": example_forward_input, + "weighted_kernel_sum": example_weight, + } + module = torch.jit.trace_module(n, inputs) + + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.trace_method` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.trace_method` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return mod + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=2, + ) + + var_lookup_fn = _create_interpreter_name_lookup_fn(0) + + if not isinstance(mod, torch.nn.Module): + raise AttributeError("expected torch.nn.Module as the first argument") + + if not isinstance(inputs, dict): + raise AttributeError("expected a dictionary of (method_name, input) pairs") + + old_module_map = torch.jit._trace._trace_module_map + try: + trace_module_map: dict[Any, Any] = {} + + def register_submods(mod, prefix): + for name, child in mod.named_children(): + submod_qualname = prefix + "." + name + trace_module_map[child] = submod_qualname + register_submods(child, submod_qualname) + + trace_module_map["__module"] = mod + torch.jit._trace._trace_module_map = trace_module_map + register_submods(mod, "__module") + + module = make_module(mod, _module_class, _compilation_unit) + + for method_name, example_inputs in inputs.items(): + if method_name == "forward": + # "forward" is a special case because we need to trace + # `Module.__call__`, which sets up some extra tracing, but uses + # argument names of the real `Module.forward` method. + func = mod + forward_method = getattr(mod, method_name) + argument_names = get_callable_argument_names(forward_method) + else: + func = getattr(mod, method_name) + argument_names = get_callable_argument_names(func) + + if isinstance(example_inputs, dict) and example_inputs_is_kwarg: + # Raise exception when the user provided key names are not aligned with forward() method's arguments' name/ + for key in example_inputs: + if key not in argument_names: + valid_arguments = "[" + ",".join(argument_names) + "]" + raise NameError( + f"""'{key}' is not in forward() method's arguments, + valid arguments name are {valid_arguments}""" + ) + module._c._create_method_from_trace_with_dict( + method_name, + func, + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + argument_names, + _store_inputs, + ) + else: + example_inputs = make_tuple(example_inputs) + module._c._create_method_from_trace( + method_name, + func, + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + argument_names, + _store_inputs, + ) + + check_trace_method = module._c._get_method(method_name) + + # Check the trace against new traces created from user-specified inputs + if check_trace: + if check_inputs is not None: + _check_trace( + check_inputs, + func, + check_trace_method, + check_tolerance, + strict, + _force_outplace, + True, + _module_class, + example_inputs_is_kwarg=example_inputs_is_kwarg, + ) + else: + _check_trace( + [inputs], + func, + check_trace_method, + check_tolerance, + strict, + _force_outplace, + True, + _module_class, + example_inputs_is_kwarg=example_inputs_is_kwarg, + ) + finally: + torch.jit._trace._trace_module_map = old_module_map + + return module + + +def is_tracing(): + """Return a boolean value. + + Returns ``True`` in tracing (if a function is called during the + tracing of code with ``torch.jit.trace``) and ``False`` otherwise. + """ + if is_scripting(): + return False + return torch._C._is_tracing() + + +class TracedModule(ScriptModule): + _disable_script_meta = True + + def __init__(self, orig, id_set=None, _compilation_unit=None): + # XXX: orig can be a nn.Module or a function! + super().__init__() + assert isinstance(orig, torch.nn.Module) + + # Copy a subset of `orig` to a temporary nn.Module. + # This is a way to customize what will actually get compiled by create_script_module + id_set = set() + + # This allows us to preserve the original module's qualified name by defining a new + # type with the attribute _jit_override_qualname. In torch._jit_internal._qualified_name + # we have a special case that will look up this attribute to override whatever qualname + # we would get from the python type system + class QualnameWrapper(torch.nn.Module): + pass + + QualnameWrapper._jit_override_qualname = torch._jit_internal._qualified_name( # type: ignore[attr-defined] + type(orig) + ) + + tmp_module = QualnameWrapper() + + def check_unique(param): + if param in id_set: + raise ValueError( + "TracedModules don't support parameter sharing between modules" + ) + id_set.add(param) + + tmp_module.training = orig.training + + for name, param in orig._parameters.items(): + if param is not None: + tmp_module._parameters[name] = param + check_unique(param) + for name, buf in orig._buffers.items(): + if buf is not None: + tmp_module._buffers[name] = buf + check_unique(buf) + for name, val in orig.__dict__.items(): + if ( + torch._C._jit_is_script_object(val) + and name not in orig._parameters + and name not in orig._buffers + ): + setattr(tmp_module, name, val) + + if orig._backward_hooks: + raise ValueError( + "Modules that have backward hooks assigned can't be compiled: " + + str(orig) + ) + + for name, submodule in orig._modules.items(): + if submodule is None: + continue + tmp_module._modules[name] = make_module( + submodule, TracedModule, _compilation_unit=None + ) + + script_module = torch.jit._recursive.create_script_module( + tmp_module, lambda module: (), share_types=False, is_tracing=True + ) + + self.__dict__["_name"] = type(orig).__name__ + self.__dict__["_actual_script_module"] = script_module + for name in ("_parameters", "_buffers", "_modules", "training"): + delattr(self, name) + + def forward(self, *args, **kwargs): + raise RuntimeError("Trace submodules cannot be called.") + + def __getattr__(self, attr): + if "_actual_script_module" not in self.__dict__: + return super().__getattr__(attr) + return getattr(self._actual_script_module, attr) + + def __setattr__(self, attr, value): + if "_actual_script_module" not in self.__dict__: + return super().__setattr__(attr, value) + setattr(self._actual_script_module, attr, value) + + def _get_name(self): + return self._name + + def extra_repr(self): + return f"original_name={self._name}" + + +class TopLevelTracedModule(TracedModule): + forward: Callable[..., Any] = _CachedForward() # type: ignore[assignment] + + def _reconstruct(self, cpp_module): + """ + Re-construct an instance of TopLevelTracedModule using an instance of a C++ module. + + Args: + cpp_module: The C++ module that this TopLevelTracedModule will be rebuilt around. + """ + self.__dict__["_actual_script_module"]._reconstruct(cpp_module) + + +def _script_if_tracing(fn: Callable[P, R]) -> Callable[P, R]: + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + if not is_tracing(): + # Not tracing, don't do anything + return fn(*args, **kwargs) + + compiled_fn: Callable[P, R] = script(wrapper.__original_fn) # type: ignore[attr-defined] + return compiled_fn(*args, **kwargs) + + wrapper.__original_fn = fn # type: ignore[attr-defined] + wrapper.__script_if_tracing_wrapper = True # type: ignore[attr-defined] + + return wrapper + + +def _get_trace_graph( + f, + args=(), + kwargs=None, + strict=True, + _force_outplace=False, + return_inputs=False, + _return_inputs_states=False, +): + """Return a tuple on tracing a function or model. + + .. warning:: + This function is internal-only and should only be used by the ONNX + exporter. If you are trying to get a graph through tracing, please go + through the public API instead:: + + trace = torch.jit.trace(nn.LSTMCell(), (input, hidden)) + trace_graph = trace.graph + + Trace a function or model, returning a tuple consisting of the both the + *trace* of an execution, as well as the original return value. If return_inputs, + also returns the trace inputs as part of the tuple + + Tracing is guaranteed not to change the semantics of the function/module + that is traced. + + Args: + f (torch.nn.Module or function): the function or module + to be traced. + args (tuple or Tensor): the positional arguments to pass to the + function/module to be traced. A non-tuple is assumed to + be a single positional argument to be passed to the model. + kwargs (dict): the keyword arguments to pass to the function/module + to be traced. + + Example (trace a cell): + + .. testcode:: + + trace = torch.jit.trace(nn.LSTMCell(), (input, hidden)) + """ + if kwargs is None: + kwargs = {} + if not isinstance(args, tuple): + args = (args,) + outs = ONNXTracedModule( + f, strict, _force_outplace, return_inputs, _return_inputs_states + )(*args, **kwargs) + return outs diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/annotations.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/annotations.py new file mode 100644 index 0000000000000000000000000000000000000000..61e4ffb7801bfcfc64d109911e50bd33a5ba3fe5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/annotations.py @@ -0,0 +1,555 @@ +# mypy: allow-untyped-defs +import ast +import builtins +import dis +import enum +import inspect +import re +import typing +import warnings +from textwrap import dedent + +import torch +from torch._C import ( + _GeneratorType, + AnyType, + AwaitType, + BoolType, + ComplexType, + DeviceObjType, + DictType, + EnumType, + FloatType, + FutureType, + InterfaceType, + IntType, + ListType, + NoneType, + NumberType, + OptionalType, + StreamObjType, + StringType, + TensorType, + TupleType, + UnionType, +) +from torch._jit_internal import ( # type: ignore[attr-defined] + _Await, + _qualified_name, + Any, + BroadcastingList1, + BroadcastingList2, + BroadcastingList3, + Dict, + Future, + is_await, + is_dict, + is_future, + is_ignored_fn, + is_list, + is_optional, + is_tuple, + is_union, + List, + Optional, + Tuple, + Union, +) +from torch._sources import get_source_lines_and_file + +from ._state import _get_script_class + + +if torch.distributed.rpc.is_available(): + from torch._C import RRefType + from torch._jit_internal import is_rref, RRef + +from torch._ops import OpOverloadPacket + + +class Module: + def __init__(self, name, members) -> None: + self.name = name + self.members = members + + def __getattr__(self, name): + try: + return self.members[name] + except KeyError: + raise RuntimeError( + f"Module {self.name} has no member called {name}" + ) from None + + +class EvalEnv: + env = { + "torch": Module("torch", {"Tensor": torch.Tensor}), + "Tensor": torch.Tensor, + "typing": Module("typing", {"Tuple": Tuple}), + "Tuple": Tuple, + "List": List, + "Dict": Dict, + "Optional": Optional, + "Union": Union, + "Future": Future, + "Await": _Await, + } + + def __init__(self, rcb) -> None: + self.rcb = rcb + if torch.distributed.rpc.is_available(): + # pyrefly: ignore [unsupported-operation] + self.env["RRef"] = RRef + + def __getitem__(self, name): + if name in self.env: + return self.env[name] + if self.rcb is not None: + return self.rcb(name) + return getattr(builtins, name, None) + + +def get_signature(fn, rcb, loc, is_method): + if isinstance(fn, OpOverloadPacket): + signature = try_real_annotations(fn.op, loc) + else: + signature = try_real_annotations(fn, loc) + if signature is not None and is_method: + # If this is a method, then the signature will include a type for + # `self`, but type comments do not contain a `self`. So strip it + # away here so everything is consistent (`inspect.ismethod` does + # not work here since `fn` is unbound at this point) + param_types, return_type = signature + param_types = param_types[1:] + signature = (param_types, return_type) + + if signature is None: + type_line, source = None, None + try: + source = dedent("".join(get_source_lines_and_file(fn)[0])) + type_line = get_type_line(source) + except TypeError: + pass + # This might happen both because we failed to get the source of fn, or + # because it didn't have any annotations. + if type_line is not None: + signature = parse_type_line(type_line, rcb, loc) + + return signature + + +def is_function_or_method(the_callable): + # A stricter version of `inspect.isroutine` that does not pass for built-in + # functions + return inspect.isfunction(the_callable) or inspect.ismethod(the_callable) + + +def is_vararg(the_callable): + if not is_function_or_method(the_callable) and callable(the_callable): # noqa: B004 + # If `the_callable` is a class, de-sugar the call so we can still get + # the signature + the_callable = the_callable.__call__ + + if is_function_or_method(the_callable): + return inspect.getfullargspec(the_callable).varargs is not None + else: + return False + + +def get_param_names(fn, n_args): + if isinstance(fn, OpOverloadPacket): + fn = fn.op + + if ( + not is_function_or_method(fn) + and callable(fn) + and is_function_or_method(fn.__call__) + ): # noqa: B004 + # De-sugar calls to classes + fn = fn.__call__ + + if is_function_or_method(fn): + if is_ignored_fn(fn): + fn = inspect.unwrap(fn) + return inspect.getfullargspec(fn).args + else: + # The `fn` was not a method or function (maybe a class with a __call__ + # method, so use a default param name list) + return [str(i) for i in range(n_args)] + + +def check_fn(fn, loc) -> None: + # Make sure the function definition is not a class instantiation + try: + source = dedent("".join(get_source_lines_and_file(fn)[0])) + except (OSError, TypeError): + return + if source is None: + return + + py_ast = ast.parse(source) + if len(py_ast.body) == 1 and isinstance(py_ast.body[0], ast.ClassDef): + raise torch.jit.frontend.FrontendError( + loc, + f"Cannot instantiate class '{py_ast.body[0].name}' in a script function", + ) + if len(py_ast.body) != 1 or not isinstance(py_ast.body[0], ast.FunctionDef): + raise torch.jit.frontend.FrontendError( + loc, "Expected a single top-level function" + ) + + +def _eval_no_call(stmt, glob, loc): + """Evaluate statement as long as it does not contain any method/function calls.""" + bytecode = compile(stmt, "", mode="eval") + for insn in dis.get_instructions(bytecode): + if "CALL" in insn.opname: + raise RuntimeError( + f"Type annotation should not contain calls, but '{stmt}' does" + ) + return eval(bytecode, glob, loc) # type: ignore[arg-type] # noqa: P204 + + +def parse_type_line(type_line, rcb, loc): + """Parse a type annotation specified as a comment. + + Example inputs: + # type: (Tensor, torch.Tensor) -> Tuple[Tensor] + # type: (Tensor, Tuple[Tensor, Tensor]) -> Tensor + """ + arg_ann_str, ret_ann_str = split_type_line(type_line) + + try: + arg_ann = _eval_no_call(arg_ann_str, {}, EvalEnv(rcb)) + except (NameError, SyntaxError) as e: + raise RuntimeError( + "Failed to parse the argument list of a type annotation" + ) from e + + if not isinstance(arg_ann, tuple): + arg_ann = (arg_ann,) + + try: + ret_ann = _eval_no_call(ret_ann_str, {}, EvalEnv(rcb)) + except (NameError, SyntaxError) as e: + raise RuntimeError( + "Failed to parse the return type of a type annotation" + ) from e + + arg_types = [ann_to_type(ann, loc) for ann in arg_ann] + return arg_types, ann_to_type(ret_ann, loc) + + +def get_type_line(source): + """Try to find the line containing a comment with the type annotation.""" + type_comment = "# type:" + + lines = source.split("\n") + lines = list(enumerate(lines)) + type_lines = list(filter(lambda line: type_comment in line[1], lines)) + # `type: ignore` comments may be needed in JIT'ed functions for mypy, due + # to the hack in torch/_VF.py. + + # An ignore type comment can be of following format: + # 1) type: ignore + # 2) type: ignore[rule-code] + # This ignore statement must be at the end of the line + + # adding an extra backslash before the space, to avoid triggering + # one of the checks in .github/workflows/lint.yml + type_pattern = re.compile("# type:\\ ignore(\\[[a-zA-Z-]+\\])?$") + type_lines = list(filter(lambda line: not type_pattern.search(line[1]), type_lines)) + + if len(type_lines) == 0: + # Catch common typo patterns like extra spaces, typo in 'ignore', etc. + wrong_type_pattern = re.compile("#[\t ]*type[\t ]*(?!: ignore(\\[.*\\])?$):") + wrong_type_lines = list( + filter(lambda line: wrong_type_pattern.search(line[1]), lines) + ) + if len(wrong_type_lines) > 0: + raise RuntimeError( + "The annotation prefix in line " + + str(wrong_type_lines[0][0]) + + " is probably invalid.\nIt must be '# type:'" + + "\nSee PEP 484 (https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)" # noqa: B950 + + "\nfor examples" + ) + return None + elif len(type_lines) == 1: + # Only 1 type line, quit now + return type_lines[0][1].strip() + + # Parse split up argument types according to PEP 484 + # https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code + return_line = None + parameter_type_lines = [] + for line_num, line in type_lines: + if "# type: (...) -> " in line: + return_line = (line_num, line) + break + elif type_comment in line: + parameter_type_lines.append(line) + if return_line is None: + raise RuntimeError( + "Return type line '# type: (...) -> ...' not found on multiline " + "type annotation\nfor type lines:\n" + + "\n".join([line[1] for line in type_lines]) + + "\n(See PEP 484 https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)" + ) + + def get_parameter_type(line): + item_type = line[line.find(type_comment) + len(type_comment) :] + return item_type.strip() + + types = map(get_parameter_type, parameter_type_lines) + parameter_types = ", ".join(types) + + return return_line[1].replace("...", parameter_types) + + +def split_type_line(type_line): + """Split the comment with the type annotation into parts for argument and return types. + + For example, for an input of: + # type: (Tensor, torch.Tensor) -> Tuple[Tensor, Tensor] + + This function will return: + ("(Tensor, torch.Tensor)", "Tuple[Tensor, Tensor]") + + """ + start_offset = len("# type:") + try: + arrow_pos = type_line.index("->") + except ValueError: + raise RuntimeError( + "Syntax error in type annotation (couldn't find `->`)" + ) from None + return type_line[start_offset:arrow_pos].strip(), type_line[arrow_pos + 2 :].strip() + + +def try_real_annotations(fn, loc): + """Try to use the Py3.5+ annotation syntax to get the type.""" + try: + # Note: anything annotated as `Optional[T]` will automatically + # be returned as `Union[T, None]` per + # https://github.com/python/cpython/blob/main/Lib/typing.py#L732 + sig = inspect.signature(fn) + except ValueError: + return None + + all_annots = [sig.return_annotation] + [ + p.annotation for p in sig.parameters.values() + ] + if all(ann is sig.empty for ann in all_annots): + return None + + arg_types = [ann_to_type(p.annotation, loc) for p in sig.parameters.values()] + return_type = ann_to_type(sig.return_annotation, loc) + return arg_types, return_type + + +# Finds common type for enum values belonging to an Enum class. If not all +# values have the same type, AnyType is returned. +def get_enum_value_type(e: type[enum.Enum], loc): + enum_values: List[enum.Enum] = list(e) + if not enum_values: + raise ValueError(f"No enum values defined for: '{e.__class__}'") + + types = {type(v.value) for v in enum_values} + ir_types = [try_ann_to_type(t, loc) for t in types] + + # If Enum values are of different types, an exception will be raised here. + # Even though Python supports this case, we chose to not implement it to + # avoid overcomplicate logic here for a rare use case. Please report a + # feature request if you find it necessary. + res = torch._C.unify_type_list(ir_types) + if not res: + return AnyType.get() + return res + + +def is_tensor(ann) -> bool: + if issubclass(ann, torch.Tensor): + return True + + if issubclass( + ann, + ( + torch.LongTensor, + torch.DoubleTensor, + torch.FloatTensor, + torch.IntTensor, + torch.ShortTensor, + torch.HalfTensor, + torch.CharTensor, + torch.ByteTensor, + torch.BoolTensor, + ), + ): + warnings.warn( + "TorchScript will treat type annotations of Tensor " + "dtype-specific subtypes as if they are normal Tensors. " + "dtype constraints are not enforced in compilation either.", + stacklevel=2, + ) + return True + + return False + + +def _fake_rcb(inp) -> None: + return None + + +def try_ann_to_type(ann, loc, rcb=None): + ann_args = typing.get_args(ann) # always returns a tuple! + + if ann is inspect.Signature.empty: + return TensorType.getInferred() + if ann is None: + return NoneType.get() + if inspect.isclass(ann) and is_tensor(ann): + return TensorType.get() + if is_tuple(ann): + # Special case for the empty Tuple type annotation `Tuple[()]` + if len(ann_args) == 1 and ann_args[0] == (): + return TupleType([]) + return TupleType([try_ann_to_type(a, loc) for a in ann_args]) + if is_list(ann): + elem_type = try_ann_to_type(ann_args[0], loc) + if elem_type: + return ListType(elem_type) + if is_dict(ann): + key = try_ann_to_type(ann_args[0], loc) + value = try_ann_to_type(ann_args[1], loc) + # Raise error if key or value is None + if key is None: + raise ValueError( + f"Unknown type annotation: '{ann_args[0]}' at {loc.highlight()}" + ) + if value is None: + raise ValueError( + f"Unknown type annotation: '{ann_args[1]}' at {loc.highlight()}" + ) + return DictType(key, value) + if is_optional(ann): + if issubclass(ann_args[1], type(None)): + contained = ann_args[0] + else: + contained = ann_args[1] + valid_type = try_ann_to_type(contained, loc) + msg = "Unsupported annotation {} could not be resolved because {} could not be resolved. At\n{}" + assert valid_type, msg.format(repr(ann), repr(contained), repr(loc)) + return OptionalType(valid_type) + if is_union(ann): + # TODO: this is hack to recognize NumberType + if set(ann_args) == {int, float, complex}: + return NumberType.get() + inner: List = [] + # We need these extra checks because both `None` and invalid + # values will return `None` + # TODO: Determine if the other cases need to be fixed as well + for a in typing.get_args(ann): + if a is None: + inner.append(NoneType.get()) + maybe_type = try_ann_to_type(a, loc) + msg = "Unsupported annotation {} could not be resolved because {} could not be resolved. At\n{}" + assert maybe_type, msg.format(repr(ann), repr(maybe_type), repr(loc)) + inner.append(maybe_type) + return UnionType(inner) # type: ignore[arg-type] + if torch.distributed.rpc.is_available() and is_rref(ann): + return RRefType(try_ann_to_type(ann_args[0], loc)) + if is_future(ann): + return FutureType(try_ann_to_type(ann_args[0], loc)) + if is_await(ann): + elementType = try_ann_to_type(ann_args[0], loc) if ann_args else AnyType.get() + return AwaitType(elementType) + if ann is float: + return FloatType.get() + if ann is complex: + return ComplexType.get() + if ann is int or ann is torch.SymInt: + return IntType.get() + if ann is str: + return StringType.get() + if ann is bool: + return BoolType.get() + if ann is Any: + return AnyType.get() + if ann is type(None): + return NoneType.get() + if inspect.isclass(ann) and hasattr(ann, "__torch_script_interface__"): + return InterfaceType(ann.__torch_script_interface__) + if ann is torch.device: + return DeviceObjType.get() + if ann is torch.Generator: + return _GeneratorType.get() + if ann is torch.Stream: + return StreamObjType.get() + if ann is torch.dtype: + return IntType.get() # dtype not yet bound in as its own type + if ann is torch.qscheme: + return IntType.get() # qscheme not yet bound in as its own type + + if inspect.isclass(ann) and issubclass(ann, enum.Enum): + if _get_script_class(ann) is None: + scripted_class = torch.jit._script._recursive_compile_class(ann, loc) + name = scripted_class.qualified_name() + else: + name = _qualified_name(ann) + return EnumType(name, get_enum_value_type(ann, loc), list(ann)) + if inspect.isclass(ann): + maybe_script_class = _get_script_class(ann) + if maybe_script_class is not None: + return maybe_script_class + if torch._jit_internal.can_compile_class(ann): + return torch.jit._script._recursive_compile_class(ann, loc) + + # Maybe resolve a NamedTuple to a Tuple Type + if rcb is None: + rcb = _fake_rcb + return torch._C._resolve_type_from_object(ann, loc, rcb) + + +def ann_to_type(ann, loc, rcb=None): + the_type = try_ann_to_type(ann, loc, rcb) + if the_type is not None: + return the_type + raise ValueError(f"Unknown type annotation: '{ann}' at {loc.highlight()}") + + +__all__ = [ + "Any", + "List", + "BroadcastingList1", + "BroadcastingList2", + "BroadcastingList3", + "Tuple", + "is_tuple", + "is_list", + "Dict", + "is_dict", + "is_optional", + "is_union", + "TensorType", + "TupleType", + "FloatType", + "ComplexType", + "IntType", + "ListType", + "StringType", + "DictType", + "AnyType", + "Module", + # TODO: Consider not exporting these during wildcard import (reserve + # that for the types; for idiomatic typing code.) + "get_signature", + "check_fn", + "get_param_names", + "parse_type_line", + "get_type_line", + "split_type_line", + "try_real_annotations", + "try_ann_to_type", + "ann_to_type", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/frontend.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/frontend.py new file mode 100644 index 0000000000000000000000000000000000000000..b07d5905a7b9d50c5fc05f18c11cf625d88b2688 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/frontend.py @@ -0,0 +1,1284 @@ +# mypy: allow-untyped-defs +import ast +import copy +import dataclasses +import inspect +import re +import string +from collections import namedtuple +from textwrap import dedent + +import torch +import torch.jit.annotations +from torch import _jit_internal +from torch._C._jit_tree_views import ( + Apply, + Assert, + Assign, + Attribute, + AugAssign, + BinOp, + Break, + ClassDef, + Const, + Continue, + Decl, + Def, + Delete, + DictComp, + DictLiteral, + Dots, + EmptyTypeAnnotation, + ExprStmt, + FalseLiteral, + For, + Ident, + If, + ListComp, + ListLiteral, + NoneLiteral, + Param, + Pass, + Property, + Raise, + Return, + Select, + SliceExpr, + Starred, + Stmt, + StringLiteral, + Subscript, + TernaryIf, + TrueLiteral, + TupleLiteral, + UnaryOp, + Var, + While, + With, + WithItem, +) +from torch._jit_internal import ( # noqa: F401 + _is_drop_fn, + FunctionModifiers, + is_static_fn, + should_drop, +) +from torch._sources import ( + get_source_lines_and_file, + make_source_context, + parse_def, + ParsedDef as _ParsedDef, +) +from torch.jit._dataclass_impls import DATACLASS_MAGIC_METHODS +from torch.jit._monkeytype_config import get_qualified_name, monkeytype_trace + + +# Borrowed from cPython implementation +# https://github.com/python/cpython/blob/561612d8456cfab5672c9b445521113b847bd6b3/Lib/textwrap.py#L411# + +_reserved_prefix = "__jit" +_reserved_names = {"print"} +_identifier_chars = set(string.ascii_lowercase + string.ascii_uppercase + string.digits) + + +def is_reserved_name(name): + return name.startswith(_reserved_prefix) or name in _reserved_names + + +pretty_node_names = { + ast.FunctionDef: "function definitions", + ast.For: "for loops", + ast.Delete: "del statements", + ast.ClassDef: "class definitions", + ast.With: "with statements", + ast.Raise: "raise statements", + ast.Assert: "assertions", + ast.Import: "import statements", + ast.ImportFrom: "import statements", + ast.Global: "global variables", + ast.Break: "break statements", + ast.Continue: "continue statements", +} + +node_start_tokens = { + ast.FunctionDef: "def", + ast.For: "for", + ast.Delete: "del", + ast.ClassDef: "class", + ast.With: "with", + ast.Raise: "raise", + ast.Assert: "assert", + ast.Import: "import", + ast.ImportFrom: "from", + ast.Global: "global", + ast.Break: "break", + ast.Continue: "continue", +} + +# pyrefly: ignore [no-matching-overload] +pretty_node_names.update( + { + ast.AsyncFunctionDef: "async function definitions", + ast.AsyncFor: "async for loops", + ast.AsyncWith: "async with statements", + ast.Try: "try blocks", + ast.Nonlocal: "nonlocal variables", + } +) + +# pyrefly: ignore [no-matching-overload] +node_start_tokens.update( + { + ast.AsyncFunctionDef: "async def", + ast.AsyncFor: "async for", + ast.AsyncWith: "async with", + ast.Try: "try", + ast.Nonlocal: "nonlocal", + } +) + +# pyrefly: ignore [no-matching-overload] +pretty_node_names.update( + { + ast.AnnAssign: "annotated assignments", + } +) +# NB: no specific token for AnnAssign + + +class FrontendError(Exception): + def __init__(self, source_range, msg) -> None: + self.source_range = source_range + self.msg = msg + + # This has to be instantiated here so the ErrorReport is accurate to the + # call stack when the FrontendError was raised + self.error_report = torch._C.ErrorReport(self.source_range) + + def __str__(self) -> str: + return self.msg + self.error_report.what().lstrip() + + +class NotSupportedError(FrontendError): + pass + + +class UnsupportedNodeError(NotSupportedError): + def __init__(self, ctx, offending_node, reason="") -> None: + # If we don't have a specific token, we default to length of 1 + node_type = type(offending_node) + range_len = len(node_start_tokens.get(node_type, " ")) + source_range = ctx.make_range( + offending_node.lineno, + offending_node.col_offset, + offending_node.col_offset + range_len, + ) + feature_name = pretty_node_names.get(node_type, node_type.__name__) + msg = f"{feature_name} {reason + ' ' if reason else ''}aren't supported" + super().__init__(source_range, msg) + + +class FrontendTypeError(FrontendError): + pass + + +def build_withitems(ctx, items): + items = [build_withitem(ctx, i) for i in items] + return list(items) + + +def build_stmts(ctx, stmts): + stmts = [build_stmt(ctx, s) for s in stmts] + return list(filter(None, stmts)) + + +def get_class_properties(cls, self_name): + """ + Get a list of Property objects representing the properties of a class. + + Args: + cls: The class to get properties of. + self_name: The name of the class that the properties should belong to. + Returns: + A list of Property objects corresponding to the properties of cls. Property + here refers to the subclass of TreeView. + """ + props = inspect.getmembers(cls, predicate=lambda m: isinstance(m, property)) + # Any property that should not compiled must be in this list on the Module. + unused_properties = getattr(cls, "__jit_unused_properties__", []) + + # Create Property TreeView objects from inspected property objects. + properties = [] + for prop in props: + if prop[0] not in unused_properties and not should_drop(prop[1].fget): + getter = get_jit_def( + prop[1].fget, f"__{prop[0]}_getter", self_name=self_name + ) + setter = ( + get_jit_def(prop[1].fset, f"__{prop[0]}_setter", self_name=self_name) + if prop[1].fset + else None + ) + properties.append( + Property(getter.range(), Ident(getter.range(), prop[0]), getter, setter) + ) + + return properties + + +def get_class_assigns(ctx, cls_ast): + assigns = [] + + def maybe_build_assign(builder, entry) -> None: + nonlocal assigns + try: + assigns.append(builder(ctx, entry)) + except NotSupportedError: + pass + + for entry in cls_ast.body: + if isinstance(entry, ast.Assign): + maybe_build_assign(StmtBuilder.build_Assign, entry) + elif isinstance(entry, ast.AnnAssign): + maybe_build_assign(StmtBuilder.build_AnnAssign, entry) + return assigns + + +def get_jit_class_def(cls, self_name): + """Get definitions for each method within the current class independently. + + Args: + cls: The class to get definition of. + self_name: The name of the class that the properties should belong to. + + Returns: + torch._C._jit_tree_views.ClassDef: A representation of the class, + the methods in the class and their definition as a tree. + """ + # TODO: proper overriding analysis when implementing class inheritance + methods = inspect.getmembers( + cls, + predicate=lambda m: (inspect.ismethod(m) or inspect.isfunction(m)) + and not is_static_fn(cls, m.__name__) + and m.__name__ in cls.__dict__ + and not _is_drop_fn(m), + ) + + def is_classmethod(fn): + return inspect.ismethod(fn) and getattr(fn, "__self__", None) == cls + + # Get and parse the source code for this class + sourcelines, file_lineno, filename = get_source_lines_and_file( + cls, torch._C.ErrorReport.call_stack() + ) + source = "".join(sourcelines) + + dedent_src = dedent(source) + py_ast = ast.parse(dedent_src) + + class_ast = py_ast.body[0] + assert isinstance(class_ast, ast.ClassDef) + + # Special case for dataclasses. In general we need access to the source code for + # an object in order to JIT compile it. But the dataclasses module dynamically synthesizes + # magic methods for classes, and we can't get the source code for these methods. As a + # workaround, we synthesize TorchScript-friendly implementations ourselves. + if dataclasses.is_dataclass(cls): + # Detect whether the user manually implemented any of the magic methods. If they did, + # we don't want to synthesize/override them. + overrides = { + method.name + for method in class_ast.body + if isinstance(method, ast.FunctionDef) + and method.name in DATACLASS_MAGIC_METHODS + } + for i, (name, _) in enumerate(methods): + # Is this a magic method we can synthesize? + synthesizer_fn = DATACLASS_MAGIC_METHODS.get(name) + if synthesizer_fn and name not in overrides: + parsed_def = synthesizer_fn(cls) + methods[i] = name, parsed_def + func = getattr(cls, name) + _jit_internal.loader.cache(func, parsed_def.source) + + method_defs = [ + get_jit_def(obj, name, self_name=self_name, is_classmethod=is_classmethod(obj)) + for (name, obj) in methods + ] + properties = get_class_properties(cls, self_name) + + leading_whitespace_len = len(source.split("\n", 1)[0]) - len( + dedent_src.split("\n", 1)[0] + ) + ctx = make_source_context( + source, filename, file_lineno, leading_whitespace_len, False + ) + assigns = get_class_assigns(ctx, class_ast) + + return build_class_def(ctx, class_ast, method_defs, properties, self_name, assigns) + + +def get_jit_def(fn, def_name, self_name=None, is_classmethod=False): + """ + Build a JIT AST (TreeView) from the given function. + + Args: + fn: A function object to compile or a pre-parsed ParsedDef object + def_name: The name to give to the resulting AST object. This is not + always the same as `fn.__name__`, for example: + def _forward(self): + ... + forward = _forward + In this case, the `__name__` attribute of the function object is "_forward", + but we want the result AST to have the name "forward". + self_name: If this function is a method, what the type name of `self` is. + """ + parsed_def = parse_def(fn) if not isinstance(fn, _ParsedDef) else fn + type_line = torch.jit.annotations.get_type_line(parsed_def.source) + fn_def = parsed_def.ast.body[0] + + if is_classmethod: + arg_name = fn_def.args.args[0].arg # type:ignore[union-attr] + # Insert a statement that assigns the first argument to the class + assign_stmt = ast.parse(f"{arg_name} = {self_name}").body[0] + fn_def.body.insert(0, assign_stmt) # type:ignore[union-attr] + + # Swap out the function signature and body if it is unused + if should_drop(fn): + unused_fn_def = ast.parse( + 'def unused_fn(self: Any):\n\traise RuntimeError("Cannot call @unused methods")' + ) + if len(unused_fn_def.body) != 1 or not isinstance( + unused_fn_def.body[0], ast.FunctionDef + ): + raise RuntimeError( + f"Expected a single top-level function: {parsed_def.filename}:{parsed_def.file_lineno}" + ) + unused_def = unused_fn_def.body[0] + fn_def.body = unused_def.body # type:ignore[union-attr] + # kwarg/vararg not supported by `build_def` + fn_def.args.kwarg = fn_def.args.vararg = None # type:ignore[union-attr] + for arg in fn_def.args.args + fn_def.args.kwonlyargs: # type:ignore[union-attr] + # Replace potentially unsupported type annotations by "Any" + arg.annotation = unused_def.args.args[0].annotation + if _is_drop_fn(fn): + # Dropping potentially unsupported return type annotation for jit._drop + fn_def.returns = None # type:ignore[union-attr] + fn_def.type_comment = None # type:ignore[union-attr] + + # If MonkeyType is installed, get all the consolidated type traces + # for the arguments from type_trace_db + type_trace_db = torch.jit._script._get_type_trace_db() + pdt_arg_types = None + if monkeytype_trace and not isinstance(fn, _ParsedDef): # type: ignore[truthy-function] + qualname = get_qualified_name(fn) + pdt_arg_types = type_trace_db.get_args_types(qualname) + + return build_def( + parsed_def.ctx, + fn_def, + type_line, + def_name, + self_name=self_name, + pdt_arg_types=pdt_arg_types, + ) + + +# TODO: more robust handling of recognizing ignore context manager +def is_torch_jit_ignore_context_manager(stmt) -> bool: + # checks if the statement is torch.jit.ignore context manager + if isinstance(stmt.items[0].context_expr, ast.Call): + # extract torch part + function = stmt.items[0].context_expr.func + if isinstance(function, ast.Attribute): + attr_name = function.attr + attr_value = function.value + if attr_name == "_IgnoreContextManager" and isinstance( + attr_value, ast.Attribute + ): + # there should be at most two nested attributes (e.g torch.jit._IgnoreContextManager) + if attr_value.attr == "jit" and isinstance(attr_value.value, ast.Name): + if attr_value.value.id == "torch": + return True + return False + + +class Builder: + def __call__(self, ctx, node): + method = getattr(self, "build_" + node.__class__.__name__, None) + if method is None: + raise UnsupportedNodeError(ctx, node) + return method(ctx, node) + + +def build_class_def(ctx, py_def, methods, properties, self_name, assigns): + r = ctx.make_range( + py_def.lineno, py_def.col_offset, py_def.col_offset + len("class") + ) + return ClassDef( + Ident(r, self_name), [Stmt(method) for method in methods], properties, assigns + ) + + +def build_def(ctx, py_def, type_line, def_name, self_name=None, pdt_arg_types=None): + body = py_def.body + r = ctx.make_range(py_def.lineno, py_def.col_offset, py_def.col_offset + len("def")) + + param_list = build_param_list(ctx, py_def.args, self_name, pdt_arg_types) + return_type = None + if getattr(py_def, "returns", None) is not None: + return_type = build_expr(ctx, py_def.returns) + + decl = Decl(r, param_list, return_type) + is_method = self_name is not None + if type_line is not None: + type_comment_decl = torch._C.parse_type_comment(type_line) + decl = torch._C.merge_type_from_type_comment( + decl, # type: ignore[arg-type] + type_comment_decl, + is_method, # type: ignore[assignment] + ) + + return Def(Ident(r, def_name), decl, build_stmts(ctx, body)) + + +_vararg_kwarg_err = ( + "Compiled functions can't take variable number of arguments " + "or use keyword-only arguments with defaults" +) + + +def build_param_list(ctx, py_args, self_name, pdt_arg_types=None): + if py_args.kwarg is not None: + expr = py_args.kwarg + ctx_range = ctx.make_range( + expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg) + ) + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + if py_args.vararg is not None: + expr = py_args.vararg + ctx_range = ctx.make_range( + expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg) + ) + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + if len(py_args.kw_defaults) > 0: + # kw_defaults is a list of the values for the kwargs (which default to None), + # so they don't actually have line numbers. + for arg in py_args.kw_defaults: + if arg is not None: + ctx_range = build_expr(ctx, arg).range() + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + + # List of Tuple of args and type as inferred by profile directed typing + arg_and_types = [ + ( + arg, + pdt_arg_types[arg.arg] + if pdt_arg_types and bool(pdt_arg_types[arg.arg]) + else None, + ) + for arg in py_args.args + ] + arg_and_types_kwonlyargs = [ + ( + arg, + pdt_arg_types[arg.arg] + if pdt_arg_types and bool(pdt_arg_types[arg.arg]) + else None, + ) + for arg in py_args.kwonlyargs + ] + + result = [ + build_param(ctx, arg, self_name, kwarg_only=False, pdt_arg_type=arg_type) + for arg, arg_type in arg_and_types + ] + result += [ + build_param(ctx, arg, self_name, kwarg_only=True, pdt_arg_type=arg_type) + for arg, arg_type in arg_and_types_kwonlyargs + ] + return result + + +def build_param(ctx, py_arg, self_name, kwarg_only, pdt_arg_type=None): + # NB: In Python3 py_arg is a pair of (str arg, expr? annotation) + name = py_arg.arg + r = ctx.make_range(py_arg.lineno, py_arg.col_offset, py_arg.col_offset + len(name)) + if getattr(py_arg, "annotation", None) is not None: + annotation_expr = build_expr(ctx, py_arg.annotation) + elif pdt_arg_type: + annotation_expr = Var(Ident(r, pdt_arg_type)) + elif self_name is not None and name == "self": + annotation_expr = Var(Ident(r, self_name)) + else: + annotation_expr = EmptyTypeAnnotation(r) + return Param(annotation_expr, Ident(r, name), kwarg_only) + + +def build_ignore_context_manager(ctx, stmt): + InputType = namedtuple("InputType", ["name", "ann"]) + OutputType = namedtuple("OutputType", ["name", "ann"]) + + def process_ins_outs(args): + # parse the context manager to figure out inputs and outputs + # with their annotated types + # TODO: add input, output validator + inputs = [] + outputs = [] + for arg in args: + var_name = arg.arg + var_ann = arg.value.value + var_decl_type, var_ann = var_ann.split(":") + if var_decl_type == "inp": + inputs.append(InputType(var_name, var_ann)) + if var_decl_type == "out": + outputs.append(OutputType(var_name, var_ann)) + return inputs, outputs + + def create_unique_name_ext(ctx, stmt) -> str: + # extension will be based on the full path filename plus + # the line number of original context manager + fn = re.sub(r"[^a-zA-Z0-9_]", "_", ctx.filename) + return f"{fn}_{stmt.lineno}" + + def build_return_ann_stmt(outputs): + return_type_ann = "" + return_statement_str = "return " + if len(outputs) == 0: + return_type_ann += " -> None" + if len(outputs) == 1: + return_type_ann = " -> " + outputs[0].ann + return_statement_str += outputs[0].name + if len(outputs) > 1: + return_type_ann = " -> tuple" + return_type_ann += "[" + ", ".join([var.ann for var in outputs]) + "]" + return_statement_str += ", ".join([var.name for var in outputs]) + return return_type_ann, return_statement_str + + def build_args(args): + return ", ".join([arg.name for arg in args]) + + inputs, outputs = process_ins_outs(stmt.items[0].context_expr.keywords) + + # build the replacement function str with given inputs and outputs + ignore_function_name = "func_ignore_" + create_unique_name_ext(ctx, stmt) + ignore_function_str = "\ndef " + ignore_function_name + ignore_function_str += ( + "(" + ", ".join([var.name + " :" + var.ann for var in inputs]) + ")" + ) + + return_ann, return_stmt = build_return_ann_stmt(outputs) + ignore_function_str += return_ann + ": pass" + + # first create the functionDef object from just declaration + ignore_function = ast.parse(ignore_function_str).body[0] + + # dump the body of context manager to dummy function + ignore_function.body = stmt.body # type: ignore[attr-defined] + + # insert return statement to the function + return_stmt = ast.parse(return_stmt).body[0] + ignore_function.body.append(return_stmt) # type: ignore[attr-defined] + + ignore_func_str = f"""\ +# Backward compat: These used to be imported into the outer global scope so some +# code may still expect them. +from typing import List, Dict, Tuple + +@torch.jit.ignore +{ast.unparse(ignore_function)} +""" + g = copy.copy(globals()) + exec(ignore_func_str, g) # noqa: P204 + # registers the custom function in the global context + globals()[ignore_function_name] = g[ignore_function_name] + + # build the statements as: + # , , ... = torch.jit.frontend.(, ) + assign_str_lhs = build_args(outputs) + # this function will be registered in torch.jit.frontend module by default + assign_str_rhs = ( + f"torch.jit.frontend.{ignore_function_name}(" + build_args(inputs) + ")" + ) + + if len(outputs) > 0: + assign_str = assign_str_lhs + " = " + assign_str_rhs + else: + assign_str = assign_str_rhs + assign_ast = ast.parse(assign_str).body[0] + return assign_ast + + +def get_default_args(fn): + """ + Get a dictionary of default arguments for a function. + + Args: + fn: Callable - The function to inspect for default arguments. + Returns: + (Dict[str, Any]): mapping argument names to their default values if + :attr:`fn` is not None, else empty dictionary. + """ + if fn is None: + return {} + + signature = inspect.signature(fn) + + return { + k: v.default + for k, v in signature.parameters.items() + if v.default is not inspect.Parameter.empty + } + + +def get_default_args_for_class(cls): + """ + Get default arguments for all methods in a class (except for static methods). + + Args: + cls: type - The class type to inspect for default arguments. + Returns: + A Dict[str, Dict[str, Any]] which maps each method name to a Dict[str, Any] + that maps each argument name to its default value. + """ + # Get methods (except static methods because those are compiled separately as + # if they were independent script functions). + methods = inspect.getmembers( + cls, + predicate=lambda m: (inspect.ismethod(m) or inspect.isfunction(m)) + and not is_static_fn(cls, m.__name__) + and m.__name__ in cls.__dict__, + ) + + # Get method defaults. Property defaults do not need to be considered + # because setters cannot be invoked without a value. + defaults = { + method_name: get_default_args(method_impl) + for method_name, method_impl in methods + } + + return defaults + + +class WithItemBuilder(Builder): + @staticmethod + def build_withitem(ctx, item): + lineno = item.context_expr.lineno + start = item.context_expr.col_offset + end = start + len(pretty_node_names[ast.With]) + op_vars = item.optional_vars + r = ctx.make_range(lineno, start, end) + + return WithItem( + r, + build_expr(ctx, item.context_expr), + build_expr(ctx, op_vars) if op_vars else None, + ) + + +class StmtBuilder(Builder): + augassign_map = { + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.Mod: "%", + ast.BitOr: "|", + ast.BitAnd: "&", + ast.BitXor: "^", + ast.LShift: "<<", + ast.RShift: ">>", + ast.Pow: "**", + } + + @staticmethod + def build_Expr(ctx, stmt): + value = stmt.value + if value.__class__.__name__ == "Str": + # If a statement is a string literal expression, + # then it is a docstring. Just ignore it. + return None + else: + return ExprStmt(build_expr(ctx, value)) + + @staticmethod + def build_Assign(ctx, stmt): + rhs = build_expr(ctx, stmt.value) + lhs = [build_expr(ctx, x) for x in stmt.targets] + return Assign(lhs, rhs) + + @staticmethod + def build_AnnAssign(ctx, stmt): + if stmt.value is None: + raise UnsupportedNodeError(ctx, stmt, reason="without assigned value") + + # Disallow type annotations on instance attributes outside of __init__ + if ( + type(stmt.target) is ast.Attribute + and stmt.target.value.id == "self" # type: ignore[attr-defined] + and ctx.funcname != "__init__" + ): + start = stmt.col_offset + end = start + len(f"self.{stmt.target.attr}") + if hasattr(stmt.annotation, "id"): + end += len(f": {stmt.annotation.id}") + sr = ctx.make_range(stmt.lineno, start, end) + raise ValueError( + "Type annotations on instance attributes must be declared in " + f"__init__, not '{ctx.funcname}': {sr}" + ) + + rhs = build_expr(ctx, stmt.value) + lhs = build_expr(ctx, stmt.target) + the_type = build_expr(ctx, stmt.annotation) + return Assign([lhs], rhs, the_type) + + @staticmethod + def build_Delete(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("del")) + + return Delete(r, [build_expr(ctx, target) for target in stmt.targets]) + + @staticmethod + def build_Return(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("return") + ) + return Return(r, None if stmt.value is None else build_expr(ctx, stmt.value)) + + @staticmethod + def build_Raise(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("raise")) + expr = build_expr(ctx, stmt.exc) + return Raise(r, expr) + + @staticmethod + def build_Assert(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("assert") + ) + test = build_expr(ctx, stmt.test) + msg = build_expr(ctx, stmt.msg) if stmt.msg is not None else None + return Assert(r, test, msg) + + @staticmethod + def build_AugAssign(ctx, stmt): + lhs = build_expr(ctx, stmt.target) + rhs = build_expr(ctx, stmt.value) + op = type(stmt.op) + if op in StmtBuilder.augassign_map: + op_token = StmtBuilder.augassign_map[op] + else: + raise NotSupportedError( + find_before(ctx, rhs.range().start, "=", offsets=(-1, 0)), + "unsupported kind of augmented assignment: " + op.__name__, + ) + return AugAssign(lhs, op_token, rhs) + + @staticmethod + def build_While(ctx, stmt): + if stmt.orelse: + # TODO: try to recover the location of else:? Python doesn't give us useful + # annotations in this case + raise NotSupportedError( + None, "else branches of while loops aren't supported" + ) + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("while")) + return While(r, build_expr(ctx, stmt.test), build_stmts(ctx, stmt.body)) + + @staticmethod + def build_For(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("for")) + if stmt.orelse: + raise NotSupportedError(r, "else branches of for loops aren't supported") + + return For( + r, + [build_expr(ctx, stmt.target)], + [build_expr(ctx, stmt.iter)], + build_stmts(ctx, stmt.body), + ) + + @staticmethod + def build_If(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("if")) + return If( + r, + build_expr(ctx, stmt.test), + build_stmts(ctx, stmt.body), + build_stmts(ctx, stmt.orelse), + ) + + @staticmethod + def build_Print(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("print")) + if stmt.dest: + raise NotSupportedError( + r, "print statements with non-default destinations aren't supported" + ) + args = [build_expr(ctx, val) for val in stmt.values] + return ExprStmt(Apply(Var(Ident(r, "print")), args, [])) + + @staticmethod + def build_Pass(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("pass")) + return Pass(r) + + @staticmethod + def build_Break(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("break")) + return Break(r) + + @staticmethod + def build_Continue(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("continue") + ) + return Continue(r) + + @staticmethod + def build_With(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("with")) + # Handle ignore context manager + if is_torch_jit_ignore_context_manager(stmt): + assign_ast = build_ignore_context_manager(ctx, stmt) + return build_stmt(ctx, assign_ast) + return With(r, build_withitems(ctx, stmt.items), build_stmts(ctx, stmt.body)) + + +class ExprBuilder(Builder): + binop_map = { + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.Pow: "**", + ast.Mod: "%", + ast.FloorDiv: "//", + ast.BitAnd: "&", + ast.BitXor: "^", + ast.BitOr: "|", + ast.LShift: "<<", + ast.RShift: ">>", + } + + # pyrefly: ignore [unsupported-operation] + binop_map[ast.MatMult] = "@" + + unop_map = { + ast.Not: "not", + ast.USub: "-", + ast.Invert: "~", + } + + boolop_map = { + ast.And: "and", + ast.Or: "or", + } + + cmpop_map = { + ast.Eq: "==", + ast.NotEq: "!=", + ast.LtE: "<=", + ast.Lt: "<", + ast.GtE: ">=", + ast.Gt: ">", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in", + } + + @staticmethod + def build_Attribute(ctx, expr): + base = build_expr(ctx, expr.value) + # expr.attr is just a string, so it's not annotated in any way, so we have + # to build the range manually + source = ctx.source.encode("utf-8") + + def get_char(index): + return chr(source[index]) + + start_pos = base.range().end + 1 + while get_char(start_pos) in string.whitespace: # Skip whitespace + start_pos += 1 + end_pos = start_pos + len(expr.attr) + name_range = ctx.make_raw_range(start_pos, end_pos) + return Select(base, Ident(name_range, expr.attr)) + + @staticmethod + def build_Call(ctx, expr): + func = build_expr(ctx, expr.func) + args = [build_expr(ctx, py_arg) for py_arg in expr.args] + if hasattr(expr, "starargs") and expr.starargs: + stararg_expr = build_expr(ctx, expr.starargs) + args += [Starred(stararg_expr.range(), stararg_expr)] + kwargs = [] + for kw in expr.keywords: + kw_expr = build_expr(ctx, kw.value) + # XXX: we could do a better job at figuring out the range for the name here + if not kw.arg: + raise NotSupportedError( + kw_expr.range(), "keyword-arg expansion is not supported" + ) + kwargs.append(Attribute(Ident(kw_expr.range(), kw.arg), kw_expr)) + return Apply(func, args, kwargs) + + @staticmethod + def build_Ellipsis(ctx, expr): + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + 3 + ) # len("...") == 3 + return Dots(r) + + @staticmethod + def build_Name(ctx, expr): + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(expr.id)) + if expr.id.startswith(_reserved_prefix): + raise NotSupportedError( + r, + "names of variables used in JIT-ed functions " + "can't start with " + _reserved_prefix, + ) + if expr.id == "True": + return TrueLiteral(r) + elif expr.id == "False": + return FalseLiteral(r) + elif expr.id == "None": + return NoneLiteral(r) + elif expr.id == "Ellipsis": + return Dots(r) + return Var(Ident(r, expr.id)) + + @staticmethod + def build_NameConstant(ctx, expr): + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(str(expr.value)) + ) + if expr.value is True: + return TrueLiteral(r) + elif expr.value is False: + return FalseLiteral(r) + elif expr.value is None: + return NoneLiteral(r) + elif expr.value == Ellipsis: + return Dots(r) + else: + raise ValueError("Name constant value unsupported: " + str(expr.value)) + + @staticmethod + def build_BinOp(ctx, expr): + lhs = build_expr(ctx, expr.left) + rhs = build_expr(ctx, expr.right) + op = type(expr.op) + + if op == ast.Div and not ctx.uses_true_division: + err_range = ctx.make_raw_range(lhs.range().end, rhs.range().start) + raise FrontendError( + err_range, + "Division of ints in TorchScript uses Python 3 true " + "division semantics. Please put `from __future__ " + "import division` at the top of your file", + ) + op_token = ExprBuilder.binop_map.get(op) + if op_token is None: + err_range = ctx.make_raw_range(lhs.range().end, rhs.range().start) + raise NotSupportedError( + err_range, "unsupported binary operator: " + op.__name__ + ) + return BinOp(op_token, lhs, rhs) + + @staticmethod + def build_UnaryOp(ctx, expr): + sub_expr = build_expr(ctx, expr.operand) + op = type(expr.op) + op_token = ExprBuilder.unop_map.get(op) + if op_token is None: + raise NotSupportedError( + expr.range(), "unsupported unary operator: " + op.__name__ + ) + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(op_token) + ) + return UnaryOp(r, op_token, sub_expr) + + @staticmethod + def build_BoolOp(ctx, expr): + if len(expr.values) < 2: + raise AssertionError( + "expected at least 2 values in BoolOp, but got " + str(len(expr.values)) + ) + sub_exprs = [build_expr(ctx, sub_expr) for sub_expr in expr.values] + op = type(expr.op) + op_token = ExprBuilder.boolop_map.get(op) + if op_token is None: + err_range = ctx.make_raw_range( + sub_exprs[0].range().end, sub_exprs[1].range().start + ) + raise NotSupportedError( + err_range, "unsupported boolean operator: " + op.__name__ + ) + lhs = sub_exprs[0] + for rhs in sub_exprs[1:]: + lhs = BinOp(op_token, lhs, rhs) + return lhs + + @staticmethod + def build_IfExp(ctx, expr): + return TernaryIf( + build_expr(ctx, expr.test), + build_expr(ctx, expr.body), + build_expr(ctx, expr.orelse), + ) + + @staticmethod + def build_Compare(ctx, expr): + operands = [build_expr(ctx, e) for e in [expr.left] + list(expr.comparators)] + result = None + for lhs, op_, rhs in zip(operands, expr.ops, operands[1:]): + op = type(op_) + op_token = ExprBuilder.cmpop_map.get(op) + r = ctx.make_raw_range(lhs.range().end, rhs.range().start) + if op_token is None: + raise NotSupportedError( + r, "unsupported comparison operator: " + op.__name__ + ) + + if op == ast.NotIn: + # NB: `not in` is just `not( in )`, so we don't introduce new tree view + # but just make it a nested call in our tree view structure + in_expr = BinOp("in", lhs, rhs) + cmp_expr = UnaryOp(r, "not", in_expr) + else: + cmp_expr = BinOp(op_token, lhs, rhs) # type: ignore[assignment] + + if result is None: + result = cmp_expr + else: + result = BinOp("and", result, cmp_expr) # type: ignore[assignment] + return result + + @staticmethod + def build_Subscript(ctx, expr): + def build_SliceExpr(ctx, base, slice_expr): + lower = ( + build_expr(ctx, slice_expr.lower) + if slice_expr.lower is not None + else None + ) + upper = ( + build_expr(ctx, slice_expr.upper) + if slice_expr.upper is not None + else None + ) + step = ( + build_expr(ctx, slice_expr.step) + if slice_expr.step is not None + else None + ) + return SliceExpr(base.range(), lower, upper, step) + + def build_Index(ctx, base, index_expr): + if isinstance(index_expr.value, ast.Tuple): + raise NotSupportedError( + base.range(), + "slicing multiple dimensions with tuples not supported yet", + ) + return build_expr(ctx, index_expr.value) + + def build_ExtSlice(ctx, base, extslice): + sub_exprs = [] + for expr in extslice.dims: + sub_type = type(expr) + if sub_type is ast.Index: + sub_exprs.append(build_Index(ctx, base, expr)) + elif sub_type is ast.Slice: + sub_exprs.append(build_SliceExpr(ctx, base, expr)) + elif sub_type is ast.Constant and expr.value is Ellipsis: + sub_exprs.append(Dots(base.range())) + else: + raise NotSupportedError( + base.range(), + f"slicing multiple dimensions with {sub_type} not supported", + ) + return sub_exprs + + base = build_expr(ctx, expr.value) + sub_type = type(expr.slice) + if sub_type is ast.Index: + if isinstance(expr.slice.value, ast.Tuple): + # N-dimensional indexing using Tuple: x[(i, j, k)] is equivalent to x[i, j, k] + # XXX: Indexing using a list is **different**! It triggers advanced indexing. + indices = [ + build_expr(ctx, index_expr) for index_expr in expr.slice.value.elts + ] + if not indices: + # `col_offset` is an int, but `end_col_offset` is + # `Optional[int]`. The magic number is here to make + # sure we can parse `()` on any machine + r = ctx.make_range( + expr.lineno, + expr.slice.value.col_offset, + expr.slice.value.col_offset + 2, + ) + tup = TupleLiteral(r, []) + indices.append(tup) + return Subscript(base, indices) + else: + return Subscript(base, [build_expr(ctx, expr.slice.value)]) + elif sub_type is ast.Slice: + return Subscript(base, [build_SliceExpr(ctx, base, expr.slice)]) + elif sub_type is ast.ExtSlice: + return Subscript(base, build_ExtSlice(ctx, base, expr.slice)) + else: # In Python3.9 array indices are not wrapped in ast.Index + if sub_type is ast.Tuple: + # N-dimensional indexing using Tuple: x[(i, j, k)] is equivalent to x[i, j, k] + indices = [] + for index_expr in expr.slice.elts: + if isinstance(index_expr, ast.Slice): + indices.append(build_SliceExpr(ctx, base, index_expr)) + else: + indices.append(build_expr(ctx, index_expr)) + # Special-case logic for `typing.Tuple[()]` + if not indices: + # See note above r.e. magic number + r = ctx.make_range( + expr.lineno, expr.slice.col_offset, expr.slice.col_offset + 2 + ) + tup = TupleLiteral(r, []) + indices.append(tup) + return Subscript(base, indices) + return Subscript(base, [build_expr(ctx, expr.slice)]) + + @staticmethod + def build_List(ctx, expr): + return ListLiteral( + ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1), + [build_expr(ctx, e) for e in expr.elts], + ) + + @staticmethod + def build_Tuple(ctx, expr): + return TupleLiteral( + ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1), + [build_expr(ctx, e) for e in expr.elts], + ) + + @staticmethod + def build_Dict(ctx, expr): + range = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + if expr.keys and not expr.keys[0]: + raise NotSupportedError( + range, "Dict expansion (e.g. `{**dict}`) is not supported" + ) + return DictLiteral( + range, + [build_expr(ctx, e) for e in expr.keys], + [build_expr(ctx, e) for e in expr.values], + ) + + @staticmethod + def build_Num(ctx, expr): + value = str(expr.value) + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(value)) + return Const(r, value) + + @staticmethod + def build_Constant(ctx, expr): + value = expr.value + if value is None or isinstance(value, bool): + # NB: this check has to happen before the int check because bool is + # a subclass of int + return ExprBuilder.build_NameConstant(ctx, expr) + if isinstance(value, (int, float, complex)): + return ExprBuilder.build_Num(ctx, expr) + elif isinstance(value, str): + return ExprBuilder.build_Str(ctx, expr) + elif isinstance(value, type(Ellipsis)): + return ExprBuilder.build_Ellipsis(ctx, expr) + else: + error_range = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(str(value)) + ) + raise FrontendError(error_range, "Unknown Constant expression type") + + @staticmethod + def build_Str(ctx, expr): + value = str(expr.value) + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(value) + 1 + ) + return StringLiteral(r, value) + + @staticmethod + def build_JoinedStr(ctx, expr): + s = "" + args = [] + for value in expr.values: + r = ctx.make_range(value.lineno, value.col_offset, value.col_offset + 1) + if isinstance(value, ast.FormattedValue): + if value.conversion != -1: + raise NotSupportedError(r, "Don't support conversion in JoinedStr") + if value.format_spec is not None: + raise NotSupportedError(r, "Don't support formatting in JoinedStr") + s += "{}" + args.append(build_expr(ctx, value.value)) + elif isinstance(value, ast.Constant): + # pyrefly: ignore [unsupported-operation] + s += value.value + else: + raise NotSupportedError(r, "Unsupported value in JoinedStr") + + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return Apply(Select(StringLiteral(r, s), Ident(r, "format")), args, []) + + @staticmethod + def build_ListComp(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset) + if len(stmt.generators) != 1: + raise NotSupportedError(r, "Only a single generator is currently supported") + + if len(stmt.generators[0].ifs) != 0: + raise NotSupportedError(r, "Comprehension ifs are not supported yet") + + elt_expr = build_expr(ctx, stmt.elt) + target_expr = build_expr(ctx, stmt.generators[0].target) + iter_expr = build_expr(ctx, stmt.generators[0].iter) + + return ListComp(r, elt_expr, target_expr, iter_expr) + + @staticmethod + def build_GeneratorExp(ctx, stmt): + # Convert Generator expression to ListComp + return ExprBuilder.build_ListComp(ctx, stmt) + + @staticmethod + def build_DictComp(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset) + if len(stmt.generators) != 1: + raise NotSupportedError(r, "Only a single generator is currently supported") + + if len(stmt.generators[0].ifs) != 0: + raise NotSupportedError(r, "Comprehension ifs are not supported yet") + + key_expr = build_expr(ctx, stmt.key) + value_expr = build_expr(ctx, stmt.value) + target_expr = build_expr(ctx, stmt.generators[0].target) + iter_expr = build_expr(ctx, stmt.generators[0].iter) + + return DictComp(r, key_expr, value_expr, target_expr, iter_expr) + + @staticmethod + def build_Starred(ctx, expr): + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return Starred(r, build_expr(ctx, expr.value)) + + +build_expr = ExprBuilder() +build_stmt = StmtBuilder() +build_withitem = WithItemBuilder() + + +def find_before(ctx, pos, substr, offsets=(0, 0)): + new_pos = ctx.source[:pos].rindex(substr) + return ctx.make_raw_range(new_pos + offsets[0], new_pos + len(substr) + offsets[1]) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/generate_bytecode.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/generate_bytecode.py new file mode 100644 index 0000000000000000000000000000000000000000..79f37d794ba8e1a90623e68916fca973e52897c9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/generate_bytecode.py @@ -0,0 +1,33 @@ +# mypy: allow-untyped-defs + +from torch._C import _compile_graph_to_code_table, _generate_upgraders_graph + + +def format_bytecode(table): + # given a nested tuple, convert it to nested list + def listify(content): + if not isinstance(content, tuple): + return content + return [listify(i) for i in content] + + formatted_table = {} + for entry in table: + identifier = entry[0] + content = entry[1] + content = listify(content) + formatted_table[identifier] = content + return formatted_table + + +def generate_upgraders_bytecode() -> list: + yaml_content = [] + upgraders_graph_map = _generate_upgraders_graph() + for upgrader_name, upgrader_graph in upgraders_graph_map.items(): + bytecode_table = _compile_graph_to_code_table(upgrader_name, upgrader_graph) + entry = {upgrader_name: format_bytecode(bytecode_table)} + yaml_content.append(entry) + return yaml_content + + +if __name__ == "__main__": + raise RuntimeError("This file is not meant to be run directly") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/quantized.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/quantized.py new file mode 100644 index 0000000000000000000000000000000000000000..3b455e5673c98605afbb07fb53d2c01d3b4a3bc4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/quantized.py @@ -0,0 +1,100 @@ +# mypy: allow-untyped-defs +import torch + + +class QuantizedLinear(torch.jit.ScriptModule): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedLinear is no longer supported. Please use " + "torch.ao.nn.quantized.dynamic.Linear instead." + ) + + +# FP16 weights +class QuantizedLinearFP16(torch.jit.ScriptModule): + def __init__(self, other): + super().__init__() + raise RuntimeError( + "torch.jit.QuantizedLinearFP16 is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.Linear instead." + ) + + +# Quantized RNN cell implementations +class QuantizedRNNCellBase(torch.jit.ScriptModule): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedRNNCellBase is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.RNNCell instead." + ) + + +class QuantizedRNNCell(QuantizedRNNCellBase): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedRNNCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.RNNCell instead." + ) + + +class QuantizedLSTMCell(QuantizedRNNCellBase): + def __init__(self, other): + super().__init__(other) + raise RuntimeError( + "torch.jit.QuantizedLSTMCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.LSTMCell instead." + ) + + +class QuantizedGRUCell(QuantizedRNNCellBase): + def __init__(self, other): + super().__init__(other) + raise RuntimeError( + "torch.jit.QuantizedGRUCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.GRUCell instead." + ) + + +class QuantizedRNNBase(torch.jit.ScriptModule): + def __init__(self, other, dtype=torch.int8): + raise RuntimeError( + "torch.jit.QuantizedRNNBase is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic instead." + ) + + +class QuantizedLSTM(QuantizedRNNBase): + def __init__(self, other, dtype): + raise RuntimeError( + "torch.jit.QuantizedLSTM is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.LSTM instead." + ) + + +class QuantizedGRU(QuantizedRNNBase): + def __init__(self, *args, **kwargs): + raise RuntimeError( + "torch.jit.QuantizedGRU is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.GRU instead." + ) + + +def quantize_rnn_cell_modules(module): + raise RuntimeError( + "quantize_rnn_cell_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) + + +def quantize_linear_modules(module, dtype=torch.int8): + raise RuntimeError( + "quantize_linear_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) + + +def quantize_rnn_modules(module, dtype=torch.int8): + raise RuntimeError( + "quantize_rnn_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/supported_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/supported_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..302b4cbea0cb29ab3966bc119c4c1a7f7690d972 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/supported_ops.py @@ -0,0 +1,346 @@ +# mypy: allow-untyped-defs +import inspect +import textwrap + +import torch.jit +from torch.jit._builtins import _find_builtin + + +# this file is for generating documentation using sphinx autodoc +# > help(torch.jit.supported_ops) will also give a nice listed of the +# supported ops programmatically + + +def _hidden(name): + return name.startswith("_") and not name.startswith("__") + + +def _emit_type(type): + return str(type) + + +def _emit_arg(indent, i, arg): + v = f"{arg.name} : {_emit_type(arg.type)}" + default = arg.default_value + if default is not None: + v = f"{v}={str(default)}" + if i > 0: + v = f"\n{' ' * indent}{v}" + return v + + +def _emit_args(indent, arguments): + return ",".join(_emit_arg(indent, i, arg) for i, arg in enumerate(arguments)) + + +def _emit_ret(ret): + return _emit_type(ret.type) + + +def _emit_rets(returns): + if len(returns) == 1: + return _emit_ret(returns[0]) + return f"Tuple[{', '.join(_emit_ret(r) for r in returns)}]" + + +def _emit_schema(mod, name, schema, arg_start=0, padding=4): + if mod is None: + qualified_name = name + else: + qualified_name = f"{mod}.{name}" + schema_str = ( + f"{qualified_name}" + f"({_emit_args(len(qualified_name) + 1 + padding, schema.arguments[arg_start:])}) " + f"-> {_emit_rets(schema.returns)}" + ) + return schema_str + + +def _get_tensor_ops(): + def is_tensor_method(schema) -> bool: + if len(schema.arguments) == 0: + return False + self = schema.arguments[0] + if self.name != "self": + return False + if not self.type.isSubtypeOf(torch._C.TensorType.get()): + return False + return True + + methods = [] + # discover methods + for elem in dir(torch.Tensor): + if not _hidden(elem): + schemas = torch._C._jit_get_schemas_for_operator("aten::" + elem) + for schema in schemas: + if is_tensor_method(schema): + methods.append(_emit_schema("Tensor", elem, schema, arg_start=1)) + + return "Supported Tensor Methods", methods + + +def _get_nn_functional_ops(): + functions = [] + + # Iterate over torch.nn.functional + mod = torch.nn.functional + name = mod.__name__ + for elem in dir(torch.nn.functional): + attr = getattr(mod, elem) + if not inspect.isfunction(attr) or _hidden(elem[0]): + # Ignore non-functions and internal methods + continue + + attr_module = inspect.getmodule(attr) + if not attr_module: + raise RuntimeError(f"Module for {attr} not found") + + if "torch.nn.functional" not in attr_module.__name__: + # Ignore functions from outside torch.nn.functional + continue + + try: + # compile fn, get schema + scripted = torch.jit.script(attr) + scripted_schema = scripted.schema + functions.append(_emit_schema(name, elem, scripted_schema)) + except: # noqa: B001,E722 + # Skip interpolate / boolean dispatched things + pass + + # Iterate over modules that we know contain a lot of builtins + for mod in torch.jit._builtins._modules_containing_builtins: + name = mod.__name__ + for elem in dir(mod): + builtin = _find_builtin(getattr(mod, elem)) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + # remove _tan but not __and__ + if not _hidden(elem): + functions.append(_emit_schema(name, elem, schema)) + return "Supported PyTorch Functions", functions + + +def _get_builtins_helper(): + builtins = [] + for fn, _builtin_name in torch.jit._builtins._builtin_ops: + mod = inspect.getmodule(fn) + + if not hasattr(fn, "__name__"): + # typing classes + continue + if not mod: + continue + if _hidden(fn.__name__) or _hidden(fn.__qualname__) or _hidden(mod.__name__): + # skip internal-only methods + continue + + if "torch._C" in mod.__name__: + continue + + builtins.append((fn, _builtin_name)) + + return builtins + + +def _is_math_fn(fn): + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + + return mod.__name__ == "math" + + +def _get_torchscript_builtins(): + functions = [] + builtins = filter(lambda fn: not _is_math_fn(fn[0]), _get_builtins_helper()) + builtins_list = list(builtins) + # Iterate over the specially added builtins + for fn, _builtin_name in builtins_list: + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + builtin = _find_builtin(fn) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + functions.append(_emit_schema(mod.__name__, fn.__name__, schema)) + + return "TorchScript Builtin Functions", functions + + +def _get_math_builtins(): + functions = [] + builtins = filter(lambda fn: _is_math_fn(fn[0]), _get_builtins_helper()) + builtins_list = list(builtins) + # Iterate over the specially added builtins + for fn, _builtin_name in builtins_list: + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + builtin = _find_builtin(fn) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + schema_str = _emit_schema(mod.__name__, fn.__name__, schema) + if "Tensor" in schema_str: + # Skip Tensor ops that have the same name as math functions + # (they will show up in the tensor methods section) + continue + functions.append(schema) + + return "``math`` Module", functions + + +def _get_global_builtins(): + # Taken from the 'globals' map in torch/csrc/jit/frontend/ir_emitter.cpp + supported_builtins = [ + "print", + "tuple", + "float", + "complex", + "int", + "bool", + "str", + "getattr", + "hasattr", + "isinstance", + "len", + "hex", + "oct", + "round", + "hash", + "min", + "max", + "abs", + "all", + "divmod", + "list", + "ord", + "chr", + "bin", + "range", + "zip", + "enumerate", + "sorted", + ] + + op_renames = { + "bool": "aten::Bool", + "int": "aten::Int", + "float": "aten::Float", + "complex": "aten::Complex", + "abs": "prim::abs", + "max": "prim::max", + "min": "prim::min", + "range": "fake::does_not_exist", + } + + schemaless_op_explanations = { + "print": "Print any value", + "tuple": "Lists cannot be converted to tuples with this method since their size is not statically known", + "getattr": "Attribute name must be a literal string", + "hasattr": "Attribute name must be a literal string", + "isinstance": "Result is static", + "zip": "Arguments must be iterable.", + "enumerate": "Arguments must be iterable.", + "range": "Can only be used as an iterator in a for loop", + } + + magic_methods = [ + ("complex", "__complex__"), + ("float", "__float__"), + ("int", "__int__"), + ("bool", "__bool__"), + ("str", "__str__"), + ("len", "__len__"), + ("hex", "__hex__"), + ("oct", "__oct__"), + ] + + magic_methods_rows = [] + for fn, magic_method in magic_methods: + # pyrefly: ignore [bad-argument-type] + magic_methods_rows.append(f'"{fn}", "``{magic_method}``"') + + schematized_ops = [] + schemaless_ops = [] + + for fn in supported_builtins: + op_name = f"aten::{fn}" + if fn in op_renames: + op_name = op_renames[fn] + schemas = torch._C._jit_get_schemas_for_operator(op_name) + for s in schemas: + schematized_ops.append(_emit_schema(None, fn, s, padding=0)) + if len(schemas) > 0: + schematized_ops.append("") + else: + table_row = ( + f'":external+python:py:obj:`{fn}`", "{schemaless_op_explanations[fn]}"' + ) + # pyrefly: ignore [bad-argument-type] + schemaless_ops.append(table_row) + + schematized_ops_str = "\n".join(schematized_ops) + schemaless_ops_str = "\n".join(schemaless_ops) + magic_methods_rows_str = "\n".join(magic_methods_rows) + schematized_ops_str = textwrap.indent(schematized_ops_str, "\t") + schemaless_ops_str = textwrap.indent(schemaless_ops_str, "\t") + magic_methods_rows_str = textwrap.indent(magic_methods_rows_str, "\t") + section = f""" +The functions in the following table are supported but do not have a static schema + +.. csv-table:: + :header: "Function", "Note" + +{schemaless_ops_str} + +The following functions will use the corresponding magic method on TorchScript classes + +.. csv-table:: + :header: "Function", "Magic Method" + +{magic_methods_rows_str} + +These built-in functions use the schema + +.. rst-class:: codeblock-height-limiter + +:: + +{schematized_ops_str} + """ + + return "Python Built-in Functions", section + + +def _list_supported_ops(): + def emit_block(decls): + return "\n.. rst-class:: codeblock-height-limiter\n\n::\n\n{}\n".format( + "".join(f" {d}\n\n" for d in decls) + ) + + body = "" + op_gathering_fns = ( + _get_tensor_ops, + _get_nn_functional_ops, + _get_torchscript_builtins, + _get_global_builtins, + _get_math_builtins, + ) + for fn in op_gathering_fns: + header, items = fn() + link_target = header.replace("`", "").replace("-", "").lower().replace(" ", "-") + if isinstance(items, str): + section = f"{header}\n{'~' * len(header)}\n{items}\n" + else: + section = f"{header}\n{'~' * len(header)}\n{emit_block(items)}" + section = f".. _{link_target}:" + "\n\n" + section + body += section + + return body + + +__doc__ = _list_supported_ops() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/unsupported_tensor_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/unsupported_tensor_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..edcba6ab100cd41dbf724fa87a435c699c1bfed3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/jit/unsupported_tensor_ops.py @@ -0,0 +1,78 @@ +# mypy: allow-untyped-defs +from textwrap import dedent +from typing import Any + +import torch.jit + + +def execWrapper(code, glob, loc) -> None: + exec(code, glob, loc) + + +def _gen_unsupported_methods_properties(): + tensor_attrs = set(filter(lambda x: x[0] != "_", dir(torch.Tensor))) + tensor = torch.tensor([2]) + funcs_template = dedent( + """ + def func(x): + return x.{op}() + """ + ) + + deprecated_apis = { + "volatile", + "resize", + "reinforce", + "new", + "name", + "map2_", + "has_names", + "grad_fn", + "resize_as", + } + tensor_attrs = tensor_attrs - deprecated_apis + + properties = [] + methods = [] + sorted_tensor_attrs = sorted(tensor_attrs, key=lambda x: x.lower()) + for attr in sorted_tensor_attrs: + funcs_str = funcs_template.format(op=attr) + scope: dict[str, Any] = {} + execWrapper(funcs_str, globals(), scope) + try: + torch.jit.CompilationUnit(funcs_str) + except Exception as e: + if "nonexistent attribute" not in repr(e): + continue + attr_repr = repr(getattr(tensor, attr)) + if "bound method" in attr_repr or "built-in method" in attr_repr: + methods.append(attr) + else: + properties.append(attr) + + mapped_methods = ("\t* :meth:`~torch.Tensor." + x + r"`" for x in methods) + mapped_properties = ("\t* :attr:`~torch.Tensor." + x + r"`" for x in properties) + return "\n".join(mapped_methods), "\n".join(mapped_properties) + + +def _list_unsupported_tensor_ops(): + header = """\n\n +Unsupported Tensor Methods +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + """ + methods, properties = _gen_unsupported_methods_properties() + return ( + header + + "\n" + + methods + + """ + +Unsupported Tensor Properties +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + """ + + "\n" + + properties + ) + + +__doc__ = _list_unsupported_tensor_ops() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/_C.lib b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/_C.lib new file mode 100644 index 0000000000000000000000000000000000000000..4d0e31e8c57ae85d97c5963151a4aa49c1d8456c Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/_C.lib differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/libiompstubs5md.dll b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/libiompstubs5md.dll new file mode 100644 index 0000000000000000000000000000000000000000..850a7c651f818f6b717ad30d3ed1172abf16e829 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/libiompstubs5md.dll differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.dll b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.dll new file mode 100644 index 0000000000000000000000000000000000000000..7d1d796687ecb660da6c61a46e89b0b37ddec526 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.dll differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.lib b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.lib new file mode 100644 index 0000000000000000000000000000000000000000..aa619f91f2c604d5f006882572143525780b14a2 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/shm.lib differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.dll b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.dll new file mode 100644 index 0000000000000000000000000000000000000000..38dc60a8fbbffd5b0c5e4c88db9a5e43c1dea6f5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.dll differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.lib b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.lib new file mode 100644 index 0000000000000000000000000000000000000000..bebfd7fb14a001254d0b7b64982dbd9a1a96183a Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch.lib differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch_global_deps.dll b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch_global_deps.dll new file mode 100644 index 0000000000000000000000000000000000000000..a33b056d13874c82e5edf4a75763b2b9d7545cc5 Binary files /dev/null and b/TTS/Kokoro/.venv/Lib/site-packages/torch/lib/torch_global_deps.dll differ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/linalg/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/linalg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f4ecc06a1317c5ef045f00802984101b53482683 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/linalg/__init__.py @@ -0,0 +1,3016 @@ +from torch._C import ( # type: ignore[attr-defined] # pyrefly: ignore # missing-module-attribute + _add_docstr, + _linalg, + _LinAlgError as LinAlgError, # pyrefly: ignore # missing-module-attribute +) + + +common_notes = { + "experimental_warning": """This function is "experimental" and it may change in a future PyTorch release.""", + "sync_note": "When inputs are on a CUDA device, this function synchronizes that device with the CPU.", + "sync_note_ex": r"When the inputs are on a CUDA device, this function synchronizes only when :attr:`check_errors`\ `= True`.", + "sync_note_has_ex": ( + "When inputs are on a CUDA device, this function synchronizes that device with the CPU. " + "For a version of this function that does not synchronize, see :func:`{}`." + ), +} + + +# Note: This not only adds doc strings for functions in the linalg namespace, but +# also connects the torch.linalg Python namespace to the torch._C._linalg builtins. + +cross = _add_docstr( + _linalg.linalg_cross, + r""" +linalg.cross(input, other, *, dim=-1, out=None) -> Tensor + + +Computes the cross product of two 3-dimensional vectors. + +Supports input of float, double, cfloat and cdouble dtypes. Also supports batches +of vectors, for which it computes the product along the dimension :attr:`dim`. +It broadcasts over the batch dimensions. + +Args: + input (Tensor): the first input tensor. + other (Tensor): the second input tensor. + dim (int, optional): the dimension along which to take the cross-product. Default: `-1`. + +Keyword args: + out (Tensor, optional): the output tensor. Ignored if `None`. Default: `None`. + +Example: + >>> a = torch.randn(4, 3) + >>> a + tensor([[-0.3956, 1.1455, 1.6895], + [-0.5849, 1.3672, 0.3599], + [-1.1626, 0.7180, -0.0521], + [-0.1339, 0.9902, -2.0225]]) + >>> b = torch.randn(4, 3) + >>> b + tensor([[-0.0257, -1.4725, -1.2251], + [-1.1479, -0.7005, -1.9757], + [-1.3904, 0.3726, -1.1836], + [-0.9688, -0.7153, 0.2159]]) + >>> torch.linalg.cross(a, b) + tensor([[ 1.0844, -0.5281, 0.6120], + [-2.4490, -1.5687, 1.9792], + [-0.8304, -1.3037, 0.5650], + [-1.2329, 1.9883, 1.0551]]) + >>> a = torch.randn(1, 3) # a is broadcast to match shape of b + >>> a + tensor([[-0.9941, -0.5132, 0.5681]]) + >>> torch.linalg.cross(a, b) + tensor([[ 1.4653, -1.2325, 1.4507], + [ 1.4119, -2.6163, 0.1073], + [ 0.3957, -1.9666, -1.0840], + [ 0.2956, -0.3357, 0.2139]]) +""", +) + +cholesky = _add_docstr( + _linalg.linalg_cholesky, + r""" +linalg.cholesky(A, *, upper=False, out=None) -> Tensor + +Computes the Cholesky decomposition of a complex Hermitian or real symmetric positive-definite matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **Cholesky decomposition** of a complex Hermitian or real symmetric positive-definite matrix +:math:`A \in \mathbb{K}^{n \times n}` is defined as + +.. math:: + + A = LL^{\text{H}}\mathrlap{\qquad L \in \mathbb{K}^{n \times n}} + +where :math:`L` is a lower triangular matrix with real positive diagonal (even in the complex case) and +:math:`L^{\text{H}}` is the conjugate transpose when :math:`L` is complex, and the transpose when :math:`L` is real-valued. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +""" + + rf""" +.. note:: {common_notes["sync_note_has_ex"].format("torch.linalg.cholesky_ex")} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.cholesky_ex` for a version of this operation that + skips the (slow) error checking by default and instead returns the debug + information. This makes it a faster way to check if a matrix is + positive-definite. + + :func:`torch.linalg.eigh` for a different decomposition of a Hermitian matrix. + The eigenvalue decomposition gives more information about the matrix but it + slower to compute than the Cholesky decomposition. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of symmetric or Hermitian positive-definite matrices. + +Keyword args: + upper (bool, optional): whether to return an upper triangular matrix. + The tensor returned with upper=True is the conjugate transpose of the tensor + returned with upper=False. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if the :attr:`A` matrix or any matrix in a batched :attr:`A` is not Hermitian + (resp. symmetric) positive-definite. If :attr:`A` is a batch of matrices, + the error message will include the batch index of the first matrix that fails + to meet this condition. + +Examples:: + + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> A = A @ A.T.conj() + torch.eye(2) # creates a Hermitian positive-definite matrix + >>> A + tensor([[2.5266+0.0000j, 1.9586-2.0626j], + [1.9586+2.0626j, 9.4160+0.0000j]], dtype=torch.complex128) + >>> L = torch.linalg.cholesky(A) + >>> L + tensor([[1.5895+0.0000j, 0.0000+0.0000j], + [1.2322+1.2976j, 2.4928+0.0000j]], dtype=torch.complex128) + >>> torch.dist(L @ L.T.conj(), A) + tensor(4.4692e-16, dtype=torch.float64) + + >>> A = torch.randn(3, 2, 2, dtype=torch.float64) + >>> A = A @ A.mT + torch.eye(2) # batch of symmetric positive-definite matrices + >>> L = torch.linalg.cholesky(A) + >>> torch.dist(L @ L.mT, A) + tensor(5.8747e-16, dtype=torch.float64) +""", +) + +cholesky_ex = _add_docstr( + _linalg.linalg_cholesky_ex, + r""" +linalg.cholesky_ex(A, *, upper=False, check_errors=False, out=None) -> (Tensor, Tensor) + +Computes the Cholesky decomposition of a complex Hermitian or real +symmetric positive-definite matrix. + +This function skips the (slow) error checking and error message construction +of :func:`torch.linalg.cholesky`, instead directly returning the LAPACK +error codes as part of a named tuple ``(L, info)``. This makes this function +a faster way to check if a matrix is positive-definite, and it provides an +opportunity to handle decomposition errors more gracefully or performantly +than :func:`torch.linalg.cholesky` does. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +If :attr:`A` is not a Hermitian positive-definite matrix, or if it's a batch of matrices +and one or more of them is not a Hermitian positive-definite matrix, +then ``info`` stores a positive integer for the corresponding matrix. +The positive integer indicates the order of the leading minor that is not positive-definite, +and the decomposition could not be completed. +``info`` filled with zeros indicates that the decomposition was successful. +If ``check_errors=True`` and ``info`` contains positive integers, then a RuntimeError is thrown. + +""" + + rf""" +.. note:: {common_notes["sync_note_ex"]} + +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +.. seealso:: + :func:`torch.linalg.cholesky` is a NumPy compatible variant that always checks for errors. + +Args: + A (Tensor): the Hermitian `n \times n` matrix or the batch of such matrices of size + `(*, n, n)` where `*` is one or more batch dimensions. + +Keyword args: + upper (bool, optional): whether to return an upper triangular matrix. + The tensor returned with upper=True is the conjugate transpose of the tensor + returned with upper=False. + check_errors (bool, optional): controls whether to check the content of ``infos``. Default: `False`. + out (tuple, optional): tuple of two tensors to write the output to. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> A = A @ A.t().conj() # creates a Hermitian positive-definite matrix + >>> L, info = torch.linalg.cholesky_ex(A) + >>> A + tensor([[ 2.3792+0.0000j, -0.9023+0.9831j], + [-0.9023-0.9831j, 0.8757+0.0000j]], dtype=torch.complex128) + >>> L + tensor([[ 1.5425+0.0000j, 0.0000+0.0000j], + [-0.5850-0.6374j, 0.3567+0.0000j]], dtype=torch.complex128) + >>> info + tensor(0, dtype=torch.int32) + +""", +) + +inv = _add_docstr( + _linalg.linalg_inv, + r""" +linalg.inv(A, *, out=None) -> Tensor + +Computes the inverse of a square matrix if it exists. +Throws a `RuntimeError` if the matrix is not invertible. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +for a matrix :math:`A \in \mathbb{K}^{n \times n}`, +its **inverse matrix** :math:`A^{-1} \in \mathbb{K}^{n \times n}` (if it exists) is defined as + +.. math:: + + A^{-1}A = AA^{-1} = \mathrm{I}_n + +where :math:`\mathrm{I}_n` is the `n`-dimensional identity matrix. + +The inverse matrix exists if and only if :math:`A` is `invertible`_. In this case, +the inverse is unique. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices +then the output has the same batch dimensions. + +""" + + rf""" +.. note:: {common_notes["sync_note_has_ex"].format("torch.linalg.inv_ex")} +""" + + r""" + +.. note:: + Consider using :func:`torch.linalg.solve` if possible for multiplying a matrix on the left by + the inverse, as:: + + linalg.solve(A, B) == linalg.inv(A) @ B # When B is a matrix + + It is always preferred to use :func:`~solve` when possible, as it is faster and more + numerically stable than computing the inverse explicitly. + +.. seealso:: + + :func:`torch.linalg.pinv` computes the pseudoinverse (Moore-Penrose inverse) of matrices + of any shape. + + :func:`torch.linalg.solve` computes :attr:`A`\ `.inv() @ \ `:attr:`B` with a + numerically stable algorithm. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of invertible matrices. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if the matrix :attr:`A` or any matrix in the batch of matrices :attr:`A` is not invertible. + +Examples:: + + >>> A = torch.randn(4, 4) + >>> Ainv = torch.linalg.inv(A) + >>> torch.dist(A @ Ainv, torch.eye(4)) + tensor(1.1921e-07) + + >>> A = torch.randn(2, 3, 4, 4) # Batch of matrices + >>> Ainv = torch.linalg.inv(A) + >>> torch.dist(A @ Ainv, torch.eye(4)) + tensor(1.9073e-06) + + >>> A = torch.randn(4, 4, dtype=torch.complex128) # Complex matrix + >>> Ainv = torch.linalg.inv(A) + >>> torch.dist(A @ Ainv, torch.eye(4)) + tensor(7.5107e-16, dtype=torch.float64) + +.. _invertible: + https://en.wikipedia.org/wiki/Invertible_matrix#The_invertible_matrix_theorem +""", +) + +solve_ex = _add_docstr( + _linalg.linalg_solve_ex, + r""" +linalg.solve_ex(A, B, *, left=True, check_errors=False, out=None) -> (Tensor, Tensor) + +A version of :func:`~solve` that does not perform error checks unless :attr:`check_errors`\ `= True`. +It also returns the :attr:`info` tensor returned by `LAPACK's getrf`_. + +""" + + rf""" +.. note:: {common_notes["sync_note_ex"]} + +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + +Keyword args: + left (bool, optional): whether to solve the system :math:`AX=B` or :math:`XA = B`. Default: `True`. + check_errors (bool, optional): controls whether to check the content of ``infos`` and raise + an error if it is non-zero. Default: `False`. + out (tuple, optional): tuple of two tensors to write the output to. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(result, info)`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> Ainv, info = torch.linalg.solve_ex(A) + >>> torch.dist(torch.linalg.inv(A), Ainv) + tensor(0.) + >>> info + tensor(0, dtype=torch.int32) + +.. _LAPACK's getrf: + https://www.netlib.org/lapack/explore-html-3.6.1/dd/d9a/group__double_g_ecomputational_ga0019443faea08275ca60a734d0593e60.html +""", +) + +inv_ex = _add_docstr( + _linalg.linalg_inv_ex, + r""" +linalg.inv_ex(A, *, check_errors=False, out=None) -> (Tensor, Tensor) + +Computes the inverse of a square matrix if it is invertible. + +Returns a namedtuple ``(inverse, info)``. ``inverse`` contains the result of +inverting :attr:`A` and ``info`` stores the LAPACK error codes. + +If :attr:`A` is not an invertible matrix, or if it's a batch of matrices +and one or more of them is not an invertible matrix, +then ``info`` stores a positive integer for the corresponding matrix. +The positive integer indicates the diagonal element of the LU decomposition of +the input matrix that is exactly zero. +``info`` filled with zeros indicates that the inversion was successful. +If ``check_errors=True`` and ``info`` contains positive integers, then a RuntimeError is thrown. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +""" + + rf""" +.. note:: {common_notes["sync_note_ex"]} + +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.inv` is a NumPy compatible variant that always checks for errors. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of square matrices. + check_errors (bool, optional): controls whether to check the content of ``info``. Default: `False`. + +Keyword args: + out (tuple, optional): tuple of two tensors to write the output to. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> Ainv, info = torch.linalg.inv_ex(A) + >>> torch.dist(torch.linalg.inv(A), Ainv) + tensor(0.) + >>> info + tensor(0, dtype=torch.int32) + +""", +) + +det = _add_docstr( + _linalg.linalg_det, + r""" +linalg.det(A, *, out=None) -> Tensor + +Computes the determinant of a square matrix. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +.. seealso:: + + :func:`torch.linalg.slogdet` computes the sign and natural logarithm of the absolute + value of the determinant of square matrices. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> torch.linalg.det(A) + tensor(0.0934) + + >>> A = torch.randn(3, 2, 2) + >>> torch.linalg.det(A) + tensor([1.1990, 0.4099, 0.7386]) +""", +) + +slogdet = _add_docstr( + _linalg.linalg_slogdet, + r""" +linalg.slogdet(A, *, out=None) -> (Tensor, Tensor) + +Computes the sign and natural logarithm of the absolute value of the determinant of a square matrix. + +For complex :attr:`A`, it returns the sign and the natural logarithm of the modulus of the +determinant, that is, a logarithmic polar decomposition of the determinant. + +The determinant can be recovered as `sign * exp(logabsdet)`. +When a matrix has a determinant of zero, it returns `(0, -inf)`. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +.. seealso:: + + :func:`torch.linalg.det` computes the determinant of square matrices. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + +Keyword args: + out (tuple, optional): output tuple of two tensors. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(sign, logabsdet)`. + + `sign` will have the same dtype as :attr:`A`. + + `logabsdet` will always be real-valued, even when :attr:`A` is complex. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> A + tensor([[ 0.0032, -0.2239, -1.1219], + [-0.6690, 0.1161, 0.4053], + [-1.6218, -0.9273, -0.0082]]) + >>> torch.linalg.det(A) + tensor(-0.7576) + >>> torch.logdet(A) + tensor(nan) + >>> torch.linalg.slogdet(A) + torch.return_types.linalg_slogdet(sign=tensor(-1.), logabsdet=tensor(-0.2776)) +""", +) + +eig = _add_docstr( + _linalg.linalg_eig, + r""" +linalg.eig(A, *, out=None) -> (Tensor, Tensor) + +Computes the eigenvalue decomposition of a square matrix if it exists. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **eigenvalue decomposition** of a square matrix +:math:`A \in \mathbb{K}^{n \times n}` (if it exists) is defined as + +.. math:: + + A = V \operatorname{diag}(\Lambda) V^{-1}\mathrlap{\qquad V \in \mathbb{C}^{n \times n}, \Lambda \in \mathbb{C}^n} + +This decomposition exists if and only if :math:`A` is `diagonalizable`_. +This is the case when all its eigenvalues are different. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The returned eigenvalues are not guaranteed to be in any specific order. + +.. note:: The eigenvalues and eigenvectors of a real matrix may be complex. + +""" + + rf""" +.. note:: {common_notes["sync_note"]} +""" + + r""" + +.. warning:: This function assumes that :attr:`A` is `diagonalizable`_ (for example, when all the + eigenvalues are different). If it is not diagonalizable, the returned + eigenvalues will be correct but :math:`A \neq V \operatorname{diag}(\Lambda)V^{-1}`. + +.. warning:: The returned eigenvectors are normalized to have norm `1`. + Even then, the eigenvectors of a matrix are not unique, nor are they continuous with respect to + :attr:`A`. Due to this lack of uniqueness, different hardware and software may compute + different eigenvectors. + + This non-uniqueness is caused by the fact that multiplying an eigenvector by + by :math:`e^{i \phi}, \phi \in \mathbb{R}` produces another set of valid eigenvectors + of the matrix. For this reason, the loss function shall not depend on the phase of the + eigenvectors, as this quantity is not well-defined. + This is checked when computing the gradients of this function. As such, + when inputs are on a CUDA device, the computation of the gradients + of this function synchronizes that device with the CPU. + + +.. warning:: Gradients computed using the `eigenvectors` tensor will only be finite when + :attr:`A` has distinct eigenvalues. + Furthermore, if the distance between any two eigenvalues is close to zero, + the gradient will be numerically unstable, as it depends on the eigenvalues + :math:`\lambda_i` through the computation of + :math:`\frac{1}{\min_{i \neq j} \lambda_i - \lambda_j}`. + +.. seealso:: + + :func:`torch.linalg.eigvals` computes only the eigenvalues. + Unlike :func:`torch.linalg.eig`, the gradients of :func:`~eigvals` are always + numerically stable. + + :func:`torch.linalg.eigh` for a (faster) function that computes the eigenvalue decomposition + for Hermitian and symmetric matrices. + + :func:`torch.linalg.svd` for a function that computes another type of spectral + decomposition that works on matrices of any shape. + + :func:`torch.linalg.qr` for another (much faster) decomposition that works on matrices of + any shape. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of diagonalizable matrices. + +Keyword args: + out (tuple, optional): output tuple of two tensors. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(eigenvalues, eigenvectors)` which corresponds to :math:`\Lambda` and :math:`V` above. + + `eigenvalues` and `eigenvectors` will always be complex-valued, even when :attr:`A` is real. The eigenvectors + will be given by the columns of `eigenvectors`. + +Examples:: + + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> A + tensor([[ 0.9828+0.3889j, -0.4617+0.3010j], + [ 0.1662-0.7435j, -0.6139+0.0562j]], dtype=torch.complex128) + >>> L, V = torch.linalg.eig(A) + >>> L + tensor([ 1.1226+0.5738j, -0.7537-0.1286j], dtype=torch.complex128) + >>> V + tensor([[ 0.9218+0.0000j, 0.1882-0.2220j], + [-0.0270-0.3867j, 0.9567+0.0000j]], dtype=torch.complex128) + >>> torch.dist(V @ torch.diag(L) @ torch.linalg.inv(V), A) + tensor(7.7119e-16, dtype=torch.float64) + + >>> A = torch.randn(3, 2, 2, dtype=torch.float64) + >>> L, V = torch.linalg.eig(A) + >>> torch.dist(V @ torch.diag_embed(L) @ torch.linalg.inv(V), A) + tensor(3.2841e-16, dtype=torch.float64) + +.. _diagonalizable: + https://en.wikipedia.org/wiki/Diagonalizable_matrix#Definition +""", +) + +eigvals = _add_docstr( + _linalg.linalg_eigvals, + r""" +linalg.eigvals(A, *, out=None) -> Tensor + +Computes the eigenvalues of a square matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **eigenvalues** of a square matrix :math:`A \in \mathbb{K}^{n \times n}` are defined +as the roots (counted with multiplicity) of the polynomial `p` of degree `n` given by + +.. math:: + + p(\lambda) = \operatorname{det}(A - \lambda \mathrm{I}_n)\mathrlap{\qquad \lambda \in \mathbb{C}} + +where :math:`\mathrm{I}_n` is the `n`-dimensional identity matrix. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The returned eigenvalues are not guaranteed to be in any specific order. + +.. note:: The eigenvalues of a real matrix may be complex, as the roots of a real polynomial may be complex. + + The eigenvalues of a matrix are always well-defined, even when the matrix is not diagonalizable. + +""" + + rf""" +.. note:: {common_notes["sync_note"]} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.eig` computes the full eigenvalue decomposition. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Returns: + A complex-valued tensor containing the eigenvalues even when :attr:`A` is real. + +Examples:: + + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> L = torch.linalg.eigvals(A) + >>> L + tensor([ 1.1226+0.5738j, -0.7537-0.1286j], dtype=torch.complex128) + + >>> torch.dist(L, torch.linalg.eig(A).eigenvalues) + tensor(2.4576e-07) +""", +) + +eigh = _add_docstr( + _linalg.linalg_eigh, + r""" +linalg.eigh(A, UPLO='L', *, out=None) -> (Tensor, Tensor) + +Computes the eigenvalue decomposition of a complex Hermitian or real symmetric matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **eigenvalue decomposition** of a complex Hermitian or real symmetric matrix +:math:`A \in \mathbb{K}^{n \times n}` is defined as + +.. math:: + + A = Q \operatorname{diag}(\Lambda) Q^{\text{H}}\mathrlap{\qquad Q \in \mathbb{K}^{n \times n}, \Lambda \in \mathbb{R}^n} + +where :math:`Q^{\text{H}}` is the conjugate transpose when :math:`Q` is complex, and the transpose when :math:`Q` is real-valued. +:math:`Q` is orthogonal in the real case and unitary in the complex case. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +:attr:`A` is assumed to be Hermitian (resp. symmetric), but this is not checked internally, instead: + +- If :attr:`UPLO`\ `= 'L'` (default), only the lower triangular part of the matrix is used in the computation. +- If :attr:`UPLO`\ `= 'U'`, only the upper triangular part of the matrix is used. + +The eigenvalues are returned in ascending order. + +""" + + rf""" +.. note:: {common_notes["sync_note"]} +""" + + r""" + +.. note:: The eigenvalues of real symmetric or complex Hermitian matrices are always real. + +.. warning:: The eigenvectors of a symmetric matrix are not unique, nor are they continuous with + respect to :attr:`A`. Due to this lack of uniqueness, different hardware and + software may compute different eigenvectors. + + This non-uniqueness is caused by the fact that multiplying an eigenvector by + `-1` in the real case or by :math:`e^{i \phi}, \phi \in \mathbb{R}` in the complex + case produces another set of valid eigenvectors of the matrix. + For this reason, the loss function shall not depend on the phase of the eigenvectors, as + this quantity is not well-defined. + This is checked for complex inputs when computing the gradients of this function. As such, + when inputs are complex and are on a CUDA device, the computation of the gradients + of this function synchronizes that device with the CPU. + +.. warning:: Gradients computed using the `eigenvectors` tensor will only be finite when + :attr:`A` has distinct eigenvalues. + Furthermore, if the distance between any two eigenvalues is close to zero, + the gradient will be numerically unstable, as it depends on the eigenvalues + :math:`\lambda_i` through the computation of + :math:`\frac{1}{\min_{i \neq j} \lambda_i - \lambda_j}`. + +.. warning:: User may see pytorch crashes if running `eigh` on CUDA devices with CUDA versions before 12.1 update 1 + with large ill-conditioned matrices as inputs. + Refer to :ref:`Linear Algebra Numerical Stability` for more details. + If this is the case, user may (1) tune their matrix inputs to be less ill-conditioned, + or (2) use :func:`torch.backends.cuda.preferred_linalg_library` to + try other supported backends. + +.. seealso:: + + :func:`torch.linalg.eigvalsh` computes only the eigenvalues of a Hermitian matrix. + Unlike :func:`torch.linalg.eigh`, the gradients of :func:`~eigvalsh` are always + numerically stable. + + :func:`torch.linalg.cholesky` for a different decomposition of a Hermitian matrix. + The Cholesky decomposition gives less information about the matrix but is much faster + to compute than the eigenvalue decomposition. + + :func:`torch.linalg.eig` for a (slower) function that computes the eigenvalue decomposition + of a not necessarily Hermitian square matrix. + + :func:`torch.linalg.svd` for a (slower) function that computes the more general SVD + decomposition of matrices of any shape. + + :func:`torch.linalg.qr` for another (much faster) decomposition that works on general + matrices. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of symmetric or Hermitian matrices. + UPLO ('L', 'U', optional): controls whether to use the upper or lower triangular part + of :attr:`A` in the computations. Default: `'L'`. + +Keyword args: + out (tuple, optional): output tuple of two tensors. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(eigenvalues, eigenvectors)` which corresponds to :math:`\Lambda` and :math:`Q` above. + + `eigenvalues` will always be real-valued, even when :attr:`A` is complex. + It will also be ordered in ascending order. + + `eigenvectors` will have the same dtype as :attr:`A` and will contain the eigenvectors as its columns. + +Examples:: + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> A = A + A.T.conj() # creates a Hermitian matrix + >>> A + tensor([[2.9228+0.0000j, 0.2029-0.0862j], + [0.2029+0.0862j, 0.3464+0.0000j]], dtype=torch.complex128) + >>> L, Q = torch.linalg.eigh(A) + >>> L + tensor([0.3277, 2.9415], dtype=torch.float64) + >>> Q + tensor([[-0.0846+-0.0000j, -0.9964+0.0000j], + [ 0.9170+0.3898j, -0.0779-0.0331j]], dtype=torch.complex128) + >>> torch.dist(Q @ torch.diag(L.cdouble()) @ Q.T.conj(), A) + tensor(6.1062e-16, dtype=torch.float64) + + >>> A = torch.randn(3, 2, 2, dtype=torch.float64) + >>> A = A + A.mT # creates a batch of symmetric matrices + >>> L, Q = torch.linalg.eigh(A) + >>> torch.dist(Q @ torch.diag_embed(L) @ Q.mH, A) + tensor(1.5423e-15, dtype=torch.float64) +""", +) + +eigvalsh = _add_docstr( + _linalg.linalg_eigvalsh, + r""" +linalg.eigvalsh(A, UPLO='L', *, out=None) -> Tensor + +Computes the eigenvalues of a complex Hermitian or real symmetric matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **eigenvalues** of a complex Hermitian or real symmetric matrix :math:`A \in \mathbb{K}^{n \times n}` +are defined as the roots (counted with multiplicity) of the polynomial `p` of degree `n` given by + +.. math:: + + p(\lambda) = \operatorname{det}(A - \lambda \mathrm{I}_n)\mathrlap{\qquad \lambda \in \mathbb{R}} + +where :math:`\mathrm{I}_n` is the `n`-dimensional identity matrix. +The eigenvalues of a real symmetric or complex Hermitian matrix are always real. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The eigenvalues are returned in ascending order. + +:attr:`A` is assumed to be Hermitian (resp. symmetric), but this is not checked internally, instead: + +- If :attr:`UPLO`\ `= 'L'` (default), only the lower triangular part of the matrix is used in the computation. +- If :attr:`UPLO`\ `= 'U'`, only the upper triangular part of the matrix is used. + +""" + + rf""" +.. note:: {common_notes["sync_note"]} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.eigh` computes the full eigenvalue decomposition. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of symmetric or Hermitian matrices. + UPLO ('L', 'U', optional): controls whether to use the upper or lower triangular part + of :attr:`A` in the computations. Default: `'L'`. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Returns: + A real-valued tensor containing the eigenvalues even when :attr:`A` is complex. + The eigenvalues are returned in ascending order. + +Examples:: + + >>> A = torch.randn(2, 2, dtype=torch.complex128) + >>> A = A + A.T.conj() # creates a Hermitian matrix + >>> A + tensor([[2.9228+0.0000j, 0.2029-0.0862j], + [0.2029+0.0862j, 0.3464+0.0000j]], dtype=torch.complex128) + >>> torch.linalg.eigvalsh(A) + tensor([0.3277, 2.9415], dtype=torch.float64) + + >>> A = torch.randn(3, 2, 2, dtype=torch.float64) + >>> A = A + A.mT # creates a batch of symmetric matrices + >>> torch.linalg.eigvalsh(A) + tensor([[ 2.5797, 3.4629], + [-4.1605, 1.3780], + [-3.1113, 2.7381]], dtype=torch.float64) +""", +) + +householder_product = _add_docstr( + _linalg.linalg_householder_product, + r""" +householder_product(A, tau, *, out=None) -> Tensor + +Computes the first `n` columns of a product of Householder matrices. + +Let :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, and +let :math:`A \in \mathbb{K}^{m \times n}` be a matrix with columns :math:`a_i \in \mathbb{K}^m` +for :math:`i=1,\ldots,m` with :math:`m \geq n`. Denote by :math:`b_i` the vector resulting from +zeroing out the first :math:`i-1` components of :math:`a_i` and setting to `1` the :math:`i`-th. +For a vector :math:`\tau \in \mathbb{K}^k` with :math:`k \leq n`, this function computes the +first :math:`n` columns of the matrix + +.. math:: + + H_1H_2 ... H_k \qquad\text{with}\qquad H_i = \mathrm{I}_m - \tau_i b_i b_i^{\text{H}} + +where :math:`\mathrm{I}_m` is the `m`-dimensional identity matrix and :math:`b^{\text{H}}` is the +conjugate transpose when :math:`b` is complex, and the transpose when :math:`b` is real-valued. +The output matrix is the same size as the input matrix :attr:`A`. + +See `Representation of Orthogonal or Unitary Matrices`_ for further details. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +.. seealso:: + + :func:`torch.geqrf` can be used together with this function to form the `Q` from the + :func:`~qr` decomposition. + + :func:`torch.ormqr` is a related function that computes the matrix multiplication + of a product of Householder matrices with another matrix. + However, that function is not supported by autograd. + +.. warning:: + Gradient computations are only well-defined if :math:`\tau_i \neq \frac{1}{||a_i||^2}`. + If this condition is not met, no error will be thrown, but the gradient produced may contain `NaN`. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + tau (Tensor): tensor of shape `(*, k)` where `*` is zero or more batch dimensions. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if :attr:`A` doesn't satisfy the requirement `m >= n`, + or :attr:`tau` doesn't satisfy the requirement `n >= k`. + +Examples:: + + >>> A = torch.randn(2, 2) + >>> h, tau = torch.geqrf(A) + >>> Q = torch.linalg.householder_product(h, tau) + >>> torch.dist(Q, torch.linalg.qr(A).Q) + tensor(0.) + + >>> h = torch.randn(3, 2, 2, dtype=torch.complex128) + >>> tau = torch.randn(3, 1, dtype=torch.complex128) + >>> Q = torch.linalg.householder_product(h, tau) + >>> Q + tensor([[[ 1.8034+0.4184j, 0.2588-1.0174j], + [-0.6853+0.7953j, 2.0790+0.5620j]], + + [[ 1.4581+1.6989j, -1.5360+0.1193j], + [ 1.3877-0.6691j, 1.3512+1.3024j]], + + [[ 1.4766+0.5783j, 0.0361+0.6587j], + [ 0.6396+0.1612j, 1.3693+0.4481j]]], dtype=torch.complex128) + +.. _Representation of Orthogonal or Unitary Matrices: + https://www.netlib.org/lapack/lug/node128.html +""", +) + +ldl_factor = _add_docstr( + _linalg.linalg_ldl_factor, + r""" +linalg.ldl_factor(A, *, hermitian=False, out=None) -> (Tensor, Tensor) + +Computes a compact representation of the LDL factorization of a Hermitian or symmetric (possibly indefinite) matrix. + +When :attr:`A` is complex valued it can be Hermitian (:attr:`hermitian`\ `= True`) +or symmetric (:attr:`hermitian`\ `= False`). + +The factorization is of the form the form :math:`A = L D L^T`. +If :attr:`hermitian` is `True` then transpose operation is the conjugate transpose. + +:math:`L` (or :math:`U`) and :math:`D` are stored in compact form in ``LD``. +They follow the format specified by `LAPACK's sytrf`_ function. +These tensors may be used in :func:`torch.linalg.ldl_solve` to solve linear systems. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +""" + + rf""" +.. note:: {common_notes["sync_note_has_ex"].format("torch.linalg.ldl_factor_ex")} +""" + + r""" + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of symmetric or Hermitian matrices. + +Keyword args: + hermitian (bool, optional): whether to consider the input to be Hermitian or symmetric. + For real-valued matrices, this switch has no effect. Default: `False`. + out (tuple, optional): tuple of two tensors to write the output to. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(LD, pivots)`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> A = A @ A.mT # make symmetric + >>> A + tensor([[7.2079, 4.2414, 1.9428], + [4.2414, 3.4554, 0.3264], + [1.9428, 0.3264, 1.3823]]) + >>> LD, pivots = torch.linalg.ldl_factor(A) + >>> LD + tensor([[ 7.2079, 0.0000, 0.0000], + [ 0.5884, 0.9595, 0.0000], + [ 0.2695, -0.8513, 0.1633]]) + >>> pivots + tensor([1, 2, 3], dtype=torch.int32) + +.. _LAPACK's sytrf: + https://www.netlib.org/lapack/explore-html-3.6.1/d3/db6/group__double_s_ycomputational_gad91bde1212277b3e909eb6af7f64858a.html +""", +) + +ldl_factor_ex = _add_docstr( + _linalg.linalg_ldl_factor_ex, + r""" +linalg.ldl_factor_ex(A, *, hermitian=False, check_errors=False, out=None) -> (Tensor, Tensor, Tensor) + +This is a version of :func:`~ldl_factor` that does not perform error checks unless :attr:`check_errors`\ `= True`. +It also returns the :attr:`info` tensor returned by `LAPACK's sytrf`_. +``info`` stores integer error codes from the backend library. +A positive integer indicates the diagonal element of :math:`D` that is zero. +Division by 0 will occur if the result is used for solving a system of linear equations. +``info`` filled with zeros indicates that the factorization was successful. +If ``check_errors=True`` and ``info`` contains positive integers, then a `RuntimeError` is thrown. + +""" + + rf""" +.. note:: {common_notes["sync_note_ex"]} + +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of symmetric or Hermitian matrices. + +Keyword args: + hermitian (bool, optional): whether to consider the input to be Hermitian or symmetric. + For real-valued matrices, this switch has no effect. Default: `False`. + check_errors (bool, optional): controls whether to check the content of ``info`` and raise + an error if it is non-zero. Default: `False`. + out (tuple, optional): tuple of three tensors to write the output to. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(LD, pivots, info)`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> A = A @ A.mT # make symmetric + >>> A + tensor([[7.2079, 4.2414, 1.9428], + [4.2414, 3.4554, 0.3264], + [1.9428, 0.3264, 1.3823]]) + >>> LD, pivots, info = torch.linalg.ldl_factor_ex(A) + >>> LD + tensor([[ 7.2079, 0.0000, 0.0000], + [ 0.5884, 0.9595, 0.0000], + [ 0.2695, -0.8513, 0.1633]]) + >>> pivots + tensor([1, 2, 3], dtype=torch.int32) + >>> info + tensor(0, dtype=torch.int32) + +.. _LAPACK's sytrf: + https://www.netlib.org/lapack/explore-html-3.6.1/d3/db6/group__double_s_ycomputational_gad91bde1212277b3e909eb6af7f64858a.html +""", +) + +ldl_solve = _add_docstr( + _linalg.linalg_ldl_solve, + r""" +linalg.ldl_solve(LD, pivots, B, *, hermitian=False, out=None) -> Tensor + +Computes the solution of a system of linear equations using the LDL factorization. + +:attr:`LD` and :attr:`pivots` are the compact representation of the LDL factorization and +are expected to be computed by :func:`torch.linalg.ldl_factor_ex`. +:attr:`hermitian` argument to this function should be the same +as the corresponding arguments in :func:`torch.linalg.ldl_factor_ex`. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +""" + + rf""" +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +Args: + LD (Tensor): the `n \times n` matrix or the batch of such matrices of size + `(*, n, n)` where `*` is one or more batch dimensions. + pivots (Tensor): the pivots corresponding to the LDL factorization of :attr:`LD`. + B (Tensor): right-hand side tensor of shape `(*, n, k)`. + +Keyword args: + hermitian (bool, optional): whether to consider the decomposed matrix to be Hermitian or symmetric. + For real-valued matrices, this switch has no effect. Default: `False`. + out (tuple, optional): output tensor. `B` may be passed as `out` and the result is computed in-place on `B`. + Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(2, 3, 3) + >>> A = A @ A.mT # make symmetric + >>> LD, pivots, info = torch.linalg.ldl_factor_ex(A) + >>> B = torch.randn(2, 3, 4) + >>> X = torch.linalg.ldl_solve(LD, pivots, B) + >>> torch.linalg.norm(A @ X - B) + >>> tensor(0.0001) +""", +) + +lstsq = _add_docstr( + _linalg.linalg_lstsq, + r""" +torch.linalg.lstsq(A, B, rcond=None, *, driver=None) -> (Tensor, Tensor, Tensor, Tensor) + +Computes a solution to the least squares problem of a system of linear equations. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **least squares problem** for a linear system :math:`AX = B` with +:math:`A \in \mathbb{K}^{m \times n}, B \in \mathbb{K}^{m \times k}` is defined as + +.. math:: + + \min_{X \in \mathbb{K}^{n \times k}} \|AX - B\|_F + +where :math:`\|-\|_F` denotes the Frobenius norm. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +:attr:`driver` chooses the backend function that will be used. +For CPU inputs the valid values are `'gels'`, `'gelsy'`, `'gelsd`, `'gelss'`. +To choose the best driver on CPU consider: + +- If :attr:`A` is well-conditioned (its `condition number`_ is not too large), or you do not mind some precision loss. + + - For a general matrix: `'gelsy'` (QR with pivoting) (default) + - If :attr:`A` is full-rank: `'gels'` (QR) + +- If :attr:`A` is not well-conditioned. + + - `'gelsd'` (tridiagonal reduction and SVD) + - But if you run into memory issues: `'gelss'` (full SVD). + +For CUDA input, the only valid driver is `'gels'`, which assumes that :attr:`A` is full-rank. + +See also the `full description of these drivers`_ + +:attr:`rcond` is used to determine the effective rank of the matrices in :attr:`A` +when :attr:`driver` is one of (`'gelsy'`, `'gelsd'`, `'gelss'`). +In this case, if :math:`\sigma_i` are the singular values of `A` in decreasing order, +:math:`\sigma_i` will be rounded down to zero if :math:`\sigma_i \leq \text{rcond} \cdot \sigma_1`. +If :attr:`rcond`\ `= None` (default), :attr:`rcond` is set to the machine precision of the dtype of :attr:`A` times `max(m, n)`. + +This function returns the solution to the problem and some extra information in a named tuple of +four tensors `(solution, residuals, rank, singular_values)`. For inputs :attr:`A`, :attr:`B` +of shape `(*, m, n)`, `(*, m, k)` respectively, it contains + +- `solution`: the least squares solution. It has shape `(*, n, k)`. +- `residuals`: the squared residuals of the solutions, that is, :math:`\|AX - B\|_F^2`. + It has shape `(*, k)`. + It is computed when `m > n` and every matrix in :attr:`A` is full-rank, + otherwise, it is an empty tensor. + If :attr:`A` is a batch of matrices and any matrix in the batch is not full rank, + then an empty tensor is returned. This behavior may change in a future PyTorch release. +- `rank`: tensor of ranks of the matrices in :attr:`A`. + It has shape equal to the batch dimensions of :attr:`A`. + It is computed when :attr:`driver` is one of (`'gelsy'`, `'gelsd'`, `'gelss'`), + otherwise it is an empty tensor. +- `singular_values`: tensor of singular values of the matrices in :attr:`A`. + It has shape `(*, min(m, n))`. + It is computed when :attr:`driver` is one of (`'gelsd'`, `'gelss'`), + otherwise it is an empty tensor. + +.. note:: + This function computes `X = \ `:attr:`A`\ `.pinverse() @ \ `:attr:`B` in a faster and + more numerically stable way than performing the computations separately. + +.. warning:: + The default value of :attr:`rcond` may change in a future PyTorch release. + It is therefore recommended to use a fixed value to avoid potential + breaking changes. + +Args: + A (Tensor): lhs tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + B (Tensor): rhs tensor of shape `(*, m, k)` where `*` is zero or more batch dimensions. + rcond (float, optional): used to determine the effective rank of :attr:`A`. + If :attr:`rcond`\ `= None`, :attr:`rcond` is set to the machine + precision of the dtype of :attr:`A` times `max(m, n)`. Default: `None`. + +Keyword args: + driver (str, optional): name of the LAPACK/MAGMA method to be used. + If `None`, `'gelsy'` is used for CPU inputs and `'gels'` for CUDA inputs. + Default: `None`. + +Returns: + A named tuple `(solution, residuals, rank, singular_values)`. + +Examples:: + + >>> A = torch.randn(1,3,3) + >>> A + tensor([[[-1.0838, 0.0225, 0.2275], + [ 0.2438, 0.3844, 0.5499], + [ 0.1175, -0.9102, 2.0870]]]) + >>> B = torch.randn(2,3,3) + >>> B + tensor([[[-0.6772, 0.7758, 0.5109], + [-1.4382, 1.3769, 1.1818], + [-0.3450, 0.0806, 0.3967]], + [[-1.3994, -0.1521, -0.1473], + [ 1.9194, 1.0458, 0.6705], + [-1.1802, -0.9796, 1.4086]]]) + >>> X = torch.linalg.lstsq(A, B).solution # A is broadcasted to shape (2, 3, 3) + >>> torch.dist(X, torch.linalg.pinv(A) @ B) + tensor(1.5152e-06) + + >>> S = torch.linalg.lstsq(A, B, driver='gelsd').singular_values + >>> torch.dist(S, torch.linalg.svdvals(A)) + tensor(2.3842e-07) + + >>> A[:, 0].zero_() # Decrease the rank of A + >>> rank = torch.linalg.lstsq(A, B).rank + >>> rank + tensor([2]) + +.. _condition number: + https://pytorch.org/docs/main/linalg.html#torch.linalg.cond +.. _full description of these drivers: + https://www.netlib.org/lapack/lug/node27.html +""", +) + +matrix_power = _add_docstr( + _linalg.linalg_matrix_power, + r""" +matrix_power(A, n, *, out=None) -> Tensor + +Computes the `n`-th power of a square matrix for an integer `n`. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +If :attr:`n`\ `= 0`, it returns the identity matrix (or batch) of the same shape +as :attr:`A`. If :attr:`n` is negative, it returns the inverse of each matrix +(if invertible) raised to the power of `abs(n)`. + +.. note:: + Consider using :func:`torch.linalg.solve` if possible for multiplying a matrix on the left by + a negative power as, if :attr:`n`\ `> 0`:: + + torch.linalg.solve(matrix_power(A, n), B) == matrix_power(A, -n) @ B + + It is always preferred to use :func:`~solve` when possible, as it is faster and more + numerically stable than computing :math:`A^{-n}` explicitly. + +.. seealso:: + + :func:`torch.linalg.solve` computes :attr:`A`\ `.inverse() @ \ `:attr:`B` with a + numerically stable algorithm. + +Args: + A (Tensor): tensor of shape `(*, m, m)` where `*` is zero or more batch dimensions. + n (int): the exponent. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if :attr:`n`\ `< 0` and the matrix :attr:`A` or any matrix in the + batch of matrices :attr:`A` is not invertible. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> torch.linalg.matrix_power(A, 0) + tensor([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + >>> torch.linalg.matrix_power(A, 3) + tensor([[ 1.0756, 0.4980, 0.0100], + [-1.6617, 1.4994, -1.9980], + [-0.4509, 0.2731, 0.8001]]) + >>> torch.linalg.matrix_power(A.expand(2, -1, -1), -2) + tensor([[[ 0.2640, 0.4571, -0.5511], + [-1.0163, 0.3491, -1.5292], + [-0.4899, 0.0822, 0.2773]], + [[ 0.2640, 0.4571, -0.5511], + [-1.0163, 0.3491, -1.5292], + [-0.4899, 0.0822, 0.2773]]]) +""", +) + +matrix_rank = _add_docstr( + _linalg.linalg_matrix_rank, + r""" +linalg.matrix_rank(A, *, atol=None, rtol=None, hermitian=False, out=None) -> Tensor + +Computes the numerical rank of a matrix. + +The matrix rank is computed as the number of singular values +(or eigenvalues in absolute value when :attr:`hermitian`\ `= True`) +that are greater than :math:`\max(\text{atol}, \sigma_1 * \text{rtol})` threshold, +where :math:`\sigma_1` is the largest singular value (or eigenvalue). + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +If :attr:`hermitian`\ `= True`, :attr:`A` is assumed to be Hermitian if complex or +symmetric if real, but this is not checked internally. Instead, just the lower +triangular part of the matrix is used in the computations. + +If :attr:`rtol` is not specified and :attr:`A` is a matrix of dimensions `(m, n)`, +the relative tolerance is set to be :math:`\text{rtol} = \max(m, n) \varepsilon` +and :math:`\varepsilon` is the epsilon value for the dtype of :attr:`A` (see :class:`.finfo`). +If :attr:`rtol` is not specified and :attr:`atol` is specified to be larger than zero then +:attr:`rtol` is set to zero. + +If :attr:`atol` or :attr:`rtol` is a :class:`torch.Tensor`, its shape must be broadcastable to that +of the singular values of :attr:`A` as returned by :func:`torch.linalg.svdvals`. + +.. note:: + This function has NumPy compatible variant `linalg.matrix_rank(A, tol, hermitian=False)`. + However, use of the positional argument :attr:`tol` is deprecated in favor of :attr:`atol` and :attr:`rtol`. + +""" + + rf""" +.. note:: The matrix rank is computed using a singular value decomposition + :func:`torch.linalg.svdvals` if :attr:`hermitian`\ `= False` (default) and the eigenvalue + decomposition :func:`torch.linalg.eigvalsh` when :attr:`hermitian`\ `= True`. + {common_notes["sync_note"]} +""" + + r""" + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + tol (float, Tensor, optional): [NumPy Compat] Alias for :attr:`atol`. Default: `None`. + +Keyword args: + atol (float, Tensor, optional): the absolute tolerance value. When `None` it's considered to be zero. + Default: `None`. + rtol (float, Tensor, optional): the relative tolerance value. See above for the value it takes when `None`. + Default: `None`. + hermitian(bool): indicates whether :attr:`A` is Hermitian if complex + or symmetric if real. Default: `False`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.eye(10) + >>> torch.linalg.matrix_rank(A) + tensor(10) + >>> B = torch.eye(10) + >>> B[0, 0] = 0 + >>> torch.linalg.matrix_rank(B) + tensor(9) + + >>> A = torch.randn(4, 3, 2) + >>> torch.linalg.matrix_rank(A) + tensor([2, 2, 2, 2]) + + >>> A = torch.randn(2, 4, 2, 3) + >>> torch.linalg.matrix_rank(A) + tensor([[2, 2, 2, 2], + [2, 2, 2, 2]]) + + >>> A = torch.randn(2, 4, 3, 3, dtype=torch.complex64) + >>> torch.linalg.matrix_rank(A) + tensor([[3, 3, 3, 3], + [3, 3, 3, 3]]) + >>> torch.linalg.matrix_rank(A, hermitian=True) + tensor([[3, 3, 3, 3], + [3, 3, 3, 3]]) + >>> torch.linalg.matrix_rank(A, atol=1.0, rtol=0.0) + tensor([[3, 2, 2, 2], + [1, 2, 1, 2]]) + >>> torch.linalg.matrix_rank(A, atol=1.0, rtol=0.0, hermitian=True) + tensor([[2, 2, 2, 1], + [1, 2, 2, 2]]) +""", +) + +norm = _add_docstr( + _linalg.linalg_norm, + r""" +linalg.norm(A, ord=None, dim=None, keepdim=False, *, out=None, dtype=None) -> Tensor + +Computes a vector or matrix norm. + +Supports input of float, double, cfloat and cdouble dtypes. + +Whether this function computes a vector or matrix norm is determined as follows: + +- If :attr:`dim` is an `int`, the vector norm will be computed. +- If :attr:`dim` is a `2`-`tuple`, the matrix norm will be computed. +- If :attr:`dim`\ `= None` and :attr:`ord`\ `= None`, + :attr:`A` will be flattened to 1D and the `2`-norm of the resulting vector will be computed. +- If :attr:`dim`\ `= None` and :attr:`ord` `!= None`, :attr:`A` must be 1D or 2D. + +:attr:`ord` defines the norm that is computed. The following norms are supported: + +====================== ========================== ====================================================== +:attr:`ord` norm for matrices norm for vectors +====================== ========================== ====================================================== +`None` (default) Frobenius norm `2`-norm (see below) +`'fro'` Frobenius norm -- not supported -- +`'nuc'` nuclear norm -- not supported -- +`inf` `max(sum(abs(x), dim=1))` `max(abs(x))` +`-inf` `min(sum(abs(x), dim=1))` `min(abs(x))` +`0` -- not supported -- `sum(x != 0)` +`1` `max(sum(abs(x), dim=0))` as below +`-1` `min(sum(abs(x), dim=0))` as below +`2` largest `singular value`_ as below +`-2` smallest `singular value`_ as below +other `int` or `float` -- not supported -- `sum(abs(x)^{ord})^{(1 / ord)}` +====================== ========================== ====================================================== + +where `inf` refers to `float('inf')`, NumPy's `inf` object, or any equivalent object. + +.. seealso:: + + :func:`torch.linalg.vector_norm` computes a vector norm. + + :func:`torch.linalg.matrix_norm` computes a matrix norm. + + The above functions are often clearer and more flexible than using :func:`torch.linalg.norm`. + For example, `torch.linalg.norm(A, ord=1, dim=(0, 1))` always + computes a matrix norm, but with `torch.linalg.vector_norm(A, ord=1, dim=(0, 1))` it is possible + to compute a vector norm over the two dimensions. + +Args: + A (Tensor): tensor of shape `(*, n)` or `(*, m, n)` where `*` is zero or more batch dimensions + ord (int, float, inf, -inf, 'fro', 'nuc', optional): order of norm. Default: `None` + dim (int, Tuple[int], optional): dimensions over which to compute + the vector or matrix norm. See above for the behavior when :attr:`dim`\ `= None`. + Default: `None` + keepdim (bool, optional): If set to `True`, the reduced dimensions are retained + in the result as dimensions with size one. Default: `False` + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + dtype (:class:`torch.dtype`, optional): If specified, the input tensor is cast to + :attr:`dtype` before performing the operation, and the returned tensor's type + will be :attr:`dtype`. Default: `None` + +Returns: + A real-valued tensor, even when :attr:`A` is complex. + +Examples:: + + >>> from torch import linalg as LA + >>> a = torch.arange(9, dtype=torch.float) - 4 + >>> a + tensor([-4., -3., -2., -1., 0., 1., 2., 3., 4.]) + >>> B = a.reshape((3, 3)) + >>> B + tensor([[-4., -3., -2.], + [-1., 0., 1.], + [ 2., 3., 4.]]) + + >>> LA.norm(a) + tensor(7.7460) + >>> LA.norm(B) + tensor(7.7460) + >>> LA.norm(B, 'fro') + tensor(7.7460) + >>> LA.norm(a, float('inf')) + tensor(4.) + >>> LA.norm(B, float('inf')) + tensor(9.) + >>> LA.norm(a, -float('inf')) + tensor(0.) + >>> LA.norm(B, -float('inf')) + tensor(2.) + + >>> LA.norm(a, 1) + tensor(20.) + >>> LA.norm(B, 1) + tensor(7.) + >>> LA.norm(a, -1) + tensor(0.) + >>> LA.norm(B, -1) + tensor(6.) + >>> LA.norm(a, 2) + tensor(7.7460) + >>> LA.norm(B, 2) + tensor(7.3485) + + >>> LA.norm(a, -2) + tensor(0.) + >>> LA.norm(B.double(), -2) + tensor(1.8570e-16, dtype=torch.float64) + >>> LA.norm(a, 3) + tensor(5.8480) + >>> LA.norm(a, -3) + tensor(0.) + +Using the :attr:`dim` argument to compute vector norms:: + + >>> c = torch.tensor([[1., 2., 3.], + ... [-1, 1, 4]]) + >>> LA.norm(c, dim=0) + tensor([1.4142, 2.2361, 5.0000]) + >>> LA.norm(c, dim=1) + tensor([3.7417, 4.2426]) + >>> LA.norm(c, ord=1, dim=1) + tensor([6., 6.]) + +Using the :attr:`dim` argument to compute matrix norms:: + + >>> A = torch.arange(8, dtype=torch.float).reshape(2, 2, 2) + >>> LA.norm(A, dim=(1,2)) + tensor([ 3.7417, 11.2250]) + >>> LA.norm(A[0, :, :]), LA.norm(A[1, :, :]) + (tensor(3.7417), tensor(11.2250)) + +.. _singular value: + https://en.wikipedia.org/wiki/Singular_value_decomposition#Singular_values,_singular_vectors,_and_their_relation_to_the_SVD +""", +) + +vector_norm = _add_docstr( + _linalg.linalg_vector_norm, + r""" +linalg.vector_norm(x, ord=2, dim=None, keepdim=False, *, dtype=None, out=None) -> Tensor + +Computes a vector norm. + +If :attr:`x` is complex valued, it computes the norm of :attr:`x`\ `.abs()` + +Supports input of float, double, cfloat and cdouble dtypes. + +This function does not necessarily treat multidimensional :attr:`x` as a batch of +vectors, instead: + +- If :attr:`dim`\ `= None`, :attr:`x` will be flattened before the norm is computed. +- If :attr:`dim` is an `int` or a `tuple`, the norm will be computed over these dimensions + and the other dimensions will be treated as batch dimensions. + +This behavior is for consistency with :func:`torch.linalg.norm`. + +:attr:`ord` defines the vector norm that is computed. The following norms are supported: + +====================== =============================== +:attr:`ord` vector norm +====================== =============================== +`2` (default) `2`-norm (see below) +`inf` `max(abs(x))` +`-inf` `min(abs(x))` +`0` `sum(x != 0)` +other `int` or `float` `sum(abs(x)^{ord})^{(1 / ord)}` +====================== =============================== + +where `inf` refers to `float('inf')`, NumPy's `inf` object, or any equivalent object. + +:attr:`dtype` may be used to perform the computation in a more precise dtype. +It is semantically equivalent to calling ``linalg.vector_norm(x.to(dtype))`` +but it is faster in some cases. + +.. seealso:: + + :func:`torch.linalg.matrix_norm` computes a matrix norm. + +Args: + x (Tensor): tensor, flattened by default, but this behavior can be + controlled using :attr:`dim`. (Note: the keyword argument + `input` can also be used as an alias for `x`.) + ord (int, float, inf, -inf, 'fro', 'nuc', optional): order of norm. Default: `2` + dim (int, Tuple[int], optional): dimensions over which to compute + the norm. See above for the behavior when :attr:`dim`\ `= None`. + Default: `None` + keepdim (bool, optional): If set to `True`, the reduced dimensions are retained + in the result as dimensions with size one. Default: `False` + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + dtype (:class:`torch.dtype`, optional): type used to perform the accumulation and the return. + If specified, :attr:`x` is cast to :attr:`dtype` before performing the operation, + and the returned tensor's type will be :attr:`dtype` if real and of its real counterpart if complex. + :attr:`dtype` may be complex if :attr:`x` is complex, otherwise it must be real. + :attr:`x` should be convertible without narrowing to :attr:`dtype`. Default: None + +Returns: + A real-valued tensor, even when :attr:`x` is complex. + +Examples:: + + >>> from torch import linalg as LA + >>> a = torch.arange(9, dtype=torch.float) - 4 + >>> a + tensor([-4., -3., -2., -1., 0., 1., 2., 3., 4.]) + >>> B = a.reshape((3, 3)) + >>> B + tensor([[-4., -3., -2.], + [-1., 0., 1.], + [ 2., 3., 4.]]) + >>> LA.vector_norm(a, ord=3.5) + tensor(5.4345) + >>> LA.vector_norm(B, ord=3.5) + tensor(5.4345) +""", +) + +matrix_norm = _add_docstr( + _linalg.linalg_matrix_norm, + r""" +linalg.matrix_norm(A, ord='fro', dim=(-2, -1), keepdim=False, *, dtype=None, out=None) -> Tensor + +Computes a matrix norm. + +If :attr:`A` is complex valued, it computes the norm of :attr:`A`\ `.abs()` + +Support input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices: the norm will be computed over the +dimensions specified by the 2-tuple :attr:`dim` and the other dimensions will +be treated as batch dimensions. The output will have the same batch dimensions. + +:attr:`ord` defines the matrix norm that is computed. The following norms are supported: + +====================== ======================================================== +:attr:`ord` matrix norm +====================== ======================================================== +`'fro'` (default) Frobenius norm +`'nuc'` nuclear norm +`inf` `max(sum(abs(x), dim=1))` +`-inf` `min(sum(abs(x), dim=1))` +`1` `max(sum(abs(x), dim=0))` +`-1` `min(sum(abs(x), dim=0))` +`2` largest singular value +`-2` smallest singular value +====================== ======================================================== + +where `inf` refers to `float('inf')`, NumPy's `inf` object, or any equivalent object. + +Args: + A (Tensor): tensor with two or more dimensions. By default its + shape is interpreted as `(*, m, n)` where `*` is zero or more + batch dimensions, but this behavior can be controlled using :attr:`dim`. + ord (int, inf, -inf, 'fro', 'nuc', optional): order of norm. Default: `'fro'` + dim (Tuple[int, int], optional): dimensions over which to compute the norm. Default: `(-2, -1)` + keepdim (bool, optional): If set to `True`, the reduced dimensions are retained + in the result as dimensions with size one. Default: `False` + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + dtype (:class:`torch.dtype`, optional): If specified, the input tensor is cast to + :attr:`dtype` before performing the operation, and the returned tensor's type + will be :attr:`dtype`. Default: `None` + +Returns: + A real-valued tensor, even when :attr:`A` is complex. + +Examples:: + + >>> from torch import linalg as LA + >>> A = torch.arange(9, dtype=torch.float).reshape(3, 3) + >>> A + tensor([[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]) + >>> LA.matrix_norm(A) + tensor(14.2829) + >>> LA.matrix_norm(A, ord=-1) + tensor(9.) + >>> B = A.expand(2, -1, -1) + >>> B + tensor([[[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]], + + [[0., 1., 2.], + [3., 4., 5.], + [6., 7., 8.]]]) + >>> LA.matrix_norm(B) + tensor([14.2829, 14.2829]) + >>> LA.matrix_norm(B, dim=(0, 2)) + tensor([ 3.1623, 10.0000, 17.2627]) +""", +) + +matmul = _add_docstr( + _linalg.linalg_matmul, + r""" +linalg.matmul(input, other, *, out=None) -> Tensor + +Alias for :func:`torch.matmul` +""", +) + +diagonal = _add_docstr( + _linalg.linalg_diagonal, + r""" +linalg.diagonal(A, *, offset=0, dim1=-2, dim2=-1) -> Tensor + +Alias for :func:`torch.diagonal` with defaults :attr:`dim1`\ `= -2`, :attr:`dim2`\ `= -1`. +""", +) + +multi_dot = _add_docstr( + _linalg.linalg_multi_dot, + r""" +linalg.multi_dot(tensors, *, out=None) + +Efficiently multiplies two or more matrices by reordering the multiplications so that +the fewest arithmetic operations are performed. + +Supports inputs of float, double, cfloat and cdouble dtypes. +This function does not support batched inputs. + +Every tensor in :attr:`tensors` must be 2D, except for the first and last which +may be 1D. If the first tensor is a 1D vector of shape `(n,)` it is treated as a row vector +of shape `(1, n)`, similarly if the last tensor is a 1D vector of shape `(n,)` it is treated +as a column vector of shape `(n, 1)`. + +If the first and last tensors are matrices, the output will be a matrix. +However, if either is a 1D vector, then the output will be a 1D vector. + +Differences with `numpy.linalg.multi_dot`: + +- Unlike `numpy.linalg.multi_dot`, the first and last tensors must either be 1D or 2D + whereas NumPy allows them to be nD + +.. warning:: This function does not broadcast. + +.. note:: This function is implemented by chaining :func:`torch.mm` calls after + computing the optimal matrix multiplication order. + +.. note:: The cost of multiplying two matrices with shapes `(a, b)` and `(b, c)` is + `a * b * c`. Given matrices `A`, `B`, `C` with shapes `(10, 100)`, + `(100, 5)`, `(5, 50)` respectively, we can calculate the cost of different + multiplication orders as follows: + + .. math:: + + \begin{align*} + \operatorname{cost}((AB)C) &= 10 \times 100 \times 5 + 10 \times 5 \times 50 = 7500 \\ + \operatorname{cost}(A(BC)) &= 10 \times 100 \times 50 + 100 \times 5 \times 50 = 75000 + \end{align*} + + In this case, multiplying `A` and `B` first followed by `C` is 10 times faster. + +Args: + tensors (Sequence[Tensor]): two or more tensors to multiply. The first and last + tensors may be 1D or 2D. Every other tensor must be 2D. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> from torch.linalg import multi_dot + + >>> multi_dot([torch.tensor([1, 2]), torch.tensor([2, 3])]) + tensor(8) + >>> multi_dot([torch.tensor([[1, 2]]), torch.tensor([2, 3])]) + tensor([8]) + >>> multi_dot([torch.tensor([[1, 2]]), torch.tensor([[2], [3]])]) + tensor([[8]]) + + >>> A = torch.arange(2 * 3).view(2, 3) + >>> B = torch.arange(3 * 2).view(3, 2) + >>> C = torch.arange(2 * 2).view(2, 2) + >>> multi_dot((A, B, C)) + tensor([[ 26, 49], + [ 80, 148]]) +""", +) + +svd = _add_docstr( + _linalg.linalg_svd, + r""" +linalg.svd(A, full_matrices=True, *, driver=None, out=None) -> (Tensor, Tensor, Tensor) + +Computes the singular value decomposition (SVD) of a matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **full SVD** of a matrix +:math:`A \in \mathbb{K}^{m \times n}`, if `k = min(m,n)`, is defined as + +.. math:: + + A = U \operatorname{diag}(S) V^{\text{H}} + \mathrlap{\qquad U \in \mathbb{K}^{m \times m}, S \in \mathbb{R}^k, V \in \mathbb{K}^{n \times n}} + +where :math:`\operatorname{diag}(S) \in \mathbb{K}^{m \times n}`, +:math:`V^{\text{H}}` is the conjugate transpose when :math:`V` is complex, and the transpose when :math:`V` is real-valued. +The matrices :math:`U`, :math:`V` (and thus :math:`V^{\text{H}}`) are orthogonal in the real case, and unitary in the complex case. + +When `m > n` (resp. `m < n`) we can drop the last `m - n` (resp. `n - m`) columns of `U` (resp. `V`) to form the **reduced SVD**: + +.. math:: + + A = U \operatorname{diag}(S) V^{\text{H}} + \mathrlap{\qquad U \in \mathbb{K}^{m \times k}, S \in \mathbb{R}^k, V \in \mathbb{K}^{n \times k}} + +where :math:`\operatorname{diag}(S) \in \mathbb{K}^{k \times k}`. +In this case, :math:`U` and :math:`V` also have orthonormal columns. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The returned decomposition is a named tuple `(U, S, Vh)` +which corresponds to :math:`U`, :math:`S`, :math:`V^{\text{H}}` above. + +The singular values are returned in descending order. + +The parameter :attr:`full_matrices` chooses between the full (default) and reduced SVD. + +The :attr:`driver` kwarg may be used in CUDA with a cuSOLVER backend to choose the algorithm used to compute the SVD. +The choice of a driver is a trade-off between accuracy and speed. + +- If :attr:`A` is well-conditioned (its `condition number`_ is not too large), or you do not mind some precision loss. + + - For a general matrix: `'gesvdj'` (Jacobi method) + - If :attr:`A` is tall or wide (`m >> n` or `m << n`): `'gesvda'` (Approximate method) + +- If :attr:`A` is not well-conditioned or precision is relevant: `'gesvd'` (QR based) + +By default (:attr:`driver`\ `= None`), we call `'gesvdj'` and, if it fails, we fallback to `'gesvd'`. + +Differences with `numpy.linalg.svd`: + +- Unlike `numpy.linalg.svd`, this function always returns a tuple of three tensors + and it doesn't support `compute_uv` argument. + Please use :func:`torch.linalg.svdvals`, which computes only the singular values, + instead of `compute_uv=False`. + +.. note:: When :attr:`full_matrices`\ `= True`, the gradients with respect to `U[..., :, min(m, n):]` + and `Vh[..., min(m, n):, :]` will be ignored, as those vectors can be arbitrary bases + of the corresponding subspaces. + +.. warning:: The returned tensors `U` and `V` are not unique, nor are they continuous with + respect to :attr:`A`. + Due to this lack of uniqueness, different hardware and software may compute + different singular vectors. + + This non-uniqueness is caused by the fact that multiplying any pair of singular + vectors :math:`u_k, v_k` by `-1` in the real case or by + :math:`e^{i \phi}, \phi \in \mathbb{R}` in the complex case produces another two + valid singular vectors of the matrix. + For this reason, the loss function shall not depend on this :math:`e^{i \phi}` quantity, + as it is not well-defined. + This is checked for complex inputs when computing the gradients of this function. As such, + when inputs are complex and are on a CUDA device, the computation of the gradients + of this function synchronizes that device with the CPU. + +.. warning:: Gradients computed using `U` or `Vh` will only be finite when + :attr:`A` does not have repeated singular values. If :attr:`A` is rectangular, + additionally, zero must also not be one of its singular values. + Furthermore, if the distance between any two singular values is close to zero, + the gradient will be numerically unstable, as it depends on the singular values + :math:`\sigma_i` through the computation of + :math:`\frac{1}{\min_{i \neq j} \sigma_i^2 - \sigma_j^2}`. + In the rectangular case, the gradient will also be numerically unstable when + :attr:`A` has small singular values, as it also depends on the computation of + :math:`\frac{1}{\sigma_i}`. + +.. seealso:: + + :func:`torch.linalg.svdvals` computes only the singular values. + Unlike :func:`torch.linalg.svd`, the gradients of :func:`~svdvals` are always + numerically stable. + + :func:`torch.linalg.eig` for a function that computes another type of spectral + decomposition of a matrix. The eigendecomposition works just on square matrices. + + :func:`torch.linalg.eigh` for a (faster) function that computes the eigenvalue decomposition + for Hermitian and symmetric matrices. + + :func:`torch.linalg.qr` for another (much faster) decomposition that works on general + matrices. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + full_matrices (bool, optional): controls whether to compute the full or reduced + SVD, and consequently, + the shape of the returned tensors + `U` and `Vh`. Default: `True`. + +Keyword args: + driver (str, optional): name of the cuSOLVER method to be used. This keyword argument only works on CUDA inputs. + Available options are: `None`, `gesvd`, `gesvdj`, and `gesvda`. + Default: `None`. + out (tuple, optional): output tuple of three tensors. Ignored if `None`. + +Returns: + A named tuple `(U, S, Vh)` which corresponds to :math:`U`, :math:`S`, :math:`V^{\text{H}}` above. + + `S` will always be real-valued, even when :attr:`A` is complex. + It will also be ordered in descending order. + + `U` and `Vh` will have the same dtype as :attr:`A`. The left / right singular vectors will be given by + the columns of `U` and the rows of `Vh` respectively. + +Examples:: + + >>> A = torch.randn(5, 3) + >>> U, S, Vh = torch.linalg.svd(A, full_matrices=False) + >>> U.shape, S.shape, Vh.shape + (torch.Size([5, 3]), torch.Size([3]), torch.Size([3, 3])) + >>> torch.dist(A, U @ torch.diag(S) @ Vh) + tensor(1.0486e-06) + + >>> U, S, Vh = torch.linalg.svd(A) + >>> U.shape, S.shape, Vh.shape + (torch.Size([5, 5]), torch.Size([3]), torch.Size([3, 3])) + >>> torch.dist(A, U[:, :3] @ torch.diag(S) @ Vh) + tensor(1.0486e-06) + + >>> A = torch.randn(7, 5, 3) + >>> U, S, Vh = torch.linalg.svd(A, full_matrices=False) + >>> torch.dist(A, U @ torch.diag_embed(S) @ Vh) + tensor(3.0957e-06) + +.. _condition number: + https://pytorch.org/docs/main/linalg.html#torch.linalg.cond +.. _the resulting vectors will span the same subspace: + https://en.wikipedia.org/wiki/Singular_value_decomposition#Singular_values,_singular_vectors,_and_their_relation_to_the_SVD +""", +) + +svdvals = _add_docstr( + _linalg.linalg_svdvals, + r""" +linalg.svdvals(A, *, driver=None, out=None) -> Tensor + +Computes the singular values of a matrix. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The singular values are returned in descending order. + +.. note:: This function is equivalent to NumPy's `linalg.svd(A, compute_uv=False)`. + +""" + + rf""" +.. note:: {common_notes["sync_note"]} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.svd` computes the full singular value decomposition. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + +Keyword args: + driver (str, optional): name of the cuSOLVER method to be used. This keyword argument only works on CUDA inputs. + Available options are: `None`, `gesvd`, `gesvdj`, and `gesvda`. + Check :func:`torch.linalg.svd` for details. + Default: `None`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Returns: + A real-valued tensor, even when :attr:`A` is complex. + +Examples:: + + >>> A = torch.randn(5, 3) + >>> S = torch.linalg.svdvals(A) + >>> S + tensor([2.5139, 2.1087, 1.1066]) + + >>> torch.dist(S, torch.linalg.svd(A, full_matrices=False).S) + tensor(2.4576e-07) +""", +) + +cond = _add_docstr( + _linalg.linalg_cond, + r""" +linalg.cond(A, p=None, *, out=None) -> Tensor + +Computes the condition number of a matrix with respect to a matrix norm. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **condition number** :math:`\kappa` of a matrix +:math:`A \in \mathbb{K}^{n \times n}` is defined as + +.. math:: + + \kappa(A) = \|A\|_p\|A^{-1}\|_p + +The condition number of :attr:`A` measures the numerical stability of the linear system `AX = B` +with respect to a matrix norm. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +:attr:`p` defines the matrix norm that is computed. The following norms are supported: + +========= ================================= +:attr:`p` matrix norm +========= ================================= +`None` `2`-norm (largest singular value) +`'fro'` Frobenius norm +`'nuc'` nuclear norm +`inf` `max(sum(abs(x), dim=1))` +`-inf` `min(sum(abs(x), dim=1))` +`1` `max(sum(abs(x), dim=0))` +`-1` `min(sum(abs(x), dim=0))` +`2` largest singular value +`-2` smallest singular value +========= ================================= + +where `inf` refers to `float('inf')`, NumPy's `inf` object, or any equivalent object. + +For :attr:`p` is one of `('fro', 'nuc', inf, -inf, 1, -1)`, this function uses +:func:`torch.linalg.norm` and :func:`torch.linalg.inv`. +As such, in this case, the matrix (or every matrix in the batch) :attr:`A` has to be square +and invertible. + +For :attr:`p` in `(2, -2)`, this function can be computed in terms of the singular values +:math:`\sigma_1 \geq \ldots \geq \sigma_n` + +.. math:: + + \kappa_2(A) = \frac{\sigma_1}{\sigma_n}\qquad \kappa_{-2}(A) = \frac{\sigma_n}{\sigma_1} + +In these cases, it is computed using :func:`torch.linalg.svdvals`. For these norms, the matrix +(or every matrix in the batch) :attr:`A` may have any shape. + +.. note :: When inputs are on a CUDA device, this function synchronizes that device with the CPU + if :attr:`p` is one of `('fro', 'nuc', inf, -inf, 1, -1)`. + +.. seealso:: + + :func:`torch.linalg.solve` for a function that solves linear systems of square matrices. + + :func:`torch.linalg.lstsq` for a function that solves linear systems of general matrices. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions + for :attr:`p` in `(2, -2)`, and of shape `(*, n, n)` where every matrix + is invertible for :attr:`p` in `('fro', 'nuc', inf, -inf, 1, -1)`. + p (int, inf, -inf, 'fro', 'nuc', optional): + the type of the matrix norm to use in the computations (see above). Default: `None` + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Returns: + A real-valued tensor, even when :attr:`A` is complex. + +Raises: + RuntimeError: + if :attr:`p` is one of `('fro', 'nuc', inf, -inf, 1, -1)` + and the :attr:`A` matrix or any matrix in the batch :attr:`A` is not square + or invertible. + +Examples:: + + >>> A = torch.randn(3, 4, 4, dtype=torch.complex64) + >>> torch.linalg.cond(A) + >>> A = torch.tensor([[1., 0, -1], [0, 1, 0], [1, 0, 1]]) + >>> torch.linalg.cond(A) + tensor([1.4142]) + >>> torch.linalg.cond(A, 'fro') + tensor(3.1623) + >>> torch.linalg.cond(A, 'nuc') + tensor(9.2426) + >>> torch.linalg.cond(A, float('inf')) + tensor(2.) + >>> torch.linalg.cond(A, float('-inf')) + tensor(1.) + >>> torch.linalg.cond(A, 1) + tensor(2.) + >>> torch.linalg.cond(A, -1) + tensor(1.) + >>> torch.linalg.cond(A, 2) + tensor([1.4142]) + >>> torch.linalg.cond(A, -2) + tensor([0.7071]) + + >>> A = torch.randn(2, 3, 3) + >>> torch.linalg.cond(A) + tensor([[9.5917], + [3.2538]]) + >>> A = torch.randn(2, 3, 3, dtype=torch.complex64) + >>> torch.linalg.cond(A) + tensor([[4.6245], + [4.5671]]) +""", +) + +pinv = _add_docstr( + _linalg.linalg_pinv, + r""" +linalg.pinv(A, *, atol=None, rtol=None, hermitian=False, out=None) -> Tensor + +Computes the pseudoinverse (Moore-Penrose inverse) of a matrix. + +The pseudoinverse may be `defined algebraically`_ +but it is more computationally convenient to understand it `through the SVD`_ + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +If :attr:`hermitian`\ `= True`, :attr:`A` is assumed to be Hermitian if complex or +symmetric if real, but this is not checked internally. Instead, just the lower +triangular part of the matrix is used in the computations. + +The singular values (or the norm of the eigenvalues when :attr:`hermitian`\ `= True`) +that are below :math:`\max(\text{atol}, \sigma_1 \cdot \text{rtol})` threshold are +treated as zero and discarded in the computation, +where :math:`\sigma_1` is the largest singular value (or eigenvalue). + +If :attr:`rtol` is not specified and :attr:`A` is a matrix of dimensions `(m, n)`, +the relative tolerance is set to be :math:`\text{rtol} = \max(m, n) \varepsilon` +and :math:`\varepsilon` is the epsilon value for the dtype of :attr:`A` (see :class:`.finfo`). +If :attr:`rtol` is not specified and :attr:`atol` is specified to be larger than zero then +:attr:`rtol` is set to zero. + +If :attr:`atol` or :attr:`rtol` is a :class:`torch.Tensor`, its shape must be broadcastable to that +of the singular values of :attr:`A` as returned by :func:`torch.linalg.svd`. + +.. note:: This function uses :func:`torch.linalg.svd` if :attr:`hermitian`\ `= False` and + :func:`torch.linalg.eigh` if :attr:`hermitian`\ `= True`. + For CUDA inputs, this function synchronizes that device with the CPU. + +.. note:: + Consider using :func:`torch.linalg.lstsq` if possible for multiplying a matrix on the left by + the pseudoinverse, as:: + + torch.linalg.lstsq(A, B).solution == A.pinv() @ B + + It is always preferred to use :func:`~lstsq` when possible, as it is faster and more + numerically stable than computing the pseudoinverse explicitly. + +.. note:: + This function has NumPy compatible variant `linalg.pinv(A, rcond, hermitian=False)`. + However, use of the positional argument :attr:`rcond` is deprecated in favor of :attr:`rtol`. + +.. warning:: + This function uses internally :func:`torch.linalg.svd` (or :func:`torch.linalg.eigh` + when :attr:`hermitian`\ `= True`), so its derivative has the same problems as those of these + functions. See the warnings in :func:`torch.linalg.svd` and :func:`torch.linalg.eigh` for + more details. + +.. seealso:: + + :func:`torch.linalg.inv` computes the inverse of a square matrix. + + :func:`torch.linalg.lstsq` computes :attr:`A`\ `.pinv() @ \ `:attr:`B` with a + numerically stable algorithm. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + rcond (float, Tensor, optional): [NumPy Compat]. Alias for :attr:`rtol`. Default: `None`. + +Keyword args: + atol (float, Tensor, optional): the absolute tolerance value. When `None` it's considered to be zero. + Default: `None`. + rtol (float, Tensor, optional): the relative tolerance value. See above for the value it takes when `None`. + Default: `None`. + hermitian(bool, optional): indicates whether :attr:`A` is Hermitian if complex + or symmetric if real. Default: `False`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(3, 5) + >>> A + tensor([[ 0.5495, 0.0979, -1.4092, -0.1128, 0.4132], + [-1.1143, -0.3662, 0.3042, 1.6374, -0.9294], + [-0.3269, -0.5745, -0.0382, -0.5922, -0.6759]]) + >>> torch.linalg.pinv(A) + tensor([[ 0.0600, -0.1933, -0.2090], + [-0.0903, -0.0817, -0.4752], + [-0.7124, -0.1631, -0.2272], + [ 0.1356, 0.3933, -0.5023], + [-0.0308, -0.1725, -0.5216]]) + + >>> A = torch.randn(2, 6, 3) + >>> Apinv = torch.linalg.pinv(A) + >>> torch.dist(Apinv @ A, torch.eye(3)) + tensor(8.5633e-07) + + >>> A = torch.randn(3, 3, dtype=torch.complex64) + >>> A = A + A.T.conj() # creates a Hermitian matrix + >>> Apinv = torch.linalg.pinv(A, hermitian=True) + >>> torch.dist(Apinv @ A, torch.eye(3)) + tensor(1.0830e-06) + +.. _defined algebraically: + https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse#Existence_and_uniqueness +.. _through the SVD: + https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse#Singular_value_decomposition_(SVD) +""", +) + +matrix_exp = _add_docstr( + _linalg.linalg_matrix_exp, + r""" +linalg.matrix_exp(A) -> Tensor + +Computes the matrix exponential of a square matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +this function computes the **matrix exponential** of :math:`A \in \mathbb{K}^{n \times n}`, which is defined as + +.. math:: + \mathrm{matrix\_exp}(A) = \sum_{k=0}^\infty \frac{1}{k!}A^k \in \mathbb{K}^{n \times n}. + +If the matrix :math:`A` has eigenvalues :math:`\lambda_i \in \mathbb{C}`, +the matrix :math:`\mathrm{matrix\_exp}(A)` has eigenvalues :math:`e^{\lambda_i} \in \mathbb{C}`. + +Supports input of bfloat16, float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + +Example:: + + >>> A = torch.empty(2, 2, 2) + >>> A[0, :, :] = torch.eye(2, 2) + >>> A[1, :, :] = 2 * torch.eye(2, 2) + >>> A + tensor([[[1., 0.], + [0., 1.]], + + [[2., 0.], + [0., 2.]]]) + >>> torch.linalg.matrix_exp(A) + tensor([[[2.7183, 0.0000], + [0.0000, 2.7183]], + + [[7.3891, 0.0000], + [0.0000, 7.3891]]]) + + >>> import math + >>> A = torch.tensor([[0, math.pi/3], [-math.pi/3, 0]]) # A is skew-symmetric + >>> torch.linalg.matrix_exp(A) # matrix_exp(A) = [[cos(pi/3), sin(pi/3)], [-sin(pi/3), cos(pi/3)]] + tensor([[ 0.5000, 0.8660], + [-0.8660, 0.5000]]) +""", +) + + +solve = _add_docstr( + _linalg.linalg_solve, + r""" +linalg.solve(A, B, *, left=True, out=None) -> Tensor + +Computes the solution of a square system of linear equations with a unique solution. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +this function computes the solution :math:`X \in \mathbb{K}^{n \times k}` of the **linear system** associated to +:math:`A \in \mathbb{K}^{n \times n}, B \in \mathbb{K}^{n \times k}`, which is defined as + +.. math:: AX = B + +If :attr:`left`\ `= False`, this function returns the matrix :math:`X \in \mathbb{K}^{n \times k}` that solves the system + +.. math:: + + XA = B\mathrlap{\qquad A \in \mathbb{K}^{k \times k}, B \in \mathbb{K}^{n \times k}.} + +This system of linear equations has one solution if and only if :math:`A` is `invertible`_. +This function assumes that :math:`A` is invertible. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +Letting `*` be zero or more batch dimensions, + +- If :attr:`A` has shape `(*, n, n)` and :attr:`B` has shape `(*, n)` (a batch of vectors) or shape + `(*, n, k)` (a batch of matrices or "multiple right-hand sides"), this function returns `X` of shape + `(*, n)` or `(*, n, k)` respectively. +- Otherwise, if :attr:`A` has shape `(*, n, n)` and :attr:`B` has shape `(n,)` or `(n, k)`, :attr:`B` + is broadcasted to have shape `(*, n)` or `(*, n, k)` respectively. + This function then returns the solution of the resulting batch of systems of linear equations. + +.. note:: + This function computes `X = \ `:attr:`A`\ `.inverse() @ \ `:attr:`B` in a faster and + more numerically stable way than performing the computations separately. + +.. note:: + It is possible to compute the solution of the system :math:`XA = B` by passing the inputs + :attr:`A` and :attr:`B` transposed and transposing the output returned by this function. + +.. note:: + :attr:`A` is allowed to be a non-batched `torch.sparse_csr_tensor`, but only with `left=True`. + +""" + + rf""" +.. note:: {common_notes["sync_note_has_ex"].format("torch.linalg.solve_ex")} +""" + + r""" + +.. seealso:: + + :func:`torch.linalg.solve_triangular` computes the solution of a triangular system of linear + equations with a unique solution. + +Args: + A (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions. + B (Tensor): right-hand side tensor of shape `(*, n)` or `(*, n, k)` or `(n,)` or `(n, k)` + according to the rules described above + +Keyword args: + left (bool, optional): whether to solve the system :math:`AX=B` or :math:`XA = B`. Default: `True`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if the :attr:`A` matrix is not invertible or any matrix in a batched :attr:`A` + is not invertible. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> b = torch.randn(3) + >>> x = torch.linalg.solve(A, b) + >>> torch.allclose(A @ x, b) + True + >>> A = torch.randn(2, 3, 3) + >>> B = torch.randn(2, 3, 4) + >>> X = torch.linalg.solve(A, B) + >>> X.shape + torch.Size([2, 3, 4]) + >>> torch.allclose(A @ X, B) + True + + >>> A = torch.randn(2, 3, 3) + >>> b = torch.randn(3, 1) + >>> x = torch.linalg.solve(A, b) # b is broadcasted to size (2, 3, 1) + >>> x.shape + torch.Size([2, 3, 1]) + >>> torch.allclose(A @ x, b) + True + >>> b = torch.randn(3) + >>> x = torch.linalg.solve(A, b) # b is broadcasted to size (2, 3) + >>> x.shape + torch.Size([2, 3]) + >>> Ax = A @ x.unsqueeze(-1) + >>> torch.allclose(Ax, b.unsqueeze(-1).expand_as(Ax)) + True + +.. _invertible: + https://en.wikipedia.org/wiki/Invertible_matrix#The_invertible_matrix_theorem +""", +) + +solve_triangular = _add_docstr( + _linalg.linalg_solve_triangular, + r""" +linalg.solve_triangular(A, B, *, upper, left=True, unitriangular=False, out=None) -> Tensor + +Computes the solution of a triangular system of linear equations with a unique solution. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +this function computes the solution :math:`X \in \mathbb{K}^{n \times k}` of the **linear system** +associated to the triangular matrix :math:`A \in \mathbb{K}^{n \times n}` without zeros on the diagonal +(that is, it is `invertible`_) and the rectangular matrix , :math:`B \in \mathbb{K}^{n \times k}`, +which is defined as + +.. math:: AX = B + +The argument :attr:`upper` signals whether :math:`A` is upper or lower triangular. + +If :attr:`left`\ `= False`, this function returns the matrix :math:`X \in \mathbb{K}^{n \times k}` that +solves the system + +.. math:: + + XA = B\mathrlap{\qquad A \in \mathbb{K}^{k \times k}, B \in \mathbb{K}^{n \times k}.} + +If :attr:`upper`\ `= True` (resp. `False`) just the upper (resp. lower) triangular half of :attr:`A` +will be accessed. The elements below the main diagonal will be considered to be zero and will not be accessed. + +If :attr:`unitriangular`\ `= True`, the diagonal of :attr:`A` is assumed to be ones and will not be accessed. + +The result may contain `NaN` s if the diagonal of :attr:`A` contains zeros or elements that +are very close to zero and :attr:`unitriangular`\ `= False` (default) or if the input matrix +has very small eigenvalues. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +.. seealso:: + + :func:`torch.linalg.solve` computes the solution of a general square system of linear + equations with a unique solution. + +Args: + A (Tensor): tensor of shape `(*, n, n)` (or `(*, k, k)` if :attr:`left`\ `= False`) + where `*` is zero or more batch dimensions. + B (Tensor): right-hand side tensor of shape `(*, n, k)`. + +Keyword args: + upper (bool): whether :attr:`A` is an upper or lower triangular matrix. + left (bool, optional): whether to solve the system :math:`AX=B` or :math:`XA = B`. Default: `True`. + unitriangular (bool, optional): if `True`, the diagonal elements of :attr:`A` are assumed to be + all equal to `1`. Default: `False`. + out (Tensor, optional): output tensor. `B` may be passed as `out` and the result is computed in-place on `B`. + Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(3, 3).triu_() + >>> B = torch.randn(3, 4) + >>> X = torch.linalg.solve_triangular(A, B, upper=True) + >>> torch.allclose(A @ X, B) + True + + >>> A = torch.randn(2, 3, 3).tril_() + >>> B = torch.randn(2, 3, 4) + >>> X = torch.linalg.solve_triangular(A, B, upper=False) + >>> torch.allclose(A @ X, B) + True + + >>> A = torch.randn(2, 4, 4).tril_() + >>> B = torch.randn(2, 3, 4) + >>> X = torch.linalg.solve_triangular(A, B, upper=False, left=False) + >>> torch.allclose(X @ A, B) + True + +.. _invertible: + https://en.wikipedia.org/wiki/Invertible_matrix#The_invertible_matrix_theorem +""", +) + +lu_factor = _add_docstr( + _linalg.linalg_lu_factor, + r""" +linalg.lu_factor(A, *, bool pivot=True, out=None) -> (Tensor, Tensor) + +Computes a compact representation of the LU factorization with partial pivoting of a matrix. + +This function computes a compact representation of the decomposition given by :func:`torch.linalg.lu`. +If the matrix is square, this representation may be used in :func:`torch.linalg.lu_solve` +to solve system of linear equations that share the matrix :attr:`A`. + +The returned decomposition is represented as a named tuple `(LU, pivots)`. +The ``LU`` matrix has the same shape as the input matrix ``A``. Its upper and lower triangular +parts encode the non-constant elements of ``L`` and ``U`` of the LU decomposition of ``A``. + +The returned permutation matrix is represented by a 1-indexed vector. `pivots[i] == j` represents +that in the `i`-th step of the algorithm, the `i`-th row was permuted with the `j-1`-th row. + +On CUDA, one may use :attr:`pivot`\ `= False`. In this case, this function returns the LU +decomposition without pivoting if it exists. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +""" + + rf""" +.. note:: {common_notes["sync_note_has_ex"].format("torch.linalg.lu_factor_ex")} +""" + + r""" +.. warning:: The LU decomposition is almost never unique, as often there are different permutation + matrices that can yield different LU decompositions. + As such, different platforms, like SciPy, or inputs on different devices, + may produce different valid decompositions. + + Gradient computations are only supported if the input matrix is full-rank. + If this condition is not met, no error will be thrown, but the gradient may not be finite. + This is because the LU decomposition with pivoting is not differentiable at these points. + +.. seealso:: + + :func:`torch.linalg.lu_solve` solves a system of linear equations given the output of this + function provided the input matrix was square and invertible. + + :func:`torch.lu_unpack` unpacks the tensors returned by :func:`~lu_factor` into the three + matrices `P, L, U` that form the decomposition. + + :func:`torch.linalg.lu` computes the LU decomposition with partial pivoting of a possibly + non-square matrix. It is a composition of :func:`~lu_factor` and :func:`torch.lu_unpack`. + + :func:`torch.linalg.solve` solves a system of linear equations. It is a composition + of :func:`~lu_factor` and :func:`~lu_solve`. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + +Keyword args: + pivot (bool, optional): Whether to compute the LU decomposition with partial pivoting, or the regular LU + decomposition. :attr:`pivot`\ `= False` not supported on CPU. Default: `True`. + out (tuple, optional): tuple of two tensors to write the output to. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(LU, pivots)`. + +Raises: + RuntimeError: if the :attr:`A` matrix is not invertible or any matrix in a batched :attr:`A` + is not invertible. + +Examples:: + + >>> A = torch.randn(2, 3, 3) + >>> B1 = torch.randn(2, 3, 4) + >>> B2 = torch.randn(2, 3, 7) + >>> LU, pivots = torch.linalg.lu_factor(A) + >>> X1 = torch.linalg.lu_solve(LU, pivots, B1) + >>> X2 = torch.linalg.lu_solve(LU, pivots, B2) + >>> torch.allclose(A @ X1, B1) + True + >>> torch.allclose(A @ X2, B2) + True + +.. _invertible: + https://en.wikipedia.org/wiki/Invertible_matrix#The_invertible_matrix_theorem +""", +) + +lu_factor_ex = _add_docstr( + _linalg.linalg_lu_factor_ex, + r""" +linalg.lu_factor_ex(A, *, pivot=True, check_errors=False, out=None) -> (Tensor, Tensor, Tensor) + +This is a version of :func:`~lu_factor` that does not perform error checks unless :attr:`check_errors`\ `= True`. +It also returns the :attr:`info` tensor returned by `LAPACK's getrf`_. + +""" + + rf""" +.. note:: {common_notes["sync_note_ex"]} + +.. warning:: {common_notes["experimental_warning"]} +""" + + r""" + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + +Keyword args: + pivot (bool, optional): Whether to compute the LU decomposition with partial pivoting, or the regular LU + decomposition. :attr:`pivot`\ `= False` not supported on CPU. Default: `True`. + check_errors (bool, optional): controls whether to check the content of ``infos`` and raise + an error if it is non-zero. Default: `False`. + out (tuple, optional): tuple of three tensors to write the output to. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(LU, pivots, info)`. + +.. _LAPACK's getrf: + https://www.netlib.org/lapack/explore-html-3.6.1/dd/d9a/group__double_g_ecomputational_ga0019443faea08275ca60a734d0593e60.html +""", +) + +lu_solve = _add_docstr( + _linalg.linalg_lu_solve, + r""" +linalg.lu_solve(LU, pivots, B, *, left=True, adjoint=False, out=None) -> Tensor + +Computes the solution of a square system of linear equations with a unique solution given an LU decomposition. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +this function computes the solution :math:`X \in \mathbb{K}^{n \times k}` of the **linear system** associated to +:math:`A \in \mathbb{K}^{n \times n}, B \in \mathbb{K}^{n \times k}`, which is defined as + +.. math:: AX = B + +where :math:`A` is given factorized as returned by :func:`~lu_factor`. + +If :attr:`left`\ `= False`, this function returns the matrix :math:`X \in \mathbb{K}^{n \times k}` that solves the system + +.. math:: + + XA = B\mathrlap{\qquad A \in \mathbb{K}^{k \times k}, B \in \mathbb{K}^{n \times k}.} + +If :attr:`adjoint`\ `= True` (and :attr:`left`\ `= True`), given an LU factorization of :math:`A` +this function function returns the :math:`X \in \mathbb{K}^{n \times k}` that solves the system + +.. math:: + + A^{\text{H}}X = B\mathrlap{\qquad A \in \mathbb{K}^{k \times k}, B \in \mathbb{K}^{n \times k}.} + +where :math:`A^{\text{H}}` is the conjugate transpose when :math:`A` is complex, and the +transpose when :math:`A` is real-valued. The :attr:`left`\ `= False` case is analogous. + +Supports inputs of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if the inputs are batches of matrices then +the output has the same batch dimensions. + +Args: + LU (Tensor): tensor of shape `(*, n, n)` (or `(*, k, k)` if :attr:`left`\ `= True`) + where `*` is zero or more batch dimensions as returned by :func:`~lu_factor`. + pivots (Tensor): tensor of shape `(*, n)` (or `(*, k)` if :attr:`left`\ `= True`) + where `*` is zero or more batch dimensions as returned by :func:`~lu_factor`. + B (Tensor): right-hand side tensor of shape `(*, n, k)`. + +Keyword args: + left (bool, optional): whether to solve the system :math:`AX=B` or :math:`XA = B`. Default: `True`. + adjoint (bool, optional): whether to solve the system :math:`AX=B` or :math:`A^{\text{H}}X = B`. Default: `False`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> A = torch.randn(3, 3) + >>> LU, pivots = torch.linalg.lu_factor(A) + >>> B = torch.randn(3, 2) + >>> X = torch.linalg.lu_solve(LU, pivots, B) + >>> torch.allclose(A @ X, B) + True + + >>> B = torch.randn(3, 3, 2) # Broadcasting rules apply: A is broadcasted + >>> X = torch.linalg.lu_solve(LU, pivots, B) + >>> torch.allclose(A @ X, B) + True + + >>> B = torch.randn(3, 5, 3) + >>> X = torch.linalg.lu_solve(LU, pivots, B, left=False) + >>> torch.allclose(X @ A, B) + True + + >>> B = torch.randn(3, 3, 4) # Now solve for A^T + >>> X = torch.linalg.lu_solve(LU, pivots, B, adjoint=True) + >>> torch.allclose(A.mT @ X, B) + True + +.. _invertible: + https://en.wikipedia.org/wiki/Invertible_matrix#The_invertible_matrix_theorem +""", +) + +lu = _add_docstr( + _linalg.linalg_lu, + r""" +lu(A, *, pivot=True, out=None) -> (Tensor, Tensor, Tensor) + +Computes the LU decomposition with partial pivoting of a matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **LU decomposition with partial pivoting** of a matrix +:math:`A \in \mathbb{K}^{m \times n}` is defined as + +.. math:: + + A = PLU\mathrlap{\qquad P \in \mathbb{K}^{m \times m}, L \in \mathbb{K}^{m \times k}, U \in \mathbb{K}^{k \times n}} + +where `k = min(m,n)`, :math:`P` is a `permutation matrix`_, :math:`L` is lower triangular with ones on the diagonal +and :math:`U` is upper triangular. + +If :attr:`pivot`\ `= False` and :attr:`A` is on GPU, then the **LU decomposition without pivoting** is computed + +.. math:: + + A = LU\mathrlap{\qquad L \in \mathbb{K}^{m \times k}, U \in \mathbb{K}^{k \times n}} + +When :attr:`pivot`\ `= False`, the returned matrix :attr:`P` will be empty. +The LU decomposition without pivoting `may not exist`_ if any of the principal minors of :attr:`A` is singular. +In this case, the output matrix may contain `inf` or `NaN`. + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +.. seealso:: + + :func:`torch.linalg.solve` solves a system of linear equations using the LU decomposition + with partial pivoting. + +.. warning:: The LU decomposition is almost never unique, as often there are different permutation + matrices that can yield different LU decompositions. + As such, different platforms, like SciPy, or inputs on different devices, + may produce different valid decompositions. + +.. warning:: Gradient computations are only supported if the input matrix is full-rank. + If this condition is not met, no error will be thrown, but the gradient + may not be finite. + This is because the LU decomposition with pivoting is not differentiable at these points. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + pivot (bool, optional): Controls whether to compute the LU decomposition with partial pivoting or + no pivoting. Default: `True`. + +Keyword args: + out (tuple, optional): output tuple of three tensors. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(P, L, U)`. + +Examples:: + + >>> A = torch.randn(3, 2) + >>> P, L, U = torch.linalg.lu(A) + >>> P + tensor([[0., 1., 0.], + [0., 0., 1.], + [1., 0., 0.]]) + >>> L + tensor([[1.0000, 0.0000], + [0.5007, 1.0000], + [0.0633, 0.9755]]) + >>> U + tensor([[0.3771, 0.0489], + [0.0000, 0.9644]]) + >>> torch.dist(A, P @ L @ U) + tensor(5.9605e-08) + + >>> A = torch.randn(2, 5, 7, device="cuda") + >>> P, L, U = torch.linalg.lu(A, pivot=False) + >>> P + tensor([], device='cuda:0') + >>> torch.dist(A, L @ U) + tensor(1.0376e-06, device='cuda:0') + +.. _permutation matrix: + https://en.wikipedia.org/wiki/Permutation_matrix +.. _may not exist: + https://en.wikipedia.org/wiki/LU_decomposition#Definitions +""", +) + +tensorinv = _add_docstr( + _linalg.linalg_tensorinv, + r""" +linalg.tensorinv(A, ind=2, *, out=None) -> Tensor + +Computes the multiplicative inverse of :func:`torch.tensordot`. + +If `m` is the product of the first :attr:`ind` dimensions of :attr:`A` and `n` is the product of +the rest of the dimensions, this function expects `m` and `n` to be equal. +If this is the case, it computes a tensor `X` such that +`tensordot(\ `:attr:`A`\ `, X, \ `:attr:`ind`\ `)` is the identity matrix in dimension `m`. +`X` will have the shape of :attr:`A` but with the first :attr:`ind` dimensions pushed back to the end + +.. code:: text + + X.shape == A.shape[ind:] + A.shape[:ind] + +Supports input of float, double, cfloat and cdouble dtypes. + +.. note:: When :attr:`A` is a `2`-dimensional tensor and :attr:`ind`\ `= 1`, + this function computes the (multiplicative) inverse of :attr:`A` + (see :func:`torch.linalg.inv`). + +.. note:: + Consider using :func:`torch.linalg.tensorsolve` if possible for multiplying a tensor on the left + by the tensor inverse, as:: + + linalg.tensorsolve(A, B) == torch.tensordot(linalg.tensorinv(A), B) # When B is a tensor with shape A.shape[:B.ndim] + + It is always preferred to use :func:`~tensorsolve` when possible, as it is faster and more + numerically stable than computing the pseudoinverse explicitly. + +.. seealso:: + + :func:`torch.linalg.tensorsolve` computes + `torch.tensordot(tensorinv(\ `:attr:`A`\ `), \ `:attr:`B`\ `)`. + +Args: + A (Tensor): tensor to invert. Its shape must satisfy + `prod(\ `:attr:`A`\ `.shape[:\ `:attr:`ind`\ `]) == + prod(\ `:attr:`A`\ `.shape[\ `:attr:`ind`\ `:])`. + ind (int): index at which to compute the inverse of :func:`torch.tensordot`. Default: `2`. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if the reshaped :attr:`A` is not invertible or the product of the first + :attr:`ind` dimensions is not equal to the product of the rest. + +Examples:: + + >>> A = torch.eye(4 * 6).reshape((4, 6, 8, 3)) + >>> Ainv = torch.linalg.tensorinv(A, ind=2) + >>> Ainv.shape + torch.Size([8, 3, 4, 6]) + >>> B = torch.randn(4, 6) + >>> torch.allclose(torch.tensordot(Ainv, B), torch.linalg.tensorsolve(A, B)) + True + + >>> A = torch.randn(4, 4) + >>> Atensorinv = torch.linalg.tensorinv(A, ind=1) + >>> Ainv = torch.linalg.inv(A) + >>> torch.allclose(Atensorinv, Ainv) + True +""", +) + +tensorsolve = _add_docstr( + _linalg.linalg_tensorsolve, + r""" +linalg.tensorsolve(A, B, dims=None, *, out=None) -> Tensor + +Computes the solution `X` to the system `torch.tensordot(A, X) = B`. + +If `m` is the product of the first :attr:`B`\ `.ndim` dimensions of :attr:`A` and +`n` is the product of the rest of the dimensions, this function expects `m` and `n` to be equal. + +The returned tensor `x` satisfies +`tensordot(\ `:attr:`A`\ `, x, dims=x.ndim) == \ `:attr:`B`. +`x` has shape :attr:`A`\ `[B.ndim:]`. + +If :attr:`dims` is specified, :attr:`A` will be reshaped as + +.. code:: text + + A = movedim(A, dims, range(len(dims) - A.ndim + 1, 0)) + +Supports inputs of float, double, cfloat and cdouble dtypes. + +.. seealso:: + + :func:`torch.linalg.tensorinv` computes the multiplicative inverse of + :func:`torch.tensordot`. + +Args: + A (Tensor): tensor to solve for. Its shape must satisfy + `prod(\ `:attr:`A`\ `.shape[:\ `:attr:`B`\ `.ndim]) == + prod(\ `:attr:`A`\ `.shape[\ `:attr:`B`\ `.ndim:])`. + B (Tensor): tensor of shape :attr:`A`\ `.shape[:\ `:attr:`B`\ `.ndim]`. + dims (Tuple[int], optional): dimensions of :attr:`A` to be moved. + If `None`, no dimensions are moved. Default: `None`. + +Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Raises: + RuntimeError: if the reshaped :attr:`A`\ `.view(m, m)` with `m` as above is not + invertible or the product of the first :attr:`ind` dimensions is not equal + to the product of the rest of the dimensions. + +Examples:: + + >>> A = torch.eye(2 * 3 * 4).reshape((2 * 3, 4, 2, 3, 4)) + >>> B = torch.randn(2 * 3, 4) + >>> X = torch.linalg.tensorsolve(A, B) + >>> X.shape + torch.Size([2, 3, 4]) + >>> torch.allclose(torch.tensordot(A, X, dims=X.ndim), B) + True + + >>> A = torch.randn(6, 4, 4, 3, 2) + >>> B = torch.randn(4, 3, 2) + >>> X = torch.linalg.tensorsolve(A, B, dims=(0, 2)) + >>> X.shape + torch.Size([6, 4]) + >>> A = A.permute(1, 3, 4, 0, 2) + >>> A.shape[B.ndim:] + torch.Size([6, 4]) + >>> torch.allclose(torch.tensordot(A, X, dims=X.ndim), B, atol=1e-6) + True +""", +) + +qr = _add_docstr( + _linalg.linalg_qr, + r""" +qr(A, mode='reduced', *, out=None) -> (Tensor, Tensor) + +Computes the QR decomposition of a matrix. + +Letting :math:`\mathbb{K}` be :math:`\mathbb{R}` or :math:`\mathbb{C}`, +the **full QR decomposition** of a matrix +:math:`A \in \mathbb{K}^{m \times n}` is defined as + +.. math:: + + A = QR\mathrlap{\qquad Q \in \mathbb{K}^{m \times m}, R \in \mathbb{K}^{m \times n}} + +where :math:`Q` is orthogonal in the real case and unitary in the complex case, +and :math:`R` is upper triangular with real diagonal (even in the complex case). + +When `m > n` (tall matrix), as `R` is upper triangular, its last `m - n` rows are zero. +In this case, we can drop the last `m - n` columns of `Q` to form the +**reduced QR decomposition**: + +.. math:: + + A = QR\mathrlap{\qquad Q \in \mathbb{K}^{m \times n}, R \in \mathbb{K}^{n \times n}} + +The reduced QR decomposition agrees with the full QR decomposition when `n >= m` (wide matrix). + +Supports input of float, double, cfloat and cdouble dtypes. +Also supports batches of matrices, and if :attr:`A` is a batch of matrices then +the output has the same batch dimensions. + +The parameter :attr:`mode` chooses between the full and reduced QR decomposition. +If :attr:`A` has shape `(*, m, n)`, denoting `k = min(m, n)` + +- :attr:`mode`\ `= 'reduced'` (default): Returns `(Q, R)` of shapes `(*, m, k)`, `(*, k, n)` respectively. + It is always differentiable. +- :attr:`mode`\ `= 'complete'`: Returns `(Q, R)` of shapes `(*, m, m)`, `(*, m, n)` respectively. + It is differentiable for `m <= n`. +- :attr:`mode`\ `= 'r'`: Computes only the reduced `R`. Returns `(Q, R)` with `Q` empty and `R` of shape `(*, k, n)`. + It is never differentiable. + +Differences with `numpy.linalg.qr`: + +- :attr:`mode`\ `= 'raw'` is not implemented. +- Unlike `numpy.linalg.qr`, this function always returns a tuple of two tensors. + When :attr:`mode`\ `= 'r'`, the `Q` tensor is an empty tensor. + +.. warning:: The elements in the diagonal of `R` are not necessarily positive. + As such, the returned QR decomposition is only unique up to the sign of the diagonal of `R`. + Therefore, different platforms, like NumPy, or inputs on different devices, + may produce different valid decompositions. + +.. warning:: The QR decomposition is only well-defined if the first `k = min(m, n)` columns + of every matrix in :attr:`A` are linearly independent. + If this condition is not met, no error will be thrown, but the QR produced + may be incorrect and its autodiff may fail or produce incorrect results. + +Args: + A (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + mode (str, optional): one of `'reduced'`, `'complete'`, `'r'`. + Controls the shape of the returned tensors. Default: `'reduced'`. + +Keyword args: + out (tuple, optional): output tuple of two tensors. Ignored if `None`. Default: `None`. + +Returns: + A named tuple `(Q, R)`. + +Examples:: + + >>> A = torch.tensor([[12., -51, 4], [6, 167, -68], [-4, 24, -41]]) + >>> Q, R = torch.linalg.qr(A) + >>> Q + tensor([[-0.8571, 0.3943, 0.3314], + [-0.4286, -0.9029, -0.0343], + [ 0.2857, -0.1714, 0.9429]]) + >>> R + tensor([[ -14.0000, -21.0000, 14.0000], + [ 0.0000, -175.0000, 70.0000], + [ 0.0000, 0.0000, -35.0000]]) + >>> (Q @ R).round() + tensor([[ 12., -51., 4.], + [ 6., 167., -68.], + [ -4., 24., -41.]]) + >>> (Q.T @ Q).round() + tensor([[ 1., 0., 0.], + [ 0., 1., -0.], + [ 0., -0., 1.]]) + >>> Q2, R2 = torch.linalg.qr(A, mode='r') + >>> Q2 + tensor([]) + >>> torch.equal(R, R2) + True + >>> A = torch.randn(3, 4, 5) + >>> Q, R = torch.linalg.qr(A, mode='complete') + >>> torch.dist(Q @ R, A) + tensor(1.6099e-06) + >>> torch.dist(Q.mT @ Q, torch.eye(4)) + tensor(6.2158e-07) +""", +) + +vander = _add_docstr( + _linalg.linalg_vander, + r""" +vander(x, N=None) -> Tensor + +Generates a Vandermonde matrix. + +Returns the Vandermonde matrix :math:`V` + +.. math:: + + V = \begin{pmatrix} + 1 & x_1 & x_1^2 & \dots & x_1^{N-1}\\ + 1 & x_2 & x_2^2 & \dots & x_2^{N-1}\\ + 1 & x_3 & x_3^2 & \dots & x_3^{N-1}\\ + \vdots & \vdots & \vdots & \ddots &\vdots \\ + 1 & x_n & x_n^2 & \dots & x_n^{N-1} + \end{pmatrix}. + +for `N > 1`. +If :attr:`N`\ `= None`, then `N = x.size(-1)` so that the output is a square matrix. + +Supports inputs of float, double, cfloat, cdouble, and integral dtypes. +Also supports batches of vectors, and if :attr:`x` is a batch of vectors then +the output has the same batch dimensions. + +Differences with `numpy.vander`: + +- Unlike `numpy.vander`, this function returns the powers of :attr:`x` in ascending order. + To get them in the reverse order call ``linalg.vander(x, N).flip(-1)``. + +Args: + x (Tensor): tensor of shape `(*, n)` where `*` is zero or more batch dimensions + consisting of vectors. + +Keyword args: + N (int, optional): Number of columns in the output. Default: `x.size(-1)` + +Example:: + + >>> x = torch.tensor([1, 2, 3, 5]) + >>> linalg.vander(x) + tensor([[ 1, 1, 1, 1], + [ 1, 2, 4, 8], + [ 1, 3, 9, 27], + [ 1, 5, 25, 125]]) + >>> linalg.vander(x, N=3) + tensor([[ 1, 1, 1], + [ 1, 2, 4], + [ 1, 3, 9], + [ 1, 5, 25]]) +""", +) + +vecdot = _add_docstr( + _linalg.linalg_vecdot, + r""" +linalg.vecdot(x, y, *, dim=-1, out=None) -> Tensor + +Computes the dot product of two batches of vectors along a dimension. + +In symbols, this function computes + +.. math:: + + \sum_{i=1}^n \overline{x_i}y_i. + +over the dimension :attr:`dim` where :math:`\overline{x_i}` denotes the conjugate for complex +vectors, and it is the identity for real vectors. + +Supports input of half, bfloat16, float, double, cfloat, cdouble and integral dtypes. +It also supports broadcasting. + +Args: + x (Tensor): first batch of vectors of shape `(*, n)`. + y (Tensor): second batch of vectors of shape `(*, n)`. + +Keyword args: + dim (int): Dimension along which to compute the dot product. Default: `-1`. + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> v1 = torch.randn(3, 2) + >>> v2 = torch.randn(3, 2) + >>> linalg.vecdot(v1, v2) + tensor([ 0.3223, 0.2815, -0.1944]) + >>> torch.vdot(v1[0], v2[0]) + tensor(0.3223) +""", +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..341d9f9d276fb978b1f1ad35664311ed07157328 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/__init__.py @@ -0,0 +1,57 @@ +from torch.masked._ops import ( + _canonical_dim, + _combine_input_and_mask, + _generate_docstring, + _input_mask, + _output_mask, + _reduction_identity, + _where, + amax, + amin, + argmax, + argmin, + cumprod, + cumsum, + log_softmax, + logaddexp, + logsumexp, + mean, + median, + norm, + normalize, + prod, + softmax, + softmin, + std, + sum, + var, +) +from torch.masked.maskedtensor.core import is_masked_tensor, MaskedTensor +from torch.masked.maskedtensor.creation import as_masked_tensor, masked_tensor + + +__all__ = [ + "amax", + "amin", + "argmax", + "argmin", + "as_masked_tensor", + "cumprod", + "cumsum", + "is_masked_tensor", + "log_softmax", + "logaddexp", + "logsumexp", + "masked_tensor", + "MaskedTensor", + "mean", + "median", + "norm", + "normalize", + "prod", + "softmax", + "softmin", + "std", + "sum", + "var", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_docs.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..89799863b462af40c3d33721b794c236cb40ecde --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_docs.py @@ -0,0 +1,1177 @@ +# This file is generated, do not modify it! +# +# To update this file, run the update masked docs script as follows: +# +# python tools/update_masked_docs.py +# +# The script must be called from an environment where the development +# version of torch package can be imported and is functional. +# + +amax_docstring = """amax(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns maximum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of maximum operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``-inf``, ``0``, and ``-2147483648``, respectively. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in maximum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of maximum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.amax(input, 1, mask=mask) + tensor([ -1, -9223372036854775808]) +""" + +amin_docstring = """amin(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns minimum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of minimum operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``inf``, ``255``, and ``2147483647``, respectively. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in minimum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of minimum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.amin(input, 1, mask=mask) + tensor([ -3, 9223372036854775807]) +""" + +argmax_docstring = """argmax(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns argmax of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of argmax operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``-inf``, ``0``, and ``-2147483648``, respectively. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in argmax computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of argmax operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which argmax is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.argmax(input, 1, mask=mask) + tensor([2, 0]) +""" + +argmin_docstring = """argmin(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns argmin of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of argmin operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``inf``, ``255``, and ``2147483647``, respectively. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in argmin computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of argmin operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which argmin is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.argmin(input, 1, mask=mask) + tensor([0, 0]) +""" + +cumprod_docstring = """cumprod(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns cumulative_prod of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is +defined as ``prod(x[:i])``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +cumulative_prod computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the cumulative_prod output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which cumulative_prod is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.cumprod(input, 1, mask=mask) + tensor([[-3., -3., 3.], + [ 1., 1., 1.]]) +""" + +cumsum_docstring = """cumsum(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns cumulative_sum of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is +defined as ``sum(x[:i])``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +cumulative_sum computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the cumulative_sum output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which cumulative_sum is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.cumsum(input, 1, mask=mask) + tensor([[-3., -3., -4.], + [ 0., 0., 0.]]) +""" + +log_softmax_docstring = """log_softmax(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns log_softmax of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. LogSoftmax of i-th element in ``x`` is +defined as ``log(exp(x[i])/sum(exp(x)))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +log_softmax computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the log_softmax output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which log_softmax is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.log_softmax(input, 1, mask=mask) + tensor([[-2.1269, -inf, -0.1269], + [ nan, nan, nan]]) +""" + +logsumexp_docstring = """logsumexp(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns logsumexp of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of logsumexp operation, which is used to start the reduction, is ``-2147483648``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in logsumexp computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of logsumexp operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.logsumexp(input, 1, mask=mask) + tensor([ 0, -9223372036854775808]) +""" + +mean_docstring = """mean(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns mean of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +By definition, the identity value of a mean operation is the mean +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +mean is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in mean computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of mean operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.mean(input, 1, mask=mask) + tensor([-2., nan]) +""" + +median_docstring = """median(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns median of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +By definition, the identity value of a median operation is the median +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +median is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in median computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of median operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which median is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.median(input, 1, mask=mask) + tensor([-3., nan]) +""" + +norm_docstring = """norm(input, ord, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns norm of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of norm operation, which is used to start the +reduction, is ``0.0``, except for ``ord=-inf`` it is +``inf``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in norm computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of norm operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + ord (int, float, optional): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms. + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.norm(input, 2.0, 1, mask=mask) + tensor([3.1623, 0.0000]) +""" + +normalize_docstring = """normalize(input, ord, dim, *, eps=1e-12, dtype=None, mask=None) -> Tensor + +Returns normalize of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Normalize of i-th element in ``x`` is +defined as ``x[i]/max(norm(x, p), eps)``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +normalize computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the normalize output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + ord (int, float): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms. + dim (int): the dimension along which normalize is computed. + +Keyword args: + eps (float, optional): small value to avoid division by zero. Default: 1e-12. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.normalize(input, 2.0, 1, mask=mask) + tensor([[-0.9487, 0.0000, -0.3162], + [ 0.0000, 0.0000, 0.0000]]) +""" + +prod_docstring = """prod(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns product of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of product operation, which is used to start the reduction, is ``1``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in product computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of product operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.prod(input, 1, mask=mask) + tensor([3, 1]) +""" + +softmax_docstring = """softmax(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns softmax of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Softmax of i-th element in ``x`` is +defined as ``exp(x[i])/sum(exp(x))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +softmax computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the softmax output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which softmax is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.softmax(input, 1, mask=mask) + tensor([[0.1192, 0.0000, 0.8808], + [ nan, nan, nan]]) +""" + +softmin_docstring = """softmin(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns softmin of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Softmin of i-th element in ``x`` is +defined as ``exp(-x[i])/sum(exp(-x))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +softmin computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the softmin output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which softmin is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.softmin(input, 1, mask=mask) + tensor([[0.8808, 0.0000, 0.1192], + [ nan, nan, nan]]) +""" + +std_docstring = """std(input, dim, unbiased, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns standard_deviation of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of sample standard deviation operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in standard_deviation computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of standard_deviation operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.std(input, 1, False, mask=mask) + tensor([1., nan]) +""" + +sum_docstring = """sum(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns sum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of sum operation, which is used to start the reduction, is ``0``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in sum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of sum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.sum(input, 1, mask=mask) + tensor([-4, 0]) +""" + +var_docstring = """var(input, dim, unbiased, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns variance of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of sample variance operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in variance computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of variance operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.var(input, 1, False, mask=mask) + tensor([1., nan]) +""" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_ops.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a3e75cb13805070c43c55769ed34b1ca0b22f337 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/masked/_ops.py @@ -0,0 +1,1821 @@ +# mypy: allow-untyped-defs +import warnings +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, TypeAlias, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch import sym_float, Tensor +from torch._prims_common import corresponding_real_dtype +from torch.masked import _docs +from torch.masked.maskedtensor.core import is_masked_tensor, MaskedTensor +from torch.masked.maskedtensor.creation import as_masked_tensor + + +if TYPE_CHECKING: + from torch._prims_common import DimsType + from torch.types import _dtype as DType + + DimOrDims: TypeAlias = DimsType | None +else: + # The JIT doesn't understand Union, nor torch.dtype here + DType = int + DimOrDims = Optional[tuple[int, ...]] + + +__all__: list[str] = [] + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +# All masked reduction/normalization operations have the same +# signatures. Here we introduce docstring templates that are applied +# to docstrings of reduction/normalization functions via +# _apply_docstring_templates decorator. + + +def _apply_docstring_templates(func: Callable[_P, _T]) -> Callable[_P, _T]: + """Decorator that applies docstring templates to function docstring + and returns the function instance. + """ + + doc_string = getattr(_docs, f"{func.__name__}_docstring", None) + if doc_string is None: + warnings.warn( + f"No documentation string available for {func.__name__}." + " PyTorch team should run `python tools/update_masked_docs.py`" + " to generate the missing docstrings.", + stacklevel=2, + ) + else: + func.__doc__ = doc_string + + # Expose function as public symbol + __all__.append(func.__name__) + + return func + + +def _generate_docstring(func): + """A utility function called from tools/update_masked_docs.py + script to update the module torch.masked._docs.py + """ + docstring_templates = dict( + reduction_signature="""\ +{function_name}(input, {operation_args}, *, {operation_kwargs}) -> Tensor""", + reduction_descr="""\ +Returns {operation name} of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`.""", + reduction_args="""\ +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in {operation name} computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of {operation name} operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + {args_declarations} + +Keyword args: + {kwargs_declarations}""", + reduction_example="""\ +Example:: + + >>> input = {example_input} + >>> input + {indent_example_input} + >>> mask = {example_mask} + >>> mask + {indent_example_mask} + >>> {full_function_name}(input, {example_args}, mask=mask) + {indent_example_output} +""", + reduction_identity="""\ +The identity value of {operation name} operation, which is used to start the reduction, is ``{identity_int32}``.""", + reduction_identity_dtype="""\ +The identity value of {operation name} operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``{identity_float32}``, ``{identity_uint8}``, and ``{identity_int32}``, respectively.""", + normalization_signature="""\ +{function_name}(input, {operation_args}, *, {operation_kwargs}) -> Tensor""", + normalization_descr="""\ +Returns {operation name} of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +{definition}""", + normalization_args="""\ +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +{operation name} computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the {operation name} output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + {args_declarations} + +Keyword args: + {kwargs_declarations}""", + normalization_example="""\ +Example:: + + >>> input = {example_input} + >>> input + {indent_example_input} + >>> mask = {example_mask} + >>> mask + {indent_example_mask} + >>> {full_function_name}(input, {example_args}, mask=mask) + {indent_example_output} +""", + ) + + args_and_kwargs = { + # argument name sufficies separated by double underscore will + # be removed in the final documentation string. + "sum": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "prod": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "cumsum": (("dim__as_int",), ("dtype=None", "mask=None")), + "cumprod": (("dim__as_int",), ("dtype=None", "mask=None")), + "amin": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "amax": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "argmin": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "argmax": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "mean": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "median": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "norm": ( + ( + "ord", + "dim", + ), + ("keepdim=False", "dtype=None", "mask=None"), + ), + "var": (("dim", "unbiased"), ("keepdim=False", "dtype=None", "mask=None")), + "std": (("dim", "unbiased"), ("keepdim=False", "dtype=None", "mask=None")), + "logsumexp": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "softmax": (("dim__as_int",), ("dtype=None", "mask=None")), + "log_softmax": (("dim__as_int",), ("dtype=None", "mask=None")), + "softmin": (("dim__as_int",), ("dtype=None", "mask=None")), + "normalize": ( + ( + "ord__required", + "dim__as_int", + ), + ("eps=1e-12", "dtype=None", "mask=None"), + ), + } + + argument_declarations = { + "dim": """\ + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``.""", + "dim__as_int": """\ + dim (int): the dimension along which {operation name} is computed.""", + "ord": """\ + ord (int, float, optional): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms.""", + "ord__required": """\ + ord (int, float): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms.""", + "unbiased": """\ + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance.""", + "eps": """\ + eps (float, optional): small value to avoid division by zero. Default: {default}.""", + "keepdim": """\ + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: {default}.""", + "dtype": """\ + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: {default}.""", + "mask": """\ + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``.""", + } + + definitions = { + "softmax": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Softmax of i-th element in ``x`` is + defined as ``exp(x[i])/sum(exp(x))``.""", + "log_softmax": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. LogSoftmax of i-th element in ``x`` is + defined as ``log(exp(x[i])/sum(exp(x)))``.""", + "softmin": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Softmin of i-th element in ``x`` is + defined as ``exp(-x[i])/sum(exp(-x))``.""", + "normalize": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Normalize of i-th element in ``x`` is + defined as ``x[i]/max(norm(x, p), eps)``.""", + "cumsum": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is + defined as ``sum(x[:i])``.""", + "cumprod": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is + defined as ``prod(x[:i])``.""", + } + + reduction_names = { + "sum": "sum", + "prod": "product", + "amax": "maximum", + "amin": "minimum", + "argmax": "argmax", + "argmin": "argmin", + "mean": "mean", + "median": "median", + "norm": "norm", + "var": "variance", + "std": "standard_deviation", + "logsumexp": "logsumexp", + } + + normalization_names = { + "softmax": "softmax", + "log_softmax": "log_softmax", + "softmin": "softmin", + "normalize": "normalize", + "cumsum": "cumulative_sum", + "cumprod": "cumulative_prod", + } + + operation_names = {} + operation_names.update(reduction_names) + operation_names.update(normalization_names) + + # Default example data: + example_dim = 1 + example_input = torch.tensor([[-3, -2, -1], [0, 1, 2]]) + example_mask = torch.tensor([[True, False, True], [False, False, False]]) + example_args: tuple[Any, ...] + if func.__name__ in {"norm", "normalize"}: + example_args = (2.0, example_dim) + example_input = example_input.to(dtype=torch.float32) + elif func.__name__ in {"var", "std"}: + example_args = (example_dim, False) + elif func.__name__ == "median": + example_args = (example_dim,) + example_input = example_input.to(dtype=torch.float32) + else: + example_args = (example_dim,) + + operation_args: tuple[str, ...] + operation_kwargs: tuple[str, ...] + operation_args, operation_kwargs = args_and_kwargs[func.__name__] + arg_declarations = [ + "\n ".join( + argument_declarations.get(a, f"{a.split('__', 1)[0]}: TBD.").splitlines() + ) + for a in operation_args + ] + kwarg_declarations = [ + "\n ".join( + argument_declarations.get( + a.split("=", 1)[0], f"{a.split('__', 1)[0]}: TBD." + ) + .format(default=a.split("=", 1)[1]) + .splitlines() + ) + for a in operation_kwargs + ] + + if func.__name__ in reduction_names: + op_kind = "reduction" + doc_sections = ["signature", "descr", "identity", "args", "example"] + elif func.__name__ in normalization_names: + op_kind = "normalization" + doc_sections = ["signature", "descr", "args", "example"] + example_input = example_input.to(dtype=torch.float32) + else: + assert 0 # add function name to operation names dictionaries + example_output = func(example_input, *example_args, mask=example_mask) + + template_data = { + "function_name": func.__name__, + "full_function_name": func.__module__ + "." + func.__name__, + "operation name": operation_names[func.__name__], + "operation_args": ", ".join(a.split("__", 1)[0] for a in operation_args), + "operation_kwargs": ", ".join(a.split("__", 1)[0] for a in operation_kwargs), + # one-line representation of a tensor: + "example_input": " ".join(str(example_input).split()), + "example_args": ", ".join(map(str, example_args)), + "example_mask": " ".join(str(example_mask).split()), + # multi-line representation of a tensor with indent + "indent_example_input": ("\n ").join(str(example_input).splitlines()), + "indent_example_mask": ("\n ").join(str(example_mask).splitlines()), + "indent_example_output": ("\n ").join(str(example_output).splitlines()), + } + + if func.__name__ in reduction_names: + template_data.update( + identity_uint8=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.uint8) + ), + identity_int32=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.int32) + ), + identity_float32=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.float32) + ), + ) + if func.__name__ == "norm": + template_data.update( + identity_ord_ninf=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.float32), float("-inf") + ) + ) + elif func.__name__ in normalization_names: + template_data.update(definition=definitions[func.__name__]) + else: + assert 0 # add function name to operation names dictionaries + template_data.update( + args_declarations=("\n ".join(arg_declarations)).format_map(template_data) + ) + template_data.update( + kwargs_declarations=("\n ".join(kwarg_declarations)).format_map( + template_data + ) + ) + + # Apply function name info to docstring templates: + templates = { + k: v.format_map(template_data) + for k, v in docstring_templates.items() + if k.startswith(op_kind) + } + templates.update( + (k, v.format_map(template_data) if isinstance(v, str) else v) + for k, v in template_data.items() + ) + + # Apply docstring templates to function doctring: + if func.__doc__ is None: + doc_template = "\n\n".join([f"{{{op_kind}_{sec}}}" for sec in doc_sections]) + else: + doc_template = func.__doc__ + return doc_template.format_map(templates) + + +def _reduction_identity(op_name: str, input: Tensor, *args): + """Return identity value as scalar tensor of a reduction operation on + given input, or None, if the identity value cannot be uniquely + defined for the given input. + + The identity value of the operation is defined as the initial + value to reduction operation that has a property ``op(op_identity, + value) == value`` for any value in the domain of the operation. + Or put it another way, including or excluding the identity value in + a list of operands will not change the reduction result. + + See https://github.com/pytorch/rfcs/pull/27 for more information. + + """ + dtype: DType = input.dtype + device = input.device + op_name = op_name.rsplit(".", 1)[-1] # lstrip module name when present + if op_name in {"sum", "cumsum"}: + return torch.tensor(0, dtype=dtype, device=device) + elif op_name in {"prod", "cumprod"}: + return torch.tensor(1, dtype=dtype, device=device) + elif op_name in {"amax", "argmax", "logaddexp"}: + if torch.is_floating_point(input): + return torch.tensor(-torch.inf, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).min, dtype=dtype, device=device) + elif op_name == "logsumexp": + if torch.is_floating_point(input): + return torch.tensor(-torch.inf, dtype=dtype, device=device) + elif torch.is_complex(input): + return torch.tensor(-torch.inf + 0j, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).min, dtype=dtype, device=device) + elif op_name in {"amin", "argmin"}: + if torch.is_floating_point(input): + return torch.tensor(torch.inf, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).max, dtype=dtype, device=device) + elif op_name == "mean": + # Strictly speaking, the identity value of the mean operation + # is the mean of the input. Since the mean value depends on + # the dim argument and it may be a non-scalar tensor, we + # consider the identity value of the mean operation ambiguous. + # Moreover, the mean value of empty input is undefined. + return None + elif op_name == "norm": + ord = args[0] if args else 2 + if ord == float("-inf"): + assert torch.is_floating_point(input), input.dtype + return torch.tensor(torch.inf, dtype=dtype, device=device) + return torch.tensor(0, dtype=dtype, device=device) + elif op_name == "median": + # We use NaN for now because the implementation is currently using torch.nanmedian + # and NaN is the identity for that function since it gets ignored + dtype = input.dtype if torch.is_floating_point(input) else torch.float + return torch.tensor(torch.nan, dtype=dtype, device=device) + elif op_name in {"var", "std"}: + return None + raise NotImplementedError(f"identity of {op_name} on {dtype} input") + + +def _canonical_dim(dim: DimOrDims, ndim: int) -> tuple[int, ...]: + """Return dim argument as a tuple of sorted dim values.""" + dims: list[int] = [] + if dim == (): + # Currently, `dim=()` in reductions operations means "reduce + # over all dimensions" while in future, it will read "no + # reduce". See https://github.com/pytorch/pytorch/issues/29137 + # When gh-29137 is resolved, this if-block must be deleted. + dim = None + if dim is None: + return tuple(range(ndim)) + ndim = max(ndim, 1) + dim_ = (dim,) if isinstance(dim, (int, torch.SymInt)) else dim + for d in dim_: + if d in dims: + raise RuntimeError(f"dim={d} appears multiple times in the list of dims") + if d >= ndim or d < -ndim: + raise IndexError( + f"Dimension out of range (expected to be in range of [{-ndim}, {ndim - 1}], but got {d})" + ) + # pyrefly: ignore [bad-argument-type] + dims.append(d % ndim) + return tuple(sorted(dims)) + + +def _sparse_coo_flatten_indices(indices: Tensor, shape: tuple): + # Flatted N-D indices to 1-D indices + flat_indices = indices.new_zeros(indices.size(1)) + for d, sz in enumerate(shape): + flat_indices.mul_(sz) + flat_indices.add_(indices[d]) + return flat_indices + + +def _any(input: Tensor, dim: tuple, keepdim: bool): + # Support torch.any with tuple dim argument. + # Workaround of https://github.com/pytorch/pytorch/issues/56586 + r = input + for d in reversed(dim): + r = r.any(dim=d, keepdim=keepdim) + return r + + +def _sparse_coo_where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """Sparse variant of torch.where. Supports sparse COO and hybrid sparse COO tensors. + + _sparse_coo_where implements the following invariant: + + _sparse_coo_where(mask, input, fill_value).to_dense(fill_value) == + torch.where(mask.to_dense(), input.to_dense(), torch.full(input.shape, fill_value)) + + where `a == b` means `assertEqual(a, b)`, mask is boolean sparse + tensor, and `to_dense(fill_value)` is like `to_dense()` except + that the unspecified elements are mapped to `fill_value` rather + than to `0`. + + Returns a sparse COO tensor with the following features: + + - all specified elements correspond to masked-in elements that + have the values of the input tensor. If there exists a masked-in + element (as specified by mask) that is not specified in the + input, in the result tensor, the corresponding element has value + 0. In the dense part of the sparse tensor, the masked-out + elements are replaced with fill_value. + + - all unspecified elements correspond to masked-out elements. + """ + + assert input.layout == torch.sparse_coo + assert mask.layout == input.layout + assert mask.shape == input.shape + assert mask.dense_dim() == input.dense_dim() # TODO: eliminate this restriction + + input = input.coalesce() + + # For set operations on sparse tensor indices, we'll convert + # multi-dimensional indices to 1-D indices for efficiency. + input_flat_indices = _sparse_coo_flatten_indices( + input.indices(), input.shape[: input.sparse_dim()] + ) + mask_flat_indices = _sparse_coo_flatten_indices( + mask.indices(), mask.shape[: mask.sparse_dim()] + ) + + # the set of mask flat indices that define masked-in elements: + if mask.dense_dim() > 0: + mask_values = _any( + mask.values(), tuple(range(1, input.sparse_dim() + 1)), False + ) + else: + mask_values = mask.values() + maskin_flat_indices = mask_flat_indices[mask_values.nonzero()[:, 0]] + + def intersection(i1, i2): + union, counts = torch.cat([i1, i2]).unique(return_counts=True) + return union, torch.where(counts.gt(1)) + + def minus(i1, i2): + union, counts = torch.cat([i1, i2]).unique(return_counts=True) + return intersection(union[torch.where(counts.eq(1))], i1) + + def _apply(a): + obj, w = a + return obj[w] + + # the set of input flat indices of specified and masked-in elements: + maskin_input_flat_indices = _apply( + intersection(maskin_flat_indices, input_flat_indices) + ) + _, w = intersection(input_flat_indices, maskin_input_flat_indices) + + # the indices and values of masked-in elements + where_input_indices = input.indices()[(slice(None),) + w] + where_input_values = input.values()[w] + + if mask.dense_dim() > 0: + # apply mask to the dense part of the input values: + _, w1 = intersection(mask_flat_indices, maskin_input_flat_indices) + where_mask_values = mask.values()[w1] + where_input_values = torch.where( + where_mask_values, where_input_values, fill_value + ) + + # the set of flat indices of unspecified input and masked-in elements: + maskin_zero_flat_indices = _apply( + minus(maskin_flat_indices, maskin_input_flat_indices) + ) + + # the indices of masked-in zero elements + _, w = intersection(mask_flat_indices, maskin_zero_flat_indices) + where_zero_indices = mask.indices()[(slice(None),) + w] + + # construct result + n = where_zero_indices.size(1) + if n == 0: + # the input is coalesced, hence input_flat_indices are ordered + # and the result is guaranteed to be coalesced: + result = torch.sparse_coo_tensor( + where_input_indices, where_input_values, input.shape + ) + return result._coalesced_(True) + + where_indices = torch.cat([where_input_indices, where_zero_indices], dim=1) + where_values = torch.cat( + [ + where_input_values, + where_input_values.new_zeros((n,) + where_input_values.shape[1:]), + ] + ) + result = torch.sparse_coo_tensor(where_indices, where_values, input.shape) + + # appending zero elements leads to uncoalesced sparse tensor + return result.coalesce() + + +def _sparse_coo_scatter_reduction_helper( + op, + mask_input: Tensor, + dims: tuple[int, ...], + keepdim: bool, + dtype: DType | None = None, +) -> Tensor: + reduce = op.__name__ + valid_reductions = ["sum", "prod", "amax", "amin"] + if reduce not in valid_reductions: + raise ValueError( + f"op must be one of {' '.join(valid_reductions)}, but got {reduce} instead" + ) + + output_dtype = dtype + values, indices = mask_input._values(), mask_input._indices() + input_dims = mask_input.dim() + num_sparse_dims = mask_input.sparse_dim() + reduced_sparse_dims = [] + retained_sparse_dims = [] + reduced_dense_dims = [] + + # promote dtype if specified + if values.dtype != output_dtype: + values = values.to(output_dtype) + + if keepdim: + output_shape = tuple( + 1 if i in dims else si for (i, si) in enumerate(mask_input.shape) + ) + else: + output_shape = tuple( + si for (i, si) in enumerate(mask_input.shape) if i not in dims + ) + + for d in dims: + if d >= input_dims: + continue + + if d < num_sparse_dims: + reduced_sparse_dims.append(d) + else: + reduced_dense_dims.append(d + 1 - num_sparse_dims) + + # Reduce dense dimensions + if len(reduced_dense_dims) > 0: + if reduce == "sum": + new_values = values + new_values = op(new_values, dim=reduced_dense_dims, keepdim=bool(keepdim)) + else: + # FIXME: Implement reductions for dense dimensions for ops with non-zero reduction identities + return NotImplemented + else: + new_values = values.clone() + + # Reduce sparse dimensions + if len(reduced_sparse_dims) == num_sparse_dims: + if reduce in {"amax", "amin"} and new_values.size(0) == 0: + # IndexError: amax(): Expected reduction dim 0 to have non-zero size. + # sum()/prod() return the reduction identity when dim has size 0 but amax()/amin() do not + # See https://github.com/pytorch/pytorch/issues/61901 + new_values = _reduction_identity(reduce, new_values) + else: + new_values = op(new_values, dim=0) + if keepdim: + for _ in range(num_sparse_dims): + new_values = new_values.unsqueeze(0) + return new_values.to(dtype=output_dtype).to_sparse() + else: + new_indices = indices.clone() + if keepdim: + # zero out reduced sparse dimensions if keepdim = True + # ensures that the call to torch.unique folds duplicated indices together while preserving the dimension + new_indices[reduced_sparse_dims, :] = 0 + else: + # remove reduced sparse dimensions if keepdim = False + if len(reduced_sparse_dims) > 0: + retained_sparse_dims = [ + i + for i in range(num_sparse_dims) + if i not in set(reduced_sparse_dims) + ] + new_indices = new_indices.index_select( + 0, torch.tensor(retained_sparse_dims).to(mask_input.device) + ) + + # Use scatter_reduce to reduce items in the new_values tensor that correspond to the same indices in new_indices + if new_indices.numel() > 0: + # lexsort indices and get index tensor for scatter reduction + new_indices, inverse_indices = torch.unique( + new_indices, return_inverse=True, dim=1 + ) + out_shape = list(new_values.shape) + out_shape[0] = new_indices.shape[1] + for _ in range(new_values.ndim - 1): + inverse_indices = inverse_indices.unsqueeze(-1) + scatter_indices = inverse_indices.expand(new_values.shape) + # FIXME: temporary workaround for issue with bfloat16/float16 remove when acctype is implemented for scatter_reduce + if output_dtype in {torch.bfloat16, torch.float16}: + new_values = new_values.to(torch.float) + out = new_values.new_empty(out_shape) + new_values = out.scatter_reduce_( + 0, scatter_indices, new_values, reduce=reduce, include_self=False + ) + new_values = new_values.to(dtype=output_dtype) + else: + out = new_values.new_empty(out_shape) + new_values = out.scatter_reduce_( + 0, scatter_indices, new_values, reduce=reduce, include_self=False + ) + + return torch.sparse_coo_tensor( + new_indices, + new_values, + output_shape, + dtype=output_dtype, + device=mask_input.device, + ) + + +def _sparse_csr_segment_reduction_helper( + op, + mask_input: Tensor, + dims: tuple[int, ...], + keepdim: bool, + dtype: DType | None = None, +) -> Tensor: + # Currently, while sparse CSR is always 2D with no dense dimensions keepdim must be True + # FIXME: when dense dimensions are implemented for CSR tensors + assert keepdim, ( + "reduction operations on CSR tensors with keepdim=False is unsupported" + ) + reduce = op.__name__ + valid_reductions = ["sum", "prod", "mean", "amax", "amin"] + if reduce not in valid_reductions: + raise ValueError( + f"op must be one of {' '.join(valid_reductions)}, but got {reduce} instead" + ) + device = mask_input.device + output_dtype = dtype + values, crow_indices, col_indices = ( + mask_input.values(), + mask_input.crow_indices(), + mask_input.col_indices(), + ) + + # promote dtype if specified + if values.dtype != output_dtype: + values = values.to(output_dtype) + + if len(dims) == 0: + return mask_input + if len(dims) == 1: + if dims[0] == 0: + new_col_indices, scatter_indices = torch.unique( + col_indices, return_inverse=True + ) + new_nnz = new_col_indices.shape[0] + new_crow_indices = torch.tensor([0, new_nnz]) + new_values = values.new_empty(new_col_indices.shape) + new_values.scatter_reduce_( + 0, scatter_indices, values, reduce, include_self=False + ) + new_shape = [1, mask_input.size(1)] + else: + assert dims[0] == 1, ( + "Sparse CSR tensors are 2D and only support reduction along dim 0 or 1." + ) + # all intervals new_crow_indices[i] - new_crow_indices[i-1] are 1 + # except for where crow_indices[i] == crow_indices[i-1] where the interval remains as 0 + new_crow_indices = torch.cat( + ( + crow_indices.new_zeros(1), + torch.cumsum(torch.diff(crow_indices) != 0, 0), + ), + 0, + ) + new_nnz = new_crow_indices[-1] + new_col_indices = col_indices.new_zeros(new_nnz) # type: ignore[call-overload] + new_values = torch._segment_reduce(values, reduce, offsets=crow_indices) # type: ignore[attr-defined] + new_shape = [mask_input.size(0), 1] + else: + assert len(dims) == 2 + nnz = min(1, values.numel()) + if nnz == 1: + op_kwargs = {"keepdim": True, "dtype": output_dtype} + # amax and amin do not support dtype kwarg + if reduce in ["amax", "amin"]: + del op_kwargs["dtype"] + new_values = op(values, 0, **op_kwargs) + else: + new_values = torch.empty(0, dtype=output_dtype) + new_col_indices = col_indices.new_zeros(nnz) + new_crow_indices = torch.tensor([0, nnz]) + new_shape = [1, nnz] + + return torch.sparse_csr_tensor( + new_crow_indices, + new_col_indices, + new_values, + new_shape, + dtype=output_dtype, + device=device, + ) + + +def _sparse_csr_where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """Sparse variant of torch.where. Supports sparse CSR tensors.""" + # TODO: implement sparse CSR specific where operator for efficiency + return _sparse_coo_where( + mask.to_sparse_coo(), input.to_sparse_coo(), fill_value + ).to_sparse_csr() + + +def _where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """torch.where with sparse inputs support. + + _where implements the following invariant: + + _where(mask, input, fill_value).to_dense(fill_value) == + torch.where(mask.to_dense(), input.to_dense(), torch.full(input.shape, fill_value)) + + where `a == b` means `assertEqual(a, b)`, mask is boolean sparse + tensor, and `to_dense(fill_value)` is like `to_dense()` except + that the unspecified elements are mapped to `fill_value` rather + than to `0`. + + Returns a sparse tensor with the following features: + + - all specified elements correspond to masked-in elements that + have the values of the input tensor. If there exists a masked-in + element (as specified by mask) that is not specified in the + input, in the result tensor, the corresponding element has value + 0. In the dense part of the sparse tensor, the masked-out + elements are replaced with fill_value. + + - all unspecified elements correspond to masked-out elements. + """ + if mask.layout == torch.strided: + return torch.where(mask, input, fill_value) + elif mask.layout == torch.sparse_coo: + return _sparse_coo_where(mask, input, fill_value) + elif mask.layout == torch.sparse_csr: + return _sparse_csr_where(mask, input, fill_value) + else: + raise ValueError( + f"_where expects strided or sparse COO or sparse CSR tensor but got {mask.layout}" + ) + + +def _input_mask(input: Tensor | MaskedTensor, *args, **kwargs) -> Tensor: + """Return canonical input mask. + + A canonical input mask is defined as a boolean mask tensor that + shape and layout matches with the shape and the layout of the + input. + + The canonical input mask is computed from the :attr:`mask` tensor + content to meet the following criteria: + + 1. The shape of the canonical input mask is the same as the shape + of :attr:`input` tensor. If the mask tensor has a smaller shape + than the shape of the :attr:`input`, broadcasting rules will be + applied. Downcasting of mask is not supported. + + 2. The layout of the canonical input mask is the same as the + layout of the :attr:`input` tensor. If the mask has different + layout, it will be converted to the expected layout. In the + case of sparse COO layout, the canonical input mask will be + coalesced. + + 3. The dtype of the canonical input mask is torch.bool. If the + mask dtype is not bool then it will be converted to bool dtype + using `.to(dtype=bool)` method call. + + 4. The elements of the canonical input mask have boolean values + copied from the content of the :attr:`mask` tensor (after + possible broadcasting and dtype conversion transforms). In + general, the sparsity pattern of the sparse canonical input + mask need not to be the same as the sparsity pattern of the + sparse :attr:`input` tensor. + + """ + if input.layout not in {torch.strided, torch.sparse_coo, torch.sparse_csr}: + raise ValueError( + f"_input_mask expects strided or sparse COO or sparse CSR tensor but got {input.layout}" + ) + + mask = kwargs.get("mask") + + # default mask + if mask is None: + raise ValueError("_input_mask requires explicit mask") + + # mask shape must match with input shape + if mask.shape != input.shape: + if mask.ndim > input.ndim: + raise IndexError( + "_input_mask expected broadcastable mask (got mask dimensionality higher than of the input)" + ) + if mask.layout == torch.strided: + mask = torch.broadcast_to(mask.clone(), input.shape).to(dtype=torch.bool) + elif mask.layout == torch.sparse_coo: + mask = torch._sparse_broadcast_to(mask, input.shape) + else: + assert mask.layout == torch.sparse_csr + # Broadcasting of CSR tensors is not implemented. Working + # around by using COO layout. + mask = torch._sparse_broadcast_to( + mask.to_sparse(), input.shape + ).to_sparse_csr() + + # mask layout must match with input layout + if mask.layout != input.layout: + if input.layout == torch.strided: + mask = mask.to_dense() + elif input.layout == torch.sparse_coo: + if mask.layout == torch.strided: + mask = mask.to_sparse(input.sparse_dim()) + else: + mask = mask.to_sparse() + else: + assert input.layout == torch.sparse_csr + mask = mask.to_sparse_csr() + + # sparse mask must be coalesced + if mask.layout == torch.sparse_coo: + mask = mask.coalesce() + + # mask is a boolean tensor + mask = mask.to(dtype=torch.bool) + + return mask + + +def _output_mask(op, input: Tensor, *args, **kwargs) -> Tensor: + """Return output mask of masked operation applied to given arguments.""" + if callable(op): + is_reduction = op.__name__ in { + "sum", + "prod", + "amax", + "amin", + "argmax", + "argmin", + "mean", + "median", + "norm", + "var", + "std", + "logsumexp", + } + is_normalization = op.__name__ in { + "softmax", + "log_softmax", + "softmin", + "normalize", + "cumsum", + "cumprod", + } + if is_reduction: + if op.__name__ == "norm": + if args: + args = args[1:] # lstrip ord argument + dim = args[0] if args else kwargs.get("dim") + outmask = _input_mask(input, *args, **kwargs) + keepdim = kwargs.get("keepdim", False) + dim_ = _canonical_dim(dim, input.ndim) + return _any(outmask, dim_, bool(keepdim)) + elif is_normalization: + return _input_mask(input, *args, **kwargs) + else: + raise ValueError( + f"_output_mask expected masked operation (got callable {op.__module__}.{op.__name__})" + ) + else: + raise ValueError( + f"_output_mask expected masked operation (got {type(op).__name__} object)" + ) + + +def _combine_input_and_mask(op, input: MaskedTensor | Tensor, mask, *args) -> Tensor: + def helper(input, mask): + if mask is None: + return input + canonical_mask = _input_mask(input, mask=mask) + if callable(op): + fill_value = _reduction_identity(op.__name__, input, *args) + return _where(canonical_mask, input, fill_value) + else: + raise ValueError( + f"_combine_input_and_mask expected masked operation (got {type(op).__name__} object)" + ) + + class Combine(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, input, mask): + """Return input with masked-out elements eliminated for the given operations.""" + ctx.save_for_backward(mask) + + if mask is not None: + ctx.mark_non_differentiable(mask) + + return helper(input, mask) + + @staticmethod + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + (mask,) = ctx.saved_tensors + grad_data = ( + grad_output.get_data() if is_masked_tensor(grad_output) else grad_output + ) + result = as_masked_tensor(grad_data, mask) + return result, None + + return ( + Combine.apply(input.get_data(), input.get_mask()) # type: ignore[union-attr] + if is_masked_tensor(input) + else helper(input, mask) + ) + + +@_apply_docstring_templates +def sum( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + # __doc__ is generated by _apply_docstring_templates decorator + if dtype is None: + # promote integer types to int64 when output dtype is not specified + if input.layout == torch.sparse_csr: + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + # csr.to(dtype=torch.int64) is not implemented, so + # using coo.to on input to ensure the promoted dtype + input = input.to_sparse_coo().to(dtype=torch.int64).to_sparse_csr() + else: + dtype = input.dtype + else: + dtype = input.dtype + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + dtype = torch.int64 + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + return torch.sum(mask_input, dim_, bool(keepdim), dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + return _sparse_coo_scatter_reduction_helper( + torch.sum, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + return torch._sparse_csr_sum( + mask_input, dim=list(dim_), keepdim=bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked sum expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def prod( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + # __doc__ is generated by _apply_docstring_templates decorator + if dtype is None: + # promote integer types to int64 when output dtype is not specified + if input.layout == torch.sparse_csr: + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + # csr.to(dtype=torch.int64) is not implemented, so + # using coo.to on input to ensure the promoted dtype + input = input.to_sparse_coo().to(dtype=torch.int64).to_sparse_csr() + else: + dtype = input.dtype + else: + dtype = input.dtype + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + dtype = torch.int64 + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(prod, input, mask) + if mask_input.layout == torch.strided: + # Workaround https://github.com/pytorch/pytorch/issues/56586 + result = mask_input + result = result.to(dtype=dtype) + for d in reversed(dim_): + result = result.prod(dim=d, keepdim=bool(keepdim)) + return result + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch, the same issue arises for sparse_coo tensors + raise ValueError( + "masked prod expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.prod, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + # mask is None corresponds to all-True mask. The + # unspecified elements in the CSR tensor correspond to + # zero values. Hence, the prod reduction result is + # automatically zero unless all elements are specified. + # A semi-optimal way to take this into account is to use: + # + # masked_prod(csr, ..., mask=None) == torch._sparse_csr_prod(csr, ...) * all(csr.nonzero(), ...) + # + # but that requires implementing `all` and `nonzero` + # support for sparse csr tensors. + raise ValueError( + "masked prod expects explicit mask for sparse_csr tensor input" + ) + return torch._sparse_csr_prod( + mask_input, dim=list(dim_), keepdim=bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked prod expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def cumsum( + input: Tensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + return torch.cumsum(mask_input, dim_, dtype=dtype).to(dtype=dtype) + else: + raise ValueError( + f"masked cumsum expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def cumprod( + input: Tensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(prod, input, mask) + if mask_input.layout == torch.strided: + return torch.cumprod(mask_input, dim_, dtype=dtype).to(dtype=dtype) + else: + raise ValueError( + f"masked cumprod expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def amax( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +{reduction_identity_dtype} + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + + mask_input = _combine_input_and_mask(amax, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask_input.layout == torch.strided: + return torch.amax(mask_input, dim_, bool(keepdim)).to(dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch of prod, a similar issue arises here + # where unspecified elements along a dimension may need to be reduced with the result + raise ValueError( + "masked amax expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.amax, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + raise ValueError( + "masked amax expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.amax, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked amax expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def amin( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +{reduction_identity_dtype} + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + + mask_input = _combine_input_and_mask(amin, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask_input.layout == torch.strided: + return torch.amin(mask_input, dim_, bool(keepdim)).to(dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch of prod, a similar issue arises here + # where unspecified elements along a dimension may need to be reduced with the result + raise ValueError( + "masked amax expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.amin, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + raise ValueError( + "masked amin expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.amin, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked amin expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def argmax( + input: Tensor | MaskedTensor, + dim: int | None = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +{reduction_identity_dtype} +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(argmax, input, mask) + if mask_input.layout == torch.strided: + return torch.argmax(mask_input, dim, bool(keepdim)).to(dtype=dtype) + else: + raise ValueError( + f"masked argmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def argmin( + input: Tensor | MaskedTensor, + dim: int | None = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +{reduction_identity_dtype} +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(argmin, input, mask) + if mask_input.layout == torch.strided: + return torch.argmin(mask_input, dim, bool(keepdim)).to(dtype=dtype) + else: + raise ValueError( + f"masked argmin expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def mean( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +By definition, the identity value of a mean operation is the mean +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +mean is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. + +{reduction_args} + +{reduction_example}""" + dtype_source = "Optional" + if dtype is None: + dtype = input.dtype + dtype_source = "Input" + + if not (dtype.is_floating_point or dtype.is_complex): + raise ValueError( + f"mean(): Could not infer output dtype. {dtype_source} dtype must be either " + f"a floating point or complex dtype. Got: {dtype}" + ) + if input.layout == torch.strided: + if mask is None: + # TODO: compute count analytically + # pyrefly: ignore [no-matching-overload] + count = sum( + torch.ones(input.shape, dtype=torch.int64, device=input.device), + dim, + keepdim=keepdim, + ) + # pyrefly: ignore [no-matching-overload] + total = sum(input, dim, keepdim=keepdim, dtype=dtype) + else: + inmask = _input_mask(input, mask=mask) + count = inmask.sum(dim=dim, keepdim=bool(keepdim)) + # pyrefly: ignore [no-matching-overload] + total = sum(input, dim, keepdim=keepdim, dtype=dtype, mask=inmask) + return total / count + elif input.layout == torch.sparse_csr: + mask_input = _combine_input_and_mask(mean, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask is None: + raise ValueError( + "masked mean expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.mean, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked mean expects strided or sparse_csr tensor (got {input.layout} tensor)" + ) + + +@_apply_docstring_templates +def median( + input: Tensor | MaskedTensor, + dim: int = -1, + *, + keepdim: bool = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +By definition, the identity value of a median operation is the median +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +median is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + is_float = torch.is_floating_point(input) + if not is_float: + input = input.to(dtype=torch.float) + mask_input = _combine_input_and_mask(median, input, mask) + if mask_input.layout == torch.strided: + output = torch.nanmedian(mask_input, dim_, keepdim).values + if is_float: + return output + elif not is_float and not torch.isnan(output).any(): + return output.to(dtype=dtype) + else: + raise ValueError( + "masked median expects no fully masked out rows if dtype is not floating point" + ) + else: + raise ValueError( + f"masked median expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def logsumexp( + input: Tensor, + dim: DimOrDims = None, + *, + keepdim: bool = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(logsumexp, input, mask) + if mask_input.layout == torch.strided: + return torch.logsumexp(mask_input, dim_, keepdim=keepdim).to(dtype=dtype) + else: + raise ValueError( + f"masked logsumexp expects strided tensor (got {mask_input.layout} tensor)" + ) + + +# Cannot use _apply_docstring_templates as it is only set up for reductions and normalizations +def logaddexp( + input: Tensor | MaskedTensor, + other: Tensor | MaskedTensor, + *, + dtype: DType | None = None, + input_mask: Tensor | None = None, + other_mask: Tensor | None = None, +) -> Tensor: + """logaddexp(input, other, *, dtype=None, input_mask=None, other_mask=None) -> Tensor + + Returns logaddexp of all the elements in the :attr:`input` and the :attr:`other` + tensor. The :attr:`input` elements are masked out according to the boolean tensor + :attr:`input_mask` and the attr:`other` elements are masked out according to the boolean tensor + :attr:`other_mask`. + + The shapes of a mask tensor and the tensor to be masked + don't need to match, but they must be :ref:`broadcastable + ` and the dimensionality of the mask + tensor must not be greater than of the tensor to be masked. + + Args: + input (Tensor): the input tensor + other (Tensor): the second input tensor + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the output tensor is + casted to :attr:`dtype` after the operation is + performed. Default: None. + input_mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of :attr:`input` tensor elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + other_mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of :attr:`other` tensor elements. + Default: None that is equivalent to ``torch.ones(other.shape, dtype=torch.bool)``. + + Example:: + + >>> input = torch.tensor([-100.0, -200, -300]) + >>> input + tensor([-100., -200., -300.]) + >>> other = torch.tensor([-1.0, -2, -3]) + >>> other + tensor([-1., -2., -3.]) + >>> mask = torch.tensor([True, False, True]) + >>> mask + tensor([ True, False, True]) + >>> torch.masked._ops.logaddexp(input, other, input_mask=mask, other_mask=mask) + tensor([-1., -inf, -3.])""" + if dtype is None: + dtype = input.dtype + if input.layout == torch.strided and other.layout == torch.strided: + mask_input = _combine_input_and_mask(logaddexp, input, input_mask) + mask_other = _combine_input_and_mask(logaddexp, other, other_mask) + return torch.logaddexp(mask_input, mask_other).to(dtype=dtype) + else: + raise ValueError( + f"masked logaddexp expects strided tensors (got {input.layout} tensor for input, {other.layout} for other)" + ) + + +@_apply_docstring_templates +def norm( + input: Tensor | MaskedTensor, + ord: float | None = 2.0, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +The identity value of norm operation, which is used to start the +reduction, is ``{identity_float32}``, except for ``ord=-inf`` it is +``{identity_ord_ninf}``. + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(norm, input, mask, ord) + if mask_input.layout == torch.strided: + dim_ = _canonical_dim(dim, input.ndim) + return torch.linalg.vector_norm( + mask_input, ord, dim_, bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked norm expects strided tensor (got {mask_input.layout} tensor)" + ) + + +def _std_var( + input: Tensor | MaskedTensor, + dim: DimOrDims, + unbiased: bool | None, + *, + correction_opt: int | float | None, + keepdim: bool | None, + dtype: DType | None, + mask: Tensor | None, + take_sqrt: bool | None, +) -> Tensor: + assert unbiased is None or correction_opt is None, ( + "Only one of unbiased and correction may be given" + ) + correction = 1.0 + if unbiased is not None: + correction = 1.0 if unbiased else 0.0 + if correction_opt is not None: + correction = sym_float(correction_opt) + + if dtype is None: + dtype = input.dtype + if not (dtype.is_floating_point or dtype.is_complex): + dtype = torch.float32 + compute_dtype = dtype + if not (compute_dtype.is_floating_point or compute_dtype.is_complex): + compute_dtype = torch.float32 + if input.layout == torch.strided: + if mask is None: + # TODO: compute count analytically + # pyrefly: ignore [no-matching-overload] + count = sum( + torch.ones(input.shape, dtype=torch.int64, device=input.device), + dim, + keepdim=True, + ) + # pyrefly: ignore [no-matching-overload] + sample_total = sum(input, dim, keepdim=True, dtype=dtype) + else: + inmask = _input_mask(input, mask=mask) + count = inmask.sum(dim=dim, keepdim=True) + # pyrefly: ignore [no-matching-overload] + sample_total = sum(input, dim, keepdim=True, dtype=dtype, mask=inmask) + # TODO: replace torch.subtract/divide/square/maximum with + # masked subtract/divide/square/maximum when these will be + # available. + sample_mean = torch.divide(sample_total, count) + x = torch.subtract(input, sample_mean) + if mask is None: + # pyrefly: ignore [no-matching-overload] + total = sum(x * x.conj(), dim, keepdim=keepdim, dtype=compute_dtype) + else: + total = sum( + x * x.conj(), + dim, + keepdim=keepdim, + dtype=compute_dtype, + mask=inmask, # type: ignore[possibly-undefined] + ) + if not keepdim: + count = count.reshape(total.shape) + if correction != 0: + real_dtype = ( + corresponding_real_dtype(compute_dtype) + if compute_dtype.is_complex + else compute_dtype + ) + count = count.to(real_dtype) + count = torch.subtract(count, correction) + count = torch.maximum(count, count.new_zeros([])) + output = torch.divide(total, count).to(dtype=dtype) + if take_sqrt: + output = torch.sqrt(output) + return output + else: + raise ValueError( + f"masked std/var expects strided tensor (got {input.layout} tensor)" + ) + + +@_apply_docstring_templates +def var( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + unbiased: bool | None = None, + *, + correction: int | float | None = None, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +The identity value of sample variance operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + return _std_var( + input=input, + dim=dim, + unbiased=unbiased, + correction_opt=correction, + keepdim=keepdim, + dtype=dtype, + mask=mask, + take_sqrt=False, + ) + + +@_apply_docstring_templates +def std( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + unbiased: bool | None = None, + *, + correction: int | None = None, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +The identity value of sample standard deviation operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + return _std_var( + input=input, + dim=dim, + unbiased=unbiased, + correction_opt=correction, + keepdim=keepdim, + dtype=dtype, + mask=mask, + take_sqrt=True, + ) + + +@_apply_docstring_templates +def softmax( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amax, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.softmax(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked softmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def log_softmax( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amax, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.log_softmax(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked log_softmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def softmin( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amin, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.softmin(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked softmin expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def normalize( + input: Tensor | MaskedTensor, + ord: float, + dim: int, + *, + eps: float = 1e-12, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + # TODO: eliminate mask_input as unnecessary when using masked divide. + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + nrm_ = norm(input, ord, dim, keepdim=True, dtype=dtype, mask=mask) + # TODO: replace torch.maximum with masked maximum when available. + denom = torch.maximum(nrm_, nrm_.new_full([], eps)) + # TODO: replace torch.divide with masked divide when available. + return torch.divide(mask_input, denom) + else: + raise ValueError( + f"masked normalize expects strided tensor (got {mask_input.layout} tensor)" + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/monitor/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/monitor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d80239c86b85f4d5a234c38798b27c12e1c8cf5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/monitor/__init__.py @@ -0,0 +1,39 @@ +from typing import TYPE_CHECKING + +from torch._C._monitor import * # noqa: F403 +from torch._C._monitor import _WaitCounter, _WaitCounterTracker + + +if TYPE_CHECKING: + from torch.utils.tensorboard import SummaryWriter + +STAT_EVENT = "torch.monitor.Stat" + + +class TensorboardEventHandler: + """ + TensorboardEventHandler is an event handler that will write known events to + the provided SummaryWriter. + + This currently only supports ``torch.monitor.Stat`` events which are logged + as scalars. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_MONITOR) + >>> # xdoctest: +REQUIRES(module:tensorboard) + >>> from torch.utils.tensorboard import SummaryWriter + >>> from torch.monitor import TensorboardEventHandler, register_event_handler + >>> writer = SummaryWriter("log_dir") + >>> register_event_handler(TensorboardEventHandler(writer)) + """ + + def __init__(self, writer: "SummaryWriter") -> None: + """ + Constructs the ``TensorboardEventHandler``. + """ + self._writer = writer + + def __call__(self, event: Event) -> None: + if event.name == STAT_EVENT: + for k, v in event.data.items(): + self._writer.add_scalar(k, v, walltime=event.timestamp.timestamp()) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..40558f2ce3d274523c2724bd830789ee26babb2e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/__init__.py @@ -0,0 +1,194 @@ +# mypy: allow-untyped-defs +r""" +This package enables an interface for accessing MPS (Metal Performance Shaders) backend in Python. +Metal is Apple's API for programming metal GPU (graphics processor unit). Using MPS means that increased +performance can be achieved, by running work on the metal GPU(s). +See https://developer.apple.com/documentation/metalperformanceshaders for more details. +""" + +from typing import Union + +import torch +from torch import Tensor + + +_is_in_bad_fork = getattr(torch._C, "_mps_is_in_bad_fork", lambda: False) +_default_mps_generator: torch._C.Generator = None # type: ignore[assignment] + + +# local helper function (not public or exported) +def _get_default_mps_generator() -> torch._C.Generator: + global _default_mps_generator + if _default_mps_generator is None: + _default_mps_generator = torch._C._mps_get_default_generator() + return _default_mps_generator + + +def device_count() -> int: + r"""Returns the number of available MPS devices.""" + return int(torch._C._has_mps and torch._C._mps_is_available()) + + +def synchronize() -> None: + r"""Waits for all kernels in all streams on a MPS device to complete.""" + return torch._C._mps_deviceSynchronize() + + +def get_rng_state(device: Union[int, str, torch.device] = "mps") -> Tensor: + r"""Returns the random number generator state as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'mps'`` (i.e., ``torch.device('mps')``, the current MPS device). + """ + return _get_default_mps_generator().get_state() + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "mps" +) -> None: + r"""Sets the random number generator state. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'mps'`` (i.e., ``torch.device('mps')``, the current MPS device). + """ + new_state_copy = new_state.clone(memory_format=torch.contiguous_format) + _get_default_mps_generator().set_state(new_state_copy) + + +def manual_seed(seed: int) -> None: + r"""Sets the seed for generating random numbers. + + Args: + seed (int): The desired seed. + """ + # the torch.mps.manual_seed() can be called from the global + # torch.manual_seed() in torch/random.py. So we need to make + # sure mps is available (otherwise we just return without + # erroring out) + if not torch._C._has_mps: + return + seed = int(seed) + _get_default_mps_generator().manual_seed(seed) + + +def seed() -> None: + r"""Sets the seed for generating random numbers to a random number.""" + _get_default_mps_generator().seed() + + +def empty_cache() -> None: + r"""Releases all unoccupied cached memory currently held by the caching + allocator so that those can be used in other GPU applications. + """ + torch._C._mps_emptyCache() + + +def set_per_process_memory_fraction(fraction) -> None: + r"""Set memory fraction for limiting process's memory allocation on MPS device. + The allowed value equals the fraction multiplied by recommended maximum device memory + (obtained from Metal API device.recommendedMaxWorkingSetSize). + If trying to allocate more than the allowed value in a process, it will raise an out of + memory error in allocator. + + Args: + fraction(float): Range: 0~2. Allowed memory equals total_memory * fraction. + + .. note:: + Passing 0 to fraction means unlimited allocations + (may cause system failure if out of memory). + Passing fraction greater than 1.0 allows limits beyond the value + returned from device.recommendedMaxWorkingSetSize. + """ + + if not isinstance(fraction, float): + raise TypeError("Invalid type for fraction argument, must be `float`") + if fraction < 0 or fraction > 2: + raise ValueError(f"Invalid fraction value: {fraction}. Allowed range: 0~2") + + torch._C._mps_setMemoryFraction(fraction) + + +def current_allocated_memory() -> int: + r"""Returns the current GPU memory occupied by tensors in bytes. + + .. note:: + The returned size does not include cached allocations in + memory pools of MPSAllocator. + """ + return torch._C._mps_currentAllocatedMemory() + + +def driver_allocated_memory() -> int: + r"""Returns total GPU memory allocated by Metal driver for the process in bytes. + + .. note:: + The returned size includes cached allocations in MPSAllocator pools + as well as allocations from MPS/MPSGraph frameworks. + """ + return torch._C._mps_driverAllocatedMemory() + + +def recommended_max_memory() -> int: + r"""Returns recommended max Working set size for GPU memory in bytes. + + .. note:: + Recommended max working set size for Metal. + returned from device.recommendedMaxWorkingSetSize. + """ + return torch._C._mps_recommendedMaxMemory() + + +def compile_shader(source: str): + r"""Compiles compute shader from source and allows one to invoke kernels + defined there from the comfort of Python runtime + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_MPS) + >>> lib = torch.mps.compile_shader( + ... "kernel void full(device float* out, constant float& val, uint idx [[thread_position_in_grid]]) { out[idx] = val; }" + ... ) + >>> x = torch.zeros(16, device="mps") + >>> lib.full(x, 3.14) + """ + from pathlib import Path + + from torch.utils._cpp_embed_headers import _embed_headers + + if not hasattr(torch._C, "_mps_compileShader"): + raise RuntimeError("MPS is not available") + source = _embed_headers( + [l + "\n" for l in source.split("\n")], + [Path(__file__).parent.parent / "include"], + set(), + ) + return torch._C._mps_compileShader(source) + + +def is_available() -> bool: + return device_count() > 0 + + +from . import profiler +from .event import Event + + +__all__ = [ + "compile_shader", + "device_count", + "get_rng_state", + "manual_seed", + "seed", + "set_rng_state", + "synchronize", + "empty_cache", + "set_per_process_memory_fraction", + "current_allocated_memory", + "driver_allocated_memory", + "Event", + "profiler", + "recommended_max_memory", + "is_available", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/event.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/event.py new file mode 100644 index 0000000000000000000000000000000000000000..251fe855a6881089b4cb7e501ab1eabc151da1be --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/event.py @@ -0,0 +1,45 @@ +import torch + + +class Event: + r"""Wrapper around an MPS event. + + MPS events are synchronization markers that can be used to monitor the + device's progress, to accurately measure timing, and to synchronize MPS streams. + + Args: + enable_timing (bool, optional): indicates if the event should measure time + (default: ``False``) + """ + + def __init__(self, enable_timing: bool = False) -> None: + self.__eventId = torch._C._mps_acquireEvent(enable_timing) + + def __del__(self) -> None: + # checks if torch._C is already destroyed + if hasattr(torch._C, "_mps_releaseEvent") and self.__eventId > 0: + torch._C._mps_releaseEvent(self.__eventId) + + def record(self) -> None: + r"""Records the event in the default stream.""" + torch._C._mps_recordEvent(self.__eventId) + + def wait(self) -> None: + r"""Makes all future work submitted to the default stream wait for this event.""" + torch._C._mps_waitForEvent(self.__eventId) + + def query(self) -> bool: + r"""Returns True if all work currently captured by event has completed.""" + return torch._C._mps_queryEvent(self.__eventId) + + def synchronize(self) -> None: + r"""Waits until the completion of all work currently captured in this event. + This prevents the CPU thread from proceeding until the event completes. + """ + torch._C._mps_synchronizeEvent(self.__eventId) + + def elapsed_time(self, end_event: "Event") -> float: + r"""Returns the time elapsed in milliseconds after the event was + recorded and before the end_event was recorded. + """ + return torch._C._mps_elapsedTimeOfEvents(self.__eventId, end_event.__eventId) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/profiler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..56c3dcca1a41ca66e1c4f6e4318bb86db10e67b3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mps/profiler.py @@ -0,0 +1,100 @@ +import contextlib +from collections.abc import Iterator +from typing import Literal + +import torch + + +__all__ = [ + "start", + "stop", + "profile", + "metal_capture", + "is_metal_capture_enabled", + "is_capturing_metal", +] + + +ProfilerMode = Literal["interval", "event", "interval,event"] + + +def start(mode: ProfilerMode = "interval", wait_until_completed: bool = False) -> None: + r"""Start OS Signpost tracing from MPS backend. + + The generated OS Signposts could be recorded and viewed in + XCode Instruments Logging tool. + + Args: + mode(str): OS Signpost tracing mode could be "interval", "event", + or both "interval,event". + The interval mode traces the duration of execution of the operations, + whereas event mode marks the completion of executions. + See document `Recording Performance Data`_ for more info. + wait_until_completed(bool): Waits until the MPS Stream complete + executing each encoded GPU operation. This helps generating single + dispatches on the trace's timeline. + Note that enabling this option would affect the performance negatively. + + .. _Recording Performance Data: + https://developer.apple.com/documentation/os/logging/recording_performance_data + """ + mode_normalized = mode.lower().replace(" ", "") + torch._C._mps_profilerStartTrace( # type: ignore[attr-defined] + mode_normalized, wait_until_completed + ) + + +def stop() -> None: + r"""Stops generating OS Signpost tracing from MPS backend.""" + torch._C._mps_profilerStopTrace() # type: ignore[attr-defined] + + +@contextlib.contextmanager +def profile( + mode: ProfilerMode = "interval", wait_until_completed: bool = False +) -> Iterator[None]: + r"""Context Manager to enabling generating OS Signpost tracing from MPS backend. + + Args: + mode(str): OS Signpost tracing mode could be "interval", "event", + or both "interval,event". + The interval mode traces the duration of execution of the operations, + whereas event mode marks the completion of executions. + See document `Recording Performance Data`_ for more info. + wait_until_completed(bool): Waits until the MPS Stream complete + executing each encoded GPU operation. This helps generating single + dispatches on the trace's timeline. + Note that enabling this option would affect the performance negatively. + + .. _Recording Performance Data: + https://developer.apple.com/documentation/os/logging/recording_performance_data + """ + try: + start(mode, wait_until_completed) + yield + finally: + stop() + + +def is_metal_capture_enabled() -> bool: + """Checks if `metal_capture` context manager is usable + To enable metal capture, set MTL_CAPTURE_ENABLED envvar + """ + return torch._C._mps_isCaptureEnabled() # type: ignore[attr-defined, no-any-return] + + +def is_capturing_metal() -> bool: + """Checks if metal capture is in progress""" + return torch._C._mps_isCapturing() # type: ignore[attr-defined, no-any-return] + + +@contextlib.contextmanager +def metal_capture(fname: str) -> Iterator[None]: + """Context manager that enables capturing of Metal calls into gputrace""" + try: + torch._C._mps_startCapture(fname) # type: ignore[attr-defined] + yield + # Drain all the work that were enqueued during the context call + torch.mps.synchronize() + finally: + torch._C._mps_stopCapture() # type: ignore[attr-defined] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cd1218651cd94fb61a8acc77880274d5dfbd85ea --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/__init__.py @@ -0,0 +1,433 @@ +# mypy: allow-untyped-defs +r""" +This package enables an interface for accessing MTIA backend in python +""" + +import threading +import warnings +from collections.abc import Callable +from typing import Any, Optional, Union + +import torch +from torch import device as _device, Tensor +from torch._utils import _dummy_type, _LazySeedTracker, classproperty +from torch.types import Device + +from ._utils import _get_device_index + + +_device_t = Union[_device, str, int] + +# torch.mtia.Event/Stream is alias of torch.Event/Stream +Event = torch.Event +Stream = torch.Stream +# Default generators are initialized inside _mtia_init +default_generators: tuple[torch._C.Generator, ...] = () # type: ignore[assignment] + +_initialized = False +_queued_calls: list[ + tuple[Callable[[], None], list[str]] +] = [] # don't invoke these until initialization occurs +_tls = threading.local() +_initialization_lock = threading.Lock() +_lazy_seed_tracker = _LazySeedTracker() + + +if hasattr(torch._C, "_mtia_exchangeDevice"): + _exchange_device = torch._C._mtia_exchangeDevice +else: + + def _exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without MTIA support") + + +if hasattr(torch._C, "_mtia_maybeExchangeDevice"): + _maybe_exchange_device = torch._C._mtia_maybeExchangeDevice +else: + + def _maybe_exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without MTIA support") + + +def init(): + _lazy_init() + + +def is_initialized(): + r"""Return whether PyTorch's MTIA state has been initialized.""" + return _initialized and not _is_in_bad_fork() + + +def _is_in_bad_fork() -> bool: + return torch._C._mtia_isInBadFork() + + +def _lazy_init() -> None: + global _initialized, _queued_calls + if is_initialized() or hasattr(_tls, "is_initializing"): + return + with _initialization_lock: + # We be double-checking locking, boys! This is OK because + # the above test was GIL protected anyway. The inner test + # is for when a thread blocked on some other thread which was + # doing the initialization; when they get the lock, they will + # find there is nothing left to do. + if is_initialized(): + return + # It is important to prevent other threads from entering _lazy_init + # immediately, while we are still guaranteed to have the GIL, because some + # of the C calls we make below will release the GIL + if _is_in_bad_fork(): + raise RuntimeError( + "Cannot re-initialize MTIA in forked subprocess. To use MTIA with " + "multiprocessing, you must use the 'spawn' start method" + ) + if not _is_compiled(): + raise AssertionError( + "Torch not compiled with MTIA enabled. " + "Ensure you have `import mtia.host_runtime.torch_mtia.dynamic_library` in your python " + "src file and include `//mtia/host_runtime/torch_mtia:torch_mtia` as " + "your target dependency!" + ) + + torch._C._mtia_init() + # Some of the queued calls may reentrantly call _lazy_init(); + # we need to just return without initializing in that case. + # However, we must not let any *other* threads in! + _tls.is_initializing = True + + _queued_calls.extend(calls for calls in _lazy_seed_tracker.get_calls() if calls) + + try: + for queued_call, orig_traceback in _queued_calls: + try: + queued_call() + except Exception as e: + msg = ( + f"MTIA call failed lazily at initialization with error: {str(e)}\n\n" + f"MTIA call was originally invoked at:\n\n{''.join(orig_traceback)}" + ) + raise DeferredMtiaCallError(msg) from e + finally: + delattr(_tls, "is_initializing") + _initialized = True + + +class DeferredMtiaCallError(Exception): + pass + + +def _is_compiled() -> bool: + r"""Return true if compiled with MTIA support.""" + return torch._C._mtia_isBuilt() + + +def is_available() -> bool: + r"""Return true if MTIA device is available""" + if not _is_compiled(): + return False + # MTIA has to init devices first to know if there is any devices available. + return device_count() > 0 + + +def synchronize(device: Optional[_device_t] = None) -> None: + r"""Waits for all jobs in all streams on a MTIA device to complete.""" + with torch.mtia.device(device): + return torch._C._mtia_deviceSynchronize() + + +def device_count() -> int: + r"""Return the number of MTIA devices available.""" + # TODO: Update _accelerator_hooks_device_count to abstract a MTIA device count API + return torch._C._mtia_getDeviceCount() + + +def current_device() -> int: + r"""Return the index of a currently selected device.""" + return torch._C._accelerator_hooks_get_current_device() + + +def current_stream(device: Optional[_device_t] = None) -> Stream: + r"""Return the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the currently selected :class:`Stream` for the current device, given + by :func:`~torch.mtia.current_device`, if :attr:`device` is ``None`` + (default). + """ + return torch._C._mtia_getCurrentStream(_get_device_index(device, optional=True)) + + +def default_stream(device: Optional[_device_t] = None) -> Stream: + r"""Return the default :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the default :class:`Stream` for the current device, given by + :func:`~torch.mtia.current_device`, if :attr:`device` is ``None`` + (default). + """ + return torch._C._mtia_getDefaultStream(_get_device_index(device, optional=True)) + + +def record_memory_history( + enabled: Optional[str] = "all", stacks: str = "python", max_entries: int = 0 +) -> None: + r"""Enable/Disable the memory profiler on MTIA allocator + + Args: + enabled (all or state, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + + stacks ("python" or "cpp", optional). Select the stack trace to record. + + max_entries (int, optional). Maximum number of entries to record. + """ + if not is_initialized(): + return + torch._C._mtia_recordMemoryHistory(enabled, stacks, max_entries) + + +def snapshot() -> dict[str, Any]: + r"""Return a dictionary of MTIA memory allocator history""" + + return torch._C._mtia_memorySnapshot() + + +def attach_out_of_memory_observer( + observer: Callable[[int, int, int, int], None], +) -> None: + r"""Attach an out-of-memory observer to MTIA memory allocator""" + torch._C._mtia_attachOutOfMemoryObserver(observer) + + +def is_bf16_supported(including_emulation: bool = True): + r"""Return a bool indicating if the current MTIA device supports dtype bfloat16.""" + return True + + +def get_device_capability(device: Optional[_device_t] = None) -> tuple[int, int]: + r"""Return capability of a given device as a tuple of (major version, minor version). + + Args: + device (torch.device or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + return torch._C._mtia_getDeviceCapability(_get_device_index(device, optional=True)) + + +def empty_cache() -> None: + r"""Empty the MTIA device cache.""" + return torch._C._mtia_emptyCache() + + +def set_stream(stream: Stream): + r"""Set the current stream.This is a wrapper API to set the stream. + Usage of this function is discouraged in favor of the ``stream`` + context manager. + + Args: + stream (Stream): selected stream. This function is a no-op + if this argument is ``None``. + """ + if stream is None: + return + torch._C._mtia_setCurrentStream(stream) + + +def set_device(device: _device_t) -> None: + r"""Set the current device. + + Args: + device (torch.device or int): selected device. This function is a no-op + if this argument is negative. + """ + device = _get_device_index(device) + if device >= 0: + torch._C._accelerator_hooks_set_current_device(device) + + +def get_device_properties(device: Optional[_device_t] = None) -> dict[str, Any]: + r"""Return a dictionary of MTIA device properties + + Args: + device (torch.device or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + return torch._C._mtia_getDeviceProperties(_get_device_index(device, optional=True)) + + +class device: + r"""Context-manager that changes the selected device. + + Args: + device (torch.device or int): device index to select. It's a no-op if + this argument is a negative integer or ``None``. + """ + + def __init__(self, device: Any): + self.idx = _get_device_index(device, optional=True) + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch._C._accelerator_hooks_maybe_exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch._C._accelerator_hooks_maybe_exchange_device(self.prev_idx) + return False + + +class StreamContext: + r"""Context-manager that selects a given stream. + + All MTIA kernels queued within its context will be enqueued on a selected + stream. + + Args: + Stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: Streams are per-device. + """ + + cur_stream: Optional["torch.mtia.Stream"] + + def __init__(self, stream: Optional["torch.mtia.Stream"]): + self.cur_stream = None + self.stream = stream + self.idx = _get_device_index(None, True) + if not torch.jit.is_scripting(): + if self.idx is None: + self.idx = -1 # pyrefly: ignore [bad-assignment] + + self.src_prev_stream = ( + None if not torch.jit.is_scripting() else torch.mtia.default_stream(None) + ) + self.dst_prev_stream = ( + None if not torch.jit.is_scripting() else torch.mtia.default_stream(None) + ) + + def __enter__(self): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # Return if stream is None or MTIA device not available + if cur_stream is None or self.idx == -1: + return + self.src_prev_stream = torch.mtia.current_stream(None) + + # If the stream is not on the current device, then + # set the current stream on the device + if self.src_prev_stream.device != cur_stream.device: + with device(cur_stream.device): + self.dst_prev_stream = torch.mtia.current_stream(cur_stream.device) + torch.mtia.set_stream(cur_stream) + + def __exit__(self, type: Any, value: Any, traceback: Any): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # If stream is None or no MTIA device available, return + if cur_stream is None or self.idx == -1: + return + + # Reset the stream on the original device + # and destination device + if self.src_prev_stream.device != cur_stream.device: # type: ignore[union-attr] + torch.mtia.set_stream(self.dst_prev_stream) # type: ignore[arg-type] + torch.mtia.set_stream(self.src_prev_stream) # type: ignore[arg-type] + + +def _set_stream_by_id(stream_id, device_index, device_type): + r"""set stream specified by the stream id, device index and + device type + + Args: stream_id (int): stream id in stream pool + device_index (int): device index in topo + device_type (int): enum device type + """ + torch._C._mtia_setStream(stream_id, device_index, device_type) + + +def stream(stream: Optional["torch.mtia.Stream"]) -> StreamContext: + r"""Wrap around the Context-manager StreamContext that selects a given stream. + + Arguments: + stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: In eager mode stream is of type Stream class while in JIT it doesn't support torch.mtia.stream + """ + return StreamContext(stream) + + +def get_rng_state(device: Union[int, str, torch.device] = "mtia") -> Tensor: + r"""Returns the random number generator state as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'mtia'`` (i.e., ``torch.device('mtia')``, the current mtia device). + """ + warnings.warn( + "get_rng_state is not implemented in torch.mtia", + UserWarning, + stacklevel=2, + ) + return torch.zeros([1], dtype=torch.uint8, device=device) + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "mtia" +) -> None: + r"""Sets the random number generator state. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'mtia'`` (i.e., ``torch.device('mtia')``, the current mtia device). + """ + warnings.warn( + "set_rng_state is not implemented in torch.mtia", + UserWarning, + stacklevel=2, + ) + + +from .memory import * # noqa: F403 +from .mtia_graph import * # noqa: F403 + + +__all__ = [ + "init", + "is_available", + "is_initialized", + "synchronize", + "device_count", + "current_device", + "current_stream", + "default_stream", + "memory_stats", + "max_memory_allocated", + "memory_allocated", + "reset_peak_memory_stats", + "get_device_capability", + "get_device_properties", + "record_memory_history", + "snapshot", + "attach_out_of_memory_observer", + "empty_cache", + "set_device", + "set_stream", + "stream", + "device", + "set_rng_state", + "get_rng_state", + "is_bf16_supported", + "MTIAGraph", + "graph", + "graph_pool_handle", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c5bffe0a7b4196b17882e1bfc1c6b67cf6d8729e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/_utils.py @@ -0,0 +1,38 @@ +from typing import Any + +import torch + +# The _get_device_index has been moved to torch.utils._get_device_index +from torch._utils import _get_device_index as _torch_get_device_index + + +def _get_device_index( + device: Any, optional: bool = False, allow_cpu: bool = False +) -> int: + r"""Get the device index from :attr:`device`, which can be a torch.device object, a Python integer, or ``None``. + + If :attr:`device` is a torch.device object, returns the device index if it + is a MTIA device. Note that for a MTIA device without a specified index, + i.e., ``torch.device('mtia')``, this will return the current default MTIA + device if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``, + CPU devices will be accepted and ``-1`` will be returned in this case. + + If :attr:`device` is a Python integer, it is returned as is. + + If :attr:`device` is ``None``, this will return the current default MTIA + device if :attr:`optional` is ``True``. + """ + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + if isinstance(device, torch.device): + if allow_cpu: + if device.type not in ["mtia", "cpu"]: + raise ValueError(f"Expected a mtia or cpu device, but got: {device}") + elif device.type != "mtia": + raise ValueError(f"Expected a mtia device, but got: {device}") + if not torch.jit.is_scripting(): + if isinstance(device, torch.mtia.device): + return device.idx + return _torch_get_device_index(device, optional, allow_cpu) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/memory.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..8b741b7b3d2ca2504b3a8875155505fa1827b95a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/memory.py @@ -0,0 +1,71 @@ +# pyre-strict + +r"""This package adds support for device memory management implemented in MTIA.""" + +from typing import Any, Optional + +import torch + +from . import _device_t, is_initialized +from ._utils import _get_device_index + + +def memory_stats(device: Optional[_device_t] = None) -> dict[str, Any]: + r"""Return a dictionary of MTIA memory allocator statistics for a given device. + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return {} + return torch._C._mtia_memoryStats(_get_device_index(device, optional=True)) + + +def max_memory_allocated(device: Optional[_device_t] = None) -> int: + r"""Return the maximum memory allocated in bytes for a given device. + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return 0 + return memory_stats(device).get("dram", 0).get("peak_bytes", 0) + + +def memory_allocated(device: Optional[_device_t] = None) -> int: + r"""Return the current MTIA memory occupied by tensors in bytes for a given device. + + Args: + device (torch.device or int or str, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.mtia.current_device`, + if :attr:`device` is ``None`` (default). + """ + if not is_initialized(): + return 0 + return memory_stats(device).get("dram", 0).get("allocated_bytes", 0) + + +def reset_peak_memory_stats(device: Optional[_device_t] = None) -> None: + r"""Reset the peak memory stats for a given device. + + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return + torch._C._mtia_resetPeakMemoryStats(_get_device_index(device, optional=True)) + + +__all__ = [ + "memory_stats", + "max_memory_allocated", + "memory_allocated", + "reset_peak_memory_stats", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/mtia_graph.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/mtia_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..599abcee07397dc61b4a2fe23ab0d90aa1ab2161 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/mtia/mtia_graph.py @@ -0,0 +1,105 @@ +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +from typing import Optional, Union +from typing_extensions import Self + +import torch + + +_POOL_HANDLE = tuple[int, int] + + +def graph_pool_handle() -> _POOL_HANDLE: + """ + Return an opaque token representing the id of a graph memory pool. + """ + return torch._C._mtia_graphPoolHandle() + + +class MTIAGraph(torch._C._MTIAGraph): + """ + Wrapper around a MTIA graph. + """ + + def __new__(cls, keep_graph: bool = False) -> Self: + return super().__new__(cls, keep_graph) + + def capture_begin(self, pool: _POOL_HANDLE) -> None: + """ + Begin capturing a MTIA graph. + """ + super().capture_begin(pool) + + def capture_end(self) -> None: + """ + End the capture of a MTIA graph. + """ + super().capture_end() + + def instantiate(self) -> None: + """ + Instantiate the captured MTIA graph. + """ + super().instantiate() + + def replay(self) -> None: + """ + Replay the captured MTIA graph. + """ + super().replay() + + def reset(self) -> None: + """ + Destroy the captured graph and reset the states. + """ + super().reset() + + def pool(self) -> _POOL_HANDLE: + """ + Return an opaque token representing the id of this graph's memory pool + """ + return super().pool() + + +class graph: + default_capture_stream: Optional[torch.mtia.Stream] = None + + def __init__( + self, + mtia_graph: MTIAGraph, + pool: Optional[_POOL_HANDLE] = None, + stream: Optional[torch.mtia.Stream] = None, + ): + if self.__class__.default_capture_stream is None: + self.__class__.default_capture_stream = torch.mtia.current_stream() + + self.pool: Union[tuple[()], tuple[_POOL_HANDLE]] = ( + () if pool is None else (pool,) + ) + self.capture_stream = ( + stream if stream is not None else self.__class__.default_capture_stream + ) + assert self.capture_stream is not None + self.stream_ctx = torch.mtia.stream(self.capture_stream) + self.mtia_graph = mtia_graph + + def __enter__(self) -> None: + torch.mtia.synchronize() + torch.mtia.empty_cache() + + self.stream_ctx.__enter__() + + pool_arg = self.pool[0] if self.pool else (0, 0) + self.mtia_graph.capture_begin(pool_arg) + + def __exit__(self, *args: object) -> None: + self.mtia_graph.capture_end() + self.stream_ctx.__exit__(*args) + + +__all__ = [ + "MTIAGraph", + "graph", + "graph_pool_handle", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..31085a73022fe3b3e2e6306e5cae5a334b3d3a29 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/__init__.py @@ -0,0 +1,122 @@ +# mypy: allow-untyped-defs +"""torch.multiprocessing is a wrapper around the native :mod:`multiprocessing` module. + +It registers custom reducers, that use shared memory to provide shared +views on the same data in different processes. Once the tensor/storage is moved +to shared_memory (see :func:`~torch.Tensor.share_memory_`), it will be possible +to send it to other processes without making any copies. + +The API is 100% compatible with the original module - it's enough to change +``import multiprocessing`` to ``import torch.multiprocessing`` to have all the +tensors sent through the queues or shared via other mechanisms, moved to shared +memory. + +Because of the similarity of APIs we do not document most of this package +contents, and we recommend referring to very good docs of the original module. +""" + +import multiprocessing +import sys + +import torch + +from .reductions import init_reductions + + +__all__ = ["set_sharing_strategy", "get_sharing_strategy", "get_all_sharing_strategies"] + + +from multiprocessing import * # noqa: F403 + + +__all__ += multiprocessing.__all__ # noqa: PLE0605 type: ignore[attr-defined] + + +# This call adds a Linux specific prctl(2) wrapper function to this module. +# See https://github.com/pytorch/pytorch/pull/14391 for more information. +torch._C._multiprocessing_init() + + +"""Add helper function to spawn N processes and wait for completion of any of +them. This depends `mp.get_context` which was added in Python 3.4.""" +from .spawn import ( + ENV_VAR_PARALLEL_START, + ProcessContext, + ProcessExitedException, + ProcessRaisedException, + spawn, + SpawnContext, + start_processes, +) + + +if sys.platform == "darwin" or sys.platform == "win32": + _sharing_strategy = "file_system" + _all_sharing_strategies = {"file_system"} +else: + _sharing_strategy = "file_descriptor" + _all_sharing_strategies = {"file_descriptor", "file_system"} + + +def set_sharing_strategy(new_strategy): + """Set the strategy for sharing CPU tensors. + + Args: + new_strategy (str): Name of the selected strategy. Should be one of + the values returned by :func:`get_all_sharing_strategies()`. + """ + global _sharing_strategy + assert new_strategy in _all_sharing_strategies + _sharing_strategy = new_strategy + + +def get_sharing_strategy(): + """Return the current strategy for sharing CPU tensors.""" + return _sharing_strategy + + +def get_all_sharing_strategies(): + """Return a set of sharing strategies supported on a current system.""" + return _all_sharing_strategies + + +def _set_thread_name(name: str) -> None: + """Set the name of the current thread. + + Args: + name (str): Name of the current thread. + """ + torch._C._set_thread_name(name) + + +def _get_thread_name() -> str: + """Get the name of the current thread. + + Returns: + str: Name of the current thread. + """ + return torch._C._get_thread_name() + + +init_reductions() + +# Leak ResourceTracker at exit for Python-3.12 on MacOS +# See https://github.com/pytorch/pytorch/issues/153050 and +# https://github.com/python/cpython/issues/88887 for more details +from multiprocessing.resource_tracker import ResourceTracker as _RT + + +if ( + sys.platform == "darwin" + and sys.version_info >= (3, 12, 2) + and hasattr(_RT, "__del__") +): + import atexit + + def _leak_RT_at_exit(): + def _noop(x): + pass + + _RT.__del__ = _noop # type: ignore[attr-defined] + + atexit.register(_leak_RT_at_exit) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/_atfork.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/_atfork.py new file mode 100644 index 0000000000000000000000000000000000000000..79fb33e5a5a7c329d1723ed05a55c17f964a9739 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/_atfork.py @@ -0,0 +1,35 @@ +# mypy: allow-untyped-defs +import sys + + +__all__ = ["register_after_fork"] + +if sys.platform == "win32": + import multiprocessing.util as _util + + def _register(func): + def wrapper(arg): + func() + + _util.register_after_fork(_register, wrapper) + +else: + import os + + def _register(func): + os.register_at_fork(after_in_child=func) + + +def register_after_fork(func): + """Register a callable to be executed in the child process after a fork. + + Note: + In python < 3.7 this will only work with processes created using the + ``multiprocessing`` module. In python >= 3.7 it also works with + ``os.fork()``. + + Args: + func (function): Function taking no arguments to be called in the child after fork + + """ + _register(func) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/pool.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/pool.py new file mode 100644 index 0000000000000000000000000000000000000000..ef895aa7249c2e99ea9d97534cba0cb5b636c34d --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/pool.py @@ -0,0 +1,52 @@ +import multiprocessing.pool +import multiprocessing.util as util + +from .queue import SimpleQueue + + +def clean_worker(*args, **kwargs): + import gc + + multiprocessing.pool.worker(*args, **kwargs) + # Regular multiprocessing workers don't fully clean up after themselves, + # so we have to explicitly trigger garbage collection to make sure that all + # destructors are called... + gc.collect() + + +class Pool(multiprocessing.pool.Pool): + """Pool implementation which uses our version of SimpleQueue. + + This lets us pass tensors in shared memory across processes instead of + serializing the underlying data. + """ + + def _setup_queues(self): + self._inqueue = SimpleQueue() + self._outqueue = SimpleQueue() + self._quick_put = self._inqueue._writer.send + self._quick_get = self._outqueue._reader.recv + + def _repopulate_pool(self): + """Increase the number of pool processes to the specified number. + + Bring the number of pool processes up to the specified number, for use after + reaping workers which have exited. + """ + for _ in range(self._processes - len(self._pool)): + # changed worker -> clean_worker + args = ( + self._inqueue, + self._outqueue, + self._initializer, + self._initargs, + self._maxtasksperchild, + ) + if hasattr(self, "_wrap_exception"): + args += (self._wrap_exception,) + w = self.Process(target=clean_worker, args=args) + self._pool.append(w) + w.name = w.name.replace("Process", "PoolWorker") + w.daemon = True + w.start() + util.debug("added worker") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/queue.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/queue.py new file mode 100644 index 0000000000000000000000000000000000000000..2f86b743821e0f92945a05468136fe6ee21d3d9b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/queue.py @@ -0,0 +1,43 @@ +# mypy: allow-untyped-defs +import io +import multiprocessing.queues +import pickle +from multiprocessing.reduction import ForkingPickler + + +class ConnectionWrapper: + """Proxy class for _multiprocessing.Connection which uses ForkingPickler for object serialization.""" + + def __init__(self, conn): + self.conn = conn + + def send(self, obj): + buf = io.BytesIO() + ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj) + self.send_bytes(buf.getvalue()) + + def recv(self): + buf = self.recv_bytes() + return pickle.loads(buf) + + def __getattr__(self, name): + if "conn" in self.__dict__: + return getattr(self.conn, name) + raise AttributeError(f"'{type(self).__name__}' object has no attribute 'conn'") + + +class Queue(multiprocessing.queues.Queue): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) + self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) + self._send = self._writer.send + self._recv = self._reader.recv + + +class SimpleQueue(multiprocessing.queues.SimpleQueue): + def _make_methods(self): + if not isinstance(self._reader, ConnectionWrapper): + self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) + self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) + super()._make_methods() # type: ignore[misc] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/reductions.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/reductions.py new file mode 100644 index 0000000000000000000000000000000000000000..36bed03073de9d1a286e12f6acf735ee7c4881e8 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/reductions.py @@ -0,0 +1,644 @@ +# mypy: allow-untyped-defs +import multiprocessing +import os +import threading +from multiprocessing import reduction +from multiprocessing.util import register_after_fork + +import torch +from torch._namedtensor_internals import check_serializing_named_tensor + + +try: + # Early load resource_sharer to prevent a partially initialized instance + # from being inherited in a forked child process. The reduce_storage method + # requires this module indirectly through DupFd(). The built-in mp.Queue + # class pickles arguments in a background thread which may overlap with the + # fork. + import multiprocessing.resource_sharer +except ImportError: + pass + + +class StorageWeakRef: + r"""A weak reference to a Storage. + + The cdata member is a Python number containing the integer representation of + the Storage pointer. + """ + + __slots__ = ["cdata", "_free_weak_ref"] + + def __init__(self, storage): + self.cdata = storage._weak_ref() + # Save a direct reference to _free_weak_ref because the `torch` module + # might be cleared during Python shutdown before this module is cleared. + self._free_weak_ref = torch.Storage._free_weak_ref # type: ignore[attr-defined] + + @classmethod + def from_weakref(cls, cdata): + instance = cls.__new__(cls) + instance.cdata = cdata + instance._free_weak_ref = torch.Storage._free_weak_ref # type: ignore[attr-defined] + return instance + + def expired(self): + return torch.Storage._expired(self.cdata) # type: ignore[attr-defined] + + def __del__(self): + self._free_weak_ref(self.cdata) + + def __hash__(self): + return self.cdata + + def __eq__(self, other): + if id(self) == id(other): + return True + return self.cdata == other.cdata + + +class SharedCache(dict): + """Dictionary from multiprocessing handles to StorageWeakRef.""" + + def __init__(self) -> None: + # free_dead_references() is called if the len exceeds the current + # limit. The limit scales with the number of remaining live objects. + self.limit = 128 + # `fork` inherits lock state, so in case we fork when the lock is held, + # we register a function to reset the lock to a new object to avoid + # possible deadlocks, following python multiprocessing library design. + self._after_fork() + register_after_fork(self, SharedCache._after_fork) + + def _after_fork(self): + self.lock = threading.Lock() + + def get(self, key): # type: ignore[override] + with self.lock: + return dict.get(self, key) + + def __setitem__(self, key, storage_ref): + with self.lock: + dict.__setitem__(self, key, storage_ref) + if len(self) > self.limit: + self.free_dead_references() + + def free_dead_references(self): + live = 0 + for key, storage_ref in list(self.items()): + if storage_ref.expired(): + del self[key] + else: + live += 1 + self.limit = max(128, live * 2) + + +# mapping from handles to StorageWeakRef objects +shared_cache = SharedCache() + + +def rebuild_event(device, handle): + return torch.cuda.Event.from_ipc_handle(device, handle) + + +def reduce_event(event): + handle = event.ipc_handle() + return (rebuild_event, (event.device, handle)) + + +def rebuild_tensor(cls, storage, metadata): + storage_offset, size, stride, requires_grad = metadata + t = torch._utils._rebuild_tensor(storage, storage_offset, size, stride) + if cls == torch.nn.parameter.Parameter: + # we have to pass requires_grad into constructor, rather than set it as an + # attribute later, because it's an important check for Integer Tensors to + # have requires_grad=False (or else they raise an error) + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + return t + + +def rebuild_meta_tensor( + tensor_cls, + tensor_size, + tensor_stride, + tensor_offset, + dtype, + storage_size_bytes, + requires_grad, +): + untyped_storage = torch.UntypedStorage(storage_size_bytes, device="meta") + + typed_storage = torch.TypedStorage( + wrap_storage=untyped_storage, dtype=dtype, _internal=True + ) + + t = torch._utils._rebuild_tensor( + typed_storage, + tensor_offset, + tensor_size, + tensor_stride, + ) + + if tensor_cls == torch.nn.parameter.Parameter: + # It is crucial for integer tensors to receive + # the requires_grad=False as an argument in the constructor + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + + return t + + +def rebuild_cuda_tensor( + tensor_cls, + tensor_size, + tensor_stride, + tensor_offset, + storage_cls, + dtype, + storage_device, + storage_handle, + storage_size_bytes, + storage_offset_bytes, + requires_grad, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, +): + # If storage_handle is None, storage points to nullptr. + if storage_handle is None or storage_size_bytes == 0: + storage = storage_cls(0, dtype=dtype, device=storage_device, _internal=True) + else: + storage = storage_from_cache( + storage_cls, (storage_handle, storage_offset_bytes) + ) + if storage is None: + torch.cuda._lazy_init() + storage = storage_cls._new_shared_cuda( + storage_device, + storage_handle, + storage_size_bytes, + storage_offset_bytes, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ) + shared_cache[(storage_handle, storage_offset_bytes)] = StorageWeakRef( + storage + ) + else: + # We already ref counting this Storage, but producer needs new ref-counters to be released. + storage_cls._release_ipc_counter( + ref_counter_handle, ref_counter_offset, device=storage_device + ) + + _storage = ( + storage + if isinstance(storage, torch.UntypedStorage) + else storage._untyped_storage + ) + + t = torch._utils._rebuild_tensor( + torch.storage.TypedStorage(wrap_storage=_storage, dtype=dtype, _internal=True), + tensor_offset, + tensor_size, + tensor_stride, + ) + + if tensor_cls == torch.nn.parameter.Parameter: + # It is crucial for integer tensors to receive + # the requires_grad=False as an argument in the constructor + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + + return t + + +def reduce_tensor(tensor): + if tensor.requires_grad and not tensor.is_leaf: + raise RuntimeError( + "Cowardly refusing to serialize non-leaf tensor which requires_grad, " + "since autograd does not support crossing process boundaries. " + "If you just want to transfer the data, call detach() on the tensor " + "before serializing (e.g., putting it on the queue)." + ) + + check_serializing_named_tensor(tensor) + torch.utils.hooks.warn_if_has_hooks(tensor) + + # Note [CUDA IPC and the caching allocator] + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # When you send a CUDA tensor over IPC, you might expect that you will + # get out the same storage from the other end. However, the CUDA caching + # allocator makes it difficult to preserve this invariant. Consider + # the following situation: a tensor of size 0x100 points to offset 0x20 of + # a storage at 0xA100 of size 0x100. (For simplicity, all of these + # sizes are given in bytes). HOWEVER, with the caching allocator, this storage + # might be part of a larger cudaMalloc allocation 0xA000 of size 0x4000. + # + # When we want to send this CUDA tensor over IPC, we must send the + # *entire* cudaMalloc allocation, i.e., the 0xA000 region, not just + # the storage 0xA100 (because that is what CUDA supports). So, on the + # other end, there simply isn't any way to say, "Wait, you gave me + # a bigger region (0xA000) than the one I wanted (0xA100)". + # + # OK, so if you sent the cudaMalloc allocation, can you just wrap that up as + # one storage itself? No, because this cudaMalloc allocation might contain + # storages of mixed types: float, bytes, double... If you make the entire + # allocation a single storage of a type A, we'll hit an error when constructing + # a tensor of type B on the storage. + # + # cudaIpcMemHandle is an identifier to access the sender cudaMalloc allocation on the + # receiver side. However, cudaIpcMemHandles from each device in a given process may + # only be opened by one context per device per other process. + # If we open and close a memory handle multiples times in a process, CUDA is allowed + # to give it a different address; similarly, once we close the memory, we're not + # allowed to access it(and the storage/tensor built on top of it), even if it is + # still live in the original process. As we cannot make a cudaMalloc allocation + # to a single storage in one go, this requires us to cache the device pointer for + # each cudaIpcMemHandle on C++ side to reconstruct types of storages, while keep + # the old ones alives. + # See [https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__DEVICE.html] + # + # This is fine, because all we need to do is to save our position in the allocation, + # and reconstruct storage and tensor from it. + # 0xA000 -> -------CUDA Allocation------ + # | | + # | | + # | | + # | | + # 0xA100 -> --------storage1 begin------ + # | | + # 0xA120 -> --------tensor1 begin ------ + # | | + # | | + # | | + # | | + # | | + # 0xA160 -> --------tensor1 end--------- + # | | + # | | + # | | + # 0xA200 -> --------storage1 end-------- + # | | + # 0xE000 -> --------CUDA allocation----- + # + # To send tensor1, the following info are required from sender to receiver for + # storage reconstruction. + # 1. cudaIpcMemHandle of 0xA000(which can be mapped to a basePtr in receiver process). + # basePtr may not be exactly 0xA000 since it's a different process. + # 2. offset(0xA100) of storage1 in the CUDA allocation. + # 3. size of storage1(0x100). + # + # On receiver side: + # 1. Get the devPtr of the MemHandle to access the memory, reconstruct a storage + # of the same type using (basePtr, offset, size). + # 2. we can reconstruct the tensor on top of the reconstructed storage + # Tensor(size=0x040, offset=0x020, storage=Storage(data=basePtr+0xA100, size=0x0100)) + # + # This strategy has a few implications: + # + # 1. When we serialize a CUDA tensor for IPC, we cannot do it all in one + # go (non-compositionally), and this requires to have a global map + # memHandle -> devPtr for each process. + # + # 2. We MUST NOT let the new IPC tensor be resizable. Originally, a resize + # of the storage beyond 0x100 would merely have caused us to do a + # reallocation. You don't really want to do this, but if you did, + # all that would happen is that you would lose IPC sharing. But if + # you do this in the new world, we will happily let you write out of + # bounds of your "allocation", clobbering unrelated data in the cached + # allocator block. BAD! + # + # By the way, in old versions of PyTorch, we supported this situation + # natively using a "storage view", which permitted multiple storages to be + # views on each other. But this was the *only* use of storage views, so we + # eliminated it so that we could just use tensor views to implement the same + # thing. + # + + # TODO: Handle distinguishing between subclass and non-subclass versions of NT better + # https://github.com/pytorch/pytorch/issues/110543 + from torch.nested._internal.nested_tensor import NestedTensor + + if tensor.is_nested and not isinstance(tensor, NestedTensor): + return reduce_nested_tensor(tensor) + + if tensor.layout in { + torch.sparse_coo, + torch.sparse_csr, + torch.sparse_bsr, + torch.sparse_csc, + torch.sparse_bsc, + }: + return reduce_sparse_tensor(tensor) + + storage = tensor._typed_storage() + + if storage._untyped_storage.device.type == "cuda": + ( + device, + handle, + storage_size_bytes, + storage_offset_bytes, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ) = storage._share_cuda_() + tensor_offset = tensor.storage_offset() + shared_cache[handle] = StorageWeakRef(storage) + # _backward_hooks purposely omitted here, see + # Note [Don't serialize hooks] + return ( + rebuild_cuda_tensor, + ( + type(tensor), + tensor.size(), + tensor.stride(), + tensor_offset, # tensor offset in its storage + type(storage), + tensor.dtype, + device, + handle, # identifier which CUDA allocation is the storage in. + storage_size_bytes, # size(in bytes) of the storage + storage_offset_bytes, # offset(in bytes) of the storage in the CUDA allocation + tensor.requires_grad, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ), + ) + elif storage._untyped_storage.device.type == "meta": + return ( + rebuild_meta_tensor, + ( + type(tensor), + tensor.size(), + tensor.stride(), + tensor.storage_offset(), + tensor.dtype, + tensor.untyped_storage().size(), + tensor.requires_grad, + ), + ) + + # _backward_hooks purposely omitted here, see Note [Don't serialize hooks] + metadata = ( + tensor.storage_offset(), + tensor.size(), + tensor.stride(), + tensor.requires_grad, + ) + return (rebuild_tensor, (type(tensor), storage, metadata)) + + +def rebuild_nested_tensor( + rebuild_buffer_func, + rebuild_buffer_args, + rebuild_sizes_func, + rebuild_sizes_args, + rebuild_strides_func, + rebuild_strides_args, + rebuild_offsets_func, + rebuild_offsets_args, +): + buffer = rebuild_buffer_func(*rebuild_buffer_args) + sizes = rebuild_sizes_func(*rebuild_sizes_args) + strides = rebuild_strides_func(*rebuild_strides_args) + offsets = rebuild_offsets_func(*rebuild_offsets_args) + return torch._nested_view_from_buffer_copy(buffer, sizes, strides, offsets) + + +def reduce_nested_tensor(nt): + rebuild_buffer_func, rebuild_buffer_args = reduce_tensor(nt.values()) + rebuild_sizes_func, rebuild_sizes_args = reduce_tensor(nt._nested_tensor_size()) + rebuild_strides_func, rebuild_strides_args = reduce_tensor( + nt._nested_tensor_strides() + ) + rebuild_offsets_func, rebuild_offsets_args = reduce_tensor( + nt._nested_tensor_storage_offsets() + ) + + return ( + rebuild_nested_tensor, + ( + rebuild_buffer_func, + rebuild_buffer_args, + rebuild_sizes_func, + rebuild_sizes_args, + rebuild_strides_func, + rebuild_strides_args, + rebuild_offsets_func, + rebuild_offsets_args, + ), + ) + + +def rebuild_sparse_coo_tensor( + rebuild_indices_func, + rebuild_indices_args, + rebuild_values_func, + rebuild_values_args, + shape, + is_coalesced, +): + indices = rebuild_indices_func(*rebuild_indices_args) + values = rebuild_values_func(*rebuild_values_args) + return torch.sparse_coo_tensor(indices, values, shape, is_coalesced=is_coalesced) + + +def rebuild_sparse_compressed_tensor( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + rebuild_plain_indices_func, + rebuild_plain_indices_args, + rebuild_values_func, + rebuild_values_args, + shape, + layout, +): + compressed_indices = rebuild_compressed_indices_func( + *rebuild_compressed_indices_args + ) + plain_indices = rebuild_plain_indices_func(*rebuild_plain_indices_args) + values = rebuild_values_func(*rebuild_values_args) + return torch.sparse_compressed_tensor( + compressed_indices, plain_indices, values, shape, layout=layout + ) + + +def reduce_sparse_tensor(sparse): + if sparse.layout is torch.sparse_coo: + rebuild_indices_func, rebuild_indices_args = reduce_tensor(sparse._indices()) + rebuild_values_func, rebuild_values_args = reduce_tensor(sparse._values()) + return ( + rebuild_sparse_coo_tensor, + ( + rebuild_indices_func, + rebuild_indices_args, + rebuild_values_func, + rebuild_values_args, + sparse.shape, + sparse.is_coalesced(), + ), + ) + else: + if sparse.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices = sparse.crow_indices() + plain_indices = sparse.col_indices() + elif sparse.layout in {torch.sparse_csc, torch.sparse_bsc}: + compressed_indices = sparse.ccol_indices() + plain_indices = sparse.row_indices() + else: + raise NotImplementedError(sparse.layout) + ( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + ) = reduce_tensor(compressed_indices) + rebuild_plain_indices_func, rebuild_plain_indices_args = reduce_tensor( + plain_indices + ) + rebuild_values_func, rebuild_values_args = reduce_tensor(sparse.values()) + return ( + rebuild_sparse_compressed_tensor, + ( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + rebuild_plain_indices_func, + rebuild_plain_indices_args, + rebuild_values_func, + rebuild_values_args, + sparse.shape, + sparse.layout, + ), + ) + + +def fd_id(fd): + # Returns a tuple which uniquely identifies a file descriptor. In Mac OS, + # this doesn't work with shared memory handles, which is why we don't + # support the "file_descriptor" sharing method on that platform. + stat = os.fstat(fd) + return (stat.st_ino, stat.st_dev) + + +def storage_from_cache(cls, key): + storage_ref = shared_cache.get(key) + if storage_ref is None: + return None + return torch.UntypedStorage._new_with_weak_ptr(storage_ref.cdata) + + +def rebuild_storage_fd(cls, df, size): + fd = df.detach() + try: + storage = storage_from_cache(cls, fd_id(fd)) + if storage is not None: + return storage + storage = cls._new_shared_fd_cpu(fd, size) + shared_cache[fd_id(fd)] = StorageWeakRef(storage) + return storage + finally: + os.close(fd) + + +def rebuild_storage_filename(cls, manager, handle, size, dtype=None): + storage: torch.TypedStorage | torch.UntypedStorage = storage_from_cache(cls, handle) + if storage is not None: + return storage._shared_decref() + if dtype is None: + storage = torch.UntypedStorage._new_shared_filename_cpu(manager, handle, size) + else: + byte_size = size * torch._utils._element_size(dtype) + untyped_storage: torch.UntypedStorage = ( + torch.UntypedStorage._new_shared_filename_cpu(manager, handle, byte_size) + ) + storage = torch.TypedStorage( + wrap_storage=untyped_storage, dtype=dtype, _internal=True + ) + shared_cache[handle] = StorageWeakRef(storage) + return storage._shared_decref() + + +def rebuild_storage_empty(cls): + return cls() + + +def rebuild_typed_storage(storage, dtype): + return torch.storage.TypedStorage(wrap_storage=storage, dtype=dtype, _internal=True) + + +# Use for torch.storage.TypedStorage +def reduce_typed_storage(storage): + return (rebuild_typed_storage, (storage._untyped_storage, storage.dtype)) + + +def rebuild_typed_storage_child(storage, storage_type): + return storage_type(wrap_storage=storage, _internal=True) + + +# Use for child classes of torch.storage.TypedStorage, like torch.FloatStorage +def reduce_typed_storage_child(storage): + return (rebuild_typed_storage_child, (storage._untyped_storage, type(storage))) + + +def reduce_storage(storage): + from . import get_sharing_strategy + + if storage.is_cuda: + raise RuntimeError( + "Cannot pickle CUDA storage; try pickling a CUDA tensor instead" + ) + elif storage.device.type == "meta": + raise RuntimeError( + "Cannot pickle meta storage; try pickling a meta tensor instead" + ) + elif get_sharing_strategy() == "file_system": + metadata = storage._share_filename_cpu_() + cache_key = metadata[1] + rebuild = rebuild_storage_filename + if isinstance(storage, torch.TypedStorage): + metadata += (storage.dtype,) + storage._shared_incref() + elif storage.size() == 0: + # This is special cased because Empty tensors + # (with size 0) cannot be mmapped. + return (rebuild_storage_empty, (type(storage),)) + else: + fd, size = storage._share_fd_cpu_() + df = multiprocessing.reduction.DupFd(fd) + cache_key = fd_id(fd) + metadata = (df, size) + rebuild = rebuild_storage_fd # type: ignore[assignment] + + shared_cache[cache_key] = StorageWeakRef(storage) + return (rebuild, (type(storage),) + metadata) + + +def init_reductions(): + reduction.register(torch.cuda.Event, reduce_event) + + for t in torch._storage_classes: + if t.__name__ == "UntypedStorage": + reduction.register(t, reduce_storage) + else: + reduction.register(t, reduce_typed_storage_child) + + reduction.register(torch.storage.TypedStorage, reduce_typed_storage) + + for t in torch._tensor_classes: + reduction.register(t, reduce_tensor) + + # TODO: Maybe this should be in tensor_classes? :) + reduction.register(torch.Tensor, reduce_tensor) + + from torch.nn.parameter import Parameter + + reduction.register(Parameter, reduce_tensor) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/spawn.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/spawn.py new file mode 100644 index 0000000000000000000000000000000000000000..e4216cdc03cb8d7825150f5e502eb78a9a20aefa --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/multiprocessing/spawn.py @@ -0,0 +1,340 @@ +# mypy: allow-untyped-defs +import logging +import multiprocessing +import multiprocessing.connection +import os +import pickle +import signal +import sys +import tempfile +import time +import warnings +from concurrent.futures import as_completed, ThreadPoolExecutor + +from . import _prctl_pr_set_pdeathsig # type: ignore[attr-defined] + + +ENV_VAR_PARALLEL_START = "TORCH_MP_PARALLEL_START" + +log = logging.getLogger(__name__) + +__all__ = [ + "ProcessContext", + "ProcessException", + "ProcessExitedException", + "ProcessRaisedException", + "spawn", + "SpawnContext", + "start_processes", +] + + +class ProcessException(Exception): + __slots__ = ["error_index", "error_pid"] + + def __init__(self, msg: str, error_index: int, error_pid: int): + super().__init__(msg) + self.msg = msg + self.error_index = error_index + self.error_pid = error_pid + + def __reduce__(self): + return type(self), (self.msg, self.error_index, self.error_pid) + + +class ProcessRaisedException(ProcessException): + """Exception raised when a process failed due to an exception raised by the code.""" + + +class ProcessExitedException(ProcessException): + """Exception raised when a process failed due to signal or exited with a specific code.""" + + __slots__ = ["exit_code"] + + def __init__( + self, + msg: str, + error_index: int, + error_pid: int, + exit_code: int, + signal_name: str | None = None, + ): + super().__init__(msg, error_index, error_pid) + self.exit_code = exit_code + self.signal_name = signal_name + + def __reduce__(self): + return ( + type(self), + ( + self.msg, + self.error_index, + self.error_pid, + self.exit_code, + self.signal_name, + ), + ) + + +def _wrap(fn, i, args, error_file): + # prctl(2) is a Linux specific system call. + # On other systems the following function call has no effect. + # This is set to ensure that non-daemonic child processes can + # terminate if their parent terminates before they do. + _prctl_pr_set_pdeathsig(signal.SIGINT) + + try: + fn(i, *args) + except KeyboardInterrupt: + pass # SIGINT; Killed by parent, do nothing + except Exception: + # Propagate exception to parent process, keeping original traceback + import traceback + + with open(error_file, "wb") as fh: + pickle.dump(traceback.format_exc(), fh) + sys.exit(1) + + +class ProcessContext: + def __init__(self, processes, error_files): + self.error_files = error_files + self.processes = processes + self.sentinels = { + process.sentinel: index for index, process in enumerate(processes) + } + + def pids(self): + return [int(process.pid) for process in self.processes] + + def _join_procs_with_timeout(self, timeout: float): + """Attempt to join all processes with a shared timeout.""" + end = time.monotonic() + timeout + for process in self.processes: + # pyrefly: ignore [no-matching-overload] + time_to_wait = max(0, end - time.monotonic()) + process.join(time_to_wait) + + def join(self, timeout: float | None = None, grace_period: float | None = None): + r"""Join one or more processes within spawn context. + + Attempt to join one or more processes in this spawn context. + If one of them exited with a non-zero exit status, this function + kills the remaining processes (optionally with a grace period) + and raises an exception with the cause of the first process exiting. + + Returns ``True`` if all processes have been joined successfully, + ``False`` if there are more processes that need to be joined. + + Args: + timeout (float): Wait this long (in seconds) before giving up on waiting. + grace_period (float): When any processes fail, wait this long (in seconds) + for others to shutdown gracefully before terminating them. If they + still don't exit, wait another grace period before killing them. + """ + # Ensure this function can be called even when we're done. + if len(self.sentinels) == 0: + return True + + # Wait for any process to fail or all of them to succeed. + ready = multiprocessing.connection.wait( + self.sentinels.keys(), + timeout=timeout, + ) + + error_index = None + for sentinel in ready: + index = self.sentinels.pop(sentinel) + process = self.processes[index] + process.join() + if process.exitcode != 0: + error_index = index + break + + # Return if there was no error. + if error_index is None: + # Return whether or not all processes have been joined. + return len(self.sentinels) == 0 + # An error occurred. Clean-up all processes before returning. + # First, allow a grace period for processes to shutdown themselves. + if grace_period is not None: + self._join_procs_with_timeout(grace_period) + # Then, terminate processes that are still alive. Try SIGTERM first. + for process in self.processes: + if process.is_alive(): + log.warning("Terminating process %s via signal SIGTERM", process.pid) + process.terminate() + + # Try SIGKILL if the process isn't going down after another grace_period. + # The reason is related to python signal handling is limited + # to main thread and if that is in c/c++ land and stuck it won't + # to handle it. We have seen processes getting stuck not handling + # SIGTERM for the above reason. + self._join_procs_with_timeout(30 if grace_period is None else grace_period) + for process in self.processes: + if process.is_alive(): + log.warning( + "Unable to shutdown process %s via SIGTERM , forcefully exiting via SIGKILL", + process.pid, + ) + process.kill() + process.join() + + # The file will only be created if the process crashed. + failed_process = self.processes[error_index] + if not os.access(self.error_files[error_index], os.R_OK): + exitcode = self.processes[error_index].exitcode + if exitcode < 0: + try: + name = signal.Signals(-exitcode).name + except ValueError: + name = f"" + raise ProcessExitedException( + f"process {error_index:d} terminated with signal {name}", + error_index=error_index, + error_pid=failed_process.pid, + exit_code=exitcode, + signal_name=name, + ) + else: + raise ProcessExitedException( + f"process {error_index:d} terminated with exit code {exitcode:d}", + error_index=error_index, + error_pid=failed_process.pid, + exit_code=exitcode, + ) + + with open(self.error_files[error_index], "rb") as fh: + original_trace = pickle.load(fh) + msg = f"\n\n-- Process {error_index:d} terminated with the following error:\n" + msg += original_trace + raise ProcessRaisedException(msg, error_index, failed_process.pid) + + +class SpawnContext(ProcessContext): + def __init__(self, processes, error_files): + warnings.warn( + "SpawnContext is renamed to ProcessContext since 1.4 release.", stacklevel=2 + ) + super().__init__(processes, error_files) + + +# Note: [start_processes] +# mp.start_processes handles both start_method='spawn' and 'fork'. It's supposed to be a +# more generalized API than mp.spawn. Currently we only document mp.spawn as it's the +# CUDA compatible start_method. However, in environments like Ipython notebooks, 'fork' +# works better than 'spawn'. Every helper function we created for mp.spawn is indeed +# general enough, and backends like XLA can reuse them in Colab notebooks as well. +# Currently we only add this API first, we can consider adding it to documentation as +# needed in the future. +def start_processes( + fn, + args=(), + nprocs=1, + join=True, + daemon=False, + start_method="spawn", +): + # To speed up performance in certain cases (see https://github.com/pytorch/pytorch/issues/133010), + # this func will start processes in parallel if start_method is 'forkserver'. + # Please opt in to this perf optimization by setting env var (TORCH_MP_PARALLEL_START) to 1. + # todo: investigate why spawn does not work with threadpool and raises SIGINT + if ( + start_method == "forkserver" + and os.environ.get(ENV_VAR_PARALLEL_START, "0") == "1" + ): + log.info("Starting processes in parallel.") + start_parallel = True + else: + # Set env var TORCH_MP_PARALLEL_START to 0 to disable parallel start + start_parallel = False + + mp = multiprocessing.get_context(start_method) + error_files = [None] * nprocs + processes = [None] * nprocs + + def start_process(i): + # Each process is assigned a file to write tracebacks to. We + # use the file being non-empty to indicate an exception + # occurred (vs an expected shutdown). Note: this previously + # used a multiprocessing.Queue but that can be prone to + # deadlocks, so we went with a simpler solution for a one-shot + # message between processes. + tf = tempfile.NamedTemporaryFile( # noqa: SIM115 + prefix="pytorch-errorfile-", suffix=".pickle", delete=False + ) + tf.close() + os.unlink(tf.name) + + process = mp.Process( # pyrefly: ignore # missing-attribute + target=_wrap, + args=(fn, i, args, tf.name), + daemon=daemon, + ) + + process.start() + return i, process, tf.name + + if not start_parallel: + for i in range(nprocs): + idx, process, tf_name = start_process(i) + error_files[idx] = tf_name + processes[idx] = process + else: + with ThreadPoolExecutor(max_workers=nprocs) as executor: + futures = [executor.submit(start_process, i) for i in range(nprocs)] + for fut in as_completed(futures): + idx, process, tf_name = fut.result() + # idx and process rank needs to be the same. + error_files[idx] = tf_name + processes[idx] = process + context = ProcessContext(processes, error_files) + if not join: + return context + + # Loop on join until it returns True or raises an exception. + while not context.join(): + pass + + +def spawn(fn, args=(), nprocs=1, join=True, daemon=False, start_method="spawn"): + r"""Spawns ``nprocs`` processes that run ``fn`` with ``args``. + + If one of the processes exits with a non-zero exit status, the + remaining processes are killed and an exception is raised with the + cause of termination. In the case an exception was caught in the + child process, it is forwarded and its traceback is included in + the exception raised in the parent process. + + Args: + fn (function): Function is called as the entrypoint of the + spawned process. This function must be defined at the top + level of a module so it can be pickled and spawned. This + is a requirement imposed by multiprocessing. + + The function is called as ``fn(i, *args)``, where ``i`` is + the process index and ``args`` is the passed through tuple + of arguments. + + args (tuple): Arguments passed to ``fn``. + nprocs (int): Number of processes to spawn. + join (bool): Perform a blocking join on all processes. + daemon (bool): The spawned processes' daemon flag. If set to True, + daemonic processes will be created. + start_method (str): (deprecated) this method will always use ``spawn`` + as the start method. To use a different start method + use ``start_processes()``. + + Returns: + None if ``join`` is ``True``, + :class:`~ProcessContext` if ``join`` is ``False`` + + """ + if start_method != "spawn": + msg = ( + f"This method only supports start_method=spawn (got: {start_method}).\n" + "To use a different start_method use:\n\t\t" + " torch.multiprocessing.start_processes(...)" + ) + warnings.warn(msg, FutureWarning, stacklevel=2) + return start_processes(fn, args, nprocs, join, daemon, start_method="spawn") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nativert/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nativert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nested/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nested/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e59ad200757ca5651bd62ba859b815b3cb993387 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nested/__init__.py @@ -0,0 +1,518 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import SymInt, Tensor +from torch._C import _add_docstr, _nested # type: ignore[attr-defined] +from torch.types import _device as Device, _dtype as DType + + +__all__ = [ + "to_padded_tensor", + "as_nested_tensor", + "nested_tensor", + "nested_tensor_from_jagged", + "narrow", + "masked_select", +] + +# Allowlist these for weights_only load of NJT +from ._internal.nested_tensor import _rebuild_njt, NestedTensor as _NestedTensor + + +torch.serialization.add_safe_globals([_NestedTensor, _rebuild_njt]) + + +def as_nested_tensor( + ts: Tensor | list[Tensor] | tuple[Tensor, ...], + dtype: DType | None = None, + device: Device | None = None, + layout=None, +) -> Tensor: + r""" + Constructs a nested tensor preserving autograd history from a tensor or a list / tuple of + tensors. + + If a nested tensor is passed, it will be returned directly unless the device / dtype / layout + differ. Note that converting device / dtype will result in a copy, while converting layout + is not currently supported by this function. + + If a non-nested tensor is passed, it is treated as a batch of constituents of consistent size. + A copy will be incurred if the passed device / dtype differ from those of the input OR if + the input is non-contiguous. Otherwise, the input's storage will be used directly. + + If a tensor list is provided, tensors in the list are always copied during construction of + the nested tensor. + + Args: + ts (Tensor or List[Tensor] or Tuple[Tensor]): a tensor to treat as a nested tensor OR a + list / tuple of tensors with the same ndim + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.as_nested_tensor([a, b]) + >>> nt.is_leaf + False + >>> fake_grad = torch.nested.nested_tensor([torch.ones_like(a), torch.zeros_like(b)]) + >>> nt.backward(fake_grad) + >>> a.grad + tensor([1., 1., 1.]) + >>> b.grad + tensor([0., 0., 0., 0., 0.]) + >>> c = torch.randn(3, 5, requires_grad=True) + >>> nt2 = torch.nested.as_nested_tensor(c) + """ + is_tensor_list = isinstance(ts, (list, tuple)) and all( + isinstance(t, Tensor) for t in ts + ) + if not isinstance(ts, Tensor) and not is_tensor_list: + raise TypeError( + "as_nested_tensor(): Expected first argument to be a tensor or a list / tuple of tensors " + ) + # convert tuple -> list if needed + if is_tensor_list and not isinstance(ts, list): + ts = list(ts) + + if isinstance(ts, Tensor) and ts.dim() < 2: + raise RuntimeError( + "as_nested_tensor(): Expected tensor argument to have dim() > 1" + ) + + if isinstance(ts, Tensor) and ts.is_nested: + if layout == ts.layout: + # return input directly or input copied to device / dtype + return ts.to(device=device, dtype=dtype) + else: + # TODO: Just use nt.to(layout=layout) when it exists. + raise RuntimeError( + "as_nested_tensor(): Converting between nested tensor layouts is not supported" + ) + + if layout is None: + layout = torch.strided + if layout == torch.strided: + if isinstance(ts, Tensor): + # contiguous() might be necessary to get flattened view. + # we could probably be more precise about when to do this as an optimization + buffer = ts.contiguous().view(-1).to(device=device, dtype=dtype) + nested_sizes = torch.tensor([t.shape for t in ts]) + return torch._nested_view_from_buffer( + buffer, + nested_sizes, + *torch._nested_compute_contiguous_strides_offsets(nested_sizes), + ) + else: + assert isinstance(ts, list) + return torch._nested_tensor_from_tensor_list(ts, dtype, None, device, None) + elif layout == torch.jagged: + if isinstance(ts, Tensor): + if device is None: + device = ts.device + + # contiguous() might be necessary to get flattened view. + # we could probably be more precise about when to do this as an optimization + values = ts.contiguous().flatten(0, 1).to(device=device, dtype=dtype) + batch_size = ts.shape[0] + seq_len = ts.shape[1] + offsets = torch.arange( + 0, batch_size * seq_len + 1, seq_len, device=device, dtype=torch.int64 + ) + + from torch.nested._internal.nested_tensor import ( + nested_view_from_values_offsets, + ) + + return nested_view_from_values_offsets( + values, offsets, min_seqlen=seq_len, max_seqlen=seq_len + ) + else: + from torch.nested._internal.nested_tensor import jagged_from_list + + assert isinstance(ts, list) + nt, _ = jagged_from_list(ts, offsets=None, device=device, dtype=dtype) + return nt + else: + raise RuntimeError( + f"Specified layout is unsupported for nested tensors: {layout}" + ) + + +# Note: This not only adds doc strings for the nested ops, but +# also connects the torch.nested Python namespace to the torch._C._nested builtins. + +to_padded_tensor = _add_docstr( + _nested.nested_to_padded_tensor, + r""" +to_padded_tensor(input, padding, output_size=None, out=None) -> Tensor + +Returns a new (non-nested) Tensor by padding the :attr:`input` nested tensor. +The leading entries will be filled with the nested data, +while the trailing entries will be padded. + +.. warning:: + + :func:`to_padded_tensor` always copies the underlying data, + since the nested and the non-nested tensors differ in memory layout. + +Args: + padding (float): The padding value for the trailing entries. + +Keyword args: + output_size (Tuple[int]): The size of the output tensor. + If given, it must be large enough to contain all nested data; + else, will infer by taking the max size of each nested sub-tensor along each dimension. + out (Tensor, optional): the output tensor. + +Example:: + + >>> nt = torch.nested.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]) + nested_tensor([ + tensor([[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995]]), + tensor([[-1.8546, -0.7194, -0.2918, -0.1846], + [ 0.2773, 0.8793, -0.5183, -0.6447], + [ 1.8009, 1.8468, -0.9832, -1.5272]]) + ]) + >>> pt_infer = torch.nested.to_padded_tensor(nt, 0.0) + tensor([[[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]], + [[-1.8546, -0.7194, -0.2918, -0.1846, 0.0000], + [ 0.2773, 0.8793, -0.5183, -0.6447, 0.0000], + [ 1.8009, 1.8468, -0.9832, -1.5272, 0.0000]]]) + >>> pt_large = torch.nested.to_padded_tensor(nt, 1.0, (2, 4, 6)) + tensor([[[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276, 1.0000], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000]], + [[-1.8546, -0.7194, -0.2918, -0.1846, 1.0000, 1.0000], + [ 0.2773, 0.8793, -0.5183, -0.6447, 1.0000, 1.0000], + [ 1.8009, 1.8468, -0.9832, -1.5272, 1.0000, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000]]]) + >>> pt_small = torch.nested.to_padded_tensor(nt, 2.0, (2, 2, 2)) + RuntimeError: Value in output_size is less than NestedTensor padded size. Truncation is not supported. + +""", +) + + +def nested_tensor( + tensor_list, + *, + dtype=None, + layout=None, + device=None, + requires_grad=False, + pin_memory=False, +) -> Tensor: + r""" + Constructs a nested tensor with no autograd history (also known as a "leaf tensor", see + :ref:`Autograd mechanics `) from :attr:`tensor_list` a list of tensors. + + Args: + tensor_list (List[array_like]): a list of tensors, or anything that can be passed to torch.tensor, + where each element of the list has the same dimensionality. + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + requires_grad (bool, optional): If autograd should record operations on the + returned nested tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned nested tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.nested_tensor([a, b], requires_grad=True) + >>> nt.is_leaf + True + """ + if layout is None: + layout = torch.strided + if layout == torch.strided: + return _nested.nested_tensor( + tensor_list, + dtype=dtype, + device=device, + requires_grad=requires_grad, + pin_memory=pin_memory, + ) + elif layout == torch.jagged: + # Need to wrap lists of scalars as tensors + list_of_tensors = [ + t if isinstance(t, Tensor) else torch.as_tensor(t) for t in tensor_list + ] + + from torch.nested._internal.nested_tensor import jagged_from_list + + with torch.no_grad(): + nt, _ = jagged_from_list( + list_of_tensors, offsets=None, device=device, dtype=dtype + ) + + nt.requires_grad_(requires_grad) + if pin_memory: + nt = nt.pin_memory() # type: ignore[assignment] + + return nt + else: + raise RuntimeError( + f"Specified layout is unsupported for nested tensors: {layout}" + ) + + +def narrow( + tensor: Tensor, + dim: int, + start: int | Tensor, + length: int | Tensor, + layout=torch.strided, +) -> Tensor: + r""" + Constructs a nested tensor (which might be a view) from :attr:`tensor`, a strided tensor. This follows + similar semantics to torch.Tensor.narrow, where in the :attr:`dim`-th dimension the new nested tensor + shows only the elements in the interval `[start, start+length)`. As nested representations + allow for a different `start` and `length` at each 'row' of that dimension, :attr:`start` and :attr:`length` + can also be tensors of shape `tensor.shape[0]`. + + There's some differences depending on the layout you use for the nested tensor. If using strided layout, + torch.narrow will do a copy of the narrowed data into a contiguous NT with strided layout, while + jagged layout narrow() will create a non-contiguous view of your original strided tensor. This particular + representation is really useful for representing kv-caches in Transformer models, as specialized + SDPA kernels can deal with format easily, resulting in performance improvements. + + + Args: + tensor (:class:`torch.Tensor`): a strided tensor, which will be used as the underlying data + for the nested tensor if using the jagged layout or will be copied for the strided layout. + dim (int): the dimension where narrow will be applied. Only `dim=1` is supported for the + jagged layout, while strided supports all dim + start (Union[int, :class:`torch.Tensor`]): starting element for the narrow operation + length (Union[int, :class:`torch.Tensor`]): number of elements taken during the narrow op + + Keyword arguments: + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + + Example:: + + >>> starts = torch.tensor([0, 1, 2, 3, 4], dtype=torch.int64) + >>> lengths = torch.tensor([3, 2, 2, 1, 5], dtype=torch.int64) + >>> narrow_base = torch.randn(5, 10, 20) + >>> nt_narrowed = torch.nested.narrow(narrow_base, 1, starts, lengths, layout=torch.jagged) + >>> nt_narrowed.is_contiguous() + False + """ + if not isinstance(start, (int, SymInt, Tensor)): + raise RuntimeError("start must be an integer or a tensor") + + if not isinstance(length, (int, SymInt, Tensor)): + raise RuntimeError("length must be an integer or a tensor") + + if layout == torch.strided: + if isinstance(start, Tensor) or isinstance(length, Tensor): + raise RuntimeError( + "start and length must be integers for the strided layout NT impl" + ) + # TODO: switch to as_nested_tensor(tensor) when it is available + nt = as_nested_tensor(torch.unbind(tensor), layout=torch.strided).narrow( + dim, start, length + ) + elif layout == torch.jagged: + if dim != 1: + raise RuntimeError("jagged layout only supports dim=1") + + from torch.nested._internal.nested_tensor import jagged_from_tensor_and_lengths + + if isinstance(start, (int, SymInt)): + start = torch.tensor([start], device=tensor.device, dtype=torch.int64) + + if isinstance(length, (int, SymInt)): + length = torch.tensor([length], device=tensor.device, dtype=torch.int64) + + nt, _, _ = jagged_from_tensor_and_lengths(tensor, start, length) + else: + raise RuntimeError( + f"Specified layout is unsupported for nested narrow: {layout}" + ) + + return nt + + +def nested_tensor_from_jagged( + values: Tensor, + offsets: Tensor | None = None, + lengths: Tensor | None = None, + jagged_dim: int | None = None, + min_seqlen: int | None = None, + max_seqlen: int | None = None, +) -> Tensor: + r""" + Constructs a jagged layout nested tensor from the given jagged components. The jagged layout + consists of a required values buffer with the jagged dimension packed into a single dimension. + The offsets / lengths metadata determines how this dimension is split into batch elements + and are expected to be allocated on the same device as the values buffer. + + Expected metadata formats: + * offsets: Indices within the packed dimension splitting it into heterogeneously-sized + batch elements. Example: [0, 2, 3, 6] indicates that a packed jagged dim of size 6 + should be conceptually split into batch elements of length [2, 1, 3]. Note that both the + beginning and ending offsets are required for kernel convenience (i.e. shape batch_size + 1). + * lengths: Lengths of the individual batch elements; shape == batch_size. Example: [2, 1, 3] + indicates that a packed jagged dim of size 6 should be conceptually split into batch + elements of length [2, 1, 3]. + + Note that it can be useful to provide both offsets and lengths. This describes a nested tensor + with "holes", where the offsets indicate the start position of each batch item and the length + specifies the total number of elements (see example below). + + The returned jagged layout nested tensor will be a view of the input values tensor. + + Args: + values (:class:`torch.Tensor`): The underlying buffer in the shape of + (sum_B(*), D_1, ..., D_N). The jagged dimension is packed into a single dimension, + with the offsets / lengths metadata used to distinguish batch elements. + offsets (optional :class:`torch.Tensor`): Offsets into the jagged dimension of shape B + 1. + lengths (optional :class:`torch.Tensor`): Lengths of the batch elements of shape B. + jagged_dim (optional int): Indicates which dimension in values is the packed jagged + dimension. Must be >= 1 as the batch dimension (dim=0) cannot be ragged. + If None, this is set to dim=1 (i.e. the dimension immediately following the batch dimension). Default: None + min_seqlen (optional int): If set, uses the specified value as the cached minimum sequence + length for the returned nested tensor. This can be a useful alternative to computing + this value on-demand, possibly avoiding a GPU -> CPU sync. Default: None + max_seqlen (optional int): If set, uses the specified value as the cached maximum sequence + length for the returned nested tensor. This can be a useful alternative to computing + this value on-demand, possibly avoiding a GPU -> CPU sync. Default: None + + Example:: + + >>> values = torch.randn(12, 5) + >>> offsets = torch.tensor([0, 3, 5, 6, 10, 12]) + >>> nt = nested_tensor_from_jagged(values, offsets) + >>> # 3D shape with the middle dimension jagged + >>> nt.shape + torch.Size([5, j2, 5]) + >>> # Length of each item in the batch: + >>> offsets.diff() + tensor([3, 2, 1, 4, 2]) + + >>> values = torch.randn(6, 5) + >>> offsets = torch.tensor([0, 2, 3, 6]) + >>> lengths = torch.tensor([1, 1, 2]) + >>> # NT with holes + >>> nt = nested_tensor_from_jagged(values, offsets, lengths) + >>> a, b, c = nt.unbind() + >>> # Batch item 1 consists of indices [0, 1) + >>> torch.equal(a, values[0:1, :]) + True + >>> # Batch item 2 consists of indices [2, 3) + >>> torch.equal(b, values[2:3, :]) + True + >>> # Batch item 3 consists of indices [3, 5) + >>> torch.equal(c, values[3:5, :]) + True + """ + from torch.fx._symbolic_trace import is_fx_tracing + + if is_fx_tracing(): + raise RuntimeError( + "torch.nested.nested_tensor_from_jagged does not support tracing with fx.symbolic_trace. " + "Use fx.wrap to wrap the function that calls nested_tensor_from_jagged." + ) + + if offsets is None: + if lengths is None: + raise RuntimeError( + "nested_tensor_from_jagged(): At least one of offsets or lengths is required." + ) + else: + # TODO: Truly support offsets=None at some point? + # For now, just convert lengths -> offsets for kernel convenience + offsets = F.pad(lengths.cumsum(0), (1, 0)) + lengths = None + + if jagged_dim is None: + jagged_dim = 1 + elif jagged_dim < 1: + raise ValueError(f"Expected jagged_dim >=1, but got {jagged_dim}.") + + from torch.nested._internal.nested_tensor import ( + nested_view_from_values_offsets_lengths, + ) + + return nested_view_from_values_offsets_lengths( + values, + offsets, + lengths, + ragged_idx=jagged_dim, + min_seqlen=min_seqlen, + max_seqlen=max_seqlen, + ) + + +def masked_select(tensor: Tensor, mask: Tensor) -> Tensor: + r""" + Constructs a nested tensor given a strided tensor input and a strided mask, the resulting jagged layout nested tensor + will have values retain values where the mask is equal to True. The dimensionality of the mask is preserved and is + represented with the offsets, this is unlike :func:`masked_select` where the output is collapsed to a 1D tensor. + + Args: + tensor (:class:`torch.Tensor`): a strided tensor from which the jagged layout nested tensor is constructed from. + mask (:class:`torch.Tensor`): a strided mask tensor which is applied to the tensor input + + Example:: + + >>> tensor = torch.randn(3, 3) + >>> mask = torch.tensor([[False, False, True], [True, False, True], [False, False, True]]) + >>> nt = torch.nested.masked_select(tensor, mask) + >>> nt.shape + torch.Size([3, j4]) + >>> # Length of each item in the batch: + >>> nt.offsets().diff() + tensor([1, 2, 1]) + + >>> tensor = torch.randn(6, 5) + >>> mask = torch.tensor([False]) + >>> nt = torch.nested.masked_select(tensor, mask) + >>> nt.shape + torch.Size([6, j5]) + >>> # Length of each item in the batch: + >>> nt.offsets().diff() + tensor([0, 0, 0, 0, 0, 0]) + """ + if tensor.layout != torch.strided: + raise RuntimeError( + f"torch.nested.masked_select requires a strided tensor, given {tensor.layout}" + ) + + if mask.layout != torch.strided: + raise RuntimeError( + f"torch.nested.masked_select requires a strided mask, given: {mask.layout}" + ) + res_values = tensor.masked_select(mask) + expanded_mask = mask.expand(tensor.shape) + res_lengths = expanded_mask.sum(dim=tensor.ndim - 1).view(-1) + + from torch.nested._internal.nested_tensor import nested_view_from_values_offsets + + return nested_view_from_values_offsets( + values=res_values, + offsets=F.pad(res_lengths.cumsum(dim=0), (1, 0)), + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2214a4723d8b4b21290531af7e582cac259fc437 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/__init__.py @@ -0,0 +1,62 @@ +# mypy: allow-untyped-defs +from torch.nn.parameter import ( # usort: skip + Buffer as Buffer, + Parameter as Parameter, + UninitializedBuffer as UninitializedBuffer, + UninitializedParameter as UninitializedParameter, +) +from torch.nn.modules import * # usort: skip # noqa: F403 +from torch.nn import ( + attention as attention, + functional as functional, + init as init, + modules as modules, + parallel as parallel, + parameter as parameter, + utils as utils, +) +from torch.nn.parallel import DataParallel as DataParallel + + +def factory_kwargs(kwargs): + r"""Return a canonicalized dict of factory kwargs. + + Given kwargs, returns a canonicalized dict of factory kwargs that can be directly passed + to factory functions like torch.empty, or errors if unrecognized kwargs are present. + + This function makes it simple to write code like this:: + + class MyModule(nn.Module): + def __init__(self, **kwargs): + factory_kwargs = torch.nn.factory_kwargs(kwargs) + self.weight = Parameter(torch.empty(10, **factory_kwargs)) + + Why should you use this function instead of just passing `kwargs` along directly? + + 1. This function does error validation, so if there are unexpected kwargs we will + immediately report an error, instead of deferring it to the factory call + 2. This function supports a special `factory_kwargs` argument, which can be used to + explicitly specify a kwarg to be used for factory functions, in the event one of the + factory kwargs conflicts with an already existing argument in the signature (e.g. + in the signature ``def f(dtype, **kwargs)``, you can specify ``dtype`` for factory + functions, as distinct from the dtype argument, by saying + ``f(dtype1, factory_kwargs={"dtype": dtype2})``) + """ + if kwargs is None: + return {} + simple_keys = {"device", "dtype", "memory_format"} + expected_keys = simple_keys | {"factory_kwargs"} + if not kwargs.keys() <= expected_keys: + raise TypeError(f"unexpected kwargs {kwargs.keys() - expected_keys}") + + # guarantee no input kwargs is untouched + r = dict(kwargs.get("factory_kwargs", {})) + for k in simple_keys: + if k in kwargs: + if k in r: + raise TypeError( + f"{k} specified twice, in **kwargs and in factory_kwargs" + ) + r[k] = kwargs[k] + + return r diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/_reduction.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/_reduction.py new file mode 100644 index 0000000000000000000000000000000000000000..bf26a8529fd94bb6cab7447c3c331b30d8caa08c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/_reduction.py @@ -0,0 +1,60 @@ +import warnings + + +# NB: Keep this file in sync with enums in aten/src/ATen/core/Reduction.h + + +def get_enum(reduction: str) -> int: + if reduction == "none": + ret = 0 + elif reduction == "mean": + ret = 1 + elif reduction == "elementwise_mean": + warnings.warn( + "reduction='elementwise_mean' is deprecated. " + "Please use reduction='mean' instead.", + stacklevel=2, + ) + ret = 1 + elif reduction == "sum": + ret = 2 + else: + ret = -1 # TODO: remove once JIT exceptions support control flow + raise ValueError(f"{reduction} is not a valid value for reduction") + return ret + + +# In order to support previous versions, accept boolean size_average and reduce +# and convert them into the new constants for now + + +# We use these functions in torch/legacy as well, in which case we'll silence the warning +def legacy_get_string( + size_average: bool | None, + reduce: bool | None, + emit_warning: bool = True, +) -> str: + warning = "size_average and reduce args will be deprecated, please use reduction='{}' instead." + + if size_average is None: + size_average = True + if reduce is None: + reduce = True + + if size_average and reduce: + ret = "mean" + elif reduce: + ret = "sum" + else: + ret = "none" + if emit_warning: + warnings.warn(warning.format(ret), stacklevel=2) + return ret + + +def legacy_get_enum( + size_average: bool | None, + reduce: bool | None, + emit_warning: bool = True, +) -> int: + return get_enum(legacy_get_string(size_average, reduce, emit_warning)) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/common_types.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/common_types.py new file mode 100644 index 0000000000000000000000000000000000000000..78ccaf026f8e32efb6b8b9800847bc325e735136 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/common_types.py @@ -0,0 +1,46 @@ +from typing import TypeAlias as _TypeAlias, TypeVar + +from torch import Tensor + + +# ruff: noqa: PYI042,PYI047 + +# Create some useful type aliases + +# Template for arguments which can be supplied as a tuple, or which can be a scalar which PyTorch will internally +# broadcast to a tuple. +# Comes in several variants: A tuple of unknown size, and a fixed-size tuple for 1d, 2d, or 3d operations. +T = TypeVar("T") +_scalar_or_tuple_any_t: _TypeAlias = T | tuple[T, ...] +_scalar_or_tuple_1_t: _TypeAlias = T | tuple[T] +_scalar_or_tuple_2_t: _TypeAlias = T | tuple[T, T] +_scalar_or_tuple_3_t: _TypeAlias = T | tuple[T, T, T] +_scalar_or_tuple_4_t: _TypeAlias = T | tuple[T, T, T, T] +_scalar_or_tuple_5_t: _TypeAlias = T | tuple[T, T, T, T, T] +_scalar_or_tuple_6_t: _TypeAlias = T | tuple[T, T, T, T, T, T] + +# For arguments which represent size parameters (eg, kernel size, padding) +_size_any_t: _TypeAlias = _scalar_or_tuple_any_t[int] +_size_1_t: _TypeAlias = _scalar_or_tuple_1_t[int] +_size_2_t: _TypeAlias = _scalar_or_tuple_2_t[int] +_size_3_t: _TypeAlias = _scalar_or_tuple_3_t[int] +_size_4_t: _TypeAlias = _scalar_or_tuple_4_t[int] +_size_5_t: _TypeAlias = _scalar_or_tuple_5_t[int] +_size_6_t: _TypeAlias = _scalar_or_tuple_6_t[int] + +# For arguments which represent optional size parameters (eg, adaptive pool parameters) +_size_any_opt_t: _TypeAlias = _scalar_or_tuple_any_t[int | None] +_size_2_opt_t: _TypeAlias = _scalar_or_tuple_2_t[int | None] +_size_3_opt_t: _TypeAlias = _scalar_or_tuple_3_t[int | None] + +# For arguments that represent a ratio to adjust each dimension of an input with (eg, upsampling parameters) +_ratio_2_t: _TypeAlias = _scalar_or_tuple_2_t[float] +_ratio_3_t: _TypeAlias = _scalar_or_tuple_3_t[float] +_ratio_any_t: _TypeAlias = _scalar_or_tuple_any_t[float] + +_tensor_list_t: _TypeAlias = _scalar_or_tuple_any_t[Tensor] + +# For the return value of max pooling operations that may or may not return indices. +# With the proposed 'Literal' feature to Python typing, it might be possible to +# eventually eliminate this. +_maybe_indices_t: _TypeAlias = _scalar_or_tuple_2_t[Tensor] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/cpp.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/cpp.py new file mode 100644 index 0000000000000000000000000000000000000000..6a8a2bbce6bf4d1eb7c974ebf34e43020b6f7386 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/cpp.py @@ -0,0 +1,90 @@ +# mypy: allow-untyped-defs +"""Functionality for Python <-> C++ frontend inter-op.""" + +from torch import nn + + +class OrderedDictWrapper: + """A wrapper around a C++ OrderedDict. + + It dynamically evaluates the OrderedDict getter on a bound C++ module, such + that new changes on the C++ side are picked up. Otherwise accessing e.g. + ``cpp_module._parameters`` just once would get a frozen copy of the parameters + at the time of access. ``torch.nn.Module`` accesses ``_parameters`` et al. via ``self.__dict__`` + so using properties does not work. + """ + + def __init__(self, cpp_module, attr) -> None: + self.cpp_module = cpp_module + self.attr = attr + + @property + def cpp_dict(self): + return getattr(self.cpp_module, self.attr) + + # Magic methods cannot be assigned dynamically and bypass ``getattr``, so we + # must manually override them. + + def items(self): + return self.cpp_dict.items() + + def keys(self): + return self.cpp_dict.keys() + + def values(self): + return self.cpp_dict.values() + + def __iter__(self): + return self.cpp_dict.__iter__() + + def __len__(self) -> int: + return self.cpp_dict.__len__() + + def __contains__(self, key) -> bool: + return self.cpp_dict.__contains__(key) + + def __getitem__(self, key): + return self.cpp_dict.__getitem__(key) + + +class ModuleWrapper(nn.Module): + """A subclass of ``torch.nn.Module`` that wraps a C++ frontend module and delegates all access.""" + + def __init__(self, cpp_module) -> None: + # Assign before the super class constructor so ``self.training`` can be + # assigned to in the super class constructor. + self.cpp_module = cpp_module + super().__init__() + self._parameters = OrderedDictWrapper(cpp_module, "_parameters") # type: ignore[assignment] + self._buffers: OrderedDictWrapper = OrderedDictWrapper(cpp_module, "_buffers") # type: ignore[assignment] + self._modules: OrderedDictWrapper = OrderedDictWrapper(cpp_module, "_modules") # type: ignore[assignment] + for attr in dir(cpp_module): + # Skip magic methods and the three attributes above. + if not attr.startswith("_"): + setattr(self, attr, getattr(self.cpp_module, attr)) + + def _apply(self, fn, recurse=True): + for param in self.parameters(): + # Tensors stored in modules are graph leaves, and we don't + # want to create copy nodes, so we have to unpack the data. + param.data = fn(param.data) + if param._grad is not None: + param._grad.data = fn(param._grad.data) + + for buf in self.buffers(): + buf.data = fn(buf.data) + + return self + + # nn.Module defines training as a boolean + @property # type: ignore[override] + # pyrefly: ignore [bad-override] + def training(self): + return self.cpp_module.training + + @training.setter + def training(self, mode) -> None: + self.cpp_module.train(mode) + + def __repr__(self) -> str: + return self.cpp_module.__repr__() diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f0199e83c6b2c6148eabfd47081c4fde6dd9fe --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.py @@ -0,0 +1,6859 @@ +"""Functional interface.""" + +import importlib +import math +import warnings +from collections.abc import Callable +from typing import Any as _Any, Optional, TYPE_CHECKING + +import torch +from torch import _VF, sym_int as _sym_int, Tensor +from torch._C import ( + _add_docstr, + _infer_size, + _ScalingType as ScalingType, + _SwizzleType as SwizzleType, +) +from torch._jit_internal import ( + _overload, + boolean_dispatch, + BroadcastingList1, + BroadcastingList2, + BroadcastingList3, +) +from torch._torch_docs import reproducibility_notes, sparse_support_notes, tf32_notes +from torch.nn import _reduction as _Reduction, grad # noqa: F401 +from torch.nn.modules.utils import _list_with_default, _pair, _single, _triple +from torch.overrides import ( + handle_torch_function, + has_torch_function, + has_torch_function_unary, + has_torch_function_variadic, +) + + +# Set visibility of the bound enums to this module +ScalingType.__module__ = "torch.nn.functional" +SwizzleType.__module__ = "torch.nn.functional" + +if TYPE_CHECKING: + from torch.types import _dtype as DType +else: + # The JIT doesn't understand Union, nor torch.dtype here + DType = int + +try: + import numpy as np +except ModuleNotFoundError: + np = None + + +conv1d = _add_docstr( + torch.conv1d, + r""" +conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 1D convolution over an input signal composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv1d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: ``None`` + stride: the stride of the convolving kernel. Can be a single number or + a one-element tuple `(sW,)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a one-element tuple `(padW,)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + dilation: the spacing between kernel elements. Can be a single number or + a one-element tuple `(dW,)`. Default: 1 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by + the number of groups. Default: 1 + +Examples:: + + >>> inputs = torch.randn(33, 16, 30) + >>> filters = torch.randn(20, 16, 5) + >>> F.conv1d(inputs, filters) +""", +) + +conv2d = _add_docstr( + torch.conv2d, + r""" +conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 2D convolution over an input image composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv2d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kH , kW)` + bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: ``None`` + stride: the stride of the convolving kernel. Can be a single number or a + tuple `(sH, sW)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a tuple `(padH, padW)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dH, dW)`. Default: 1 + groups: split input into groups, both :math:`\text{in\_channels}` and :math:`\text{out\_channels}` + should be divisible by the number of groups. Default: 1 + +Examples:: + + >>> # With square kernels and equal stride + >>> filters = torch.randn(8, 4, 3, 3) + >>> inputs = torch.randn(1, 4, 5, 5) + >>> F.conv2d(inputs, filters, padding=1) +""", +) # noqa: E501 + +conv3d = _add_docstr( + torch.conv3d, + r""" +conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 3D convolution over an input image composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv3d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kT , kH , kW)` + bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple `(sT, sH, sW)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a tuple `(padT, padH, padW)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dT, dH, dW)`. Default: 1 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by + the number of groups. Default: 1 + +Examples:: + + >>> filters = torch.randn(33, 16, 3, 3, 3) + >>> inputs = torch.randn(20, 16, 50, 10, 20) + >>> F.conv3d(inputs, filters) +""", +) # noqa: E501 + +conv_transpose1d = _add_docstr( + torch.conv_transpose1d, + r""" +conv_transpose1d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 1D transposed convolution operator over an input signal +composed of several input planes, sometimes also called "deconvolution". + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose1d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sW,)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padW,)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple ``(out_padW)``. Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple ``(dW,)``. Default: 1 + +Examples:: + + >>> inputs = torch.randn(20, 16, 50) + >>> weights = torch.randn(16, 33, 5) + >>> F.conv_transpose1d(inputs, weights) +""", +) + +conv_transpose2d = _add_docstr( + torch.conv_transpose2d, + r""" +conv_transpose2d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 2D transposed convolution operator over an input image +composed of several input planes, sometimes also called "deconvolution". + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose2d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kH , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sH, sW)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padH, padW)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple ``(out_padH, out_padW)``. + Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple ``(dH, dW)``. Default: 1 + +Examples:: + + >>> # With square kernels and equal stride + >>> inputs = torch.randn(1, 4, 5, 5) + >>> weights = torch.randn(4, 8, 3, 3) + >>> F.conv_transpose2d(inputs, weights, padding=1) +""", +) # noqa: E501 + +conv_transpose3d = _add_docstr( + torch.conv_transpose3d, + r""" +conv_transpose3d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 3D transposed convolution operator over an input image +composed of several input planes, sometimes also called "deconvolution" + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose3d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kT , kH , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sT, sH, sW)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padT, padH, padW)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple + ``(out_padT, out_padH, out_padW)``. Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dT, dH, dW)`. Default: 1 + +Examples:: + + >>> inputs = torch.randn(20, 16, 50, 10, 20) + >>> weights = torch.randn(16, 33, 3, 3, 3) + >>> F.conv_transpose3d(inputs, weights) +""", +) # noqa: E501 + +conv_tbc = _add_docstr( + torch.conv_tbc, + r""" +Applies a 1-dimensional sequence convolution over an input sequence. +Input and output dimensions are (Time, Batch, Channels) - hence TBC. + +Args: + input: input tensor of shape :math:`(\text{sequence length} \times batch \times \text{in\_channels})` + weight: filter of shape (:math:`\text{kernel width} \times \text{in\_channels} \times \text{out\_channels}`) + bias: bias of shape (:math:`\text{out\_channels}`) + pad: number of timesteps to pad. Default: 0 +""", +) + + +# Pooling +avg_pool1d = _add_docstr( + torch.avg_pool1d, + r""" +avg_pool1d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True) -> Tensor + +Applies a 1D average pooling over an input signal composed of several +input planes. + +See :class:`~torch.nn.AvgPool1d` for details and output shape. + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + kernel_size: the size of the window. Can be a single number or a + tuple `(kW,)` + stride: the stride of the window. Can be a single number or a tuple + `(sW,)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a single + number or a tuple `(padW,)`. Should be at most half of effective kernel + size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` to compute the + output shape. Default: ``False`` + count_include_pad: when True, will include the zero-padding in the + averaging calculation. Default: ``True`` + +Examples:: + + >>> # pool of square window of size=3, stride=2 + >>> input = torch.tensor([[[1, 2, 3, 4, 5, 6, 7]]], dtype=torch.float32) + >>> F.avg_pool1d(input, kernel_size=3, stride=2) + tensor([[[ 2., 4., 6.]]]) + +""", +) + + +avg_pool2d = _add_docstr( + torch._C._nn.avg_pool2d, + r""" +avg_pool2d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor + +Applies 2D average-pooling operation in :math:`kH \times kW` regions by step size +:math:`sH \times sW` steps. The number of output features is equal to the number of +input planes. + +See :class:`~torch.nn.AvgPool2d` for details and output shape. + +Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + kernel_size: size of the pooling region. Can be a single number, a single-element tuple or a + tuple `(kH, kW)` + stride: stride of the pooling operation. Can be a single number, a single-element tuple or a + tuple `(sH, sW)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a + single number, a single-element tuple or a tuple `(padH, padW)`. + Should be at most half of effective kernel size, that + is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` in the formula + to compute the output shape. Default: ``False`` + count_include_pad: when True, will include the zero-padding in the + averaging calculation. Default: ``True`` + divisor_override: if specified, it will be used as divisor, otherwise + size of the pooling region will be used. Default: None +""", +) + +avg_pool3d = _add_docstr( + torch._C._nn.avg_pool3d, + r""" +avg_pool3d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor + +Applies 3D average-pooling operation in :math:`kT \times kH \times kW` regions by step +size :math:`sT \times sH \times sW` steps. The number of output features is equal to +:math:`\lfloor\frac{\text{input planes}}{sT}\rfloor`. + +See :class:`~torch.nn.AvgPool3d` for details and output shape. + +Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iT \times iH , iW)` + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kT, kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a + single number or a tuple `(padT, padH, padW)`. Should be at most half + of effective kernel size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. + Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` in the formula + to compute the output shape + count_include_pad: when True, will include the zero-padding in the + averaging calculation + divisor_override: if specified, it will be used as divisor, otherwise + size of the pooling region will be used. Default: None +""", +) + + +def fractional_max_pool2d_with_indices( + input: Tensor, + kernel_size: BroadcastingList2[int], + output_size: Optional[BroadcastingList2[int]] = None, + output_ratio: Optional[BroadcastingList2[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + fractional_max_pool2d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) + + Applies 2D fractional max pooling over an input signal composed of several input planes. + + Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham + + The max-pooling operation is applied in :math:`kH \times kW` regions by a stochastic + step size determined by the target output size. + The number of output features is equal to the number of input planes. + + Args: + kernel_size: the size of the window to take a max over. + Can be a single number :math:`k` (for a square kernel of :math:`k \times k`) + or a tuple `(kH, kW)` + output_size: the target output size of the image of the form :math:`oH \times oW`. + Can be a tuple `(oH, oW)` or a single number :math:`oH` for a square image :math:`oH \times oH` + output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. + This has to be a number or tuple in the range (0, 1) + return_indices: if ``True``, will return the indices along with the outputs. + Useful to pass to :func:`~torch.nn.functional.max_unpool2d`. + + Examples:: + >>> input = torch.randn(20, 16, 50, 32) + >>> # pool of square window of size=3, and target output size 13x12 + >>> F.fractional_max_pool2d(input, 3, output_size=(13, 12)) + >>> # pool of square window and target output size being half of input image size + >>> F.fractional_max_pool2d(input, 3, output_ratio=(0.5, 0.5)) + + .. _Fractional MaxPooling: + http://arxiv.org/abs/1412.6071 + """ + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool2d_with_indices, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + if output_size is None and output_ratio is None: + raise ValueError( + "fractional_max_pool2d requires specifying either an output_size or an output_ratio" + ) + if output_size is None: + assert output_ratio is not None + if len(output_ratio) > 2: + raise ValueError( + "fractional_max_pool2d requires output_ratio to either be a single Int or tuple of Ints." + ) + _output_ratio = _pair(output_ratio) + output_size = [ + int(input.size(-2) * _output_ratio[0]), + int(input.size(-1) * _output_ratio[1]), + ] + + if _random_samples is None: + n_batch = 1 if input.dim() == 3 else input.size(0) + _random_samples = torch.rand( + n_batch, input.size(-3), 2, dtype=input.dtype, device=input.device + ) + return torch._C._nn.fractional_max_pool2d( + input, kernel_size, output_size, _random_samples + ) + + +def _fractional_max_pool2d( + input: Tensor, + kernel_size: BroadcastingList2[int], + output_size: Optional[BroadcastingList2[int]] = None, + output_ratio: Optional[BroadcastingList2[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> Tensor: + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool2d, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + return fractional_max_pool2d_with_indices( + input, kernel_size, output_size, output_ratio, return_indices, _random_samples + )[0] + + +fractional_max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=4, + default=False, + if_true=fractional_max_pool2d_with_indices, + if_false=_fractional_max_pool2d, + module_name=__name__, + func_name="fractional_max_pool2d", +) + + +def fractional_max_pool3d_with_indices( + input: Tensor, + kernel_size: BroadcastingList3[int], + output_size: Optional[BroadcastingList3[int]] = None, + output_ratio: Optional[BroadcastingList3[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + fractional_max_pool3d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) + + Applies 3D fractional max pooling over an input signal composed of several input planes. + + Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham + + The max-pooling operation is applied in :math:`kT \times kH \times kW` regions by a stochastic + step size determined by the target output size. + The number of output features is equal to the number of input planes. + + Args: + kernel_size: the size of the window to take a max over. + Can be a single number :math:`k` (for a square kernel of :math:`k \times k \times k`) + or a tuple `(kT, kH, kW)` + output_size: the target output size of the form :math:`oT \times oH \times oW`. + Can be a tuple `(oT, oH, oW)` or a single number :math:`oH` for a cubic output + :math:`oH \times oH \times oH` + output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. + This has to be a number or tuple in the range (0, 1) + return_indices: if ``True``, will return the indices along with the outputs. + Useful to pass to :func:`~torch.nn.functional.max_unpool3d`. + + Shape: + - Input: :math:`(N, C, T_{in}, H_{in}, W_{in})` or :math:`(C, T_{in}, H_{in}, W_{in})`. + - Output: :math:`(N, C, T_{out}, H_{out}, W_{out})` or :math:`(C, T_{out}, H_{out}, W_{out})`, where + :math:`(T_{out}, H_{out}, W_{out})=\text{output\_size}` or + :math:`(T_{out}, H_{out}, W_{out})=\text{output\_ratio} \times (T_{in}, H_{in}, W_{in})` + + Examples:: + >>> input = torch.randn(20, 16, 50, 32, 16) + >>> # pool of cubic window of size=3, and target output size 13x12x11 + >>> F.fractional_max_pool3d(input, 3, output_size=(13, 12, 11)) + >>> # pool of cubic window and target output size being half of input size + >>> F.fractional_max_pool3d(input, 3, output_ratio=(0.5, 0.5, 0.5)) + + .. _Fractional MaxPooling: + http://arxiv.org/abs/1412.6071 + """ + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool3d_with_indices, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + if output_size is None and output_ratio is None: + raise ValueError( + "fractional_max_pool3d requires specifying either an output_size or an output_ratio" + ) + if output_size is None: + assert output_ratio is not None + _output_ratio = _triple(output_ratio) + output_size = [ + int(input.size(-3) * _output_ratio[0]), + int(input.size(-2) * _output_ratio[1]), + int(input.size(-1) * _output_ratio[2]), + ] + + if _random_samples is None: + n_batch = 1 if input.dim() == 4 else input.size(0) + _random_samples = torch.rand( + n_batch, input.size(-4), 3, dtype=input.dtype, device=input.device + ) + return torch._C._nn.fractional_max_pool3d( + input, kernel_size, output_size, _random_samples + ) + + +def _fractional_max_pool3d( + input: Tensor, + kernel_size: BroadcastingList3[int], + output_size: Optional[BroadcastingList3[int]] = None, + output_ratio: Optional[BroadcastingList3[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> Tensor: + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool3d, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + return fractional_max_pool3d_with_indices( + input, kernel_size, output_size, output_ratio, return_indices, _random_samples + )[0] + + +fractional_max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=4, + default=False, + if_true=fractional_max_pool3d_with_indices, + if_false=_fractional_max_pool3d, + module_name=__name__, + func_name="fractional_max_pool3d", +) + + +def max_pool1d_with_indices( + input: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + dilation: BroadcastingList1[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool1d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 1D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool1d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool1d` for details. + + Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`, minibatch dim optional. + kernel_size: the size of the window. Can be a single number or a + tuple `(kW,)` + stride: the stride of the window. Can be a single number or a tuple + `(sW,)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool1d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool1d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool1d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool1d( + input: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + dilation: BroadcastingList1[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool1d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool1d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool1d_with_indices, + if_false=_max_pool1d, + module_name=__name__, + func_name="max_pool1d", +) + + +def max_pool2d_with_indices( + input: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + dilation: BroadcastingList2[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool2d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 2D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool2d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool2d` for details. + + Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`, minibatch dim optional. + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sH, sW)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool2d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool2d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch._C._nn.max_pool2d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool2d( + input: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + dilation: BroadcastingList2[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool2d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool2d_with_indices, + if_false=_max_pool2d, + module_name=__name__, + func_name="max_pool2d", +) + + +def max_pool3d_with_indices( + input: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + dilation: BroadcastingList3[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool3d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 3D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool3d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool3d` for details. + + Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iD, iH , iW)`, minibatch dim optional. + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kT, kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool3d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool3d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch._C._nn.max_pool3d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool3d( + input: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + dilation: BroadcastingList3[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool3d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool3d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool3d_with_indices, + if_false=_max_pool3d, + module_name=__name__, + func_name="max_pool3d", +) + + +def _unpool_output_size( + input: Tensor, + kernel_size: list[int], + stride: list[int], + padding: list[int], + output_size: Optional[list[int]], +) -> list[int]: + input_size = input.size() + default_size = torch.jit.annotate(list[int], []) + for d in range(len(kernel_size)): + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) + if output_size is None: + ret = default_size + else: + if len(output_size) == len(kernel_size) + 2: + output_size = output_size[2:] + if len(output_size) != len(kernel_size): + raise ValueError( + "output_size should be a sequence containing " + f"{len(kernel_size)} or {len(kernel_size) + 2} elements, but it has a length of '{len(output_size)}'" + ) + for d in range(len(kernel_size)): + min_size = default_size[d] - stride[d] + max_size = default_size[d] + stride[d] + if not (min_size < output_size[d] < max_size): + raise ValueError( + f'invalid output_size "{output_size}" (dim {d} must be between {min_size} and {max_size})' + ) + + ret = output_size + return ret + + +def max_unpool1d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + output_size: Optional[BroadcastingList1[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool1d`. + + See :class:`~torch.nn.MaxUnpool1d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool1d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _single(kernel_size) + if stride is not None: + _stride = _single(stride) + else: + _stride = kernel_size + padding = _single(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + if isinstance(output_size, list): + output_size = output_size + [1] + else: + output_size = output_size + (1,) + return torch._C._nn.max_unpool2d( + input.unsqueeze(-1), indices.unsqueeze(-1), output_size + ).squeeze(-1) + + +def max_unpool2d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + output_size: Optional[BroadcastingList2[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool2d`. + + See :class:`~torch.nn.MaxUnpool2d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool2d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _pair(kernel_size) + if stride is not None: + _stride = _pair(stride) + else: + _stride = kernel_size + padding = _pair(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + return torch._C._nn.max_unpool2d(input, indices, output_size) + + +def max_unpool3d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + output_size: Optional[BroadcastingList3[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool3d`. + + See :class:`~torch.nn.MaxUnpool3d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool3d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _triple(kernel_size) + if stride is not None: + _stride = _triple(stride) + else: + _stride = kernel_size + padding = _triple(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + return torch._C._nn.max_unpool3d(input, indices, output_size, _stride, padding) + + +def lp_pool3d( + input: Tensor, + norm_type: int | float, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r""" + Apply a 3D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool3d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool3d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + kd, kw, kh = _triple(kernel_size) + if stride is not None: + out = avg_pool3d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool3d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return ( + (torch.sign(out) * relu(torch.abs(out))).mul(kd * kw * kh).pow(1.0 / norm_type) + ) + + +def lp_pool2d( + input: Tensor, + norm_type: int | float, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r""" + Apply a 2D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool2d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool2d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + kw, kh = _pair(kernel_size) + if stride is not None: + out = avg_pool2d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool2d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return (torch.sign(out) * relu(torch.abs(out))).mul(kw * kh).pow(1.0 / norm_type) + + +def lp_pool1d( + input: Tensor, + norm_type: int | float, + kernel_size: int, + stride: Optional[BroadcastingList1[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r"""Apply a 1D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool1d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool1d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + if stride is not None: + out = avg_pool1d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool1d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return ( + (torch.sign(out) * relu(torch.abs(out))).mul(kernel_size).pow(1.0 / norm_type) + ) + + +def adaptive_max_pool1d_with_indices( + input: Tensor, + output_size: BroadcastingList1[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + adaptive_max_pool1d(input, output_size, return_indices=False) + + Applies a 1D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool1d` for details and output shape. + + Args: + output_size: the target output size (single integer) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool1d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return torch.adaptive_max_pool1d(input, output_size) + + +def _adaptive_max_pool1d( + input: Tensor, + output_size: BroadcastingList1[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool1d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool1d_with_indices(input, output_size)[0] + + +adaptive_max_pool1d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool1d_with_indices, + if_false=_adaptive_max_pool1d, + module_name=__name__, + func_name="adaptive_max_pool1d", +) + + +def adaptive_max_pool2d_with_indices( + input: Tensor, + output_size: BroadcastingList2[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r"""adaptive_max_pool2d(input, output_size, return_indices=False) + + Applies a 2D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool2d` for details and output shape. + + Args: + output_size: the target output size (single integer or + double-integer tuple) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool2d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + # pyrefly: ignore [bad-argument-type] + output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_max_pool2d(input, output_size) + + +def _adaptive_max_pool2d( + input: Tensor, + output_size: BroadcastingList2[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool2d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool2d_with_indices(input, output_size)[0] + + +adaptive_max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool2d_with_indices, + if_false=_adaptive_max_pool2d, + module_name=__name__, + func_name="adaptive_max_pool2d", +) + + +def adaptive_max_pool3d_with_indices( + input: Tensor, + output_size: BroadcastingList3[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + adaptive_max_pool3d(input, output_size, return_indices=False) + + Applies a 3D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool3d` for details and output shape. + + Args: + output_size: the target output size (single integer or + triple-integer tuple) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool3d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + # pyrefly: ignore [bad-argument-type] + output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_max_pool3d(input, output_size) + + +def _adaptive_max_pool3d( + input: Tensor, + output_size: BroadcastingList3[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool3d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool3d_with_indices(input, output_size)[0] + + +adaptive_max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool3d_with_indices, + if_false=_adaptive_max_pool3d, + module_name=__name__, + func_name="adaptive_max_pool3d", +) + + +adaptive_avg_pool1d = _add_docstr( + torch.adaptive_avg_pool1d, + r""" +adaptive_avg_pool1d(input, output_size) -> Tensor + +Applies a 1D adaptive average pooling over an input signal composed of +several input planes. + +See :class:`~torch.nn.AdaptiveAvgPool1d` for details and output shape. + +Args: + output_size: the target output size (single integer) +""", +) + + +def adaptive_avg_pool2d(input: Tensor, output_size: BroadcastingList2[int]) -> Tensor: + r"""Apply a 2D adaptive average pooling over an input signal composed of several input planes. + + See :class:`~torch.nn.AdaptiveAvgPool2d` for details and output shape. + + Args: + output_size: the target output size (single integer or + double-integer tuple) + """ + if has_torch_function_unary(input): + return handle_torch_function(adaptive_avg_pool2d, (input,), input, output_size) + # pyrefly: ignore [bad-argument-type] + _output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_avg_pool2d(input, _output_size) + + +def adaptive_avg_pool3d(input: Tensor, output_size: BroadcastingList3[int]) -> Tensor: + r"""Apply a 3D adaptive average pooling over an input signal composed of several input planes. + + See :class:`~torch.nn.AdaptiveAvgPool3d` for details and output shape. + + Args: + output_size: the target output size (single integer or + triple-integer tuple) + """ + if has_torch_function_unary(input): + return handle_torch_function(adaptive_avg_pool3d, (input,), input, output_size) + # pyrefly: ignore [bad-argument-type] + _output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_avg_pool3d(input, _output_size) + + +# Activation functions +def dropout( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""During training, randomly zeroes some elements of the input tensor with probability :attr:`p`. + + Uses samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout` for details. + + Args: + p: probability of an element to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training) + ) + + +def alpha_dropout( + input: Tensor, + p: float = 0.5, + training: bool = False, + inplace: bool = False, +) -> Tensor: + r"""Apply alpha dropout to the input. + + See :class:`~torch.nn.AlphaDropout` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + alpha_dropout, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.alpha_dropout_(input, p, training) + if inplace + else _VF.alpha_dropout(input, p, training) + ) + + +def dropout1d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 1D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 1D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout1d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout1d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (2, 3): + raise RuntimeError( + f"dropout1d: Expected 2D or 3D input, but received a {inp_dim}D input. " + "Note that dropout1d exists to provide channel-wise dropout on inputs with 1 " + "spatial dimension, a channel dimension, and an optional batch dimension " + "(i.e. 2D or 3D inputs)." + ) + + is_batched = inp_dim == 3 + if not is_batched: + input = input.unsqueeze_(0) if inplace else input.unsqueeze(0) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + if not is_batched: + result = result.squeeze_(0) if inplace else result.squeeze(0) + + return result + + +def dropout2d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 2D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 2D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout2d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout2d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (3, 4): + warn_msg = ( + f"dropout2d: Received a {inp_dim}-D input to dropout2d, which is deprecated " + "and will result in an error in a future release. To retain the behavior " + "and silence this warning, please use dropout instead. Note that dropout2d " + "exists to provide channel-wise dropout on inputs with 2 spatial dimensions, " + "a channel dimension, and an optional batch dimension (i.e. 3D or 4D inputs)." + ) + warnings.warn(warn_msg, stacklevel=2) + + # TODO: Properly support no-batch-dim inputs. For now, these are NOT supported; passing + # a 3D input will perform dropout1d behavior instead. This was done historically and the + # behavior is maintained here for now. + # See https://github.com/pytorch/pytorch/issues/77081 + if inp_dim == 3: + warnings.warn( + "dropout2d: Received a 3D input to dropout2d and assuming that channel-wise " + "1D dropout behavior is desired - input is interpreted as shape (N, C, L), where C " + "is the channel dim. This behavior will change in a future release to interpret the " + "input as one without a batch dimension, i.e. shape (C, H, W). To maintain the 1D " + "channel-wise dropout behavior, please switch to using dropout1d instead.", + stacklevel=2, + ) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + return result + + +def dropout3d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 3D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 3D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout3d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout3d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (4, 5): + warn_msg = ( + f"dropout3d: Received a {inp_dim}-D input to dropout3d, which is deprecated " + "and will result in an error in a future release. To retain the behavior " + "and silence this warning, please use dropout instead. Note that dropout3d " + "exists to provide channel-wise dropout on inputs with 3 spatial dimensions, " + "a channel dimension, and an optional batch dimension (i.e. 4D or 5D inputs)." + ) + warnings.warn(warn_msg, stacklevel=2) + + is_batched = inp_dim == 5 + if not is_batched: + input = input.unsqueeze_(0) if inplace else input.unsqueeze(0) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + if not is_batched: + result = result.squeeze_(0) if inplace else result.squeeze(0) + return result + + +def feature_alpha_dropout( + input: Tensor, + p: float = 0.5, + training: bool = False, + inplace: bool = False, +) -> Tensor: + r"""Randomly masks out entire channels (a channel is a feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the batch input + is a tensor :math:`\text{input}[i, j]` of the input tensor. Instead of + setting activations to zero, as in regular Dropout, the activations are set + to the negative saturation value of the SELU activation function. + + Each element will be masked independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + The elements to be masked are randomized on every forward call, and scaled + and shifted to maintain zero mean and unit variance. + + See :class:`~torch.nn.FeatureAlphaDropout` for details. + + Args: + p: dropout probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + feature_alpha_dropout, + (input,), + input, + p=p, + training=training, + inplace=inplace, + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.feature_alpha_dropout_(input, p, training) + if inplace + else _VF.feature_alpha_dropout(input, p, training) + ) + + +def _threshold( + input: Tensor, + threshold: float, + value: float, + inplace: bool = False, +) -> Tensor: + r"""Apply a threshold to each element of the input Tensor. + + See :class:`~torch.nn.Threshold` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + _threshold, (input,), input, threshold, value, inplace=inplace + ) + if inplace: + result = _VF.threshold_(input, threshold, value) + else: + result = _VF.threshold(input, threshold, value) + return result + + +# We define this function as _threshold because it takes an argument +# named threshold, which clobbers the recursive reference to the +# function needed for __torch_function__ support +threshold = _threshold + +threshold_ = _add_docstr( + _VF.threshold_, + r""" +threshold_(input, threshold, value) -> Tensor + +In-place version of :func:`~threshold`. +""", +) + + +def relu(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""relu(input, inplace=False) -> Tensor + + Applies the rectified linear unit function element-wise. See + :class:`~torch.nn.ReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(relu, (input,), input, inplace=inplace) + if inplace: + result = torch.relu_(input) + else: + result = torch.relu(input) + return result + + +relu_ = _add_docstr( + torch.relu_, + r""" +relu_(input) -> Tensor + +In-place version of :func:`~relu`. +""", +) + + +def glu(input: Tensor, dim: int = -1) -> Tensor: # noqa: D400,D402 + r""" + glu(input, dim=-1) -> Tensor + + The gated linear unit. Computes: + + .. math :: + \text{GLU}(a, b) = a \otimes \sigma(b) + + where `input` is split in half along `dim` to form `a` and `b`, :math:`\sigma` + is the sigmoid function and :math:`\otimes` is the element-wise product between matrices. + + See `Language Modeling with Gated Convolutional Networks `_. + + Args: + input (Tensor): input tensor + dim (int): dimension on which to split the input. Default: -1 + """ + if has_torch_function_unary(input): + return handle_torch_function(glu, (input,), input, dim=dim) + if input.dim() == 0: + raise RuntimeError( + "glu does not support scalars because halving size must be even" + ) + return torch._C._nn.glu(input, dim) + + +def hardtanh( + input: Tensor, + min_val: float = -1.0, + max_val: float = 1.0, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r""" + hardtanh(input, min_val=-1., max_val=1., inplace=False) -> Tensor + + Applies the HardTanh function element-wise. See :class:`~torch.nn.Hardtanh` for more + details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + hardtanh, (input,), input, min_val=min_val, max_val=max_val, inplace=inplace + ) + if min_val > max_val: + raise ValueError("min_val cannot be greater than max_val") + if inplace: + result = torch._C._nn.hardtanh_(input, min_val, max_val) + else: + result = torch._C._nn.hardtanh(input, min_val, max_val) + return result + + +hardtanh_ = _add_docstr( + torch._C._nn.hardtanh_, + r""" +hardtanh_(input, min_val=-1., max_val=1.) -> Tensor + +In-place version of :func:`~hardtanh`. +""", +) + + +def relu6(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""relu6(input, inplace=False) -> Tensor + + Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)`. + + See :class:`~torch.nn.ReLU6` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(relu6, (input,), input, inplace=inplace) + if inplace: + result = torch._C._nn.relu6_(input) + else: + result = torch._C._nn.relu6(input) + return result + + +def elu(input: Tensor, alpha: float = 1.0, inplace: bool = False) -> Tensor: + r"""Apply the Exponential Linear Unit (ELU) function element-wise. + + See :class:`~torch.nn.ELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(elu, (input,), input, alpha=alpha, inplace=inplace) + if inplace: + result = torch._C._nn.elu_(input, alpha) + else: + result = torch._C._nn.elu(input, alpha) + return result + + +elu_ = _add_docstr( + torch._C._nn.elu_, + r""" +elu_(input, alpha=1.) -> Tensor + +In-place version of :func:`~elu`. +""", +) + + +def selu(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""selu(input, inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{SELU}(x) = scale * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`, + with :math:`\alpha=1.6732632423543772848170429916717` and + :math:`scale=1.0507009873554804934193349852946`. + + See :class:`~torch.nn.SELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(selu, (input,), input, inplace=inplace) + if inplace: + result = torch.selu_(input) + else: + result = torch.selu(input) + return result + + +selu_ = _add_docstr( + torch.selu_, + r""" +selu_(input) -> Tensor + +In-place version of :func:`~selu`. +""", +) + + +def celu( + input: Tensor, + alpha: float = 1.0, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r"""celu(input, alpha=1., inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`. + + See :class:`~torch.nn.CELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + celu, (input,), input, alpha=alpha, inplace=inplace + ) + if inplace: + result = torch.celu_(input, alpha) + else: + result = torch.celu(input, alpha) + return result + + +celu_ = _add_docstr( + torch.celu_, + r""" +celu_(input, alpha=1.) -> Tensor + +In-place version of :func:`~celu`. +""", +) + + +def leaky_relu( + input: Tensor, + negative_slope: float = 0.01, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r""" + leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)` + + See :class:`~torch.nn.LeakyReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + leaky_relu, (input,), input, negative_slope=negative_slope, inplace=inplace + ) + if inplace: + result = torch._C._nn.leaky_relu_(input, negative_slope) + else: + result = torch._C._nn.leaky_relu(input, negative_slope) + return result + + +leaky_relu_ = _add_docstr( + torch._C._nn.leaky_relu_, + r""" +leaky_relu_(input, negative_slope=0.01) -> Tensor + +In-place version of :func:`~leaky_relu`. +""", +) + + +prelu = _add_docstr( + torch.prelu, + r"""prelu(input, weight) -> Tensor + +Applies element-wise the function +:math:`\text{PReLU}(x) = \max(0,x) + \text{weight} * \min(0,x)` where weight is a +learnable parameter. + +.. note:: + `weight` is expected to be a scalar or 1-D tensor. If `weight` is 1-D, + its size must match the number of input channels, determined by + `input.size(1)` when `input.dim() >= 2`, otherwise 1. + In the 1-D case, note that when `input` has dim > 2, `weight` can be expanded + to the shape of `input` in a way that is not possible using normal + :ref:`broadcasting semantics`. + +See :class:`~torch.nn.PReLU` for more details. +""", +) + + +def rrelu( + input: Tensor, + lower: float = 1.0 / 8, + upper: float = 1.0 / 3, + training: bool = False, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r"""rrelu(input, lower=1./8, upper=1./3, training=False, inplace=False) -> Tensor + + Randomized leaky ReLU. + + See :class:`~torch.nn.RReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + rrelu, + (input,), + input, + lower=lower, + upper=upper, + training=training, + inplace=inplace, + ) + if inplace: + result = torch.rrelu_(input, lower, upper, training) + else: + result = torch.rrelu(input, lower, upper, training) + return result + + +rrelu_ = _add_docstr( + torch.rrelu_, + r""" +rrelu_(input, lower=1./8, upper=1./3, training=False) -> Tensor + +In-place version of :func:`~rrelu`. +""", +) + +logsigmoid = _add_docstr( + torch._C._nn.log_sigmoid, + r""" +logsigmoid(input) -> Tensor + +Applies element-wise :math:`\text{LogSigmoid}(x_i) = \log \left(\frac{1}{1 + \exp(-x_i)}\right)` + +See :class:`~torch.nn.LogSigmoid` for more details. +""", +) + +gelu = _add_docstr( + torch._C._nn.gelu, + r""" +gelu(input, approximate = 'none') -> Tensor + +When the approximate argument is 'none', it applies element-wise the function +:math:`\text{GELU}(x) = x * \Phi(x)` + +where :math:`\Phi(x)` is the Cumulative Distribution Function for Gaussian Distribution. + +When the approximate argument is 'tanh', Gelu is estimated with + +.. math:: + \text{GELU}(x) = 0.5 * x * (1 + \text{Tanh}(\sqrt{2 / \pi} * (x + 0.044715 * x^3))) + +See `Gaussian Error Linear Units (GELUs) `_. +""", +) + +hardshrink = _add_docstr( + torch.hardshrink, + r""" +hardshrink(input, lambd=0.5) -> Tensor + +Applies the hard shrinkage function element-wise + +See :class:`~torch.nn.Hardshrink` for more details. +""", +) + + +def tanhshrink(input): # noqa: D400,D402 + r"""tanhshrink(input) -> Tensor + + Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)` + + See :class:`~torch.nn.Tanhshrink` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(tanhshrink, (input,), input) + return input - input.tanh() + + +def softsign(input): # noqa: D400,D402 + r"""softsign(input) -> Tensor + + Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{1 + |x|}` + + See :class:`~torch.nn.Softsign` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(softsign, (input,), input) + return input / (input.abs() + 1) + + +softplus = _add_docstr( + torch._C._nn.softplus, + r""" +softplus(input, beta=1, threshold=20) -> Tensor + +Applies element-wise, the function :math:`\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))`. + +For numerical stability the implementation reverts to the linear function +when :math:`input \times \beta > threshold`. + +See :class:`~torch.nn.Softplus` for more details. +""", +) + + +def _get_softmax_dim(name: str, ndim: int, stacklevel: int) -> int: + warnings.warn( + f"Implicit dimension choice for {name} has been deprecated. " + "Change the call to include dim=X as an argument.", + stacklevel=stacklevel, + ) + if ndim == 0 or ndim == 1 or ndim == 3: + ret = 0 + else: + ret = 1 + return ret + + +def softmin( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmin function. + + Note that :math:`\text{Softmin}(x) = \text{Softmax}(-x)`. See softmax definition for mathematical formula. + + See :class:`~torch.nn.Softmin` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which softmin will be computed (so every slice + along dim will sum to 1). + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + """ + if has_torch_function_unary(input): + return handle_torch_function( + softmin, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("softmin", input.dim(), _stacklevel) + if dtype is None: + ret = (-input).softmax(dim) + else: + ret = (-input).softmax(dim, dtype=dtype) + return ret + + +def softmax( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmax function. + + Softmax is defined as: + + :math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` + + It is applied to all slices along dim, and will re-scale them so that the elements + lie in the range `[0, 1]` and sum to 1. + + See :class:`~torch.nn.Softmax` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + .. note:: + This function doesn't work directly with NLLLoss, + which expects the Log to be computed between the Softmax and itself. + Use log_softmax instead (it's faster and has better numerical properties). + + """ + if has_torch_function_unary(input): + return handle_torch_function( + softmax, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("softmax", input.dim(), _stacklevel) + if dtype is None: + ret = input.softmax(dim) + else: + ret = input.softmax(dim, dtype=dtype) + return ret + + +def gumbel_softmax( + logits: Tensor, + tau: float = 1, + hard: bool = False, + eps: float = 1e-10, + dim: int = -1, +) -> Tensor: + r""" + Sample from the Gumbel-Softmax distribution (`Link 1`_ `Link 2`_) and optionally discretize. + + Args: + logits: `[..., num_features]` unnormalized log probabilities + tau: non-negative scalar temperature + hard: if ``True``, the returned samples will be discretized as one-hot vectors, + but will be differentiated as if it is the soft sample in autograd + dim (int): A dimension along which softmax will be computed. Default: -1. + + Returns: + Sampled tensor of same shape as `logits` from the Gumbel-Softmax distribution. + If ``hard=True``, the returned samples will be one-hot, otherwise they will + be probability distributions that sum to 1 across `dim`. + + .. note:: + This function is here for legacy reasons, may be removed from nn.Functional in the future. + + .. note:: + The main trick for `hard` is to do `y_hard - y_soft.detach() + y_soft` + + It achieves two things: + - makes the output value exactly one-hot + (since we add then subtract y_soft value) + - makes the gradient equal to y_soft gradient + (since we strip all other gradients) + + Examples:: + >>> logits = torch.randn(20, 32) + >>> # Sample soft categorical using reparametrization trick: + >>> F.gumbel_softmax(logits, tau=1, hard=False) + >>> # Sample hard categorical using "Straight-through" trick: + >>> F.gumbel_softmax(logits, tau=1, hard=True) + + .. _Link 1: + https://arxiv.org/abs/1611.00712 + .. _Link 2: + https://arxiv.org/abs/1611.01144 + """ + if has_torch_function_unary(logits): + return handle_torch_function( + gumbel_softmax, (logits,), logits, tau=tau, hard=hard, eps=eps, dim=dim + ) + if eps != 1e-10: + warnings.warn("`eps` parameter is deprecated and has no effect.", stacklevel=2) + + gumbels = ( + -torch.empty_like(logits, memory_format=torch.legacy_contiguous_format) + .exponential_() + .log() + ) # ~Gumbel(0,1) + gumbels = (logits + gumbels) / tau # ~Gumbel(logits,tau) + y_soft = gumbels.softmax(dim) + + if hard: + # Straight through. + index = y_soft.max(dim, keepdim=True)[1] + y_hard = torch.zeros_like( + logits, memory_format=torch.legacy_contiguous_format + ).scatter_(dim, index, 1.0) + ret = y_hard - y_soft.detach() + y_soft + else: + # Reparameterization trick. + ret = y_soft + return ret + + +def log_softmax( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmax followed by a logarithm. + + While mathematically equivalent to log(softmax(x)), doing these two + operations separately is slower and numerically unstable. This function + uses an alternative formulation to compute the output and gradient correctly. + + See :class:`~torch.nn.LogSoftmax` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which log_softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is cast to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + """ + if has_torch_function_unary(input): + return handle_torch_function( + log_softmax, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("log_softmax", input.dim(), _stacklevel) + if dtype is None: + ret = input.log_softmax(dim) + else: + ret = input.log_softmax(dim, dtype=dtype) + return ret + + +softshrink = _add_docstr( + torch._C._nn.softshrink, + r""" +softshrink(input, lambd=0.5) -> Tensor + +Applies the soft shrinkage function elementwise + +See :class:`~torch.nn.Softshrink` for more details. +""", +) + + +def tanh(input): # noqa: D400,D402 + r"""tanh(input) -> Tensor + + Applies element-wise, + :math:`\text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}` + + See :class:`~torch.nn.Tanh` for more details. + """ + return input.tanh() + + +def sigmoid(input): # noqa: D400,D402 + r"""sigmoid(input) -> Tensor + + Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}` + + See :class:`~torch.nn.Sigmoid` for more details. + """ + return input.sigmoid() + + +def hardsigmoid(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Hardsigmoid function element-wise. + + .. math:: + \text{Hardsigmoid}(x) = \begin{cases} + 0 & \text{if~} x \le -3, \\ + 1 & \text{if~} x \ge +3, \\ + x / 6 + 1 / 2 & \text{otherwise} + \end{cases} + + Args: + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + + See :class:`~torch.nn.Hardsigmoid` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(hardsigmoid, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.hardsigmoid_(input) + return torch._C._nn.hardsigmoid(input) + + +linear = _add_docstr( + torch._C._nn.linear, + r""" +linear(input, weight, bias=None) -> Tensor + +Applies a linear transformation to the incoming data: :math:`y = xA^T + b`. + +This operation supports 2-D :attr:`weight` with :ref:`sparse layout` + +{sparse_beta_warning} + +This operator supports :ref:`TensorFloat32`. + +Shape: + + - Input: :math:`(*, in\_features)` where `*` means any number of + additional dimensions, including none + - Weight: :math:`(out\_features, in\_features)` or :math:`(in\_features)` + - Bias: :math:`(out\_features)` or :math:`()` + - Output: :math:`(*, out\_features)` or :math:`(*)`, based on the shape of the weight +""".format(**sparse_support_notes), +) + + +bilinear = _add_docstr( + torch.bilinear, + r""" +bilinear(input1, input2, weight, bias=None) -> Tensor + +Applies a bilinear transformation to the incoming data: +:math:`y = x_1^T A x_2 + b` + +Shape: + + - input1: :math:`(N, *, H_{in1})` where :math:`H_{in1}=\text{in1\_features}` + and :math:`*` means any number of additional dimensions. + All but the last dimension of the inputs should be the same. + - input2: :math:`(N, *, H_{in2})` where :math:`H_{in2}=\text{in2\_features}` + - weight: :math:`(\text{out\_features}, \text{in1\_features}, + \text{in2\_features})` + - bias: :math:`(\text{out\_features})` + - output: :math:`(N, *, H_{out})` where :math:`H_{out}=\text{out\_features}` + and all but the last dimension are the same shape as the input. +""", +) + + +def silu(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Sigmoid Linear Unit (SiLU) function, element-wise. + + The SiLU function is also known as the swish function. + + .. math:: + \text{silu}(x) = x * \sigma(x), \text{where } \sigma(x) \text{ is the logistic sigmoid.} + + .. note:: + See `Gaussian Error Linear Units (GELUs) `_ + where the SiLU (Sigmoid Linear Unit) was originally coined, and see + `Sigmoid-Weighted Linear Units for Neural Network Function Approximation + in Reinforcement Learning `_ and `Swish: + a Self-Gated Activation Function `_ + where the SiLU was experimented with later. + + See :class:`~torch.nn.SiLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(silu, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.silu_(input) + return torch._C._nn.silu(input) + + +def mish(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Mish function, element-wise. + + Mish: A Self Regularized Non-Monotonic Neural Activation Function. + + .. math:: + \text{Mish}(x) = x * \text{Tanh}(\text{Softplus}(x)) + + .. note:: + See `Mish: A Self Regularized Non-Monotonic Neural Activation Function `_ + + See :class:`~torch.nn.Mish` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(mish, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.mish_(input) + return torch._C._nn.mish(input) + + +def hardswish(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply hardswish function, element-wise. + + Follows implementation as described in the paper: + `Searching for MobileNetV3`_. + + .. math:: + \text{Hardswish}(x) = \begin{cases} + 0 & \text{if~} x \le -3, \\ + x & \text{if~} x \ge +3, \\ + x \cdot (x + 3) /6 & \text{otherwise} + \end{cases} + + See :class:`~torch.nn.Hardswish` for more details. + + .. _`Searching for MobileNetV3`: + https://arxiv.org/abs/1905.02244 + """ + if has_torch_function_unary(input): + return handle_torch_function(hardswish, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.hardswish_(input) + return torch._C._nn.hardswish(input) + + +def _no_grad_embedding_renorm_( + weight: Tensor, + input: Tensor, + max_norm: float, + norm_type: float, + # pyrefly: ignore [bad-return] +) -> tuple[Tensor, Tensor]: + torch.embedding_renorm_(weight.detach(), input, max_norm, norm_type) + + +def embedding( + input: Tensor, + weight: Tensor, + padding_idx: Optional[int] = None, + max_norm: Optional[float] = None, + norm_type: float = 2.0, + scale_grad_by_freq: bool = False, + sparse: bool = False, +) -> Tensor: + r"""Generate a simple lookup table that looks up embeddings in a fixed dictionary and size. + + This module is often used to retrieve word embeddings using indices. + The input to the module is a list of indices, and the embedding matrix, + and the output is the corresponding word embeddings. + + See :class:`torch.nn.Embedding` for more details. + + .. note:: + Note that the analytical gradients of this function with respect to + entries in :attr:`weight` at the row specified by :attr:`padding_idx` + are expected to differ from the numerical ones. + + .. note:: + Note that `:class:`torch.nn.Embedding` differs from this function in + that it initializes the row of :attr:`weight` specified by + :attr:`padding_idx` to all zeros on construction. + + Args: + input (LongTensor): Tensor containing indices into the embedding matrix + weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, + and number of columns equal to the embedding size + padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the gradient; + therefore, the embedding vector at :attr:`padding_idx` is not updated during training, + i.e. it remains as a fixed "pad". + max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` + is renormalized to have norm :attr:`max_norm`. + Note: this will modify :attr:`weight` in-place. + norm_type (float, optional): The p of the p-norm to compute for the :attr:`max_norm` option. Default ``2``. + scale_grad_by_freq (bool, optional): If given, this will scale gradients by the inverse of frequency of + the words in the mini-batch. Default ``False``. + sparse (bool, optional): If ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under + :class:`torch.nn.Embedding` for more details regarding sparse gradients. + + Shape: + - Input: LongTensor of arbitrary shape containing the indices to extract + - Weight: Embedding matrix of floating point type with shape `(V, embedding_dim)`, + where V = maximum index + 1 and embedding_dim = the embedding size + - Output: `(*, embedding_dim)`, where `*` is the input shape + + Examples:: + + >>> # a batch of 2 samples of 4 indices each + >>> input = torch.tensor([[1, 2, 4, 5], [4, 3, 2, 9]]) + >>> # an embedding matrix containing 10 tensors of size 3 + >>> embedding_matrix = torch.rand(10, 3) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> F.embedding(input, embedding_matrix) + tensor([[[ 0.8490, 0.9625, 0.6753], + [ 0.9666, 0.7761, 0.6108], + [ 0.6246, 0.9751, 0.3618], + [ 0.4161, 0.2419, 0.7383]], + + [[ 0.6246, 0.9751, 0.3618], + [ 0.0237, 0.7794, 0.0528], + [ 0.9666, 0.7761, 0.6108], + [ 0.3385, 0.8612, 0.1867]]]) + + >>> # example with padding_idx + >>> weights = torch.rand(10, 3) + >>> weights[0, :].zero_() + >>> embedding_matrix = weights + >>> input = torch.tensor([[0, 2, 0, 5]]) + >>> F.embedding(input, embedding_matrix, padding_idx=0) + tensor([[[ 0.0000, 0.0000, 0.0000], + [ 0.5609, 0.5384, 0.8720], + [ 0.0000, 0.0000, 0.0000], + [ 0.6262, 0.2438, 0.7471]]]) + """ + if has_torch_function_variadic(input, weight): + return handle_torch_function( + embedding, + (input, weight), + input, + weight, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + ) + if padding_idx is not None: + if padding_idx > 0: + assert padding_idx < weight.size(0), ( + "Padding_idx must be within num_embeddings" + ) + elif padding_idx < 0: + assert padding_idx >= -weight.size(0), ( + "Padding_idx must be within num_embeddings" + ) + padding_idx = weight.size(0) + padding_idx + else: + padding_idx = -1 + if max_norm is not None: + # Note [embedding_renorm contiguous] + # `embedding_renorm_` will call .contiguous() on input anyways, so we + # call it here and take advantage of the improved locality in the + # `embedding` call below too. + input = input.contiguous() + # Note [embedding_renorm set_grad_enabled] + # XXX: equivalent to + # with torch.no_grad(): + # torch.embedding_renorm_ + # remove once script supports set_grad_enabled + _no_grad_embedding_renorm_(weight, input, max_norm, norm_type) + return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse) + + +def embedding_bag( + input: Tensor, + weight: Tensor, + offsets: Optional[Tensor] = None, + max_norm: Optional[float] = None, + norm_type: float = 2, + scale_grad_by_freq: bool = False, + mode: str = "mean", + sparse: bool = False, + per_sample_weights: Optional[Tensor] = None, + include_last_offset: bool = False, + padding_idx: Optional[int] = None, +) -> Tensor: + r"""Compute sums, means or maxes of `bags` of embeddings. + + Calculation is done without instantiating the intermediate embeddings. + See :class:`torch.nn.EmbeddingBag` for more details. + + Note: + {backward_reproducibility_note} + + Args: + input (LongTensor): Tensor containing bags of indices into the embedding matrix + weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, + and number of columns equal to the embedding size + offsets (LongTensor, optional): Only used when :attr:`input` is 1D. :attr:`offsets` determines + the starting index position of each bag (sequence) in :attr:`input`. + max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` + is renormalized to have norm :attr:`max_norm`. + Note: this will modify :attr:`weight` in-place. + norm_type (float, optional): The ``p`` in the ``p``-norm to compute for the :attr:`max_norm` option. + Default ``2``. + scale_grad_by_freq (bool, optional): if given, this will scale gradients by the inverse of frequency of + the words in the mini-batch. Default ``False``. + Note: this option is not supported when ``mode="max"``. + mode (str, optional): ``"sum"``, ``"mean"`` or ``"max"``. Specifies the way to reduce the bag. + Default: ``"mean"`` + sparse (bool, optional): if ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under + :class:`torch.nn.Embedding` for more details regarding sparse gradients. + Note: this option is not supported when ``mode="max"``. + per_sample_weights (Tensor, optional): a tensor of float / double weights, or None + to indicate all weights should be taken to be 1. If specified, :attr:`per_sample_weights` + must have exactly the same shape as input and is treated as having the same + :attr:`offsets`, if those are not None. + + include_last_offset (bool, optional): if ``True``, the size of offsets is equal to the number of bags + 1. + The last element is the size of the input, or the ending index position + of the last bag (sequence). This matches the CSR format. Ignored when + input is 2D. Default ``False``. + + padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the + gradient; therefore, the embedding vector at :attr:`padding_idx` is not updated + during training, i.e. it remains as a fixed "pad". Note that the embedding + vector at :attr:`padding_idx` is excluded from the reduction. + + Shape: + - :attr:`input` (LongTensor) and :attr:`offsets` (LongTensor, optional) + + - If :attr:`input` is 2D of shape `(B, N)`, it will be treated as ``B`` bags (sequences) + each of fixed length ``N``, and this will return ``B`` values aggregated in a way + depending on the :attr:`mode`. :attr:`offsets` is ignored and required to be ``None`` in this case. + + - If :attr:`input` is 1D of shape `(N)`, it will be treated as a concatenation of + multiple bags (sequences). :attr:`offsets` is required to be a 1D tensor containing + the starting index positions of each bag in :attr:`input`. Therefore, for :attr:`offsets` + of shape `(B)`, :attr:`input` will be viewed as having ``B`` bags. + Empty bags (i.e., having 0-length) will have returned vectors filled by zeros. + + - :attr:`weight` (Tensor): the learnable weights of the module of shape `(num_embeddings, embedding_dim)` + + - :attr:`per_sample_weights` (Tensor, optional). Has the same shape as :attr:`input`. + + - :attr:`output`: aggregated embedding values of shape `(B, embedding_dim)` + + Examples:: + + >>> # an Embedding module containing 10 tensors of size 3 + >>> embedding_matrix = torch.rand(10, 3) + >>> # a batch of 2 samples of 4 indices each + >>> input = torch.tensor([1, 2, 4, 5, 4, 3, 2, 9]) + >>> offsets = torch.tensor([0, 4]) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> F.embedding_bag(input, embedding_matrix, offsets) + tensor([[ 0.3397, 0.3552, 0.5545], + [ 0.5893, 0.4386, 0.5882]]) + + >>> # example with padding_idx + >>> embedding_matrix = torch.rand(10, 3) + >>> input = torch.tensor([2, 2, 2, 2, 4, 3, 2, 9]) + >>> offsets = torch.tensor([0, 4]) + >>> F.embedding_bag(input, embedding_matrix, offsets, padding_idx=2, mode='sum') + tensor([[ 0.0000, 0.0000, 0.0000], + [-0.7082, 3.2145, -2.6251]]) + """ + if has_torch_function_variadic(input, weight, offsets, per_sample_weights): + return handle_torch_function( + embedding_bag, + (input, weight, offsets, per_sample_weights), + input, + weight, + offsets=offsets, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + mode=mode, + sparse=sparse, + per_sample_weights=per_sample_weights, + include_last_offset=include_last_offset, + padding_idx=padding_idx, + ) + # Check for backward compatibility. + # Used to be embedding_bag(weight, input, ...) + # Now is embedding_bag(input, weight, ...) + if weight.dtype == torch.long and input.is_floating_point(): + warnings.warn( + "Argument order of nn.functional.embedding_bag was changed. " + "Usage `embedding_bag(weight, input, ...)` is deprecated, " + "and should now be `embedding_bag(input, weight, ...)`.", + stacklevel=2, + ) + weight, input = input, weight + + if per_sample_weights is not None and input.size() != per_sample_weights.size(): + raise ValueError( + f"embedding_bag: If per_sample_weights ({per_sample_weights.shape}) is not None, " + f"then it must have the same shape as the input ({input.shape})" + ) + + if not weight.dim() == 2: + raise ValueError( + f"weight has to be a 2D Tensor, but got Tensor of dimension {weight.dim()}" + ) + + if not torch.jit.is_scripting() and input.dim() == 2 and input.is_nested: + include_last_offset = True + # pyrefly: ignore [missing-attribute] + offsets = input.offsets() + input = input.values().reshape(-1) + if per_sample_weights is not None: + if not per_sample_weights.is_nested: + raise ValueError( + "If input is nested, then per_sample_weights must be nested if specified" + ) + per_sample_weights = per_sample_weights.values().reshape(-1) + elif input.dim() == 2: + if offsets is not None: + type_str = "" + # TODO: Remove this once script supports type() calls + if not torch.jit.is_scripting(): + type_str = str(type(offsets)) + raise ValueError( + "if input is 2D, then offsets has to be None" + ", as input is treated is a mini-batch of" + " fixed length sequences. However, found " + f"offsets of type {type_str}" + ) + offsets = torch.arange( + 0, input.numel(), input.size(1), dtype=input.dtype, device=input.device + ) + include_last_offset = False + input = input.reshape(-1) + if per_sample_weights is not None: + per_sample_weights = per_sample_weights.reshape(-1) + elif input.dim() == 1: + if offsets is None: + raise ValueError("offsets has to be a 1D Tensor but got None") + if offsets.dim() != 1: + raise ValueError("offsets has to be a 1D Tensor") + else: + raise ValueError( + f"input has to be 1D or 2D Tensor, but got Tensor of dimension {input.dim()}" + ) + if mode == "sum": + mode_enum = 0 + elif mode == "mean": + mode_enum = 1 + elif mode == "max": + mode_enum = 2 + + if scale_grad_by_freq: + raise ValueError( + "max mode does not support scaling the gradient by the frequency" + ) + + if sparse: + raise ValueError("max mode does not support sparse weights") + + else: + raise ValueError("mode has to be one of sum, mean or max") + + if max_norm is not None: + # XXX: equivalent to + # with torch.no_grad(): + # torch.nembedding_renorm_ + # remove once script supports set_grad_enabled + _no_grad_embedding_renorm_(weight, input, max_norm, norm_type) + + if per_sample_weights is not None and mode != "sum": + raise NotImplementedError( + "embedding_bag: per_sample_weights was not None. " + "per_sample_weights is only supported for mode='sum' " + f"(got mode='{mode}'). Please open a feature request on GitHub." + ) + + ret, _, _, _ = torch.embedding_bag( + weight, + input, + offsets, + scale_grad_by_freq, + mode_enum, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + return ret + + +if embedding_bag.__doc__: + embedding_bag.__doc__ = embedding_bag.__doc__.format(**reproducibility_notes) + + +def _verify_batch_size(size: list[int]) -> None: + # XXX: JIT script does not support the reduce from functools, and mul op is a + # builtin, which cannot be used as a value to a func yet, so rewrite this size + # check to a simple equivalent for loop + # + # TODO: make use of reduce like below when JIT is ready with the missing features: + # from operator import mul + # from functools import reduce + # + # if reduce(mul, size[2:], size[0]) == 1 + size_prods = size[0] + for i in range(len(size) - 2): + size_prods *= size[i + 2] + if size_prods == 1: + raise ValueError( + f"Expected more than 1 value per channel when training, got input size {size}" + ) + + +def batch_norm( + input: Tensor, + running_mean: Optional[Tensor], + running_var: Optional[Tensor], + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + training: bool = False, + momentum: float = 0.1, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Batch Normalization for each channel across a batch of data. + + See :class:`~torch.nn.BatchNorm1d`, :class:`~torch.nn.BatchNorm2d`, + :class:`~torch.nn.BatchNorm3d` for details. + """ + if has_torch_function_variadic(input, running_mean, running_var, weight, bias): + return handle_torch_function( + batch_norm, + (input, running_mean, running_var, weight, bias), + input, + running_mean, + running_var, + weight=weight, + bias=bias, + training=training, + momentum=momentum, + eps=eps, + ) + if training: + # pyrefly: ignore [bad-argument-type] + _verify_batch_size(input.size()) + + if eps <= 0.0: + raise ValueError(f"batch_norm eps must be positive, but got {eps}") + + return torch.batch_norm( + input, + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + torch.backends.cudnn.enabled, + ) + + +def _verify_spatial_size(size: list[int]) -> None: + # Verify that there is > 1 spatial element for instance norm calculation. + size_prods = 1 + for i in range(2, len(size)): + size_prods *= size[i] + if size_prods == 1: + raise ValueError( + f"Expected more than 1 spatial element when training, got input size {size}" + ) + + +def instance_norm( + input: Tensor, + running_mean: Optional[Tensor] = None, + running_var: Optional[Tensor] = None, + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + use_input_stats: bool = True, + momentum: float = 0.1, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Instance Normalization independently for each channel in every data sample within a batch. + + See :class:`~torch.nn.InstanceNorm1d`, :class:`~torch.nn.InstanceNorm2d`, + :class:`~torch.nn.InstanceNorm3d` for details. + """ + if has_torch_function_variadic(input, running_mean, running_var, weight, bias): + return handle_torch_function( + instance_norm, + (input, running_mean, running_var, weight, bias), + input, + running_mean=running_mean, + running_var=running_var, + weight=weight, + bias=bias, + use_input_stats=use_input_stats, + momentum=momentum, + eps=eps, + ) + if use_input_stats: + # pyrefly: ignore [bad-argument-type] + _verify_spatial_size(input.size()) + return torch.instance_norm( + input, + weight, + bias, + running_mean, + running_var, + use_input_stats, + momentum, + eps, + torch.backends.cudnn.enabled, + ) + + +def layer_norm( + input: Tensor, + normalized_shape: list[int], + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Layer Normalization for last certain number of dimensions. + + See :class:`~torch.nn.LayerNorm` for details. + """ + if has_torch_function_variadic(input, weight, bias): + return handle_torch_function( + layer_norm, + (input, weight, bias), + input, + normalized_shape, + weight=weight, + bias=bias, + eps=eps, + ) + return torch.layer_norm( + input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled + ) + + +def rms_norm( + input: Tensor, + normalized_shape: list[int], + weight: Optional[Tensor] = None, + eps: Optional[float] = None, +) -> Tensor: + r"""Apply Root Mean Square Layer Normalization. + + See :class:`~torch.nn.RMSNorm` for details. + """ + if has_torch_function_variadic(input, weight): + return handle_torch_function( + rms_norm, (input, weight), input, normalized_shape, weight=weight, eps=eps + ) + return torch.rms_norm(input, normalized_shape, weight, eps) + + +def group_norm( + input: Tensor, + num_groups: int, + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Group Normalization for last certain number of dimensions. + + See :class:`~torch.nn.GroupNorm` for details. + """ + if has_torch_function_variadic(input, weight, bias): + return handle_torch_function( + group_norm, + ( + input, + weight, + bias, + ), + input, + num_groups, + weight=weight, + bias=bias, + eps=eps, + ) + if input.dim() < 2: + raise RuntimeError( + f"Expected at least 2 dimensions for input tensor but received {input.dim()}" + ) + _verify_batch_size( + [input.size(0) * input.size(1) // num_groups, num_groups] + + list(input.size()[2:]) + ) + return torch.group_norm( + input, num_groups, weight, bias, eps, torch.backends.cudnn.enabled + ) + + +def local_response_norm( + input: Tensor, + size: int, + alpha: float = 1e-4, + beta: float = 0.75, + k: float = 1.0, +) -> Tensor: + r"""Apply local response normalization over an input signal. + + The input signal is composed of several input planes, where channels occupy the second dimension. + Normalization is applied across channels. + + See :class:`~torch.nn.LocalResponseNorm` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + local_response_norm, (input,), input, size, alpha=alpha, beta=beta, k=k + ) + dim = input.dim() + if dim < 3: + raise ValueError( + f"Expected 3D or higher dimensionality input (got {dim} dimensions)" + ) + + if input.numel() == 0: + return input + + div = input.mul(input) + if dim == 3: + div = div.unsqueeze(1) + # pyrefly: ignore [bad-argument-type] + div = pad(div, (0, 0, size // 2, (size - 1) // 2)) + div = avg_pool2d(div, (size, 1), stride=1).squeeze(1) + else: + sizes = input.size() + div = div.view(sizes[0], 1, sizes[1], sizes[2], -1) + # pyrefly: ignore [bad-argument-type] + div = pad(div, (0, 0, 0, 0, size // 2, (size - 1) // 2)) + div = avg_pool3d(div, (size, 1, 1), stride=1).squeeze(1) + div = div.view(sizes) + div = div.mul(alpha).add(k).pow(beta) + return input / div + + +# loss + + +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: int = 0, + reduction: str = "mean", + zero_infinity: bool = False, +) -> Tensor: + r"""Compute the Connectionist Temporal Classification loss. + + See :class:`~torch.nn.CTCLoss` for details. + + Note: + {cudnn_reproducibility_note} + + Note: + {backward_reproducibility_note} + + Args: + log_probs: :math:`(T, N, C)` or :math:`(T, C)` where `C = number of characters in alphabet including blank`, + `T = input length`, and `N = batch size`. + The logarithmized probabilities of the outputs + (e.g. obtained with :func:`torch.nn.functional.log_softmax`). + targets: :math:`(N, S)` or `(sum(target_lengths))`. + May be an empty tensor if all entries in `target_lengths` are zero. + In the second form, the targets are assumed to be concatenated. + input_lengths: :math:`(N)` or :math:`()`. + Lengths of the inputs (must each be :math:`\leq T`) + target_lengths: :math:`(N)` or :math:`()`. + Lengths of the targets + blank (int, optional): + Blank label. Default :math:`0`. + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output losses will be divided by the target lengths and + then the mean over the batch is taken, ``'sum'``: the output will be + summed. Default: ``'mean'`` + zero_infinity (bool, optional): + Whether to zero infinite losses and the associated gradients. + Default: ``False`` + Infinite losses mainly occur when the inputs are too short + to be aligned to the targets. + + Example:: + + >>> log_probs = torch.randn(50, 16, 20).log_softmax(2).detach().requires_grad_() + >>> targets = torch.randint(1, 20, (16, 30), dtype=torch.long) + >>> input_lengths = torch.full((16,), 50, dtype=torch.long) + >>> target_lengths = torch.randint(10, 30, (16,), dtype=torch.long) + >>> loss = F.ctc_loss(log_probs, targets, input_lengths, target_lengths) + >>> loss.backward() + """ + if has_torch_function_variadic(log_probs, targets, input_lengths, target_lengths): + return handle_torch_function( + ctc_loss, + (log_probs, targets, input_lengths, target_lengths), + log_probs, + targets, + input_lengths, + target_lengths, + blank=blank, + reduction=reduction, + zero_infinity=zero_infinity, + ) + return torch.ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + _Reduction.get_enum(reduction), + zero_infinity, + ) + + +if ctc_loss.__doc__: + ctc_loss.__doc__ = ctc_loss.__doc__.format(**reproducibility_notes) + + +def nll_loss( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + ignore_index: int = -100, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the negative log likelihood loss. + + See :class:`~torch.nn.NLLLoss` for details. + + Args: + input: :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)` + in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)` where :math:`K \geq 1` + in the case of K-dimensional loss. `input` is expected to be log-probabilities. + target: :math:`(N)` where each value is :math:`0 \leq \text{targets}[i] \leq C-1`, + or :math:`(N, d_1, d_2, ..., d_K)` where :math:`K \geq 1` for + K-dimensional loss. + weight (Tensor, optional): A manual rescaling weight given to each + class. If given, has to be a Tensor of size `C` + size_average (bool, optional): Deprecated (see :attr:`reduction`). + ignore_index (int, optional): Specifies a target value that is ignored + and does not contribute to the input gradient. When :attr:`size_average` is + ``True``, the loss is averaged over non-ignored targets. Default: -100 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + Example:: + + >>> # input is of size N x C = 3 x 5 + >>> input = torch.randn(3, 5, requires_grad=True) + >>> # each element in target has to have 0 <= value < C + >>> target = torch.tensor([1, 0, 4]) + >>> output = F.nll_loss(F.log_softmax(input, dim=1), target) + >>> output.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + nll_loss, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + ignore_index=ignore_index, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + return torch._C._nn.nll_loss_nd( + input, + target, + weight, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ignore_index, + ) + + +def poisson_nll_loss( + input: Tensor, + target: Tensor, + log_input: bool = True, + full: bool = False, + size_average: Optional[bool] = None, + eps: float = 1e-8, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the Poisson negative log likelihood loss. + + See :class:`~torch.nn.PoissonNLLLoss` for details. + + Args: + input: Expectation of underlying Poisson distribution. + target: Random sample :math:`target \sim \text{Poisson}(input)`. + log_input: If ``True`` the loss is computed as + :math:`\exp(\text{input}) - \text{target} * \text{input}`, if ``False`` then loss is + :math:`\text{input} - \text{target} * \log(\text{input}+\text{eps})`. Default: ``True`` + full: Whether to compute full loss, i. e. to add the Stirling + approximation term. Default: ``False`` + :math:`\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})`. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + eps (float, optional): Small value to avoid evaluation of :math:`\log(0)` when + :attr:`log_input`\ =\ ``False``. Default: 1e-8 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + poisson_nll_loss, + (input, target), + input, + target, + log_input=log_input, + full=full, + size_average=size_average, + eps=eps, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + if reduction != "none" and reduction != "mean" and reduction != "sum": + ret = input + raise ValueError(reduction + " is not a valid value for reduction") + + ret = torch.poisson_nll_loss( + input, target, log_input, full, eps, _Reduction.get_enum(reduction) + ) + return ret + + +def gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor | float, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + r"""Compute the Gaussian negative log likelihood loss. + + See :class:`~torch.nn.GaussianNLLLoss` for details. + + Args: + input: Expectation of the Gaussian distribution. + target: Sample from the Gaussian distribution. + var: Tensor of positive variance(s), one for each of the expectations + in the input (heteroscedastic), or a single one (homoscedastic), + or a positive scalar value to be used for all expectations. + full (bool, optional): Whether to include the constant term in the loss calculation. Default: ``False``. + eps (float, optional): Value added to var, for stability. Default: 1e-6. + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output is the average of all batch member losses, + ``'sum'``: the output is the sum of all batch member losses. + Default: ``'mean'``. + """ + if has_torch_function_variadic(input, target, var): + return handle_torch_function( + gaussian_nll_loss, + (input, target, var), + input, + target, + var, + full=full, + eps=eps, + reduction=reduction, + ) + + # Entries of var must be non-negative + if isinstance(var, float): + if var < 0: + raise ValueError("var has negative entry/entries") + var = var * torch.ones_like(input) + elif torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Check var size + # If var.size == input.size, the case is heteroscedastic and no further checks are needed. + # Otherwise: + if var.size() != input.size(): + # If var is one dimension short of input, but the sizes match otherwise, then this is a homoscedastic case. + # e.g. input.size = (10, 2, 3), var.size = (10, 2) + # -> unsqueeze var so that var.shape = (10, 2, 1) + # this is done so that broadcasting can happen in the loss calculation + if input.size()[:-1] == var.size(): + var = torch.unsqueeze(var, -1) + + # This checks if the var is broadcastable to the input and there is only one mismatched dimension. + # This is also a homoscedastic case. + # e.g. input.size = (10, 2, 3), var.size = (10, 2, 1) + # or input.size = (4, 3, 32, 32), var.size = (4, 1, 32, 32) + elif ( + input.ndim == var.ndim + and sum(y for x, y in zip(input.size(), var.size(), strict=True) if x != y) + == 1 + ): # Heteroscedastic case + pass + + # If none of the above pass, then the size of var is incorrect. + else: + raise ValueError("var is of incorrect size") + + # Check validity of reduction mode + if reduction != "none" and reduction != "mean" and reduction != "sum": + raise ValueError(reduction + " is not valid") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + # Calculate the loss + # pyrefly: ignore [unsupported-operation] + loss = 0.5 * (torch.log(var) + (input - target) ** 2 / var) + if full: + loss += 0.5 * math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss + + +def kl_div( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + log_target: bool = False, +) -> Tensor: + r"""Compute the KL Divergence loss. + + Refer - The `Kullback-Leibler divergence Loss + `__ + + See :class:`~torch.nn.KLDivLoss` for details. + + Args: + input: Tensor of arbitrary shape in log-probabilities. + target: Tensor of the same shape as input. See :attr:`log_target` for + the target's interpretation. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'batchmean'`` | ``'sum'`` | ``'mean'``. + ``'none'``: no reduction will be applied + ``'batchmean'``: the sum of the output will be divided by the batchsize + ``'sum'``: the output will be summed + ``'mean'``: the output will be divided by the number of elements in the output + Default: ``'mean'`` + log_target (bool): A flag indicating whether ``target`` is passed in the log space. + It is recommended to pass certain distributions (like ``softmax``) + in the log space to avoid numerical issues caused by explicit ``log``. + Default: ``False`` + + .. note:: + :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, + and in the meantime, specifying either of those two args will override :attr:`reduction`. + + .. warning:: + :attr:`reduction` = ``'mean'`` doesn't return the true kl divergence value, please use + :attr:`reduction` = ``'batchmean'`` which aligns with KL math definition. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + kl_div, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + log_target=log_target, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + if reduction == "mean": + warnings.warn( + "reduction: 'mean' divides the total loss by both the batch size and the support size." + "'batchmean' divides only by the batch size, and aligns with the KL div math definition." + "'mean' will be changed to behave the same as 'batchmean' in the next major release.", + stacklevel=2, + ) + + # special case for batchmean + if reduction == "batchmean": + reduction_enum = _Reduction.get_enum("sum") + else: + reduction_enum = _Reduction.get_enum(reduction) + + reduced = torch.kl_div(input, target, reduction_enum, log_target=log_target) + + if reduction == "batchmean" and input.dim() != 0: + reduced = reduced / input.size()[0] + + return reduced + + +def cross_entropy( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + ignore_index: int = -100, + reduce: Optional[bool] = None, + reduction: str = "mean", + label_smoothing: float = 0.0, +) -> Tensor: + r"""Compute the cross entropy loss between input logits and target. + + See :class:`~torch.nn.CrossEntropyLoss` for details. + + Args: + input (Tensor) : Predicted unnormalized logits; + see Shape section below for supported shapes. + target (Tensor) : Ground truth class indices or class probabilities; + see Shape section below for supported shapes. + weight (Tensor, optional): a manual rescaling weight given to each + class. If given, has to be a Tensor of size `C` + size_average (bool, optional): Deprecated (see :attr:`reduction`). + ignore_index (int, optional): Specifies a target value that is ignored + and does not contribute to the input gradient. When :attr:`size_average` is + ``True``, the loss is averaged over non-ignored targets. Note that + :attr:`ignore_index` is only applicable when the target contains class indices. + Default: -100 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + label_smoothing (float, optional): A float in [0.0, 1.0]. Specifies the amount + of smoothing when computing the loss, where 0.0 means no smoothing. The targets + become a mixture of the original ground truth and a uniform distribution as described in + `Rethinking the Inception Architecture for Computer Vision `__. Default: :math:`0.0`. + + Shape: + - Input: Shape :math:`(C)`, :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` with :math:`K \geq 1` + in the case of `K`-dimensional loss. + - Target: If containing class indices, shape :math:`()`, :math:`(N)` or :math:`(N, d_1, d_2, ..., d_K)` with + :math:`K \geq 1` in the case of K-dimensional loss where each value should be between :math:`[0, C)`. + If containing class probabilities, same shape as the input and each value should be between :math:`[0, 1]`. + + where: + + .. math:: + \begin{aligned} + C ={} & \text{number of classes} \\ + N ={} & \text{batch size} \\ + \end{aligned} + + Examples:: + + >>> # Example of target with class indices + >>> input = torch.randn(3, 5, requires_grad=True) + >>> target = torch.randint(5, (3,), dtype=torch.int64) + >>> loss = F.cross_entropy(input, target) + >>> loss.backward() + >>> + >>> # Example of target with class probabilities + >>> input = torch.randn(3, 5, requires_grad=True) + >>> target = torch.randn(3, 5).softmax(dim=1) + >>> loss = F.cross_entropy(input, target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + cross_entropy, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + ignore_index=ignore_index, + reduce=reduce, + reduction=reduction, + label_smoothing=label_smoothing, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + return torch._C._nn.cross_entropy_loss( + input, + target, + weight, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ignore_index, + label_smoothing, + ) + + +def binary_cross_entropy( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute Binary Cross Entropy between the target and input probabilities. + + See :class:`~torch.nn.BCELoss` for details. + + Args: + input: Tensor of arbitrary shape as probabilities. + target: Tensor of the same shape as input with values between 0 and 1. + weight (Tensor, optional): a manual rescaling weight + if provided it's repeated to match input tensor shape + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + Examples:: + + >>> input = torch.randn(3, 2, requires_grad=True) + >>> target = torch.rand(3, 2, requires_grad=False) + >>> loss = F.binary_cross_entropy(torch.sigmoid(input), target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + binary_cross_entropy, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if target.size() != input.size(): + raise ValueError( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}) is deprecated. " + "Please ensure they have the same size." + ) + + if weight is not None: + new_size = _infer_size(target.size(), weight.size()) + weight = weight.expand(new_size) + + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.binary_cross_entropy(input, target, weight, reduction_enum) + + +def binary_cross_entropy_with_logits( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + pos_weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute Binary Cross Entropy between target and input logits. + + See :class:`~torch.nn.BCEWithLogitsLoss` for details. + + Args: + input: Tensor of arbitrary shape as unnormalized scores (often referred to as logits). + target: Tensor of the same shape as input with values between 0 and 1 + weight (Tensor, optional): a manual rescaling weight + if provided it's repeated to match input tensor shape + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + pos_weight (Tensor, optional): a weight of positive examples to be broadcasted with target. + Must be a tensor with equal size along the class dimension to the number of classes. + Pay close attention to PyTorch's broadcasting semantics in order to achieve the desired + operations. For a target of size [B, C, H, W] (where B is batch size) pos_weight of + size [B, C, H, W] will apply different pos_weights to each element of the batch or + [C, H, W] the same pos_weights across the batch. To apply the same positive weight + along all spatial dimensions for a 2D multi-class target [C, H, W] use: [C, 1, 1]. + Default: ``None`` + + Examples:: + + >>> input = torch.randn(3, requires_grad=True) + >>> target = torch.empty(3).random_(2) + >>> loss = F.binary_cross_entropy_with_logits(input, target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight, pos_weight): + return handle_torch_function( + binary_cross_entropy_with_logits, + (input, target, weight, pos_weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + pos_weight=pos_weight, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + + if not (target.size() == input.size()): + raise ValueError( + f"Target size ({target.size()}) must be the same as input size ({input.size()})" + ) + + return torch.binary_cross_entropy_with_logits( + input, target, weight, pos_weight, reduction_enum + ) + + +def smooth_l1_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + beta: float = 1.0, +) -> Tensor: + r"""Compute the Smooth L1 loss. + + Function uses a squared term if the absolute + element-wise error falls below beta and an L1 term otherwise. + + See :class:`~torch.nn.SmoothL1Loss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + beta (float, optional): Specifies the threshold at which to change from the squared + term to the L1 term in the loss calculation. This value must be positive. + Default: 1.0. + + Returns: + Tensor: L1 loss (optionally weighted). + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + smooth_l1_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + beta=beta, + ) + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if beta == 0.0: + return torch._C._nn.l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + else: + return torch._C._nn.smooth_l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + beta, + ) + + +def huber_loss( + input: Tensor, + target: Tensor, + reduction: str = "mean", + delta: float = 1.0, + weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute the Huber loss, with optional weighting. + + Function uses a squared term if the absolute + element-wise error falls below delta and a delta-scaled L1 term otherwise. + + When delta equals 1, this loss is equivalent to SmoothL1Loss. + In general, Huber loss differs from SmoothL1Loss by a factor of delta (AKA beta in Smooth L1). + + See :class:`~torch.nn.HuberLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + delta (float, optional): The threshold at which to change between delta-scaled L1 and L2 loss. Default: 1.0. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: Huber loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + huber_loss, + (input, target, weight), + input, + target, + reduction=reduction, + delta=delta, + weight=weight, + ) + + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is None: + # Use the optimized C++ backend for standard Huber loss + return torch._C._nn.huber_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + delta, + ) + else: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + # Calculate the unweighted loss first + unweighted_loss = torch._C._nn.huber_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum("none"), + delta, + ) + + # Apply weight to the unweighted loss + weighted_loss = unweighted_loss * weight + + if reduction == "none": + return weighted_loss + elif reduction == "sum": + return torch.sum(weighted_loss) + elif reduction == "mean": + return weighted_loss.mean() + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + + +def l1_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + weight: Optional[Tensor] = None, +) -> Tensor: # noqa: D400,D402 + r"""Compute the L1 loss, with optional weighting. + + Function that takes the mean element-wise absolute value difference. + + See :class:`~torch.nn.L1Loss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: L1 loss (optionally weighted). + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + l1_loss, + (input, target, weight), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is not None: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + absolute_errors = torch.abs(expanded_input - expanded_target) + weighted_absolute_errors = absolute_errors * weight + + if reduction == "none": + return weighted_absolute_errors + elif reduction == "sum": + return torch.sum(weighted_absolute_errors) + elif reduction == "mean": + return torch.sum(weighted_absolute_errors) / torch.sum(weight) + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + else: + return torch._C._nn.l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + + +def mse_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute the element-wise mean squared error, with optional weighting. + + See :class:`~torch.nn.MSELoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: Mean Squared Error loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + mse_loss, + (input, target, weight), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + weight=weight, + ) + + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is not None: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + # Perform weighted MSE loss manually + squared_errors = torch.pow(expanded_input - expanded_target, 2) + weighted_squared_errors = squared_errors * weight + + if reduction == "none": + return weighted_squared_errors + elif reduction == "sum": + return torch.sum(weighted_squared_errors) + elif reduction == "mean": + return torch.sum(weighted_squared_errors) / torch.sum(weight) + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + else: + return torch._C._nn.mse_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + + +def margin_ranking_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = 0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the margin ranking loss. + + See :class:`~torch.nn.MarginRankingLoss` for details. + + Args: + input1 (Tensor): Predicted values. + input2 (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Margin ranking loss. + """ + if has_torch_function_variadic(input1, input2, target): + return handle_torch_function( + margin_ranking_loss, + (input1, input2, target), + input1, + input2, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if input1.dim() != input2.dim() or input1.dim() != target.dim(): + raise RuntimeError( + f"margin_ranking_loss : All input tensors should have same dimension but got sizes: " + f"input1: {input1.size()}, input2: {input2.size()}, target: {target.size()} " + ) + return torch.margin_ranking_loss(input1, input2, target, margin, reduction_enum) + + +def hinge_embedding_loss( + input: Tensor, + target: Tensor, + margin: float = 1.0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the hinge embedding loss. + + See :class:`~torch.nn.HingeEmbeddingLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + margin (float, optional): Margin for hinge loss. Has a default value of 1. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Hinge embedding loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + hinge_embedding_loss, + (input, target), + input, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + return torch.hinge_embedding_loss(input, target, margin, reduction_enum) + + +def multilabel_margin_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multilabel margin loss. + + See :class:`~torch.nn.MultiLabelMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Mutilabel margin loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + multilabel_margin_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.multilabel_margin_loss(input, target, reduction_enum) + + +def soft_margin_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the soft margin loss. + + See :class:`~torch.nn.SoftMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Soft margin loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + soft_margin_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.soft_margin_loss(input, target, reduction_enum) + + +def multilabel_soft_margin_loss( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multilabel soft margin loss. + + See :class:`~torch.nn.MultiLabelSoftMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Mutilabel soft margin loss. + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + multilabel_soft_margin_loss, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + loss = -(target * logsigmoid(input) + (1 - target) * logsigmoid(-input)) + + if weight is not None: + loss = loss * weight + + class_dim = input.dim() - 1 + C = input.size(class_dim) + loss = loss.sum(dim=class_dim) / C # only return N loss values + + if reduction == "none": + ret = loss + elif reduction == "mean": + ret = loss.mean() + elif reduction == "sum": + ret = loss.sum() + else: + ret = input + raise ValueError(reduction + " is not valid") + return ret + + +def cosine_embedding_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = 0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the cosine embedding loss. + + See :class:`~torch.nn.CosineEmbeddingLoss` for details. + + Args: + input1 (Tensor): Predicted values. + input2 (Tensor): Predicted values. + target (Tensor): Ground truth values. + margin (float, optional): Margin for cosine embedding. Has a default value of 0. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Cosine embedding loss. + """ + if has_torch_function_variadic(input1, input2, target): + return handle_torch_function( + cosine_embedding_loss, + (input1, input2, target), + input1, + input2, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + return torch.cosine_embedding_loss(input1, input2, target, margin, reduction_enum) + + +def multi_margin_loss( + input: Tensor, + target: Tensor, + p: int = 1, + margin: float = 1.0, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multi margin loss, with optional weighting. + + See :class:`~torch.nn.MultiMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + p (int, optional): Has a default value of 1. 1 and 2 are the only supported values. + margin (float, optional): Margin for multi margin loss. Has a default value of 1. + weight (Tensor, optional): Weights for each sample. Default: None. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Multi margin loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + multi_margin_loss, + (input, target, weight), + input, + target, + p=p, + margin=margin, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if p != 1 and p != 2: + raise ValueError("only p == 1 and p == 2 supported") + if weight is not None: + if weight.dim() != 1: + raise ValueError("weight must be one-dimensional") + + return torch._C._nn.multi_margin_loss( + input, + target, + p, + margin, + weight, + # pyrefly: ignore [bad-argument-type] + reduction_enum, + ) + + +pixel_shuffle = _add_docstr( + torch.pixel_shuffle, + r""" +pixel_shuffle(input, upscale_factor) -> Tensor + +Rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` to a +tensor of shape :math:`(*, C, H \times r, W \times r)`, where r is the :attr:`upscale_factor`. + +See :class:`~torch.nn.PixelShuffle` for details. + +Args: + input (Tensor): the input tensor + upscale_factor (int): factor to increase spatial resolution by + +Examples:: + + >>> input = torch.randn(1, 9, 4, 4) + >>> output = torch.nn.functional.pixel_shuffle(input, 3) + >>> print(output.size()) + torch.Size([1, 1, 12, 12]) +""", +) + +pixel_unshuffle = _add_docstr( + torch.pixel_unshuffle, + r""" +pixel_unshuffle(input, downscale_factor) -> Tensor + +Reverses the :class:`~torch.nn.PixelShuffle` operation by rearranging elements in a +tensor of shape :math:`(*, C, H \times r, W \times r)` to a tensor of shape +:math:`(*, C \times r^2, H, W)`, where r is the :attr:`downscale_factor`. + +See :class:`~torch.nn.PixelUnshuffle` for details. + +Args: + input (Tensor): the input tensor + downscale_factor (int): factor to increase spatial resolution by + +Examples:: + + >>> input = torch.randn(1, 1, 12, 12) + >>> output = torch.nn.functional.pixel_unshuffle(input, 3) + >>> print(output.size()) + torch.Size([1, 9, 4, 4]) +""", +) + +channel_shuffle = _add_docstr( + torch.channel_shuffle, + r""" +channel_shuffle(input, groups) -> Tensor + +Divide the channels in a tensor of shape :math:`(*, C , H, W)` +into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, +while keeping the original tensor shape. + +See :class:`~torch.nn.ChannelShuffle` for details. + +Args: + input (Tensor): the input tensor + groups (int): number of groups to divide channels in and rearrange. + +Examples:: + + >>> input = torch.randn(1, 4, 2, 2) + >>> print(input) + [[[[1, 2], + [3, 4]], + [[5, 6], + [7, 8]], + [[9, 10], + [11, 12]], + [[13, 14], + [15, 16]], + ]] + >>> output = torch.nn.functional.channel_shuffle(input, 2) + >>> print(output) + [[[[1, 2], + [3, 4]], + [[9, 10], + [11, 12]], + [[5, 6], + [7, 8]], + [[13, 14], + [15, 16]], + ]] +""", +) + +native_channel_shuffle = _add_docstr( + torch.native_channel_shuffle, + r""" +native_channel_shuffle(input, groups) -> Tensor + +Native kernel level implementation of the `channel_shuffle`. +This function might become private in future releases, use with caution. + +Divide the channels in a tensor of shape :math:`(*, C , H, W)` +into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, +while keeping the original tensor shape. + +See :class:`~torch.nn.ChannelShuffle` for details. + +Args: + input (Tensor): the input tensor + groups (int): number of groups to divide channels in and rearrange. + +Examples:: + + >>> input = torch.randn(1, 4, 2, 2) + >>> print(input) + [[[[1, 2], + [3, 4]], + [[5, 6], + [7, 8]], + [[9, 10], + [11, 12]], + [[13, 14], + [15, 16]], + ]] + >>> output = torch.nn.functional.native_channel_shuffle(input, 2) + >>> print(output) + [[[[1, 2], + [3, 4]], + [[9, 10], + [11, 12]], + [[5, 6], + [7, 8]], + [[13, 14], + [15, 16]], + ]] +""", +) + + +@_overload +def upsample( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def upsample( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +def upsample( # noqa: F811 + input, + size=None, + scale_factor=None, + mode="nearest", + align_corners=None, +): + r"""Upsample input. + + Provided tensor is upsampled to either the given :attr:`size` or the given + :attr:`scale_factor` + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with ``nn.functional.interpolate(...)``. + + Note: + {backward_reproducibility_note} + + The algorithm used for upsampling is determined by :attr:`mode`. + + Currently temporal, spatial and volumetric upsampling are supported, i.e. + expected inputs are 3-D, 4-D or 5-D in shape. + + The input dimensions are interpreted in the form: + `mini-batch x channels x [optional depth] x [optional height] x width`. + + The modes available for upsampling are: `nearest`, `linear` (3D-only), + `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only) + + Args: + input (Tensor): the input tensor + size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): + output spatial size. + scale_factor (float or Tuple[float]): multiplier for spatial size. Has to match input size if it is a tuple. + mode (str): algorithm used for upsampling: + ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | + ``'trilinear'``. Default: ``'nearest'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input and output as squares rather than points. + If set to ``True``, the input and output tensors are aligned by the + center points of their corner pixels, preserving the values at the corner pixels. + If set to ``False``, the input and output tensors are aligned by the corner + points of their corner pixels, and the interpolation uses edge value padding + for out-of-boundary values, making this operation *independent* of input size + when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` + is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. + Default: ``False`` + + .. note:: + With ``mode='bicubic'``, it's possible to cause overshoot, in other words it can produce + negative values or values greater than 255 for images. + Explicitly call ``result.clamp(min=0, max=255)`` if you want to reduce the overshoot + when displaying the image. + + .. warning:: + With ``align_corners = True``, the linearly interpolating modes + (`linear`, `bilinear`, and `trilinear`) don't proportionally align the + output and input pixels, and thus the output values can depend on the + input size. This was the default behavior for these modes up to version + 0.3.1. Since then, the default behavior is ``align_corners = False``. + See :class:`~torch.nn.Upsample` for concrete examples on how this + affects the outputs. + + """ + warnings.warn( + "`nn.functional.upsample` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode, align_corners) + + +if upsample.__doc__: + upsample.__doc__ = upsample.__doc__.format(**reproducibility_notes) + + +def _is_integer(x) -> bool: + r"""Type check the input number is an integer. + + Will return True for int, SymInt, Numpy integers and Tensors with integer elements. + """ + if isinstance(x, (int, torch.SymInt)): + return True + if np is not None and isinstance(x, np.integer): + return True + return isinstance(x, Tensor) and not x.is_floating_point() + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, +) -> Tensor: # noqa: B950 + r"""Down/up samples the input. + + Tensor interpolated to either the given :attr:`size` or the given + :attr:`scale_factor` + + The algorithm used for interpolation is determined by :attr:`mode`. + + Currently temporal, spatial and volumetric sampling are supported, i.e. + expected inputs are 3-D, 4-D or 5-D in shape. + + The input dimensions are interpreted in the form: + `mini-batch x channels x [optional depth] x [optional height] x width`. + + The modes available for resizing are: `nearest`, `linear` (3D-only), + `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only), `area`, `nearest-exact` + + Args: + input (Tensor): the input tensor + size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): + output spatial size. + scale_factor (float or Tuple[float]): multiplier for spatial size. If `scale_factor` is a tuple, + its length has to match the number of spatial dimensions; `input.dim() - 2`. + mode (str): algorithm used for upsampling: + ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | + ``'trilinear'`` | ``'area'`` | ``'nearest-exact'``. Default: ``'nearest'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input and output as squares rather than points. + If set to ``True``, the input and output tensors are aligned by the + center points of their corner pixels, preserving the values at the corner pixels. + If set to ``False``, the input and output tensors are aligned by the corner + points of their corner pixels, and the interpolation uses edge value padding + for out-of-boundary values, making this operation *independent* of input size + when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` + is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. + Default: ``False`` + recompute_scale_factor (bool, optional): recompute the scale_factor for use in the + interpolation calculation. If `recompute_scale_factor` is ``True``, then + `scale_factor` must be passed in and `scale_factor` is used to compute the + output `size`. The computed output `size` will be used to infer new scales for + the interpolation. Note that when `scale_factor` is floating-point, it may differ + from the recomputed `scale_factor` due to rounding and precision issues. + If `recompute_scale_factor` is ``False``, then `size` or `scale_factor` will + be used directly for interpolation. Default: ``None``. + antialias (bool, optional): flag to apply anti-aliasing. Default: ``False``. Using anti-alias + option together with ``align_corners=False``, interpolation result would match Pillow + result for downsampling operation. Supported modes: ``'bilinear'``, ``'bicubic'``. + + .. note:: + With ``mode='bicubic'``, it's possible to cause overshoot. For some dtypes, it can produce + negative values or values greater than 255 for images. Explicitly call ``result.clamp(min=0,max=255)`` + if you want to reduce the overshoot when displaying the image. + For ``uint8`` inputs, it already performs saturating cast operation. So, no manual `clamp` operation is needed. + + .. note:: + Mode ``mode='nearest-exact'`` matches Scikit-Image and PIL nearest neighbours interpolation + algorithms and fixes known issues with ``mode='nearest'``. This mode is introduced to keep + backward compatibility. + Mode ``mode='nearest'`` matches buggy OpenCV's ``INTER_NEAREST`` interpolation algorithm. + + .. note:: + The gradients for the dtype ``float16`` on CUDA may be inaccurate in the upsample operation + when using modes ``['linear', 'bilinear', 'bicubic', 'trilinear', 'area']``. + For more details, please refer to the discussion in + `issue#104157 `_. + + Note: + {backward_reproducibility_note} + """ + if has_torch_function_unary(input): + return handle_torch_function( + interpolate, + (input,), + input, + size=size, + scale_factor=scale_factor, + mode=mode, + align_corners=align_corners, + recompute_scale_factor=recompute_scale_factor, + antialias=antialias, + ) + + if mode in ("nearest", "area", "nearest-exact"): + if align_corners is not None: + raise ValueError( + "align_corners option can only be set with the " + "interpolating modes: linear | bilinear | bicubic | trilinear" + ) + else: + if align_corners is None: + align_corners = False + + dim = input.dim() - 2 # Number of spatial dimensions. + + # Process size and scale_factor. Validate that exactly one is set. + # Validate its length if it is a list, or expand it if it is a scalar. + # After this block, exactly one of output_size and scale_factors will + # be non-None, and it will be a list (or tuple). + if size is not None and scale_factor is not None: + raise ValueError("only one of size or scale_factor should be defined") + elif size is not None: + assert scale_factor is None + scale_factors = None + if isinstance(size, (list, tuple)): + if len(size) != dim: + raise ValueError( + "Input and output must have the same number of spatial dimensions, but got " + f"input with spatial dimensions of {list(input.shape[2:])} and output size of {size}. " + "Please provide input tensor in (N, C, d1, d2, ...,dK) format and " + "output size in (o1, o2, ...,oK) format." + ) + if not torch.jit.is_scripting(): + if not all(_is_integer(x) for x in size): + raise TypeError( + "expected size to be one of int or Tuple[int] or Tuple[int, int] or " + f"Tuple[int, int, int], but got size with types {[type(x) for x in size]}" + ) + output_size = size + else: + output_size = [size for _ in range(dim)] + elif scale_factor is not None: + assert size is None + output_size = None + if isinstance(scale_factor, (list, tuple)): + if len(scale_factor) != dim: + raise ValueError( + "Input and scale_factor must have the same number of spatial dimensions, but " + f"got input with spatial dimensions of {list(input.shape[2:])} and " + f"scale_factor of shape {scale_factor}. " + "Please provide input tensor in (N, C, d1, d2, ...,dK) format and " + "scale_factor in (s1, s2, ...,sK) format." + ) + scale_factors = scale_factor + else: + scale_factors = [scale_factor for _ in range(dim)] + else: + raise ValueError("either size or scale_factor should be defined") + + if ( + recompute_scale_factor is not None + and recompute_scale_factor + and size is not None + ): + raise ValueError( + "recompute_scale_factor is not meaningful with an explicit size." + ) + + # "area" mode always requires an explicit size rather than scale factor. + # Reuse the recompute_scale_factor code path. + if mode == "area" and output_size is None: + recompute_scale_factor = True + + if recompute_scale_factor is not None and recompute_scale_factor: + # We compute output_size here, then un-set scale_factors. + # The C++ code will recompute it based on the (integer) output size. + assert scale_factors is not None + if not torch.jit.is_scripting() and torch._C._get_tracing_state(): + # make scale_factor a tensor in tracing so constant doesn't get baked in + output_size = [ + ( + torch.floor( + ( + # pyrefly: ignore [missing-attribute] + input.size(i + 2).float() + * torch.tensor(scale_factors[i], dtype=torch.float32) + ).float() + ) + ) + for i in range(dim) + ] + elif torch.jit.is_scripting(): + output_size = [ + math.floor(float(input.size(i + 2)) * scale_factors[i]) + for i in range(dim) + ] + else: + output_size = [ + _sym_int(input.size(i + 2) * scale_factors[i]) for i in range(dim) + ] + scale_factors = None + + if antialias and not (mode in ("bilinear", "bicubic") and input.ndim == 4): + raise ValueError( + "Anti-alias option is restricted to bilinear and bicubic modes and requires a 4-D tensor as input" + ) + + if input.dim() == 3 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest1d(input, output_size, scale_factors) + if input.dim() == 4 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest2d(input, output_size, scale_factors) + if input.dim() == 5 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest3d(input, output_size, scale_factors) + + if input.dim() == 3 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact1d(input, output_size, scale_factors) + if input.dim() == 4 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact2d(input, output_size, scale_factors) + if input.dim() == 5 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact3d(input, output_size, scale_factors) + + if input.dim() == 3 and mode == "area": + assert output_size is not None + # pyrefly: ignore [bad-argument-type] + return adaptive_avg_pool1d(input, output_size) + if input.dim() == 4 and mode == "area": + assert output_size is not None + return adaptive_avg_pool2d(input, output_size) + if input.dim() == 5 and mode == "area": + assert output_size is not None + return adaptive_avg_pool3d(input, output_size) + + if input.dim() == 3 and mode == "linear": + assert align_corners is not None + return torch._C._nn.upsample_linear1d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 4 and mode == "bilinear": + assert align_corners is not None + if antialias: + return torch._C._nn._upsample_bilinear2d_aa( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + # Two levels are necessary to prevent TorchScript from touching + # are_deterministic_algorithms_enabled. + if not torch.jit.is_scripting(): + if not input.is_cpu and torch.are_deterministic_algorithms_enabled(): + # Use slow decomp whose backward will be in terms of index_put + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._upsample_linear_vec(input, output_size, align_corners, scale_factors) + return torch._C._nn.upsample_bilinear2d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 5 and mode == "trilinear": + assert align_corners is not None + # Two levels are necessary to prevent TorchScript from touching + # are_deterministic_algorithms_enabled. + if not torch.jit.is_scripting(): + if not input.is_cpu and torch.are_deterministic_algorithms_enabled(): + # Use slow decomp whose backward will be in terms of index_put + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._upsample_linear_vec(input, output_size, align_corners, scale_factors) + return torch._C._nn.upsample_trilinear3d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 4 and mode == "bicubic": + assert align_corners is not None + if antialias: + return torch._C._nn._upsample_bicubic2d_aa( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + return torch._C._nn.upsample_bicubic2d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + + if input.dim() == 3 and mode == "bilinear": + raise NotImplementedError("Got 3D input, but bilinear mode needs 4D input") + if input.dim() == 3 and mode == "trilinear": + raise NotImplementedError("Got 3D input, but trilinear mode needs 5D input") + if input.dim() == 4 and mode == "linear": + raise NotImplementedError("Got 4D input, but linear mode needs 3D input") + if input.dim() == 4 and mode == "trilinear": + raise NotImplementedError("Got 4D input, but trilinear mode needs 5D input") + if input.dim() == 5 and mode == "linear": + raise NotImplementedError("Got 5D input, but linear mode needs 3D input") + if input.dim() == 5 and mode == "bilinear": + raise NotImplementedError("Got 5D input, but bilinear mode needs 4D input") + + raise NotImplementedError( + "Input Error: Only 3D, 4D and 5D input Tensors supported" + f" (got {input.dim()}D) for the modes: nearest | linear | bilinear | bicubic | trilinear | area | nearest-exact" + f" (got {mode})" + ) + + +if interpolate.__doc__: + interpolate.__doc__ = interpolate.__doc__.format(**reproducibility_notes) + + +@_overload +def upsample_nearest( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_nearest( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def upsample_nearest(input, size=None, scale_factor=None): # noqa: F811 + r"""Upsamples the input, using nearest neighbours' pixel values. + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with ``nn.functional.interpolate(..., mode='nearest')``. + + Currently spatial and volumetric upsampling are supported (i.e. expected + inputs are 4 or 5 dimensional). + + Args: + input (Tensor): input + size (int or Tuple[int, int] or Tuple[int, int, int]): output spatia + size. + scale_factor (int): multiplier for spatial size. Has to be an integer. + + Note: + {backward_reproducibility_note} + """ + # DeprecationWarning is ignored by default + warnings.warn( + "`nn.functional.upsample_nearest` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode="nearest") + + +if upsample_nearest.__doc__: + upsample_nearest.__doc__ = upsample_nearest.__doc__.format(**reproducibility_notes) + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[list[float]] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def upsample_bilinear(input, size=None, scale_factor=None): # noqa: F811 + r"""Upsamples the input, using bilinear upsampling. + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with + ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``. + + Expected inputs are spatial (4 dimensional). Use `upsample_trilinear` for + volumetric (5 dimensional) inputs. + + Args: + input (Tensor): input + size (int or Tuple[int, int]): output spatial size. + scale_factor (int or Tuple[int, int]): multiplier for spatial size + + Note: + {backward_reproducibility_note} + """ + # DeprecationWarning is ignored by default + warnings.warn( + "`nn.functional.upsample_bilinear` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode="bilinear", align_corners=True) + + +if upsample_bilinear.__doc__: + upsample_bilinear.__doc__ = upsample_bilinear.__doc__.format( + **reproducibility_notes + ) + +GRID_SAMPLE_INTERPOLATION_MODES = { + "bilinear": 0, + "nearest": 1, + "bicubic": 2, +} + +GRID_SAMPLE_PADDING_MODES = { + "zeros": 0, + "border": 1, + "reflection": 2, +} + + +def grid_sample( + input: Tensor, + grid: Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: Optional[bool] = None, +) -> Tensor: + r"""Compute grid sample. + + Given an :attr:`input` and a flow-field :attr:`grid`, computes the + ``output`` using :attr:`input` values and pixel locations from :attr:`grid`. + + Currently, only spatial (4-D) and volumetric (5-D) :attr:`input` are + supported. + + In the spatial (4-D) case, for :attr:`input` with shape + :math:`(N, C, H_\text{in}, W_\text{in})` and :attr:`grid` with shape + :math:`(N, H_\text{out}, W_\text{out}, 2)`, the output will have shape + :math:`(N, C, H_\text{out}, W_\text{out})`. + + For each output location ``output[n, :, h, w]``, the size-2 vector + ``grid[n, h, w]`` specifies :attr:`input` pixel locations ``x`` and ``y``, + which are used to interpolate the output value ``output[n, :, h, w]``. + In the case of 5D inputs, ``grid[n, d, h, w]`` specifies the + ``x``, ``y``, ``z`` pixel locations for interpolating + ``output[n, :, d, h, w]``. :attr:`mode` argument specifies ``nearest`` or + ``bilinear`` interpolation method to sample the input pixels. + + :attr:`grid` specifies the sampling pixel locations normalized by the + :attr:`input` spatial dimensions. Therefore, it should have most values in + the range of ``[-1, 1]``. For example, values ``x = -1, y = -1`` is the + left-top pixel of :attr:`input`, and values ``x = 1, y = 1`` is the + right-bottom pixel of :attr:`input`. + + If :attr:`grid` has values outside the range of ``[-1, 1]``, the corresponding + outputs are handled as defined by :attr:`padding_mode`. Options are + + * ``padding_mode="zeros"``: use ``0`` for out-of-bound grid locations, + * ``padding_mode="border"``: use border values for out-of-bound grid locations, + * ``padding_mode="reflection"``: use values at locations reflected by + the border for out-of-bound grid locations. For location far away + from the border, it will keep being reflected until becoming in bound, + e.g., (normalized) pixel location ``x = -3.5`` reflects by border ``-1`` + and becomes ``x' = 1.5``, then reflects by border ``1`` and becomes + ``x'' = -0.5``. + + Note: + This function is often used in conjunction with :func:`affine_grid` + to build `Spatial Transformer Networks`_ . + + Note: + When using the CUDA backend, this operation may induce nondeterministic + behaviour in its backward pass that is not easily switched off. + Please see the notes on :doc:`/notes/randomness` for background. + + Note: + NaN values in :attr:`grid` would be interpreted as ``-1``. + + Args: + input (Tensor): input of shape :math:`(N, C, H_\text{in}, W_\text{in})` (4-D case) + or :math:`(N, C, D_\text{in}, H_\text{in}, W_\text{in})` (5-D case) + grid (Tensor): flow-field of shape :math:`(N, H_\text{out}, W_\text{out}, 2)` (4-D case) + or :math:`(N, D_\text{out}, H_\text{out}, W_\text{out}, 3)` (5-D case) + mode (str): interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'`` | ``'bicubic'``. Default: ``'bilinear'`` + Note: ``mode='bicubic'`` supports only 4-D input. + When ``mode='bilinear'`` and the input is 5-D, the interpolation mode + used internally will actually be trilinear. However, when the input is 4-D, + the interpolation mode will legitimately be bilinear. + padding_mode (str): padding mode for outside grid values + ``'zeros'`` | ``'border'`` | ``'reflection'``. Default: ``'zeros'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input as squares rather than points. + If set to ``True``, the extrema (``-1`` and ``1``) are considered as referring + to the center points of the input's corner pixels. If set to ``False``, they + are instead considered as referring to the corner points of the input's corner + pixels, making the sampling more resolution agnostic. + This option parallels the ``align_corners`` option in + :func:`interpolate`, and so whichever option is used here + should also be used there to resize the input image before grid sampling. + Default: ``False`` + + Returns: + output (Tensor): output Tensor + + .. _`Spatial Transformer Networks`: + https://arxiv.org/abs/1506.02025 + + .. warning:: + When ``align_corners = True``, the grid positions depend on the pixel + size relative to the input image size, and so the locations sampled by + :func:`grid_sample` will differ for the same input given at different + resolutions (that is, after being upsampled or downsampled). + The default behavior up to version 1.2.0 was ``align_corners = True``. + Since then, the default behavior has been changed to ``align_corners = False``, + in order to bring it in line with the default for :func:`interpolate`. + + .. note:: + ``mode='bicubic'`` is implemented using the `cubic convolution algorithm`_ with :math:`\alpha=-0.75`. + The constant :math:`\alpha` might be different from packages to packages. + For example, `PIL`_ and `OpenCV`_ use -0.5 and -0.75 respectively. + This algorithm may "overshoot" the range of values it's interpolating. + For example, it may produce negative values or values greater than 255 when interpolating input in [0, 255]. + Clamp the results with :func:`torch.clamp` to ensure they are within the valid range. + .. _`cubic convolution algorithm`: https://en.wikipedia.org/wiki/Bicubic_interpolation + .. _`PIL`: https://github.com/python-pillow/Pillow/blob/4634eafe3c695a014267eefdce830b4a825beed7/src/libImaging/Resample.c#L51 + .. _`OpenCV`: https://github.com/opencv/opencv/blob/f345ed564a06178670750bad59526cfa4033be55/modules/imgproc/src/resize.cpp#L908 + """ + if has_torch_function_variadic(input, grid): + return handle_torch_function( + grid_sample, + (input, grid), + input, + grid, + mode=mode, + padding_mode=padding_mode, + align_corners=align_corners, + ) + if mode != "bilinear" and mode != "nearest" and mode != "bicubic": + raise ValueError( + f"nn.functional.grid_sample(): expected mode to be 'bilinear', 'nearest' or 'bicubic', but got: '{mode}'" + ) + if ( + padding_mode != "zeros" + and padding_mode != "border" + and padding_mode != "reflection" + ): + raise ValueError( + "nn.functional.grid_sample(): expected padding_mode " + "to be 'zeros', 'border', or 'reflection', " + f"but got: '{padding_mode}'" + ) + + if mode == "bilinear": + mode_enum = 0 + elif mode == "nearest": + mode_enum = 1 + else: # mode == 'bicubic' + mode_enum = 2 + + if padding_mode == "zeros": + padding_mode_enum = 0 + elif padding_mode == "border": + padding_mode_enum = 1 + else: # padding_mode == 'reflection' + padding_mode_enum = 2 + + if align_corners is None: + warnings.warn( + "Default grid_sample and affine_grid behavior has changed " + "to align_corners=False since 1.3.0. Please specify " + "align_corners=True if the old behavior is desired. " + "See the documentation of grid_sample for details.", + stacklevel=2, + ) + align_corners = False + + return torch.grid_sampler(input, grid, mode_enum, padding_mode_enum, align_corners) + + +def affine_grid( + theta: Tensor, + size: list[int], + align_corners: Optional[bool] = None, +) -> Tensor: + r"""Generate 2D or 3D flow field (sampling grid), given a batch of affine matrices :attr:`theta`. + + .. note:: + This function is often used in conjunction with :func:`grid_sample` + to build `Spatial Transformer Networks`_ . + + Args: + theta (Tensor): input batch of affine matrices with shape + (:math:`N \times 2 \times 3`) for 2D or + (:math:`N \times 3 \times 4`) for 3D + size (torch.Size): the target output image size. + (:math:`N \times C \times H \times W` for 2D or + :math:`N \times C \times D \times H \times W` for 3D) + Example: torch.Size((32, 3, 24, 24)) + align_corners (bool, optional): if ``True``, consider ``-1`` and ``1`` + to refer to the centers of the corner pixels rather than the image corners. + Refer to :func:`grid_sample` for a more complete description. + A grid generated by :func:`affine_grid` should be passed to :func:`grid_sample` + with the same setting for this option. + Default: ``False`` + + Returns: + output (Tensor): output Tensor of size (:math:`N \times H \times W \times 2`) + + .. _`Spatial Transformer Networks`: + https://arxiv.org/abs/1506.02025 + + .. warning:: + When ``align_corners = True``, the grid positions depend on the pixel + size relative to the input image size, and so the locations sampled by + :func:`grid_sample` will differ for the same input given at different + resolutions (that is, after being upsampled or downsampled). + The default behavior up to version 1.2.0 was ``align_corners = True``. + Since then, the default behavior has been changed to ``align_corners = False``, + in order to bring it in line with the default for :func:`interpolate`. + .. warning:: + When ``align_corners = True``, 2D affine transforms on 1D data and + 3D affine transforms on 2D data (that is, when one of the spatial + dimensions has unit size) are ill-defined, and not an intended use case. + This is not a problem when ``align_corners = False``. + Up to version 1.2.0, all grid points along a unit dimension were + considered arbitrarily to be at ``-1``. + From version 1.3.0, under ``align_corners = True`` all grid points + along a unit dimension are considered to be at ``0`` + (the center of the input image). + """ + if has_torch_function_unary(theta): + return handle_torch_function( + affine_grid, (theta,), theta, size, align_corners=align_corners + ) + if align_corners is None: + warnings.warn( + "Default grid_sample and affine_grid behavior has changed " + "to align_corners=False since 1.3.0. Please specify " + "align_corners=True if the old behavior is desired. " + "See the documentation of grid_sample for details.", + stacklevel=2, + ) + align_corners = False + + # enforce floating point dtype on theta + if not theta.is_floating_point(): + raise ValueError( + f"Expected theta to have floating point type, but got {theta.dtype}" + ) + # check that shapes and sizes match + if len(size) == 4: + if theta.dim() != 3 or theta.shape[-2] != 2 or theta.shape[-1] != 3: + raise ValueError( + f"Expected a batch of 2D affine matrices of shape Nx2x3 for size {size}. Got {theta.shape}." + ) + spatial_size = size[-2:] # spatial dimension sizes + elif len(size) == 5: + if theta.dim() != 3 or theta.shape[-2] != 3 or theta.shape[-1] != 4: + raise ValueError( + f"Expected a batch of 3D affine matrices of shape Nx3x4 for size {size}. Got {theta.shape}." + ) + spatial_size = size[-3:] # spatial dimension sizes + else: + raise NotImplementedError( + "affine_grid only supports 4D and 5D sizes, " + "for 2D and 3D affine transforms, respectively. " + f"Got size {size}." + ) + # check for empty span + if align_corners and min(spatial_size) == 1: + warnings.warn( + "Since version 1.3.0, affine_grid behavior has changed " + "for unit-size grids when align_corners=True. " + "This is not an intended use case of affine_grid. " + "See the documentation of affine_grid for details.", + stacklevel=2, + ) + elif min(size) <= 0: + raise ValueError(f"Expected non-zero, positive output size. Got {size}") + + return torch.affine_grid_generator(theta, size, align_corners) + + +def pad( + input: Tensor, + pad: list[int], + mode: str = "constant", + value: Optional[float] = None, +) -> Tensor: + r""" + pad(input, pad, mode="constant", value=None) -> Tensor + + Pads tensor. + + Padding size: + The padding size by which to pad some dimensions of :attr:`input` + are described starting from the last dimension and moving forward. + :math:`\left\lfloor\frac{\text{len(pad)}}{2}\right\rfloor` dimensions + of ``input`` will be padded. + For example, to pad only the last dimension of the input tensor, then + :attr:`pad` has the form + :math:`(\text{padding\_left}, \text{padding\_right})`; + to pad the last 2 dimensions of the input tensor, then use + :math:`(\text{padding\_left}, \text{padding\_right},` + :math:`\text{padding\_top}, \text{padding\_bottom})`; + to pad the last 3 dimensions, use + :math:`(\text{padding\_left}, \text{padding\_right},` + :math:`\text{padding\_top}, \text{padding\_bottom}` + :math:`\text{padding\_front}, \text{padding\_back})`. + + Padding mode: + See :class:`torch.nn.CircularPad2d`, :class:`torch.nn.ConstantPad2d`, + :class:`torch.nn.ReflectionPad2d`, and :class:`torch.nn.ReplicationPad2d` + for concrete examples on how each of the padding modes works. Constant + padding is implemented for arbitrary dimensions. Circular, replicate and + reflection padding are implemented for padding the last 3 dimensions of a + 4D or 5D input tensor, the last 2 dimensions of a 3D or 4D input tensor, + or the last dimension of a 2D or 3D input tensor. + + Note: + When using the CUDA backend, this operation may induce nondeterministic + behaviour in its backward pass that is not easily switched off. + Please see the notes on :doc:`/notes/randomness` for background. + + Args: + input (Tensor): N-dimensional tensor + pad (tuple): m-elements tuple, where + :math:`\frac{m}{2} \leq` input dimensions and :math:`m` is even. + mode: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + Default: ``'constant'`` + value: fill value for ``'constant'`` padding. Default: ``0`` + + Examples:: + + >>> t4d = torch.empty(3, 3, 4, 2) + >>> p1d = (1, 1) # pad last dim by 1 on each side + >>> out = F.pad(t4d, p1d, "constant", 0) # effectively zero padding + >>> print(out.size()) + torch.Size([3, 3, 4, 4]) + >>> p2d = (1, 1, 2, 2) # pad last dim by (1, 1) and 2nd to last by (2, 2) + >>> out = F.pad(t4d, p2d, "constant", 0) + >>> print(out.size()) + torch.Size([3, 3, 8, 4]) + >>> t4d = torch.empty(3, 3, 4, 2) + >>> p3d = (0, 1, 2, 1, 3, 3) # pad by (0, 1), (2, 1), and (3, 3) + >>> out = F.pad(t4d, p3d, "constant", 0) + >>> print(out.size()) + torch.Size([3, 9, 7, 3]) + """ + if has_torch_function_unary(input): + return handle_torch_function( + torch.nn.functional.pad, (input,), input, pad, mode=mode, value=value + ) + if not torch.jit.is_scripting(): + if torch.are_deterministic_algorithms_enabled() and ( + input.is_cuda or input.is_xpu + ): + if mode == "replicate": + # Use slow decomp whose backward will be in terms of index_put. + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._replication_pad(input, pad) + return torch._C._nn.pad(input, pad, mode, value) + + +# TODO: Fix via https://github.com/pytorch/pytorch/issues/75798 +pad.__module__ = "torch.nn.functional" + +# distance + + +pairwise_distance = _add_docstr( + torch.pairwise_distance, + r""" +pairwise_distance(x1, x2, p=2.0, eps=1e-6, keepdim=False) -> Tensor + +See :class:`torch.nn.PairwiseDistance` for details +""", +) + + +pdist = _add_docstr( + torch.pdist, + r""" +pdist(input, p=2) -> Tensor + +Computes the p-norm distance between every pair of row vectors in the input. +This is identical to the upper triangular portion, excluding the diagonal, of +`torch.norm(input[:, None] - input, dim=2, p=p)`. This function will be faster +if the rows are contiguous. + +If input has shape :math:`N \times M` then the output will have shape +:math:`\frac{1}{2} N (N - 1)`. + +This function is equivalent to ``scipy.spatial.distance.pdist(input, +'minkowski', p=p)`` if :math:`p \in (0, \infty)`. When :math:`p = 0` it is +equivalent to ``scipy.spatial.distance.pdist(input, 'hamming') * M``. +When :math:`p = \infty`, the closest scipy function is +``scipy.spatial.distance.pdist(xn, lambda x, y: np.abs(x - y).max())``. + +Args: + input: input tensor of shape :math:`N \times M`. + p: p value for the p-norm distance to calculate between each vector pair + :math:`\in [0, \infty]`. +""", +) + + +cosine_similarity = _add_docstr( + torch.cosine_similarity, + r""" +cosine_similarity(x1, x2, dim=1, eps=1e-8) -> Tensor + +Returns cosine similarity between ``x1`` and ``x2``, computed along dim. ``x1`` and ``x2`` must be broadcastable +to a common shape. ``dim`` refers to the dimension in this common shape. Dimension ``dim`` of the output is +squeezed (see :func:`torch.squeeze`), resulting in the +output tensor having 1 fewer dimension. + +.. math :: + \text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2, \epsilon) \cdot \max(\Vert x_2 \Vert _2, \epsilon)} + +Supports :ref:`type promotion `. + +Args: + x1 (Tensor): First input. + x2 (Tensor): Second input. + dim (int, optional): Dimension along which cosine similarity is computed. Default: 1 + eps (float, optional): Small value to avoid division by zero. + Default: 1e-8 + +Example:: + + >>> input1 = torch.randn(100, 128) + >>> input2 = torch.randn(100, 128) + >>> output = F.cosine_similarity(input1, input2) + >>> print(output) +""", +) + + +one_hot = _add_docstr( + torch._C._nn.one_hot, + r""" +one_hot(tensor, num_classes=-1) -> LongTensor + +Takes LongTensor with index values of shape ``(*)`` and returns a tensor +of shape ``(*, num_classes)`` that have zeros everywhere except where the +index of last dimension matches the corresponding value of the input tensor, +in which case it will be 1. + +See also `One-hot on Wikipedia`_ . + +.. _One-hot on Wikipedia: + https://en.wikipedia.org/wiki/One-hot + +Arguments: + tensor (LongTensor): class values of any shape. + num_classes (int, optional): Total number of classes. If set to -1, the number + of classes will be inferred as one greater than the largest class + value in the input tensor. Default: -1 + +Returns: + LongTensor that has one more dimension with 1 values at the + index of last dimension indicated by the input, and 0 everywhere + else. + +Examples: + >>> F.one_hot(torch.arange(0, 5) % 3) + tensor([[1, 0, 0], + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + [0, 1, 0]]) + >>> F.one_hot(torch.arange(0, 5) % 3, num_classes=5) + tensor([[1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0]]) + >>> F.one_hot(torch.arange(0, 6).view(3,2) % 3) + tensor([[[1, 0, 0], + [0, 1, 0]], + [[0, 0, 1], + [1, 0, 0]], + [[0, 1, 0], + [0, 0, 1]]]) +""", +) + + +def triplet_margin_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + margin: float = 1.0, + p: float = 2, + eps: float = 1e-6, + swap: bool = False, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the triplet loss between given input tensors and a margin greater than 0. + + See :class:`~torch.nn.TripletMarginLoss` for details. + """ + if has_torch_function_variadic(anchor, positive, negative): + return handle_torch_function( + triplet_margin_loss, + (anchor, positive, negative), + anchor, + positive, + negative, + margin=margin, + p=p, + eps=eps, + swap=swap, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if margin <= 0: + raise ValueError(f"margin must be greater than 0, got {margin}") + return torch.triplet_margin_loss( + anchor, positive, negative, margin, p, eps, swap, reduction_enum + ) + + +def triplet_margin_with_distance_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + *, + distance_function: Optional[Callable[[Tensor, Tensor], Tensor]] = None, + margin: float = 1.0, + swap: bool = False, + reduction: str = "mean", +) -> Tensor: + r"""Compute the triplet margin loss for input tensors using a custom distance function. + + See :class:`~torch.nn.TripletMarginWithDistanceLoss` for details. + """ + if torch.jit.is_scripting(): + raise NotImplementedError( + "F.triplet_margin_with_distance_loss does not support JIT scripting: " + "functions requiring Callables cannot be scripted." + ) + + if has_torch_function_variadic(anchor, positive, negative): + return handle_torch_function( + triplet_margin_with_distance_loss, + (anchor, positive, negative), + anchor, + positive, + negative, + distance_function=distance_function, + margin=margin, + swap=swap, + reduction=reduction, + ) + + # Check validity of reduction mode + if reduction not in ("mean", "sum", "none"): + raise ValueError(f"{reduction} is not a valid value for reduction") + + # Check validity of margin + if margin <= 0: + raise ValueError(f"margin must be greater than 0, got {margin}") + + # Check dimensions + a_dim = anchor.ndim + p_dim = positive.ndim + n_dim = negative.ndim + if not (a_dim == p_dim and p_dim == n_dim): + raise RuntimeError( + f"The anchor, positive, and negative tensors are expected to have " + f"the same number of dimensions, but got: anchor {a_dim}D, " + f"positive {p_dim}D, and negative {n_dim}D inputs" + ) + + # Calculate loss + if distance_function is None: + distance_function = torch.pairwise_distance + + dist_pos = distance_function(anchor, positive) + dist_neg = distance_function(anchor, negative) + # The distance swap is described in the paper "Learning shallow + # convolutional feature descriptors with triplet losses" by V. Balntas, E. + # Riba et al. If True, and if the positive example is closer to the + # negative example than the anchor is, swaps the positive example and the + # anchor in the loss computation. + if swap: + dist_swap = distance_function(positive, negative) + dist_neg = torch.minimum(dist_neg, dist_swap) + loss = torch.clamp_min(margin + dist_pos - dist_neg, 0) + + # Apply reduction + if reduction == "sum": + return torch.sum(loss) + elif reduction == "mean": + return torch.mean(loss) + else: # reduction == "none" + return loss + + +def normalize( + input: Tensor, + p: float = 2.0, + dim: int = 1, + eps: float = 1e-12, + out: Optional[Tensor] = None, +) -> Tensor: + r"""Perform :math:`L_p` normalization of inputs over specified dimension. + + For a tensor :attr:`input` of sizes :math:`(n_0, ..., n_{dim}, ..., n_k)`, each + :math:`n_{dim}` -element vector :math:`v` along dimension :attr:`dim` is transformed as + + .. math:: + v = \frac{v}{\max(\lVert v \rVert_p, \epsilon)}. + + With the default arguments it uses the Euclidean norm over vectors along dimension :math:`1` for normalization. + + Args: + input: input tensor of any shape + p (float): the exponent value in the norm formulation. Default: 2 + dim (int or tuple of ints): the dimension to reduce. Default: 1 + eps (float): small value to avoid division by zero. Default: 1e-12 + out (Tensor, optional): the output tensor. If :attr:`out` is used, this + operation won't be differentiable. + """ + if has_torch_function_variadic(input, out): + return handle_torch_function( + normalize, (input, out), input, p=p, dim=dim, eps=eps, out=out + ) + if out is None: + denom = input.norm(p, dim, keepdim=True).clamp_min(eps).expand_as(input) + return input / denom + else: + denom = input.norm(p, dim, keepdim=True).clamp_min_(eps).expand_as(input) + return torch.div(input, denom, out=out) + + +def assert_int_or_pair(arg: list[int], arg_name: str, message: str) -> None: + assert isinstance(arg, int) or len(arg) == 2, message.format(arg_name) + + +def unfold( + input: Tensor, + kernel_size: BroadcastingList2[int], + dilation: BroadcastingList2[int] = 1, + padding: BroadcastingList2[int] = 0, + stride: BroadcastingList2[int] = 1, +) -> Tensor: + r"""Extract sliding local blocks from a batched input tensor. + + .. warning:: + Currently, only 4-D input tensors (batched image-like tensors) are + supported. + + .. warning:: + + More than one element of the unfolded tensor may refer to a single + memory location. As a result, in-place operations (especially ones that + are vectorized) may result in incorrect behavior. If you need to write + to the tensor, please clone it first. + + + See :class:`torch.nn.Unfold` for details + """ + if has_torch_function_unary(input): + return handle_torch_function( + unfold, + (input,), + input, + kernel_size, + dilation=dilation, + padding=padding, + stride=stride, + ) + return torch._C._nn.im2col( + input, _pair(kernel_size), _pair(dilation), _pair(padding), _pair(stride) + ) + + +def fold( + input: Tensor, + output_size: BroadcastingList2[int], + kernel_size: BroadcastingList2[int], + dilation: BroadcastingList2[int] = 1, + padding: BroadcastingList2[int] = 0, + stride: BroadcastingList2[int] = 1, +) -> Tensor: + r"""Combine an array of sliding local blocks into a large containing tensor. + + .. warning:: + Currently, only unbatched (3D) or batched (4D) image-like output tensors are supported. + + See :class:`torch.nn.Fold` for details + """ + if has_torch_function_unary(input): + return handle_torch_function( + fold, + (input,), + input, + output_size, + kernel_size, + dilation=dilation, + padding=padding, + stride=stride, + ) + return torch._C._nn.col2im( + input, + _pair(output_size), + _pair(kernel_size), + _pair(dilation), + _pair(padding), + _pair(stride), + ) + + +# +# multihead attention +# + + +def _in_projection_packed( + q: Tensor, + k: Tensor, + v: Tensor, + w: Tensor, + b: Optional[Tensor] = None, +) -> list[Tensor]: + r"""Perform the in-projection step of the attention operation, using packed weights. + + Output is a triple containing projection tensors for query, key and value. + + Args: + q, k, v: query, key and value tensors to be projected. For self-attention, + these are typically the same tensor; for encoder-decoder attention, + k and v are typically the same tensor. (We take advantage of these + identities for performance if they are present.) Regardless, q, k and v + must share a common embedding dimension; otherwise their shapes may vary. + w: projection weights for q, k and v, packed into a single tensor. Weights + are packed along dimension 0, in q, k, v order. + b: optional projection biases for q, k and v, packed into a single tensor + in q, k, v order. + + Shape: + Inputs: + - q: :math:`(..., E)` where E is the embedding dimension + - k: :math:`(..., E)` where E is the embedding dimension + - v: :math:`(..., E)` where E is the embedding dimension + - w: :math:`(E * 3, E)` where E is the embedding dimension + - b: :math:`E * 3` where E is the embedding dimension + + Output: + - in output list :math:`[q', k', v']`, each output tensor will have the + same shape as the corresponding input tensor. + """ + E = q.size(-1) + if k is v: + if q is k: + # self-attention + proj = linear(q, w, b) + # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk() + proj = ( + proj.unflatten(-1, (3, E)) + .unsqueeze(0) + .transpose(0, -2) + .squeeze(-2) + .contiguous() + ) + # pyrefly: ignore [bad-return] + return proj[0], proj[1], proj[2] + else: + # encoder-decoder attention + w_q, w_kv = w.split([E, E * 2]) + if b is None: + b_q = b_kv = None + else: + b_q, b_kv = b.split([E, E * 2]) + q_proj = linear(q, w_q, b_q) + kv_proj = linear(k, w_kv, b_kv) + # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk() + kv_proj = ( + kv_proj.unflatten(-1, (2, E)) + .unsqueeze(0) + .transpose(0, -2) + .squeeze(-2) + .contiguous() + ) + # pyrefly: ignore [bad-return] + return (q_proj, kv_proj[0], kv_proj[1]) + else: + w_q, w_k, w_v = w.chunk(3) + if b is None: + b_q = b_k = b_v = None + else: + b_q, b_k, b_v = b.chunk(3) + # pyrefly: ignore [bad-return] + return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v) + + +def _in_projection( + q: Tensor, + k: Tensor, + v: Tensor, + w_q: Tensor, + w_k: Tensor, + w_v: Tensor, + b_q: Optional[Tensor] = None, + b_k: Optional[Tensor] = None, + b_v: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor, Tensor]: + r"""Perform the in-projection step of the attention operation. + + This is simply a triple of linear projections, + with shape constraints on the weights which + ensure embedding dimension uniformity in the projected outputs. + Output is a triple containing projection tensors for query, key and value. + + Args: + q, k, v: query, key and value tensors to be projected. + w_q, w_k, w_v: weights for q, k and v, respectively. + b_q, b_k, b_v: optional biases for q, k and v, respectively. + + Shape: + Inputs: + - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any + number of leading dimensions. + - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any + number of leading dimensions. + - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any + number of leading dimensions. + - w_q: :math:`(Eq, Eq)` + - w_k: :math:`(Eq, Ek)` + - w_v: :math:`(Eq, Ev)` + - b_q: :math:`(Eq)` + - b_k: :math:`(Eq)` + - b_v: :math:`(Eq)` + + Output: in output triple :math:`(q', k', v')`, + - q': :math:`[Qdims..., Eq]` + - k': :math:`[Kdims..., Eq]` + - v': :math:`[Vdims..., Eq]` + + """ + Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1) + assert w_q.shape == ( + Eq, + Eq, + ), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}" + assert w_k.shape == ( + Eq, + Ek, + ), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}" + assert w_v.shape == ( + Eq, + Ev, + ), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}" + assert b_q is None or b_q.shape == (Eq,), ( + f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}" + ) + assert b_k is None or b_k.shape == (Eq,), ( + f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}" + ) + assert b_v is None or b_v.shape == (Eq,), ( + f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}" + ) + return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v) + + +scaled_dot_product_attention = _add_docstr( + torch._C._nn.scaled_dot_product_attention, + r"""scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, + is_causal=False, scale=None, enable_gqa=False) -> Tensor: + + Computes scaled dot product attention on query, key and value tensors, using an optional attention mask if passed, + and applying dropout if a probability greater than 0.0 is specified. The optional scale argument can only be + specified as a keyword argument. + + .. code-block:: python + + # Efficient implementation equivalent to the following: + def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, + is_causal=False, scale=None, enable_gqa=False) -> torch.Tensor: + L, S = query.size(-2), key.size(-2) + scale_factor = 1 / math.sqrt(query.size(-1)) if scale is None else scale + attn_bias = torch.zeros(L, S, dtype=query.dtype, device=query.device) + if is_causal: + assert attn_mask is None + temp_mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) + else: + attn_bias = attn_mask + attn_bias + + if enable_gqa: + key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) + value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) + + attn_weight = query @ key.transpose(-2, -1) * scale_factor + attn_weight += attn_bias + attn_weight = torch.softmax(attn_weight, dim=-1) + attn_weight = torch.dropout(attn_weight, dropout_p, train=True) + return attn_weight @ value + + .. warning:: + This function is beta and subject to change. + + .. warning:: + This function always applies dropout according to the specified ``dropout_p`` argument. + To disable dropout during evaluation, be sure to pass a value of ``0.0`` when the module + that makes the function call is not in training mode. + + For example: + + .. code-block:: python + + class MyModel(nn.Module): + def __init__(self, p=0.5): + super().__init__() + self.p = p + + def forward(self, ...): + return F.scaled_dot_product_attention(..., + dropout_p=(self.p if self.training else 0.0)) + + Note: + + There are currently three supported implementations of scaled dot product attention: + + - `FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning`_ + - `Memory-Efficient Attention`_ + - A PyTorch implementation defined in C++ matching the above formulation + + The function may call optimized kernels for improved performance when using the CUDA backend. + For all other backends, the PyTorch implementation will be used. + + All implementations are enabled by default. Scaled dot product attention attempts to automatically select the + most optimal implementation based on the inputs. In order to provide more fine-grained control over what implementation + is used, the following functions are provided for enabling and disabling implementations. + The context manager is the preferred mechanism: + + - :func:`torch.nn.attention.sdpa_kernel`: A context manager used to enable or disable any of the implementations. + - :func:`torch.backends.cuda.enable_flash_sdp`: Globally enables or disables FlashAttention. + - :func:`torch.backends.cuda.enable_mem_efficient_sdp`: Globally enables or disables Memory-Efficient Attention. + - :func:`torch.backends.cuda.enable_math_sdp`: Globally enables or disables the PyTorch C++ implementation. + + Each of the fused kernels has specific input limitations. If the user requires the use of a specific fused implementation, + disable the PyTorch C++ implementation using :func:`torch.nn.attention.sdpa_kernel`. + In the event that a fused implementation is not available, a warning will be raised with the + reasons why the fused implementation cannot run. + + Due to the nature of fusing floating point operations, the output of this function may be different + depending on what backend kernel is chosen. + The c++ implementation supports torch.float64 and can be used when higher precision is required. + For math backend, all intermediates are kept in torch.float if inputs are in torch.half or torch.bfloat16. + For more information please see :doc:`/notes/numerical_accuracy` + + Grouped Query Attention (GQA) is an experimental feature. It currently works only for Flash_attention + and math kernel on CUDA tensor, and does not support Nested tensor. + Constraints for GQA: + + - number_of_heads_query % number_of_heads_key_value == 0 and, + - number_of_heads_key == number_of_heads_value + + Note: + + {cudnn_reproducibility_note} + """.format(**reproducibility_notes) + + r""" + Args: + query (Tensor): Query tensor; shape :math:`(N, ..., Hq, L, E)`. + key (Tensor): Key tensor; shape :math:`(N, ..., H, S, E)`. + value (Tensor): Value tensor; shape :math:`(N, ..., H, S, Ev)`. + attn_mask (optional Tensor): Attention mask; shape must be broadcastable to the shape of attention weights, + which is :math:`(N,..., L, S)`. Two types of masks are supported. + A boolean mask where a value of True indicates that the element *should* take part in attention. + A float mask of the same type as query, key, value that is added to the attention score. + dropout_p (float): Dropout probability; if greater than 0.0, dropout is applied + is_causal (bool): If set to true, the attention masking is a lower triangular matrix when the mask is a + square matrix. The attention masking has the form of the upper left causal bias due to the alignment + (see :class:`torch.nn.attention.bias.CausalBias`) when the mask is a non-square matrix. + An error is thrown if both attn_mask and is_causal are set. + scale (optional float, keyword-only): Scaling factor applied prior to softmax. If None, the default value is set + to :math:`\frac{1}{\sqrt{E}}`. + enable_gqa (bool): If set to True, Grouped Query Attention (GQA) is enabled, by default it is set to False. + + Returns: + output (Tensor): Attention output; shape :math:`(N, ..., Hq, L, Ev)`. + + Shape legend: + - :math:`N: \text{Batch size} ... : \text{Any number of other batch dimensions (optional)}` + - :math:`S: \text{Source sequence length}` + - :math:`L: \text{Target sequence length}` + - :math:`E: \text{Embedding dimension of the query and key}` + - :math:`Ev: \text{Embedding dimension of the value}` + - :math:`Hq: \text{Number of heads of query}` + - :math:`H: \text{Number of heads of key and value}` + + Examples: + + >>> # Optionally use the context manager to ensure one of the fused kernels is run + >>> query = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> with sdpa_kernel(backends=[SDPBackend.FLASH_ATTENTION]): + >>> F.scaled_dot_product_attention(query,key,value) + + + >>> # Sample for GQA for llama3 + >>> query = torch.rand(32, 32, 128, 64, dtype=torch.float16, device="cuda") + >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> with sdpa_kernel(backends=[SDPBackend.MATH]): + >>> F.scaled_dot_product_attention(query,key,value,enable_gqa=True) + + + .. _FlashAttention-2\: Faster Attention with Better Parallelism and Work Partitioning: + https://arxiv.org/abs/2307.08691 + .. _Memory-Efficient Attention: + https://github.com/facebookresearch/xformers + .. _Grouped-Query Attention: + https://arxiv.org/pdf/2305.13245 + """, +) + + +def _mha_shape_check( + query: Tensor, + key: Tensor, + value: Tensor, + key_padding_mask: Optional[Tensor], + attn_mask: Optional[Tensor], + num_heads: int, +): + # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask` + # and returns if the input is batched or not. + # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor. + + # Shape check. + if query.dim() == 3: + # Batched Inputs + is_batched = True + assert key.dim() == 3 and value.dim() == 3, ( + "For batched (3-D) `query`, expected `key` and `value` to be 3-D" + f" but found {key.dim()}-D and {value.dim()}-D tensors respectively" + ) + if key_padding_mask is not None: + assert key_padding_mask.dim() == 2, ( + "For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D" + f" but found {key_padding_mask.dim()}-D tensor instead" + ) + if attn_mask is not None: + assert attn_mask.dim() in (2, 3), ( + "For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D" + f" but found {attn_mask.dim()}-D tensor instead" + ) + elif query.dim() == 2: + # Unbatched Inputs + is_batched = False + assert key.dim() == 2 and value.dim() == 2, ( + "For unbatched (2-D) `query`, expected `key` and `value` to be 2-D" + f" but found {key.dim()}-D and {value.dim()}-D tensors respectively" + ) + + if key_padding_mask is not None: + assert key_padding_mask.dim() == 1, ( + "For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D" + f" but found {key_padding_mask.dim()}-D tensor instead" + ) + + if attn_mask is not None: + assert attn_mask.dim() in (2, 3), ( + "For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D" + f" but found {attn_mask.dim()}-D tensor instead" + ) + if attn_mask.dim() == 3: + expected_shape = (num_heads, query.shape[0], key.shape[0]) + assert attn_mask.shape == expected_shape, ( + f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}" + ) + else: + raise AssertionError( + f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor" + ) + + return is_batched + + +def _canonical_mask( + mask: Optional[Tensor], + mask_name: str, + other_type: Optional[DType], + other_name: str, + target_type: DType, + check_other: bool = True, +) -> Optional[Tensor]: + if mask is not None: + _mask_dtype = mask.dtype + _mask_is_float = torch.is_floating_point(mask) + if _mask_dtype != torch.bool and not _mask_is_float: + raise AssertionError( + f"only bool and floating types of {mask_name} are supported" + ) + if check_other and other_type is not None: + if _mask_dtype != other_type: + warnings.warn( + f"Support for mismatched {mask_name} and {other_name} " + "is deprecated. Use same type for both instead.", + stacklevel=2, + ) + if not _mask_is_float: + mask = torch.zeros_like(mask, dtype=target_type).masked_fill_( + mask, float("-inf") + ) + return mask + + +def _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]: + if input is None: + return None + elif isinstance(input, torch.Tensor): + return input.dtype + raise RuntimeError("input to _none_or_dtype() must be None or torch.Tensor") + + +def _check_key_padding_mask( + key_padding_mask: torch.Tensor, src_len: int, bsz: int +) -> None: + torch._check_with( + AssertionError, + key_padding_mask.shape[0] == bsz, + lambda: f"Expected key_padded_mask.shape[0] to be {bsz}, but got {key_padding_mask.shape[0]}", + ) + torch._check_with( + AssertionError, + key_padding_mask.shape[1] == src_len, + lambda: f"Expected key_padded_mask.shape[1] to be {src_len}, but got {key_padding_mask.shape[1]}", + ) + + +def multi_head_attention_forward( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Optional[Tensor], + in_proj_bias: Optional[Tensor], + bias_k: Optional[Tensor], + bias_v: Optional[Tensor], + add_zero_attn: bool, + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Optional[Tensor], + training: bool = True, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + use_separate_proj_weight: bool = False, + q_proj_weight: Optional[Tensor] = None, + k_proj_weight: Optional[Tensor] = None, + v_proj_weight: Optional[Tensor] = None, + static_k: Optional[Tensor] = None, + static_v: Optional[Tensor] = None, + average_attn_weights: bool = True, + is_causal: bool = False, +) -> tuple[Tensor, Optional[Tensor]]: + r"""Forward method for MultiHeadAttention. + + See :class:`torch.nn.MultiheadAttention` for details. + + Args: + query, key, value: map a query and a set of key-value pairs to an output. + See "Attention Is All You Need" for more details. + embed_dim_to_check: total dimension of the model. + num_heads: parallel attention heads. + in_proj_weight, in_proj_bias: input projection weight and bias. + bias_k, bias_v: bias of the key and value sequences to be added at dim=0. + add_zero_attn: add a new batch of zeros to the key and + value sequences at dim=1. + dropout_p: probability of an element to be zeroed. + out_proj_weight, out_proj_bias: the output projection weight and bias. + training: apply dropout if is ``True``. + key_padding_mask: if provided, specified padding elements in the key will + be ignored by the attention. This is an binary mask. When the value is True, + the corresponding value on the attention layer will be filled with -inf. + need_weights: output attn_output_weights. + Default: `True` + Note: `needs_weight` defaults to `True`, but should be set to `False` + For best performance when attention weights are not needed. + *Setting needs_weights to `True` + leads to a significant performance degradation.* + attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all + the batches while a 3D mask allows to specify a different mask for the entries of each batch. + is_causal: If specified, applies a causal mask as attention mask, and ignores + attn_mask for computing scaled dot product attention. + Default: ``False``. + .. warning:: + is_causal is provides a hint that the attn_mask is the + causal mask.Providing incorrect hints can result in + incorrect execution, including forward and backward + compatibility. + use_separate_proj_weight: the function accept the proj. weights for query, key, + and value in different forms. If false, in_proj_weight will be used, which is + a combination of q_proj_weight, k_proj_weight, v_proj_weight. + q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. + static_k, static_v: static key and value used for attention operators. + average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across heads. + Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an effect + when ``need_weights=True.``. Default: True + + + Shape: + Inputs: + - query: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is + the embedding dimension. + - key: :math:`(S, E)` or :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - value: :math:`(S, E)` or :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length. + If a FloatTensor is provided, it will be directly added to the value. + If a BoolTensor is provided, the positions with the + value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. + - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. + 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, + S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked + positions. If a BoolTensor is provided, positions with ``True`` + are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor + is provided, it will be added to the attention weight. + - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + + Outputs: + - attn_output: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, + E is the embedding dimension. + - attn_output_weights: Only returned when ``need_weights=True``. If ``average_attn_weights=True``, returns + attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or + :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and + :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per + head of shape :math:`(num_heads, L, S)` when input is unbatched or :math:`(N, num_heads, L, S)`. + """ + tens_ops = ( + query, + key, + value, + in_proj_weight, + in_proj_bias, + bias_k, + bias_v, + out_proj_weight, + out_proj_bias, + ) + if has_torch_function(tens_ops): + return handle_torch_function( + multi_head_attention_forward, + tens_ops, + query, + key, + value, + embed_dim_to_check, + num_heads, + in_proj_weight, + in_proj_bias, + bias_k, + bias_v, + add_zero_attn, + dropout_p, + out_proj_weight, + out_proj_bias, + training=training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + is_causal=is_causal, + use_separate_proj_weight=use_separate_proj_weight, + q_proj_weight=q_proj_weight, + k_proj_weight=k_proj_weight, + v_proj_weight=v_proj_weight, + static_k=static_k, + static_v=static_v, + average_attn_weights=average_attn_weights, + ) + + is_batched = _mha_shape_check( + query, key, value, key_padding_mask, attn_mask, num_heads + ) + + # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input + # is batched, run the computation and before returning squeeze the + # batch dimension so that the output doesn't carry this temporary batch dimension. + if not is_batched: + # unsqueeze if the input is unbatched + query = query.unsqueeze(1) + key = key.unsqueeze(1) + value = value.unsqueeze(1) + if key_padding_mask is not None: + key_padding_mask = key_padding_mask.unsqueeze(0) + + # set up shape vars + tgt_len, bsz, embed_dim = query.shape + src_len, _, _ = key.shape + + key_padding_mask = _canonical_mask( + mask=key_padding_mask, + mask_name="key_padding_mask", + other_type=_none_or_dtype(attn_mask), + other_name="attn_mask", + target_type=query.dtype, + ) + + if is_causal and attn_mask is None: + raise RuntimeError( + "Need attn_mask if specifying the is_causal hint. " + "You may use the Transformer module method " + "`generate_square_subsequent_mask` to create this mask." + ) + + if is_causal and key_padding_mask is None and not need_weights: + # when we have a kpm or need weights, we need attn_mask + # Otherwise, we use the is_causal hint go as is_causal + # indicator to SDPA. + attn_mask = None + else: + attn_mask = _canonical_mask( + mask=attn_mask, + mask_name="attn_mask", + other_type=None, + other_name="", + target_type=query.dtype, + check_other=False, + ) + + if key_padding_mask is not None: + # We have the attn_mask, and use that to merge kpm into it. + # Turn off use of is_causal hint, as the merged mask is no + # longer causal. + is_causal = False + + assert embed_dim == embed_dim_to_check, ( + f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}" + ) + if isinstance(embed_dim, torch.Tensor): + # embed_dim can be a tensor when JIT tracing + head_dim = embed_dim.div(num_heads, rounding_mode="trunc") + else: + head_dim = embed_dim // num_heads + assert head_dim * num_heads == embed_dim, ( + f"embed_dim {embed_dim} not divisible by num_heads {num_heads}" + ) + if use_separate_proj_weight: + # allow MHA to have different embedding dimensions when separate projection weights are used + assert key.shape[:2] == value.shape[:2], ( + f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}" + ) + else: + assert key.shape == value.shape, ( + f"key shape {key.shape} does not match value shape {value.shape}" + ) + + # + # compute in-projection + # + if not use_separate_proj_weight: + assert in_proj_weight is not None, ( + "use_separate_proj_weight is False but in_proj_weight is None" + ) + q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias) + else: + assert q_proj_weight is not None, ( + "use_separate_proj_weight is True but q_proj_weight is None" + ) + assert k_proj_weight is not None, ( + "use_separate_proj_weight is True but k_proj_weight is None" + ) + assert v_proj_weight is not None, ( + "use_separate_proj_weight is True but v_proj_weight is None" + ) + if in_proj_bias is None: + b_q = b_k = b_v = None + else: + b_q, b_k, b_v = in_proj_bias.chunk(3) + q, k, v = _in_projection( + query, + key, + value, + q_proj_weight, + k_proj_weight, + v_proj_weight, + b_q, + b_k, + b_v, + ) + + # prep attention mask + + if attn_mask is not None: + # ensure attn_mask's dim is 3 + if attn_mask.dim() == 2: + correct_2d_size = (tgt_len, src_len) + if attn_mask.shape != correct_2d_size: + raise RuntimeError( + f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}." + ) + attn_mask = attn_mask.unsqueeze(0) + elif attn_mask.dim() == 3: + correct_3d_size = (bsz * num_heads, tgt_len, src_len) + if attn_mask.shape != correct_3d_size: + raise RuntimeError( + f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}." + ) + else: + raise RuntimeError( + f"attn_mask's dimension {attn_mask.dim()} is not supported" + ) + + # add bias along batch dimension (currently second) + if bias_k is not None and bias_v is not None: + assert static_k is None, "bias cannot be added to static key." + assert static_v is None, "bias cannot be added to static value." + k = torch.cat([k, bias_k.repeat(1, bsz, 1)]) + v = torch.cat([v, bias_v.repeat(1, bsz, 1)]) + if attn_mask is not None: + # pyrefly: ignore [bad-argument-type] + attn_mask = pad(attn_mask, (0, 1)) + if key_padding_mask is not None: + # pyrefly: ignore [bad-argument-type] + key_padding_mask = pad(key_padding_mask, (0, 1)) + else: + assert bias_k is None + assert bias_v is None + + # + # reshape q, k, v for multihead attention and make them batch first + # + # pyrefly: ignore [no-matching-overload] + q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1) + if static_k is None: + # pyrefly: ignore [no-matching-overload] + k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1) + else: + # TODO finish disentangling control flow so we don't do in-projections when statics are passed + assert static_k.size(0) == bsz * num_heads, ( + f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}" + ) + assert static_k.size(2) == head_dim, ( + f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}" + ) + k = static_k + if static_v is None: + # pyrefly: ignore [no-matching-overload] + v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1) + else: + # TODO finish disentangling control flow so we don't do in-projections when statics are passed + assert static_v.size(0) == bsz * num_heads, ( + f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}" + ) + assert static_v.size(2) == head_dim, ( + f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}" + ) + v = static_v + + # add zero attention along batch dimension (now first) + if add_zero_attn: + zero_attn_shape = (bsz * num_heads, 1, head_dim) + k = torch.cat( + # pyrefly: ignore [no-matching-overload] + [k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], + dim=1, + ) + v = torch.cat( + # pyrefly: ignore [no-matching-overload] + [v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], + dim=1, + ) + if attn_mask is not None: + # pyrefly: ignore [bad-argument-type] + attn_mask = pad(attn_mask, (0, 1)) + if key_padding_mask is not None: + # pyrefly: ignore [bad-argument-type] + key_padding_mask = pad(key_padding_mask, (0, 1)) + + # update source sequence length after adjustments + src_len = k.size(1) + + # merge key padding and attention masks + if key_padding_mask is not None: + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _check_key_padding_mask(key_padding_mask, src_len, bsz) + + key_padding_mask = ( + key_padding_mask.view(bsz, 1, 1, src_len) + .expand(-1, num_heads, -1, -1) + .reshape(bsz * num_heads, 1, src_len) + ) + if attn_mask is None: + attn_mask = key_padding_mask + else: + attn_mask = attn_mask + key_padding_mask + + # adjust dropout probability + if not training: + dropout_p = 0.0 + + # + # (deep breath) calculate attention and out projection + # + + if need_weights: + _B, _Nt, E = q.shape + q_scaled = q * math.sqrt(1.0 / float(E)) + + assert not (is_causal and attn_mask is None), ( + "FIXME: is_causal not implemented for need_weights" + ) + + if attn_mask is not None: + attn_output_weights = torch.baddbmm( + attn_mask, q_scaled, k.transpose(-2, -1) + ) + else: + attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1)) + attn_output_weights = softmax(attn_output_weights, dim=-1) + if dropout_p > 0.0: + attn_output_weights = dropout(attn_output_weights, p=dropout_p) + + attn_output = torch.bmm(attn_output_weights, v) + + attn_output = ( + # pyrefly: ignore [no-matching-overload] + attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim) + ) + attn_output = linear(attn_output, out_proj_weight, out_proj_bias) + attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1)) + + # optionally average attention weights over heads + attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) + if average_attn_weights: + attn_output_weights = attn_output_weights.mean(dim=1) + + if not is_batched: + # squeeze the output if input was unbatched + attn_output = attn_output.squeeze(1) + attn_output_weights = attn_output_weights.squeeze(0) + return attn_output, attn_output_weights + else: + # attn_mask can be either (L,S) or (N*num_heads, L, S) + # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S) + # in order to match the input for SDPA of (N, num_heads, L, S) + if attn_mask is not None: + if attn_mask.size(0) == 1 and attn_mask.dim() == 3: + attn_mask = attn_mask.unsqueeze(0) + else: + attn_mask = attn_mask.view(bsz, num_heads, -1, src_len) + + q = q.view(bsz, num_heads, tgt_len, head_dim) + # pyrefly: ignore [no-matching-overload] + k = k.view(bsz, num_heads, src_len, head_dim) + # pyrefly: ignore [no-matching-overload] + v = v.view(bsz, num_heads, src_len, head_dim) + + attn_output = scaled_dot_product_attention( + q, k, v, attn_mask, dropout_p, is_causal + ) + attn_output = ( + # pyrefly: ignore [no-matching-overload] + attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim) + ) + + attn_output = linear(attn_output, out_proj_weight, out_proj_bias) + attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1)) + if not is_batched: + # squeeze the output if input was unbatched + attn_output = attn_output.squeeze(1) + return attn_output, None + + +def grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + *, + offs: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + out_dtype: Optional[torch.dtype] = None, +) -> Tensor: + r""" + grouped_mm(mat_a, mat_b, *, offs=None, bias=None, out_dtype=None) + + Computes a grouped matrix multiply that shares weight shapes across experts but + allows jagged token counts per expert, which is common in Mixture-of-Experts + (MoE) layers. Both ``mat_a`` and ``mat_b`` must be 2D or 3D tensors that already + satisfy the physical layout restrictions of grouped GEMM kernels (e.g., row-major + ``mat_a`` and column-major ``mat_b`` for FP8 inputs). Inputs are currently + expected to be ``torch.bfloat16`` values on CUDA devices with :math:`SM \ge 80`. + + Args: + mat_a: Left operand. When 2D, its leading dimension is sliced into groups + according to ``offs``. When 3D, its first dimension enumerates the groups + directly and ``offs`` must be ``None``. + mat_b: Right operand. When both operands are 2D (e.g., MoE weight-gradient + updates), the trailing dimension of ``mat_a`` and the leading dimension of + ``mat_b`` are partitioned according to the same ``offs`` tensor. For the + common forward pass (``out = input @ weight.T``) ``mat_b`` is 3D with + shape ``(num_groups, N, K)``. + offs: Optional 1D tensor of monotonically increasing ``int32`` offsets that + delimit the jagged dimension of any 2D operand. ``offs[i]`` marks the end + of group ``i`` and ``offs[-1]`` must be strictly less than the total + length of that operand's sliced dimension; elements beyond ``offs[-1]`` + are ignored. + bias: Optional tensor that is added to the grouped outputs. Bias is not + jagged and must be broadcastable to the result shape of each group. + out_dtype: Optional dtype that controls the accumulation/output dtype. + Passing ``torch.float32`` accumulates BF16 inputs in FP32 while keeping + the grouped GEMM API non-differentiable. + + Returns: + A tensor containing the concatenated results of each per-group GEMM with + shape inferred from the operands and ``offs``. + """ + + return torch._grouped_mm(mat_a, mat_b, offs=offs, bias=bias, out_dtype=out_dtype) + + +def scaled_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Optional[Tensor] = None, + output_dtype: Optional[torch.dtype] = torch.bfloat16, + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: + r""" + scaled_mm(mat_a, mat_b, scale_a, scale_recipe_a, scale_b, scale_recipe_b, swizzle_a, swizzle_b, bias, output_dtype, + contraction_dim, use_fast_accum) + + Applies a scaled matrix-multiply, mm(mat_a, mat_b) where the scaling of mat_a and mat_b are described by + scale_recipe_a and scale_recipe_b respectively. + + Args: + scale_a: Tensor containing decoding scaling factors for mat_a + scale_recipe_a: Enum describing how mat_a has been scaled + scale_b: Tensor containing decoding scaling factors for mat_b + scale_recipe_b: Enum describing how mat_b has been scaled + swizzle_a: Enum describing the swizzling pattern (if any) of scale_a + swizzle_b: Enum describing the swizzling pattern (if any) of scale_b + bias: optional bias term to be added to the output + output_dtype: dtype used for the output tensor + contraction_dim: describe which dimensions are :math:`K` in the matmul. + use_fast_accum: enable/disable tensor-core fast accumulation (Hopper-GPUs only) + """ + + def expand_single_value(v: _Any | list[_Any] | None) -> list[_Any]: + if v is None: + return [] + elif not isinstance(v, (list)): + return [ + v, + ] + else: + return v + + scale_a = expand_single_value(scale_a) + scale_recipe_a = expand_single_value(scale_recipe_a) + scale_b = expand_single_value(scale_b) + scale_recipe_b = expand_single_value(scale_recipe_b) + swizzle_a = expand_single_value(swizzle_a) + swizzle_b = expand_single_value(swizzle_b) + + # native_functions has restrictions on what can be defined + # & passed through - std::optional> for instance + # *cannot* be passed, but an empty vector (list) can. + # So, we need to convert None arguments for lists in python + # explicitly into empty lists. + def list_or_empty(l: list[_Any] | None) -> list[_Any]: + return l if l else [] + + def enum_list_as_int_list(l: _Any | list[_Any]) -> list[_Any]: + if not isinstance(l, list): + l = [ + l, + ] + return [li.value for li in l] + + out = torch._scaled_mm_v2( + mat_a, + mat_b, + scale_a, + enum_list_as_int_list(scale_recipe_a), + enum_list_as_int_list(list_or_empty(swizzle_a)), + scale_b, + enum_list_as_int_list(scale_recipe_b), + enum_list_as_int_list(list_or_empty(swizzle_b)), + bias, + output_dtype, + contraction_dim, + use_fast_accum, + ) + + return out + + +def scaled_grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Optional[Tensor] = None, + offs: Optional[Tensor] = None, + output_dtype: Optional[torch.dtype] = torch.bfloat16, + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: + r""" + scaled_grouped_mm(mat_a, mat_b, scale_a, scale_recipe_a, scale_b, scale_recipe_b, swizzle_a, swizzle_b, bias, offs, + output_dtype, use_fast_accum) + + Applies a grouped scaled matrix-multiply, grouped_mm(mat_a, mat_b) where the scaling of mat_a and mat_b are described by + scale_recipe_a and scale_recipe_b respectively. + + Args: + scale_a: Tensor containing decoding scaling factors for mat_a + scale_recipe_a: Enum describing how mat_a has been scaled + scale_b: Tensor containing decoding scaling factors for mat_b + scale_recipe_b: Enum describing how mat_b has been scaled + swizzle_a: Enum describing the swizzling pattern (if any) of scale_a + swizzle_b: Enum describing the swizzling pattern (if any) of scale_b + bias: optional bias term to be added to the output + offs: optional offsets into the source tensors denoting group start indices + output_dtype: dtype used for the output tensor + contraction_dim: describe which dimensions are :math:`K` in the matmul. + use_fast_accum: enable/disable tensor-core fast accumulation (Hopper-GPUs only) + """ + + def expand_single_value(v: _Any | list[_Any] | None) -> list[_Any]: + if v is None: + return [] + elif not isinstance(v, (list)): + return [ + v, + ] + else: + return v + + scale_a = expand_single_value(scale_a) + scale_recipe_a = expand_single_value(scale_recipe_a) + scale_b = expand_single_value(scale_b) + scale_recipe_b = expand_single_value(scale_recipe_b) + swizzle_a = expand_single_value(swizzle_a) + swizzle_b = expand_single_value(swizzle_b) + + # native_functions has restrictions on what can be defined + # & passed through - std::optional> for instance + # *cannot* be passed, but an empty vector (list) can. + # So, we need to convert None arguments for lists in python + # explicitly into empty lists. + def list_or_empty(l: list[_Any] | None) -> list[_Any]: + return l if l else [] + + def enum_list_as_int_list(l: _Any | list[_Any]) -> list[_Any]: + if not isinstance(l, list): + l = [ + l, + ] + return [li.value for li in l] + + out = torch._scaled_grouped_mm_v2( + mat_a, + mat_b, + scale_a, + enum_list_as_int_list(scale_recipe_a), + enum_list_as_int_list(list_or_empty(swizzle_a)), + scale_b, + enum_list_as_int_list(scale_recipe_b), + enum_list_as_int_list(list_or_empty(swizzle_b)), + offs, + bias, + output_dtype, + contraction_dim, + use_fast_accum, + ) + + return out diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.pyi b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.pyi new file mode 100644 index 0000000000000000000000000000000000000000..23ff7e34a592720efe4e3b2d7eaf3225f54b90eb --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/functional.pyi @@ -0,0 +1,1200 @@ +# @generated by tools/pyi/gen_pyi.py from torch/nn/functional.pyi.in +# mypy: allow-untyped-defs + +from collections.abc import Callable, Sequence +from enum import Enum +from typing import Any, Literal, overload, TypeAlias + +from torch import Tensor +from torch.types import _dtype, _int, _size + +from .common_types import ( + _ratio_any_t, + _size_1_t, + _size_2_opt_t, + _size_2_t, + _size_3_opt_t, + _size_3_t, + _size_any_t, +) + +__all__ = [ + "GRID_SAMPLE_INTERPOLATION_MODES", + "GRID_SAMPLE_PADDING_MODES", +] + +# 'TypedDict' is a new accepted type that represents a dictionary with a fixed set of allowed keys. +# It is standards-track but not in `typing` yet. We leave this hear to be uncommented once the feature +# is wide-spread. + +# from mypy_extensions import TypedDict + +# GRID_SAMPLE_INTERPOLATION_MODES = TypedDict('GRID_SAMPLE_INTERPOLATION_MODES', {'bilinear': int, 'nearest': int}) +# GRID_SAMPLE_PADDING_MODES = TypedDict('GRID_SAMPLE_PADDING_MODES', {'zeros': int, 'border': int, 'reflection': int}) + +GRID_SAMPLE_INTERPOLATION_MODES: TypeAlias = dict[str, int] +GRID_SAMPLE_PADDING_MODES: TypeAlias = dict[str, int] + +# These stubs were generated by running stubgen (`stubgen --parse-only functional.py`), followed by manual cleaning. +# +# The 'BroadcastingList{1,2,3}' types were replaced by `_size` or _output_ratio, as appropriate. +# This was necessary since the JIT uses BroadcastingList* types but static checking with mypy etc requires a `Sequence` +# type. There is no way to express the expected lengths of these lists in the current Python typing system. +# +# Functions created via `_add_docstr` in `functional.py` where merely typed as `Any` by `stubgen`, so those were +# deleted from the stub and replaced by generated declarations. See `gen_pyi` for the implementation of the code +# generation logic for those functions. In the future, it might be worth looking into using the mypy plugin system +# to encode the type semantics of `_add_docstr`, should that system ever become widespread. +def _canonical_mask( + mask: Tensor | None, + mask_name: str, + other_type: _dtype | None, + other_name: str, + target_type: _dtype, + check_other: bool = True, +) -> Tensor | None: ... + +__all__ += ["_canonical_mask"] + +def _none_or_dtype(input: Tensor | None) -> _dtype | None: ... + +__all__ += ["_none_or_dtype"] + +def adaptive_avg_pool2d(input: Tensor, output_size: _size_2_opt_t) -> Tensor: ... + +__all__ += ["adaptive_avg_pool2d"] + +def adaptive_avg_pool3d(input: Tensor, output_size: _size_3_opt_t) -> Tensor: ... + +__all__ += ["adaptive_avg_pool3d"] + +def adaptive_max_pool1d_with_indices( + input: Tensor, + output_size: _size, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool1d_with_indices"] + +def adaptive_max_pool2d_with_indices( + input: Tensor, + output_size: _size_2_opt_t, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool2d_with_indices"] + +def adaptive_max_pool3d_with_indices( + input: Tensor, + output_size: _size_3_opt_t, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool3d_with_indices"] + +def affine_grid( + theta: Tensor, + size: list[int], + align_corners: Any | None = ..., +) -> Tensor: ... + +__all__ += ["affine_grid"] + +def alpha_dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["alpha_dropout"] + +def assert_int_or_pair(arg: Any, arg_name: Any, message: Any) -> None: ... + +__all__ += ["assert_int_or_pair"] + +def batch_norm( + input: Tensor, + running_mean: Tensor | None, + running_var: Tensor | None, + weight: Tensor | None = ..., + bias: Tensor | None = ..., + training: bool = ..., + momentum: float = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["batch_norm"] + +def binary_cross_entropy_with_logits( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + pos_weight: Tensor | None = ..., +) -> Tensor: ... + +__all__ += ["binary_cross_entropy_with_logits"] + +def binary_cross_entropy( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["binary_cross_entropy"] + +def celu(input: Tensor, alpha: float = ..., inplace: bool = ...) -> Tensor: ... + +__all__ += ["celu"] + +def cosine_embedding_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["cosine_embedding_loss"] + +def cross_entropy( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + ignore_index: int = ..., + reduce: bool | None = ..., + reduction: str = ..., + label_smoothing: float = ..., +) -> Tensor: ... + +__all__ += ["cross_entropy"] + +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: int = ..., + reduction: str = ..., + zero_infinity: bool = ..., +) -> Tensor: ... + +__all__ += ["ctc_loss"] + +def dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout"] + +def dropout1d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout1d"] + +def dropout2d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout2d"] + +def dropout3d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout3d"] + +def elu(input: Tensor, alpha: float = ..., inplace: bool = ...) -> Tensor: ... + +__all__ += ["elu"] + +def embedding_bag( + input: Tensor, + weight: Tensor, + offsets: Tensor | None = ..., + max_norm: float | None = ..., + norm_type: float = ..., + scale_grad_by_freq: bool = ..., + mode: str = ..., + sparse: bool = ..., + per_sample_weights: Tensor | None = ..., + include_last_offset: bool = ..., + padding_idx: int | None = ..., +) -> Tensor: ... + +__all__ += ["embedding_bag"] + +def embedding( + input: Tensor, + weight: Tensor, + padding_idx: int | None = ..., + max_norm: float | None = ..., + norm_type: float = ..., + scale_grad_by_freq: bool = ..., + sparse: bool = ..., +) -> Tensor: ... + +__all__ += ["embedding"] + +def feature_alpha_dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["feature_alpha_dropout"] + +def fold( + input: Tensor, + output_size: _size_any_t, + kernel_size: _size_any_t, + dilation: _size_any_t = ..., + padding: _size_any_t = ..., + stride: _size_any_t = ..., +) -> Tensor: ... + +__all__ += ["fold"] + +def fractional_max_pool2d_with_indices( + input: Tensor, + kernel_size: _size, + output_size: _size | None = ..., + output_ratio: _ratio_any_t | None = ..., + return_indices: bool = ..., + _random_samples: Tensor | None = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["fractional_max_pool2d_with_indices"] + +def fractional_max_pool3d_with_indices( + input: Tensor, + kernel_size: _size, + output_size: _size | None = ..., + output_ratio: _ratio_any_t | None = ..., + return_indices: bool = ..., + _random_samples: Tensor | None = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["fractional_max_pool3d_with_indices"] + +def gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor | float, + full: bool | None = ..., + eps: float | None = ..., + reduction: str | None = ..., +) -> Tensor: ... + +__all__ += ["gaussian_nll_loss"] + +def glu(input: Tensor, dim: int = ...) -> Tensor: ... + +__all__ += ["glu"] + +def grid_sample( + input: Tensor, + grid: Tensor, + mode: str = ..., + padding_mode: str = ..., + align_corners: Any | None = ..., +) -> Tensor: ... + +__all__ += ["grid_sample"] + +def group_norm( + input: Tensor, + num_groups: int, + weight: Tensor | None = ..., + bias: Tensor | None = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["group_norm"] + +def gumbel_softmax( + logits: Tensor, + tau: float = ..., + hard: bool = ..., + eps: float = ..., + dim: int = ..., +) -> Tensor: ... + +__all__ += ["gumbel_softmax"] + +def hardsigmoid(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["hardsigmoid"] + +def hardswish(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["hardswish"] + +def hardtanh( + input: Tensor, + min_val: float = ..., + max_val: float = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["hardtanh"] + +def hinge_embedding_loss( + input: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["hinge_embedding_loss"] + +def huber_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., + delta: float = ..., +) -> Tensor: ... + +__all__ += ["huber_loss"] + +def instance_norm( + input: Tensor, + running_mean: Tensor | None = ..., + running_var: Tensor | None = ..., + weight: Tensor | None = ..., + bias: Tensor | None = ..., + use_input_stats: bool = ..., + momentum: float = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["instance_norm"] + +def interpolate( + input: Tensor, + size: int | Sequence[int] | None = ..., + scale_factor: float | Sequence[float] | None = ..., + mode: str = ..., + align_corners: bool | None = ..., + recompute_scale_factor: bool | None = ..., + antialias: bool = ..., +) -> Tensor: ... + +__all__ += ["interpolate"] + +def kl_div( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + log_target: bool = ..., +) -> Tensor: ... + +__all__ += ["kl_div"] + +def l1_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["l1_loss"] + +def layer_norm( + input: Tensor, + normalized_shape: Sequence[int], + weight: Tensor | None = ..., + bias: Tensor | None = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["layer_norm"] + +def leaky_relu( + input: Tensor, + negative_slope: float = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["leaky_relu"] + +def local_response_norm( + input: Tensor, + size: int, + alpha: float = ..., + beta: float = ..., + k: float = ..., +) -> Tensor: ... + +__all__ += ["local_response_norm"] + +def log_softmax( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["log_softmax"] + +def lp_pool1d( + input: Tensor, + norm_type: float, + kernel_size: _size_1_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool1d"] + +def lp_pool2d( + input: Tensor, + norm_type: float, + kernel_size: _size_2_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool2d"] + +def lp_pool3d( + input: Tensor, + norm_type: float, + kernel_size: _size_3_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool3d"] + +def margin_ranking_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["margin_ranking_loss"] + +def max_pool1d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool1d_with_indices"] + +def max_pool2d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool2d_with_indices"] + +def max_pool3d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool3d_with_indices"] + +def max_unpool1d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool1d"] + +def max_unpool2d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool2d"] + +def max_unpool3d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool3d"] + +def mish(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["mish"] + +def mse_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["mse_loss"] + +def multi_head_attention_forward( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Tensor | None, + in_proj_bias: Tensor | None, + bias_k: Tensor | None, + bias_v: Tensor | None, + add_zero_attn: bool, + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Tensor | None, + training: bool = True, + key_padding_mask: Tensor | None = None, + need_weights: bool = True, + attn_mask: Tensor | None = None, + use_separate_proj_weight: bool = False, + q_proj_weight: Tensor | None = None, + k_proj_weight: Tensor | None = None, + v_proj_weight: Tensor | None = None, + static_k: Tensor | None = None, + static_v: Tensor | None = None, + average_attn_weights: bool = True, + is_causal: bool = False, +) -> tuple[Tensor, Tensor | None]: ... + +__all__ += ["multi_head_attention_forward"] + +def multi_margin_loss( + input: Tensor, + target: Tensor, + p: int = ..., + margin: float = ..., + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multi_margin_loss"] + +def multilabel_margin_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multilabel_margin_loss"] + +def multilabel_soft_margin_loss( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multilabel_soft_margin_loss"] + +def nll_loss( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + ignore_index: int = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["nll_loss"] + +def normalize( + input: Tensor, + p: float = ..., + dim: int = ..., + eps: float = ..., + out: Tensor | None = ..., +) -> Tensor: ... + +__all__ += ["normalize"] + +def poisson_nll_loss( + input: Tensor, + target: Tensor, + log_input: bool = ..., + full: bool = ..., + size_average: bool | None = ..., + eps: float = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["poisson_nll_loss"] + +def relu(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["relu"] + +def relu6(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["relu6"] + +def rms_norm( + input: Tensor, + normalized_shape: Sequence[int], + weight: Tensor | None = ..., + eps: float | None = ..., +) -> Tensor: ... + +__all__ += ["rms_norm"] + +def rrelu( + input: Tensor, + lower: float = ..., + upper: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["rrelu"] + +def scaled_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Tensor | None = None, + output_dtype: _dtype = ..., + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: ... + +__all__ += ["scaled_mm"] + +def grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + *, + offs: Tensor | None = None, + bias: Tensor | None = None, + out_dtype: _dtype | None = None, +) -> Tensor: ... + +__all__ += ["grouped_mm"] + +class SwizzleType(Enum): + NO_SWIZZLE = 0 + SWIZZLE_32_4_4 = 1 + +__all__ += ["SwizzleType"] + +class ScalingType(Enum): + TensorWise = 0 + RowWise = 1 + BlockWise1x16 = 2 + BlockWise1x32 = 3 + BlockWise1x128 = 4 + BlockWise128x128 = 5 + +__all__ += ["ScalingType"] + +def selu(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["selu"] + +def sigmoid(input: Any) -> Tensor: ... + +__all__ += ["sigmoid"] + +def silu(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["silu"] + +def smooth_l1_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + beta: float = ..., +) -> Tensor: ... + +__all__ += ["smooth_l1_loss"] + +def soft_margin_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["soft_margin_loss"] + +def softmax( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["softmax"] + +def softmin( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["softmin"] + +def softsign(input: Any): ... + +__all__ += ["softsign"] + +def tanh(input: Any): ... + +__all__ += ["tanh"] + +def tanhshrink(input: Any): ... + +__all__ += ["tanhshrink"] + +def threshold( + input: Tensor, + threshold: float, + value: float, + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["threshold"] + +def triplet_margin_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + margin: float = ..., + p: float = ..., + eps: float = ..., + swap: bool = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["triplet_margin_loss"] + +def triplet_margin_with_distance_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + *, + distance_function: Callable[[Tensor, Tensor], Tensor] | None = ..., + margin: float = ..., + swap: bool = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["triplet_margin_with_distance_loss"] + +def unfold( + input: Tensor, + kernel_size: _size_any_t, + dilation: _size_any_t = ..., + padding: _size_any_t = ..., + stride: _size_any_t = ..., +) -> Tensor: ... + +__all__ += ["unfold"] + +def upsample_bilinear( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., +): ... + +__all__ += ["upsample_bilinear"] + +def upsample_nearest( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., +): ... + +__all__ += ["upsample_nearest"] + +def upsample( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., + mode: str = ..., + align_corners: Any | None = ..., +): ... + +__all__ += ["upsample"] + +from torch import ( + adaptive_avg_pool1d as adaptive_avg_pool1d, + avg_pool1d as avg_pool1d, + bilinear as bilinear, + celu_ as celu_, + channel_shuffle as channel_shuffle, + conv1d as conv1d, + conv2d as conv2d, + conv3d as conv3d, + conv_tbc as conv_tbc, + conv_transpose1d as conv_transpose1d, + conv_transpose2d as conv_transpose2d, + conv_transpose3d as conv_transpose3d, + cosine_similarity as cosine_similarity, + hardshrink as hardshrink, + native_channel_shuffle as native_channel_shuffle, + pairwise_distance as pairwise_distance, + pdist as pdist, + pixel_shuffle as pixel_shuffle, + pixel_unshuffle as pixel_unshuffle, + prelu as prelu, + relu_ as relu_, + rrelu_ as rrelu_, + selu_ as selu_, +) +from torch._C._nn import ( + avg_pool2d as avg_pool2d, + avg_pool3d as avg_pool3d, + elu_ as elu_, + gelu as gelu, + hardtanh_ as hardtanh_, + leaky_relu_ as leaky_relu_, + linear as linear, + log_sigmoid as logsigmoid, + one_hot as one_hot, + pad as pad, + scaled_dot_product_attention as scaled_dot_product_attention, + softplus as softplus, + softshrink as softshrink, +) + +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + return_indices: Literal[False] = False, + _random_samples: Tensor | None = None, +) -> Tensor: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None, + output_ratio: _ratio_any_t | None, + return_indices: Literal[True], + /, + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + *, + return_indices: Literal[True], + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + return_indices: Literal[False] = False, + _random_samples: Tensor | None = None, +) -> Tensor: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None, + output_ratio: _ratio_any_t | None, + return_indices: Literal[True], + /, + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + *, + return_indices: Literal[True], + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... + +__all__ += [ + "adaptive_avg_pool1d", + "avg_pool1d", + "bilinear", + "celu_", + "channel_shuffle", + "conv_tbc", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "conv1d", + "conv2d", + "conv3d", + "cosine_similarity", + "hardshrink", + "native_channel_shuffle", + "pairwise_distance", + "pdist", + "pixel_shuffle", + "pixel_unshuffle", + "prelu", + "relu_", + "rrelu_", + "selu_", + "avg_pool2d", + "avg_pool3d", + "elu_", + "gelu", + "hardtanh_", + "leaky_relu_", + "linear", + "logsigmoid", + "one_hot", + "pad", + "scaled_dot_product_attention", + "softplus", + "softshrink", + "max_pool1d", + "adaptive_max_pool1d", + "max_pool2d", + "fractional_max_pool2d", + "adaptive_max_pool2d", + "max_pool3d", + "fractional_max_pool3d", + "adaptive_max_pool3d", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/grad.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/grad.py new file mode 100644 index 0000000000000000000000000000000000000000..5aadfdebbe95fb6f5de10ce238a991f07b3ef61e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/grad.py @@ -0,0 +1,298 @@ +# mypy: allow-untyped-defs +"""Gradient interface.""" + +import torch +from torch.nn.modules.utils import _pair, _single, _triple + + +def conv1d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv1d with respect to the input of the convolution. + + This is same as the 1D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weight tensor (out_channels x in_channels/groups x kW) + grad_output : output gradient tensor (minibatch x out_channels x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, requires_grad=True) + >>> output = F.conv1d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv1d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _single(stride), + _single(padding), + _single(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv1d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv1d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, requires_grad=True) + >>> output = F.conv1d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> # xdoctest: +SKIP + >>> grad_weight = torch.autograd.grad(output, filter, grad_output) + >>> F.grad.conv1d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _single(stride), + _single(padding), + _single(dilation), + False, + [0], + groups, + (False, True, False), + )[1] + + +def conv2d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv2d with respect to the input of the convolution. + + This is same as the 2D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weight tensor (out_channels x in_channels/groups x kH x kW) + grad_output : output gradient tensor (minibatch x out_channels x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, 2, requires_grad=True) + >>> output = F.conv2d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv2d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _pair(stride), + _pair(padding), + _pair(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv2d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv2d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iH x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, 2, requires_grad=True) + >>> output = F.conv2d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> # xdoctest: +SKIP + >>> grad_weight = torch.autograd.grad(output, filter, grad_output) + >>> F.grad.conv2d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _pair(stride), + _pair(padding), + _pair(dilation), + False, + [0], + groups, + (False, True, False), + )[1] + + +def conv3d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv3d with respect to the input of the convolution. + + This is same as the 3D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weights tensor (out_channels x in_channels/groups x kT x kH x kW) + grad_output : output gradient tensor (minibatch x out_channels x oT x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(2, 8, 10, 10, 20, requires_grad=True) + >>> weight = torch.randn(4, 8, 2, 3, 3, requires_grad=True) + >>> output = F.conv3d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv3d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _triple(stride), + _triple(padding), + _triple(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv3d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv3d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iT x iH x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oT x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(2, 8, 10, 10, 20, requires_grad=True) + >>> weight = torch.randn(4, 8, 2, 3, 3, requires_grad=True) + >>> output = F.conv3d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_weight = torch.autograd.grad(output, weight, grad_output) + >>> F.grad.conv3d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _triple(stride), + _triple(padding), + _triple(dilation), + False, + [0], + groups, + (False, True, False), + )[1] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/init.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/init.py new file mode 100644 index 0000000000000000000000000000000000000000..4e14b3f4bdd3b3d2abda51e533c20d3ad619e3f7 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/init.py @@ -0,0 +1,754 @@ +"""This file contains utilities for initializing neural network parameters.""" + +import math +import warnings +from collections.abc import Callable +from typing import Literal, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch import Tensor + + +__all__ = [ + "calculate_gain", + "uniform_", + "normal_", + "trunc_normal_", + "constant_", + "ones_", + "zeros_", + "eye_", + "dirac_", + "xavier_uniform_", + "xavier_normal_", + "kaiming_uniform_", + "kaiming_normal_", + "orthogonal_", + "sparse_", + # Deprecated aliases (for backward compatibility) + "uniform", + "normal", + "constant", + "eye", + "dirac", + "xavier_uniform", + "xavier_normal", + "kaiming_uniform", + "kaiming_normal", + "orthogonal", + "sparse", +] + + +_R = TypeVar("_R") +_P = ParamSpec("_P") + +_NonlinearityType = Literal[ + "linear", + "conv1d", + "conv2d", + "conv3d", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "sigmoid", + "tanh", + "relu", + "leaky_relu", + "selu", +] + +_FanMode = Literal["fan_in", "fan_out"] + + +# These no_grad_* functions are necessary as wrappers around the parts of these +# functions that use `with torch.no_grad()`. The JIT doesn't support context +# managers, so these need to be implemented as builtins. Using these wrappers +# lets us keep those builtins small and reusable. +def _no_grad_uniform_( + tensor: Tensor, a: float, b: float, generator: torch.Generator | None = None +) -> Tensor: + with torch.no_grad(): + return tensor.uniform_(a, b, generator=generator) + + +def _no_grad_normal_( + tensor: Tensor, + mean: float, + std: float, + generator: torch.Generator | None = None, +) -> Tensor: + with torch.no_grad(): + return tensor.normal_(mean, std, generator=generator) + + +def _no_grad_trunc_normal_( + tensor: Tensor, + mean: float, + std: float, + a: float, + b: float, + generator: torch.Generator | None = None, +) -> Tensor: + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x: float) -> float: + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1, generator=generator) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def _no_grad_fill_(tensor: Tensor, val: float) -> Tensor: + with torch.no_grad(): + return tensor.fill_(val) + + +def _no_grad_zero_(tensor: Tensor) -> Tensor: + with torch.no_grad(): + return tensor.zero_() + + +def calculate_gain( + nonlinearity: _NonlinearityType, param: int | float | None = None +) -> float: + r"""Return the recommended gain value for the given nonlinearity function. + + The values are as follows: + + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + SELU :math:`\frac{3}{4}` + ================= ==================================================== + + .. warning:: + In order to implement `Self-Normalizing Neural Networks`_ , + you should use ``nonlinearity='linear'`` instead of ``nonlinearity='selu'``. + This gives the initial weights a variance of ``1 / N``, + which is necessary to induce a stable fixed point in the forward pass. + In contrast, the default gain for ``SELU`` sacrifices the normalization + effect for more stable gradient flow in rectangular layers. + + Args: + nonlinearity: the non-linear function (`nn.functional` name) + param: optional parameter for the non-linear function + + Examples: + >>> gain = nn.init.calculate_gain( + ... "leaky_relu", 0.2 + ... ) # leaky_relu with negative_slope=0.2 + + .. _Self-Normalizing Neural Networks: https://papers.nips.cc/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html + """ + linear_fns = [ + "linear", + "conv1d", + "conv2d", + "conv3d", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + ] + if nonlinearity in linear_fns or nonlinearity == "sigmoid": + return 1 + elif nonlinearity == "tanh": + return 5.0 / 3 + elif nonlinearity == "relu": + return math.sqrt(2.0) + elif nonlinearity == "leaky_relu": + if param is None: + negative_slope = 0.01 + elif ( + not isinstance(param, bool) + and isinstance(param, int) + or isinstance(param, float) + ): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError(f"negative_slope {param} not a valid number") + return math.sqrt(2.0 / (1 + negative_slope**2)) + elif nonlinearity == "selu": + return ( + 3.0 / 4 + ) # Value found empirically (https://github.com/pytorch/pytorch/pull/50664) + else: + raise ValueError(f"Unsupported nonlinearity {nonlinearity}") + + +def uniform_( + tensor: Tensor, + a: float = 0.0, + b: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from the uniform distribution. + + :math:`\mathcal{U}(a, b)`. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the lower bound of the uniform distribution + b: the upper bound of the uniform distribution + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.uniform_(w) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + uniform_, (tensor,), tensor=tensor, a=a, b=b, generator=generator + ) + return _no_grad_uniform_(tensor, a, b, generator) + + +def normal_( + tensor: Tensor, + mean: float = 0.0, + std: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from the normal distribution. + + :math:`\mathcal{N}(\text{mean}, \text{std}^2)`. + + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.normal_(w) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + normal_, (tensor,), tensor=tensor, mean=mean, std=std, generator=generator + ) + return _no_grad_normal_(tensor, mean, std, generator) + + +def trunc_normal_( + tensor: Tensor, + mean: float = 0.0, + std: float = 1.0, + a: float = -2.0, + b: float = 2.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from a truncated normal distribution. + + The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b, generator=generator) + + +def constant_(tensor: Tensor, val: float) -> Tensor: + r"""Fill the input Tensor with the value :math:`\text{val}`. + + Args: + tensor: an n-dimensional `torch.Tensor` + val: the value to fill the tensor with + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.constant_(w, 0.3) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + constant_, (tensor,), tensor=tensor, val=val + ) + return _no_grad_fill_(tensor, val) + + +def ones_(tensor: Tensor) -> Tensor: + r"""Fill the input Tensor with the scalar value `1`. + + Args: + tensor: an n-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.ones_(w) + """ + return _no_grad_fill_(tensor, 1.0) + + +def zeros_(tensor: Tensor) -> Tensor: + r"""Fill the input Tensor with the scalar value `0`. + + Args: + tensor: an n-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.zeros_(w) + """ + return _no_grad_zero_(tensor) + + +def eye_(tensor: Tensor) -> Tensor: + r"""Fill the 2-dimensional input `Tensor` with the identity matrix. + + Preserves the identity of the inputs in `Linear` layers, where as + many inputs are preserved as possible. + + Args: + tensor: a 2-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.eye_(w) + """ + if tensor.ndimension() != 2: + raise ValueError("Only tensors with 2 dimensions are supported") + + with torch.no_grad(): + torch.eye(*tensor.shape, out=tensor, requires_grad=tensor.requires_grad) + return tensor + + +def dirac_(tensor: Tensor, groups: int = 1) -> Tensor: + r"""Fill the {3, 4, 5}-dimensional input `Tensor` with the Dirac delta function. + + Preserves the identity of the inputs in `Convolutional` + layers, where as many input channels are preserved as possible. In case + of groups>1, each group of channels preserves identity + + Args: + tensor: a {3, 4, 5}-dimensional `torch.Tensor` + groups (int, optional): number of groups in the conv layer (default: 1) + Examples: + >>> w = torch.empty(3, 16, 5, 5) + >>> nn.init.dirac_(w) + >>> w = torch.empty(3, 24, 5, 5) + >>> nn.init.dirac_(w, 3) + """ + dimensions = tensor.ndimension() + if dimensions not in [3, 4, 5]: + raise ValueError("Only tensors with 3, 4, or 5 dimensions are supported") + + sizes = tensor.size() + + if sizes[0] % groups != 0: + raise ValueError("dim 0 must be divisible by groups") + + out_chans_per_grp = sizes[0] // groups + min_dim = min(out_chans_per_grp, sizes[1]) + + with torch.no_grad(): + tensor.zero_() + + for g in range(groups): + for d in range(min_dim): + if dimensions == 3: # Temporal convolution + tensor[g * out_chans_per_grp + d, d, tensor.size(2) // 2] = 1 + elif dimensions == 4: # Spatial convolution + tensor[ + g * out_chans_per_grp + d, + d, + tensor.size(2) // 2, + tensor.size(3) // 2, + ] = 1 + else: # Volumetric convolution + tensor[ + g * out_chans_per_grp + d, + d, + tensor.size(2) // 2, + tensor.size(3) // 2, + tensor.size(4) // 2, + ] = 1 + return tensor + + +def _calculate_fan_in_and_fan_out(tensor: Tensor) -> tuple[int, int]: + dimensions = tensor.dim() + if dimensions < 2: + raise ValueError( + "Fan in and fan out can not be computed for tensor with fewer than 2 dimensions" + ) + + num_input_fmaps = tensor.size(1) + num_output_fmaps = tensor.size(0) + receptive_field_size = 1 + if tensor.dim() > 2: + # math.prod is not always available, accumulate the product manually + # we could use functools.reduce but that is not supported by TorchScript + for s in tensor.shape[2:]: + receptive_field_size *= s + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + + return fan_in, fan_out + + +def xavier_uniform_( + tensor: Tensor, + gain: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Xavier uniform distribution. + + The method is described in `Understanding the difficulty of training + deep feedforward neural networks` - Glorot, X. & Bengio, Y. (2010). + The resulting tensor will have values sampled from + :math:`\mathcal{U}(-a, a)` where + + .. math:: + a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}} + + Also known as Glorot initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + gain: an optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain("relu")) + """ + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + std = gain * math.sqrt(2.0 / float(fan_in + fan_out)) + a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + + return _no_grad_uniform_(tensor, -a, a, generator) + + +def xavier_normal_( + tensor: Tensor, + gain: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Xavier normal distribution. + + The method is described in `Understanding the difficulty of training deep feedforward + neural networks` - Glorot, X. & Bengio, Y. (2010). The resulting tensor + will have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}} + + Also known as Glorot initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + gain: an optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.xavier_normal_(w) + """ + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + std = gain * math.sqrt(2.0 / float(fan_in + fan_out)) + + return _no_grad_normal_(tensor, 0.0, std, generator) + + +def _calculate_correct_fan(tensor: Tensor, mode: _FanMode) -> int: + # pyrefly: ignore [bad-assignment] + mode = mode.lower() + valid_modes = ["fan_in", "fan_out"] + if mode not in valid_modes: + raise ValueError(f"Mode {mode} not supported, please use one of {valid_modes}") + + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + return fan_in if mode == "fan_in" else fan_out + + +def kaiming_uniform_( + tensor: Tensor, + a: float = 0, + mode: _FanMode = "fan_in", + nonlinearity: _NonlinearityType = "leaky_relu", + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Kaiming uniform distribution. + + The method is described in `Delving deep into rectifiers: Surpassing + human-level performance on ImageNet classification` - He, K. et al. (2015). + The resulting tensor will have values sampled from + :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.kaiming_uniform_(w, mode="fan_in", nonlinearity="relu") + + Note: + Be aware that ``fan_in`` and ``fan_out`` are calculated assuming + that the weight matrix is used in a transposed manner, + (i.e., ``x @ w.T`` in ``Linear`` layers, where ``w.shape = [fan_out, fan_in]``). + This is important for correct initialization. + If you plan to use ``x @ w``, where ``w.shape = [fan_in, fan_out]``, + pass in a transposed weight matrix, i.e. ``nn.init.kaiming_uniform_(w.T, ...)``. + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + kaiming_uniform_, + (tensor,), + tensor=tensor, + a=a, + mode=mode, + nonlinearity=nonlinearity, + generator=generator, + ) + + if 0 in tensor.shape: + warnings.warn("Initializing zero-element tensors is a no-op", stacklevel=2) + return tensor + fan = _calculate_correct_fan(tensor, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + with torch.no_grad(): + return tensor.uniform_(-bound, bound, generator=generator) + + +def kaiming_normal_( + tensor: Tensor, + a: float = 0, + mode: _FanMode = "fan_in", + nonlinearity: _NonlinearityType = "leaky_relu", + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Kaiming normal distribution. + + The method is described in `Delving deep into rectifiers: Surpassing + human-level performance on ImageNet classification` - He, K. et al. (2015). + The resulting tensor will have values sampled from + :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.kaiming_normal_(w, mode="fan_out", nonlinearity="relu") + + Note: + Be aware that ``fan_in`` and ``fan_out`` are calculated assuming + that the weight matrix is used in a transposed manner, + (i.e., ``x @ w.T`` in ``Linear`` layers, where ``w.shape = [fan_out, fan_in]``). + This is important for correct initialization. + If you plan to use ``x @ w``, where ``w.shape = [fan_in, fan_out]``, + pass in a transposed weight matrix, i.e. ``nn.init.kaiming_normal_(w.T, ...)``. + """ + if 0 in tensor.shape: + warnings.warn("Initializing zero-element tensors is a no-op", stacklevel=2) + return tensor + fan = _calculate_correct_fan(tensor, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + with torch.no_grad(): + return tensor.normal_(0, std, generator=generator) + + +def orthogonal_( + tensor: Tensor, + gain: float = 1, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with a (semi) orthogonal matrix. + + Described in `Exact solutions to the nonlinear dynamics of learning in deep + linear neural networks` - Saxe, A. et al. (2013). The input tensor must have + at least 2 dimensions, and for tensors with more than 2 dimensions the + trailing dimensions are flattened. + + Args: + tensor: an n-dimensional `torch.Tensor`, where :math:`n \geq 2` + gain: optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> w = torch.empty(3, 5) + >>> nn.init.orthogonal_(w) + """ + if tensor.ndimension() < 2: + raise ValueError("Only tensors with 2 or more dimensions are supported") + + if tensor.numel() == 0: + # no-op + return tensor + rows = tensor.size(0) + cols = tensor.numel() // rows + flattened = tensor.new_empty((rows, cols)).normal_(0, 1, generator=generator) + + if rows < cols: + flattened.t_() + + # Compute the qr factorization + q, r = torch.linalg.qr(flattened) + # Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf + d = torch.diag(r, 0) + ph = d.sign() + q *= ph + + if rows < cols: + q.t_() + + with torch.no_grad(): + tensor.view_as(q).copy_(q) + tensor.mul_(gain) + return tensor + + +def sparse_( + tensor: Tensor, + sparsity: float, + std: float = 0.01, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the 2D input `Tensor` as a sparse matrix. + + The non-zero elements will be drawn from the normal distribution + :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via + Hessian-free optimization` - Martens, J. (2010). + + Args: + tensor: an n-dimensional `torch.Tensor` + sparsity: The fraction of elements in each column to be set to zero + std: the standard deviation of the normal distribution used to generate + the non-zero values + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.sparse_(w, sparsity=0.1) + """ + if tensor.ndimension() != 2: + raise ValueError("Only tensors with 2 dimensions are supported") + + rows, cols = tensor.shape + num_zeros = math.ceil(sparsity * rows) + + with torch.no_grad(): + tensor.normal_(0, std, generator=generator) + for col_idx in range(cols): + row_indices = torch.randperm(rows) + zero_indices = row_indices[:num_zeros] + tensor[zero_indices, col_idx] = 0 + return tensor + + +# for backward compatibility +def _make_deprecate(meth: Callable[_P, _R]) -> Callable[_P, _R]: + new_name = meth.__name__ + old_name = new_name[:-1] + + def deprecated_init(*args: _P.args, **kwargs: _P.kwargs) -> _R: + warnings.warn( + f"`nn.init.{old_name}` is now deprecated in favor of `nn.init.{new_name}`.", + FutureWarning, + stacklevel=2, + ) + return meth(*args, **kwargs) + + deprecated_init.__doc__ = rf""" + {old_name}(...) + + .. warning:: + This method is now deprecated in favor of :func:`torch.nn.init.{new_name}`. + + See :func:`~torch.nn.init.{new_name}` for details.""" + deprecated_init.__name__ = old_name + return deprecated_init + + +uniform = _make_deprecate(uniform_) +normal = _make_deprecate(normal_) +constant = _make_deprecate(constant_) +eye = _make_deprecate(eye_) +dirac = _make_deprecate(dirac_) +xavier_uniform = _make_deprecate(xavier_uniform_) +xavier_normal = _make_deprecate(xavier_normal_) +kaiming_uniform = _make_deprecate(kaiming_uniform_) +kaiming_normal = _make_deprecate(kaiming_normal_) +orthogonal = _make_deprecate(orthogonal_) +sparse = _make_deprecate(sparse_) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.py new file mode 100644 index 0000000000000000000000000000000000000000..69899c287ffee66338b888efd26a10a98205e505 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.py @@ -0,0 +1,307 @@ +from collections import OrderedDict +from typing import Any + +import torch +from torch._C import _disabled_torch_function_impl + + +__all__ = [ + "Parameter", + "UninitializedParameter", + "is_lazy", + "Buffer", + "UninitializedBuffer", + "UninitializedTensorMixin", +] + + +# Metaclass to combine _TensorMeta and the instance check override for Parameter. +class _ParameterMeta(torch._C._TensorMeta): + # Make `isinstance(t, Parameter)` return True for custom tensor instances that have the _is_param flag. + def __instancecheck__(self, instance) -> bool: + if self is Parameter: + if isinstance(instance, torch.Tensor) and getattr( + instance, "_is_param", False + ): + return True + return super().__instancecheck__(instance) + + +class Parameter(torch.Tensor, metaclass=_ParameterMeta): + r"""A kind of Tensor that is to be considered a module parameter. + + Parameters are :class:`~torch.Tensor` subclasses, that have a + very special property when used with :class:`Module` s - when they're + assigned as Module attributes they are automatically added to the list of + its parameters, and will appear e.g. in :meth:`~Module.parameters` iterator. + Assigning a Tensor doesn't have such effect. This is because one might + want to cache some temporary state, like last hidden state of the RNN, in + the model. If there was no such class as :class:`Parameter`, these + temporaries would get registered too. + + Args: + data (Tensor): parameter tensor. + requires_grad (bool, optional): if the parameter requires gradient. Note that + the torch.no_grad() context does NOT affect the default behavior of + Parameter creation--the Parameter will still have `requires_grad=True` in + :class:`~no_grad` mode. See :ref:`locally-disable-grad-doc` for more + details. Default: `True` + """ + + def __new__(cls, data=None, requires_grad=True): + if data is None: + data = torch.empty(0) + if type(data) is torch.Tensor or type(data) is Parameter: + # For ease of BC maintenance, keep this path for standard Tensor. + # Eventually (tm), we should change the behavior for standard Tensor to match. + return torch.Tensor._make_subclass(cls, data, requires_grad) + + # Path for custom tensors: set a flag on the instance to indicate parameter-ness. + t = data.detach().requires_grad_(requires_grad) + if type(t) is not type(data): + raise RuntimeError( + f"Creating a Parameter from an instance of type {type(data).__name__} " + "requires that detach() returns an instance of the same type, but return " + f"type {type(t).__name__} was found instead. To use the type as a " + "Parameter, please correct the detach() semantics defined by " + "its __torch_dispatch__() implementation." + ) + t._is_param = True + return t + + # Note: the 3 methods below only apply to standard Tensor. Parameters of custom tensor types + # are still considered that custom tensor type and these methods will not be called for them. + def __deepcopy__(self, memo): + if id(self) in memo: + return memo[id(self)] + else: + result = type(self)( + self.data.clone(memory_format=torch.preserve_format), self.requires_grad + ) + memo[id(self)] = result + return result + + # pyrefly: ignore [bad-override] + def __repr__(self) -> str: + return "Parameter containing:\n" + super().__repr__() + + def __reduce_ex__(self, proto): + state = torch._utils._get_obj_state(self) + + # See Note [Don't serialize hooks] + hooks = OrderedDict() + if not state: + return ( + torch._utils._rebuild_parameter, + (self.data, self.requires_grad, hooks), + ) + + return ( + torch._utils._rebuild_parameter_with_state, + (self.data, self.requires_grad, hooks, state), + ) + + __torch_function__ = _disabled_torch_function_impl + + +class UninitializedTensorMixin: + _allowed_methods = [ + torch.Tensor.__hash__, + torch.Tensor.size, + torch.Tensor.copy_, + torch.Tensor.is_complex, + torch.Tensor.is_floating_point, + torch.Tensor.half, + torch.Tensor.float, + torch.Tensor.double, + torch.Tensor.char, + torch.Tensor.short, + torch.Tensor.int, + torch.Tensor.long, + torch.Tensor.cuda, + torch.Tensor.cpu, + torch.Tensor.to, + torch.Tensor.get_device, + torch._has_compatible_shallow_copy_type, + ] + + def materialize(self, shape, device=None, dtype=None) -> None: + r"""Create a Parameter or Tensor with the same properties of the uninitialized one. + + Given a shape, it materializes a parameter in the same device + and with the same `dtype` as the current one or the specified ones in the + arguments. + + Args: + shape : (tuple): the shape for the materialized tensor. + device (:class:`torch.device`): the desired device of the parameters + and buffers in this module. Optional. + dtype (:class:`torch.dtype`): the desired floating point type of + the floating point parameters and buffers in this module. Optional. + """ + if device is None: + device = self.data.device + if dtype is None: + dtype = self.data.dtype + self.data = torch.empty(shape, device=device, dtype=dtype) + # pyrefly: ignore [bad-override, missing-attribute] + self.__class__ = self.cls_to_become + + @property + def shape(self): + raise RuntimeError( + "Can't access the shape of an uninitialized parameter or buffer. " + "This error usually happens in `load_state_dict` when trying to load " + "an uninitialized parameter into an initialized one. " + "Call `forward` to initialize the parameters before accessing their attributes." + ) + + def share_memory_(self): + raise RuntimeError( + "Can't share memory on an uninitialized parameter or buffer. " + "Call `forward` to initialize the parameters before calling " + "`module.share_memory()`." + ) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}>" + + def __reduce_ex__(self, proto): + # See Note [Don't serialize hooks] + # pyrefly: ignore [missing-attribute] + return (self.__class__, (self.requires_grad,)) + + @classmethod + def __torch_function__(cls, func, types, args=(), kwargs=None): + # method-wrapper is to detect access to Tensor properties that are + # wrapped in descriptors + if func in cls._allowed_methods or func.__class__.__name__ == "method-wrapper": + if kwargs is None: + kwargs = {} + # pyrefly: ignore [missing-attribute] + return super().__torch_function__(func, types, args, kwargs) + raise ValueError( + f"Attempted to use an uninitialized parameter in {func}. " + "This error happens when you are using a `LazyModule` or " + f"explicitly manipulating `torch.nn.parameter.{cls.__name__}` " + "objects. When using LazyModules Call `forward` with a dummy batch " + "to initialize the parameters before calling torch functions" + ) + + +def is_lazy(param: Any) -> bool: + """ + Returns whether ``param`` is an ``UninitializedParameter`` or ``UninitializedBuffer``. + + Args: + param (Any): the input to check. + """ + return isinstance(param, UninitializedTensorMixin) + + +class UninitializedParameter(UninitializedTensorMixin, Parameter): + r"""A parameter that is not initialized. + + Uninitialized Parameters are a special case of :class:`torch.nn.Parameter` + where the shape of the data is still unknown. + + Unlike a :class:`torch.nn.Parameter`, uninitialized parameters + hold no data and attempting to access some properties, like their shape, + will throw a runtime error. The only operations that can be performed on a uninitialized + parameter are changing its datatype, moving it to a different device and + converting it to a regular :class:`torch.nn.Parameter`. + + The default device or dtype to use when the parameter is materialized can be set + during construction using e.g. ``device='cuda'``. + """ + + cls_to_become = Parameter + + def __new__(cls, requires_grad=True, device=None, dtype=None) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + data = torch.empty(0, **factory_kwargs) + # pyrefly: ignore [bad-return] + return torch.Tensor._make_subclass(cls, data, requires_grad) + + def __deepcopy__(self, memo): + if id(self) in memo: + return memo[id(self)] + else: + result = type(self)(self.requires_grad, self.data.device, self.data.dtype) + memo[id(self)] = result + return result + + +# Metaclass to combine _TensorMeta and the instance check override for Buffer. +class _BufferMeta(torch._C._TensorMeta): + # Make `isinstance(t, Buffer)` return True for custom tensor instances that have the _is_buffer flag. + def __instancecheck__(self, instance) -> bool: + if self is Buffer: + if isinstance(instance, torch.Tensor) and getattr( + instance, "_is_buffer", False + ): + return True + return super().__instancecheck__(instance) + + +class Buffer(torch.Tensor, metaclass=_BufferMeta): + r"""A kind of Tensor that should not be considered a model + parameter. For example, BatchNorm's ``running_mean`` is not a parameter, but is part of the module's state. + + Buffers are :class:`~torch.Tensor` subclasses, that have a + very special property when used with :class:`Module` s -- when they're + assigned as Module attributes they are automatically added to the list of + its buffers, and will appear e.g. in :meth:`~torch.nn.Module.buffers` iterator. + Assigning a Tensor doesn't have such effect. One can still assign a Tensor as explicitly by using + the :meth:`~torch.nn.Module.register_buffer` function. + + Args: + data (Tensor): buffer tensor. + persistent (bool, optional): whether the buffer is part of the module's + :attr:`state_dict`. Default: ``True`` + """ + + def __new__(cls, data=None, *, persistent=True): + if data is None: + data = torch.empty(0) + + t = data.detach().requires_grad_(data.requires_grad) + # pyrefly: ignore [missing-attribute] + t.persistent = persistent + # pyrefly: ignore [missing-attribute] + t._is_buffer = True + return t + + __torch_function__ = _disabled_torch_function_impl + + +class UninitializedBuffer(UninitializedTensorMixin, torch.Tensor): + r"""A buffer that is not initialized. + + Uninitialized Buffer is a a special case of :class:`torch.Tensor` + where the shape of the data is still unknown. + + Unlike a :class:`torch.Tensor`, uninitialized parameters + hold no data and attempting to access some properties, like their shape, + will throw a runtime error. The only operations that can be performed on a uninitialized + parameter are changing its datatype, moving it to a different device and + converting it to a regular :class:`torch.Tensor`. + + The default device or dtype to use when the buffer is materialized can be set + during construction using e.g. ``device='cuda'``. + """ + + cls_to_become = torch.Tensor + + def __new__( + cls, requires_grad=False, device=None, dtype=None, persistent=True + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + data = torch.empty(0, **factory_kwargs) + ret = torch.Tensor._make_subclass(cls, data, requires_grad) + # pyrefly: ignore [missing-attribute] + ret.persistent = persistent + # pyrefly: ignore [missing-attribute] + ret._is_buffer = True + # pyrefly: ignore [bad-return] + return ret diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.pyi b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.pyi new file mode 100644 index 0000000000000000000000000000000000000000..968869f1d52ef971f0391b532a0b63405da40df2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/nn/parameter.pyi @@ -0,0 +1,43 @@ +from typing_extensions import TypeIs + +from torch import device, dtype, Tensor + +class Parameter(Tensor): + def __init__(self, data: Tensor = ..., requires_grad: bool = ...) -> None: ... + +def is_lazy( + param: Tensor, +) -> TypeIs[UninitializedParameter | UninitializedBuffer]: ... + +class UninitializedParameter(Tensor): + def __init__(self, data: Tensor = ..., requires_grad: bool = ...) -> None: ... + def materialize( + self, + shape: tuple[int, ...], + device: device | None = None, + dtype: dtype | None = None, + ) -> None: ... + +class Buffer(Tensor): + persistent: bool + def __init__( + self, + data: Tensor = ..., + requires_grad: bool = ..., + persistent: bool = ..., + ) -> None: ... + +class UninitializedBuffer(Tensor): + persistent: bool + def __init__( + self, + data: Tensor = ..., + requires_grad: bool = ..., + persistent: bool = ..., + ) -> None: ... + def materialize( + self, + shape: tuple[int, ...], + device: device | None = None, + dtype: dtype | None = None, + ) -> None: ... diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/numa/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/numa/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/numa/binding.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/numa/binding.py new file mode 100644 index 0000000000000000000000000000000000000000..554d62ce3736a74f5faad6b512a404443b3b7249 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/numa/binding.py @@ -0,0 +1,675 @@ +import os +import shutil +import traceback +from collections import defaultdict +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass +from enum import Enum +from functools import wraps +from logging import getLogger +from typing import ParamSpec, TypeVar + +import torch +from torch._utils_internal import signpost_event + + +__all__ = [ + "AffinityMode", + "maybe_wrap_command_args_with_numa_binding", + "maybe_wrap_with_numa_binding", + "NumaOptions", +] + +logger = getLogger(__name__) + + +class AffinityMode(str, Enum): + """ + See behavior description for each affinity mode + in torch.distributed.run. + """ + + NODE = "node" + SOCKET = "socket" + EXCLUSIVE = "exclusive" + CORE_COMPLEX = "core-complex" + + +@dataclass(frozen=True) +class NumaOptions: + affinity_mode: AffinityMode + + """ + If true, we will fall back to using the original command/entrypoint if we fail to compute + NUMA bindings. + + You should avoid using this option! It is only intended as a safety mechanism for facilitating + mass rollouts of numa binding. + """ + should_fall_back_if_binding_fails: bool = False + + +def maybe_wrap_command_args_with_numa_binding( + command_args: tuple[str, ...], + *, + gpu_index: int, + numa_options: NumaOptions | None, +) -> tuple[str, ...]: + """ + Wraps command arguments with numactl to apply NUMA CPU binding. + + This function prepends numactl with appropriate CPU affinity flags to the + provided command arguments, binding the process to CPUs associated with + the specified GPU's NUMA node. + + Args: + command_args: The original command arguments to wrap. + gpu_index: The index of the GPU that will be used by the subprocess. + numa_options: Configuration for NUMA binding behavior. If None, returns + the original command_args unchanged. + + Returns: + Tuple of command arguments, potentially wrapped with numactl for NUMA binding. + Returns the original command_args if numa_options is None or if binding fails + and fallback is enabled. + """ + if numa_options is None: + return command_args + + kwargs = { + "command_args": command_args, + "gpu_index": gpu_index, + "numa_options": asdict(numa_options), + } + + try: + logical_cpu_indices = _get_validated_logical_cpus_to_bind_to( + gpu_index=gpu_index, + numa_options=numa_options, + ) + + wrapped_command_args = _assemble_numactl_command_args( + original_command_args=command_args, + logical_cpu_indices=logical_cpu_indices, + ) + signpost_event( + category="numa_binding", + name="apply_success", + parameters={ + **kwargs, + "wrapped_command": wrapped_command_args, + }, + ) + return wrapped_command_args + except Exception: + # pyrefly: ignore [bad-argument-type] + _handle_exception(numa_options=numa_options, logger_kwargs=kwargs) + return command_args + + +_TParams = ParamSpec("_TParams") +_TReturn = TypeVar("_TReturn") + + +def maybe_wrap_with_numa_binding( + func: Callable[_TParams, _TReturn], + *, + gpu_index: int, + numa_options: NumaOptions | None, +) -> Callable[_TParams, _TReturn]: + """ + Wraps a function to apply NUMA CPU binding before execution. + + This decorator applies NUMA CPU affinity to all threads in the current process + before calling the wrapped function, binding them to CPUs associated with the + specified GPU's NUMA node. + + Args: + func: The function to wrap with NUMA binding. + gpu_index: The index of the GPU that will be used. + numa_options: Configuration for NUMA binding behavior. If None, returns + the original function unchanged. + + Returns: + A wrapped function that applies NUMA binding before execution, or the + original function if numa_options is None. + """ + if numa_options is None: + return func + + @wraps(func) + def wrapped(*args: _TParams.args, **kwargs: _TParams.kwargs) -> _TReturn: + _maybe_apply_numa_binding_to_current_process( + gpu_index=gpu_index, + # pyrefly: ignore [bad-argument-type] + numa_options=numa_options, + ) + return func(*args, **kwargs) + + return wrapped + + +def _maybe_apply_numa_binding_to_current_process( + *, gpu_index: int, numa_options: NumaOptions +) -> None: + kwargs = { + "gpu_index": gpu_index, + "numa_options": asdict(numa_options), + } + + try: + logical_cpu_indices = _get_validated_logical_cpus_to_bind_to( + gpu_index=gpu_index, + numa_options=numa_options, + ) + + _bind_all_threads_in_current_process_to_logical_cpus( + logical_cpu_indices=logical_cpu_indices + ) + + signpost_event( + category="numa_binding", + name="apply_success", + parameters={ + **kwargs, + "logical_cpu_indices": _get_ranges_str_from_ints(logical_cpu_indices), + }, + ) + except Exception: + # pyrefly: ignore [bad-argument-type] + _handle_exception(numa_options=numa_options, logger_kwargs=kwargs) + + +def _assemble_numactl_command_args( + *, original_command_args: tuple[str, ...], logical_cpu_indices: set[int] +) -> tuple[str, ...]: + return ( + "numactl", + f"--physcpubind={_get_ranges_str_from_ints(logical_cpu_indices)}", + *original_command_args, + ) + + +def _handle_exception( + *, numa_options: NumaOptions, logger_kwargs: dict[str, object] +) -> None: + signpost_event( + category="numa_binding", + name="apply_exception", + parameters={ + **logger_kwargs, + "traceback": traceback.format_exc(), + }, + ) + logger.exception("Failed to apply NUMA binding for input=%r", logger_kwargs) + if numa_options.should_fall_back_if_binding_fails: + logger.warning( + "Continuing executing without applying NUMA binding, despite exception %s", + traceback.format_exc(), + ) + return + # This function is called within an except block, so silence the warning + # about raise without an exception. + raise # noqa: PLE0704 + + +def _get_validated_logical_cpus_to_bind_to( + *, + gpu_index: int, + numa_options: NumaOptions, +) -> set[int]: + logical_cpu_indices = _get_logical_cpus_to_bind_to( + gpu_index=gpu_index, numa_options=numa_options + ) + _raise_if_binding_invalid(logical_cpu_indices=logical_cpu_indices) + + return logical_cpu_indices + + +def _raise_if_binding_invalid(*, logical_cpu_indices: set[int]) -> None: + # NOTE: numactl CLI is only actually necessary for the str entrypoint path, + # but for simplicity we will just check it no matter what. + if shutil.which("numactl") is None: + raise RuntimeError("numactl CLI is required for NUMA binding") + + if not logical_cpu_indices: + raise RuntimeError("Must bind to a non-empty set of CPU indices") + + +def _bind_all_threads_in_current_process_to_logical_cpus( + *, logical_cpu_indices: set[int] +) -> None: + # Save the original affinity of the main thread before changing it + # pyrefly: ignore [missing-attribute] + original_main_thread_affinity = os.sched_getaffinity(0) # type: ignore[attr-defined] + + # 0 represents the current thread. + # This is outside the try/except because the main thread should always bind successfully. + # pyrefly: ignore [missing-attribute] + os.sched_setaffinity(0, logical_cpu_indices) # type: ignore[attr-defined] + + for tid_str in os.listdir("/proc/self/task"): + try: + tid = int(tid_str) + # pyrefly: ignore [missing-attribute] + tid_affinity = os.sched_getaffinity(tid) # type: ignore[attr-defined] + + # Defensive check to ensure we do not overwrite affinity on any threads + # that have already had their affinity set elsewhere. + if tid_affinity == original_main_thread_affinity: + # pyrefly: ignore [missing-attribute] + os.sched_setaffinity(tid, logical_cpu_indices) # type: ignore[attr-defined] + except Exception: + # Thread may have exited or otherwise become invalid + pass + + +def _get_logical_cpus_to_bind_to( + *, + gpu_index: int, + numa_options: NumaOptions, +) -> set[int]: + """ + Args: + gpu_index: The index of the GPU that will be used by the subprocess. + Example: 0 + numa_options: See NumaOptions for details. + + Returns: + Set of logical CPU indices to bind to. + """ + if numa_options.affinity_mode == AffinityMode.NODE: + logical_cpus = _node_get_logical_cpus_to_bind_to(gpu_index=gpu_index) + elif numa_options.affinity_mode == AffinityMode.SOCKET: + logical_cpus = _socket_get_logical_cpus_to_bind_to(gpu_index=gpu_index) + elif numa_options.affinity_mode == AffinityMode.EXCLUSIVE: + logical_cpus = _exclusive_get_logical_cpus_to_bind_to(gpu_index=gpu_index) + elif numa_options.affinity_mode == AffinityMode.CORE_COMPLEX: + logical_cpus = _core_complex_get_logical_cpus_to_bind_to(gpu_index=gpu_index) + else: + raise ValueError(f"Affinity mode {numa_options.affinity_mode} not supported.") + + return logical_cpus + + +def _node_get_logical_cpus_to_bind_to(*, gpu_index: int) -> set[int]: + """ + Core logic of 'node' numa strategy. + """ + numa_node_index = _get_numa_node_index_for_gpu_index(gpu_index=gpu_index) + + return _get_allowed_logical_cpu_indices_for_numa_node( + numa_node_index=numa_node_index + ) + + +def _socket_get_logical_cpus_to_bind_to(*, gpu_index: int) -> set[int]: + """ + Core logic of 'socket' numa strategy. + """ + numa_node_index_of_gpu = _get_numa_node_index_for_gpu_index(gpu_index=gpu_index) + socket_index = _get_socket_index_for_numa_node( + numa_node_index=numa_node_index_of_gpu + ) + numa_node_indices = _get_numa_node_indices_for_socket_index( + socket_index=socket_index + ) + + logical_cpus = set() + for numa_node_index in numa_node_indices: + logical_cpus.update( + _get_allowed_logical_cpu_indices_for_numa_node( + numa_node_index=numa_node_index + ) + ) + + return logical_cpus + + +def _exclusive_get_logical_cpus_to_bind_to(*, gpu_index: int) -> set[int]: + """ + Core logic of 'exclusive' numa strategy. + """ + numa_node_index = _get_numa_node_index_for_gpu_index(gpu_index=gpu_index) + + gpu_indices = _get_gpu_indices_for_numa_node(numa_node_index=numa_node_index) + gpu_indices = sorted(gpu_indices) + original_gpu_relative_index = gpu_indices.index(gpu_index) + + allowed_logical_cpu_indices = _get_allowed_logical_cpu_indices_for_numa_node( + numa_node_index=numa_node_index + ) + + # Arbitrarily use the min logical cpu index on the physical core to + # represent the physical core. + physical_core_to_allowed_logical_cpu_indices = _group_by( + allowed_logical_cpu_indices, + lambda logical_cpu_index: min( + _get_logical_cpu_indices_sharing_same_physical_core_as( + logical_cpu_index=logical_cpu_index + ) + ), + ) + # Sort the dict for consistency (dicts maintain order in Python) + physical_core_to_allowed_logical_cpu_indices = dict( + sorted(physical_core_to_allowed_logical_cpu_indices.items()) + ) + + num_physical_cores_per_gpu = len( + physical_core_to_allowed_logical_cpu_indices + ) // len(gpu_indices) + # Often, the number of physical cores will not be perfectly divisible by the number + # of GPUs. In those cases, give the lowest GPU indices an extra core + num_gpus_to_give_one_extra_physical_core = len( + physical_core_to_allowed_logical_cpu_indices + ) % len(gpu_indices) + + if num_physical_cores_per_gpu < 1: + raise RuntimeError( + f"There are only {len(physical_core_to_allowed_logical_cpu_indices)} physical cores on {numa_node_index=}," + + f" but there are {len(gpu_indices)} GPUs associated with this NUMA node." + ) + + # Compute slice indices for this GPU + start = original_gpu_relative_index * num_physical_cores_per_gpu + min( + original_gpu_relative_index, num_gpus_to_give_one_extra_physical_core + ) + end = ( + start + + num_physical_cores_per_gpu + + ( + 1 + if original_gpu_relative_index < num_gpus_to_give_one_extra_physical_core + else 0 + ) + ) + + # Slice and flatten the logical CPUs from the selected physical cores + logical_cpu_indices_for_original_gpu = { + logical_cpu_index + for logical_cpu_indices in list( + physical_core_to_allowed_logical_cpu_indices.values() + )[start:end] + for logical_cpu_index in logical_cpu_indices + } + + return logical_cpu_indices_for_original_gpu + + +def _core_complex_get_logical_cpus_to_bind_to(*, gpu_index: int) -> set[int]: + """ + Core logic of 'core-complex' numa strategy. + + Each GPU is assigned a full core complex (group of cores sharing L3 cache) + within its affined NUMA node. + """ + numa_node_index = _get_numa_node_index_for_gpu_index(gpu_index=gpu_index) + + gpu_indices = _get_gpu_indices_for_numa_node(numa_node_index=numa_node_index) + gpu_indices = sorted(gpu_indices) + original_gpu_relative_index = gpu_indices.index(gpu_index) + + allowed_logical_cpu_indices = _get_allowed_logical_cpu_indices_for_numa_node( + numa_node_index=numa_node_index + ) + + # Arbitrarily use the min logical cpu index on the max level cache + # to represent the max level cache. + max_level_cache_to_allowed_logical_cpu_indices = _group_by( + allowed_logical_cpu_indices, + lambda logical_cpu_index: min( + _get_logical_cpus_sharing_same_max_level_cache_as( + logical_cpu_index=logical_cpu_index + ) + ), + ) + + max_level_cache_to_allowed_logical_cpu_indices = dict( + sorted( + max_level_cache_to_allowed_logical_cpu_indices.items(), + # First, prioritize caches with more available cpus + # Second, prioritize lower index cpus (just for clarity/consistency) + key=lambda item: (-len(item[1]), item[0]), + ) + ) + + cache_index_for_original_gpu = original_gpu_relative_index % len( + max_level_cache_to_allowed_logical_cpu_indices + ) + logical_cpu_indices_for_original_gpu = list( + max_level_cache_to_allowed_logical_cpu_indices.values() + )[cache_index_for_original_gpu] + + return logical_cpu_indices_for_original_gpu + + +K = TypeVar("K") +V = TypeVar("V") + + +def _group_by(values: Iterable[V], get_key: Callable[[V], K]) -> dict[K, set[V]]: + """ + Groups elements with same key into sets. + """ + key_to_values: defaultdict[K, set[V]] = defaultdict(set) + for value in values: + key = get_key(value) + key_to_values[key].add(value) + return key_to_values + + +def _get_logical_cpu_indices_sharing_same_physical_core_as( + *, logical_cpu_index: int +) -> set[int]: + thread_siblings_list_absolute_path = ( + f"/sys/devices/system/cpu/cpu{logical_cpu_index}/topology/thread_siblings_list" + ) + with open(thread_siblings_list_absolute_path) as f: + return _get_set_of_int_from_ranges_str(f.read()) + + +def _get_logical_cpus_sharing_same_max_level_cache_as( + *, logical_cpu_index: int +) -> set[int]: + cpu_cache_dir_absolute_path = ( + f"/sys/devices/system/cpu/cpu{logical_cpu_index}/cache" + ) + + max_level = -1 + logical_cpus_sharing_max_level_cache = set() + for entry in os.listdir(cpu_cache_dir_absolute_path): + if not entry.startswith("index") or not entry[5:].isdecimal(): + continue + cache_index_absolute_path = os.path.join(cpu_cache_dir_absolute_path, entry) + + # Filter out other cache types like Instruction + type_absolute_path = os.path.join(cache_index_absolute_path, "type") + with open(type_absolute_path) as type_file: + if type_file.read().strip() not in {"Unified", "Data"}: + continue + + level_absolute_path = os.path.join(cache_index_absolute_path, "level") + with open(level_absolute_path) as level_file: + level = int(level_file.read()) + if level <= max_level: + continue + + max_level = level + shared_cpu_list_absolute_path = os.path.join( + cache_index_absolute_path, "shared_cpu_list" + ) + with open(shared_cpu_list_absolute_path) as share_cpu_list_file: + logical_cpus_sharing_max_level_cache = _get_set_of_int_from_ranges_str( + share_cpu_list_file.read() + ) + + return logical_cpus_sharing_max_level_cache + + +def _get_allowed_logical_cpu_indices_for_numa_node(*, numa_node_index: int) -> set[int]: + all_cpu_indices = _get_cpu_indices_for_numa_node_MAYBE_NOT_ALLOWED( + numa_node_index=numa_node_index + ) + allowed_cpu_indices = _get_allowed_cpu_indices_for_current_thread() + return all_cpu_indices & allowed_cpu_indices + + +def _get_cpu_indices_for_numa_node_MAYBE_NOT_ALLOWED( + *, numa_node_index: int +) -> set[int]: + """ + Returns: + Indices of all CPUs associated with numa_node_index. However, the list + is not filtered based on whether the thread is allowed to use them. + """ + cpulist_absolute_path = f"/sys/devices/system/node/node{numa_node_index}/cpulist" + try: + with open(cpulist_absolute_path) as f: + cpu_range_str = f.read() + except FileNotFoundError as e: + raise RuntimeError( + f"Could not determine CPUs corresponding to {numa_node_index=}." + ) from e + return _get_set_of_int_from_ranges_str(cpu_range_str) + + +def _get_gpu_count() -> int: + return torch.cuda.device_count() + + +def _get_numa_node_index_for_gpu_index(*, gpu_index: int) -> int: + device_properties = torch.cuda.get_device_properties(gpu_index) + + domain = device_properties.pci_domain_id # type: ignore[attr-defined] + bus = device_properties.pci_bus_id # type: ignore[attr-defined] + device = device_properties.pci_device_id # type: ignore[attr-defined] + + # Format to sysfs PCI address: "0000:dc:00.0" + pci_addr = f"{domain:04x}:{bus:02x}:{device:02x}.0" + + pci_numa_node_absolute_path = f"/sys/bus/pci/devices/{pci_addr}/numa_node" + with open(pci_numa_node_absolute_path) as f: + # In systems with only one NUMA node, this will + # often be saved as -1. In those cases, there is obviously + # at least one numa node, 0, so we use that. + return max(int(f.read().strip()), 0) + + +def _get_gpu_indices_for_numa_node(*, numa_node_index: int) -> set[int]: + return { + gpu_index + for gpu_index in range(_get_gpu_count()) + if _get_numa_node_index_for_gpu_index(gpu_index=gpu_index) == numa_node_index + } + + +def _get_socket_index_for_numa_node(*, numa_node_index: int) -> int: + arbitrary_cpu_index = _get_arbitrary_allowed_cpu_index_for_numa_node( + numa_node_index=numa_node_index + ) + + return _get_socket_index_for_cpu(cpu_index=arbitrary_cpu_index) + + +def _get_socket_index_for_cpu(*, cpu_index: int) -> int: + package_id_absolute_path = ( + f"/sys/devices/system/cpu/cpu{cpu_index}/topology/physical_package_id" + ) + try: + with open(package_id_absolute_path) as f: + return int(f.read().strip()) + except FileNotFoundError as e: + raise RuntimeError(f"Could not determine socket for {cpu_index=}") from e + + +def _get_arbitrary_allowed_cpu_index_for_numa_node(*, numa_node_index: int) -> int: + return min( + _get_allowed_logical_cpu_indices_for_numa_node(numa_node_index=numa_node_index) + ) + + +def _get_set_of_int_from_ranges_str(ranges_str: str) -> set[int]: + """ + Util for parsing a string of int ranges, as in a sysfs file. + + Args: + ranges_str: E.g., "0-2,4,6-7" + + Returns: + E.g., {0, 1, 2, 4, 6, 7} + """ + ints: set[int] = set() + for range_str in ranges_str.split(","): + range_str = range_str.strip() + if not range_str: + continue + if "-" in range_str: + start_str, end_str = range_str.split("-") + start, end = int(start_str), int(end_str) + ints.update(range(start, end + 1)) + else: + ints.add(int(range_str)) + return ints + + +def _get_ranges_str_from_ints(ints: Iterable[int]) -> str: + """ + Convert a set of integers to a compact string with ranges. + + Args: + ints: E.g., {0, 1, 2, 4, 6, 7} + + Returns: + E.g., "0-2,4,6-7" + """ + if not ints: + return "" + + sorted_ints = sorted(ints) + ranges = [] + start = prev = sorted_ints[0] + + for num in sorted_ints[1:]: + if num == prev + 1: + prev = num + else: + if start == prev: + ranges.append(f"{start}") + else: + ranges.append(f"{start}-{prev}") + start = prev = num + + # Append the last range + if start == prev: + ranges.append(f"{start}") + else: + ranges.append(f"{start}-{prev}") + + return ",".join(ranges) + + +def _get_systemwide_numa_node_indices() -> set[int]: + with open("/sys/devices/system/node/possible") as f: + possible_nodes_str = f.read() + + return _get_set_of_int_from_ranges_str(possible_nodes_str) + + +def _get_numa_node_indices_for_socket_index(*, socket_index: int) -> set[int]: + systemwide_numa_node_indices = _get_systemwide_numa_node_indices() + + matching_numa_node_indices = set() + for numa_node_index in systemwide_numa_node_indices: + arbitrary_cpu_index = _get_arbitrary_allowed_cpu_index_for_numa_node( + numa_node_index=numa_node_index + ) + if socket_index == _get_socket_index_for_cpu(cpu_index=arbitrary_cpu_index): + matching_numa_node_indices.add(numa_node_index) + + return matching_numa_node_indices + + +def _get_allowed_cpu_indices_for_current_thread() -> set[int]: + # 0 denotes current thread + # pyrefly: ignore [missing-attribute] + return os.sched_getaffinity(0) # type:ignore[attr-defined] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..931401f10d2966c9058e3422d2804a2a4b05d719 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/__init__.py @@ -0,0 +1,368 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + + +__all__ = [ + # Modules + "errors", + "ops", + # Public functions + "export", + "is_in_onnx_export", + # Base error + "OnnxExporterError", + "ONNXProgram", +] + +from typing import Any, TYPE_CHECKING + +import torch +from torch._C import _onnx as _C_onnx +from torch._C._onnx import ( # Deprecated members that are excluded from __all__ + OperatorExportTypes as OperatorExportTypes, + TensorProtoDataType as TensorProtoDataType, + TrainingMode as TrainingMode, +) + +from . import errors, ops +from ._internal.exporter._onnx_program import ONNXProgram +from ._internal.torchscript_exporter import ( # Deprecated members that are excluded from __all__ + symbolic_helper, + symbolic_opset10, + symbolic_opset9, + utils, +) +from ._internal.torchscript_exporter._type_utils import ( + JitScalarType, # Deprecated members that are excluded from __all__ +) +from ._internal.torchscript_exporter.utils import ( # Deprecated members that are excluded from __all__ + register_custom_op_symbolic, + select_model_mode_for_export, # pyrefly: ignore # deprecated + unregister_custom_op_symbolic, +) +from .errors import OnnxExporterError + + +if TYPE_CHECKING: + import os + from collections.abc import Callable, Collection, Mapping, Sequence + +# Set namespace for exposed private names +ONNXProgram.__module__ = "torch.onnx" +OnnxExporterError.__module__ = "torch.onnx" + +# TODO(justinchuby): Remove these two properties +producer_name = "pytorch" +producer_version = _C_onnx.PRODUCER_VERSION + + +def export( + model: torch.nn.Module + | torch.export.ExportedProgram + | torch.jit.ScriptModule + | torch.jit.ScriptFunction, + args: tuple[Any, ...] = (), + f: str | os.PathLike | None = None, + *, + kwargs: dict[str, Any] | None = None, + verbose: bool | None = None, + input_names: Sequence[str] | None = None, + output_names: Sequence[str] | None = None, + opset_version: int | None = None, + dynamo: bool = True, + # Dynamo only options + external_data: bool = True, + dynamic_shapes: dict[str, Any] | tuple[Any, ...] | list[Any] | None = None, + custom_translation_table: dict[Callable, Callable | Sequence[Callable]] + | None = None, + report: bool = False, + optimize: bool = True, + verify: bool = False, + profile: bool = False, + dump_exported_program: bool = False, + artifacts_dir: str | os.PathLike = ".", + fallback: bool = False, + # BC options + export_params: bool = True, + keep_initializers_as_inputs: bool = False, + dynamic_axes: Mapping[str, Mapping[int, str]] + | Mapping[str, Sequence[int]] + | None = None, + # Deprecated options + training: _C_onnx.TrainingMode = _C_onnx.TrainingMode.EVAL, + operator_export_type: _C_onnx.OperatorExportTypes = _C_onnx.OperatorExportTypes.ONNX, + do_constant_folding: bool = True, + custom_opsets: Mapping[str, int] | None = None, + export_modules_as_functions: bool | Collection[type[torch.nn.Module]] = False, + autograd_inlining: bool = True, +) -> ONNXProgram | None: + r"""Exports a model into ONNX format. + + Setting ``dynamo=True`` enables the new ONNX export logic + which is based on :class:`torch.export.ExportedProgram` and a more modern + set of translation logic. This is the recommended and default way to export models + to ONNX. + + When ``dynamo=True``: + + The exporter tries the following strategies to get an ExportedProgram for conversion to ONNX. + + #. If the model is already an ExportedProgram, it will be used as-is. + #. Use :func:`torch.export.export` and set ``strict=False``. + #. Use :func:`torch.export.export` and set ``strict=True``. + + Args: + model: The model to be exported. + args: Example positional inputs. Any non-Tensor arguments will be hard-coded into the + exported model; any Tensor arguments will become inputs of the exported model, + in the order they occur in the tuple. + f: Path to the output ONNX model file. E.g. "model.onnx". This argument is kept for + backward compatibility. It is recommended to leave unspecified (None) + and use the returned :class:`torch.onnx.ONNXProgram` to serialize the model + to a file instead. + kwargs: Optional example keyword inputs. + verbose: Whether to enable verbose logging. + input_names: names to assign to the input nodes of the graph, in order. + output_names: names to assign to the output nodes of the graph, in order. + opset_version: The version of the + `default (ai.onnx) opset `_ + to target. You should set ``opset_version`` according to the supported opset versions + of the runtime backend or compiler you want to run the exported model with. + Leave as default (``None``) to use the recommended version, or refer to + the ONNX operators documentation for more information. + dynamo: Whether to export the model with ``torch.export`` ExportedProgram instead of TorchScript. + external_data: Whether to save the model weights as an external data file. + This is required for models with large weights that exceed the ONNX file size limit (2GB). + When False, the weights are saved in the ONNX file with the model architecture. + dynamic_shapes: A dictionary or a tuple of dynamic shapes for the model inputs. Refer to + :func:`torch.export.export` for more details. This is only used (and preferred) when dynamo is True. + Note that dynamic_shapes is designed to be used when the model is exported with dynamo=True, while + dynamic_axes is used when dynamo=False. + custom_translation_table: A dictionary of custom decompositions for operators in the model. + The dictionary should have the callable target in the fx Node as the key (e.g. ``torch.ops.aten.stft.default``), + and the value should be a function that builds that graph using ONNX Script. This option + is only valid when dynamo is True. + report: Whether to generate a markdown report for the export process. This option + is only valid when dynamo is True. + optimize: Whether to optimize the exported model. This option + is only valid when dynamo is True. Default is True. + verify: Whether to verify the exported model using ONNX Runtime. This option + is only valid when dynamo is True. + profile: Whether to profile the export process. This option + is only valid when dynamo is True. + dump_exported_program: Whether to dump the :class:`torch.export.ExportedProgram` to a file. + This is useful for debugging the exporter. This option is only valid when dynamo is True. + artifacts_dir: The directory to save the debugging artifacts like the report and the serialized + exported program. This option is only valid when dynamo is True. + fallback: Whether to fallback to the TorchScript exporter if the dynamo exporter fails. + This option is only valid when dynamo is True. When fallback is enabled, It is + recommended to set dynamic_axes even when dynamic_shapes is provided. + export_params: **When ``f`` is specified**: If false, parameters (weights) will not be exported. + + You can also leave it unspecified and use the returned :class:`torch.onnx.ONNXProgram` + to control how initializers are treated when serializing the model. + keep_initializers_as_inputs: **When ``f`` is specified**: If True, all the + initializers (typically corresponding to model weights) in the + exported graph will also be added as inputs to the graph. If False, + then initializers are not added as inputs to the graph, and only + the user inputs are added as inputs. + + Set this to True if you intend to supply model weights at runtime. + Set it to False if the weights are static to allow for better optimizations + (e.g. constant folding) by backends/runtimes. + + You can also leave it unspecified and use the returned :class:`torch.onnx.ONNXProgram` + to control how initializers are treated when serializing the model. + dynamic_axes: + Prefer specifying ``dynamic_shapes`` when ``dynamo=True`` and when ``fallback`` + is not enabled. + + By default the exported model will have the shapes of all input and output tensors + set to exactly match those given in ``args``. To specify axes of tensors as + dynamic (i.e. known only at run-time), set ``dynamic_axes`` to a dict with schema: + + * KEY (str): an input or output name. Each name must also be provided in ``input_names`` or + ``output_names``. + * VALUE (dict or list): If a dict, keys are axis indices and values are axis names. If a + list, each element is an axis index. + + For example:: + + class SumModule(torch.nn.Module): + def forward(self, x): + return torch.sum(x, dim=1) + + + torch.onnx.export( + SumModule(), + (torch.ones(2, 2),), + "onnx.pb", + input_names=["x"], + output_names=["sum"], + ) + + Produces:: + + input { + name: "x" + ... + shape { + dim { + dim_value: 2 # axis 0 + } + dim { + dim_value: 2 # axis 1 + ... + output { + name: "sum" + ... + shape { + dim { + dim_value: 2 # axis 0 + ... + + While:: + + torch.onnx.export( + SumModule(), + (torch.ones(2, 2),), + "onnx.pb", + input_names=["x"], + output_names=["sum"], + dynamic_axes={ + # dict value: manually named axes + "x": {0: "my_custom_axis_name"}, + # list value: automatic names + "sum": [0], + }, + ) + + Produces:: + + input { + name: "x" + ... + shape { + dim { + dim_param: "my_custom_axis_name" # axis 0 + } + dim { + dim_value: 2 # axis 1 + ... + output { + name: "sum" + ... + shape { + dim { + dim_param: "sum_dynamic_axes_1" # axis 0 + ... + + training: Deprecated option. Instead, set the training mode of the model before exporting. + operator_export_type: Deprecated option. Only ONNX is supported. + do_constant_folding: Deprecated option. + custom_opsets: Deprecated option. + export_modules_as_functions: Deprecated option. + autograd_inlining: Deprecated option. + + Returns: + :class:`torch.onnx.ONNXProgram` if dynamo is True, otherwise None. + + .. versionchanged:: 2.6 + *training* is now deprecated. Instead, set the training mode of the model before exporting. + *operator_export_type* is now deprecated. Only ONNX is supported. + *do_constant_folding* is now deprecated. It is always enabled. + *export_modules_as_functions* is now deprecated. + *autograd_inlining* is now deprecated. + .. versionchanged:: 2.7 + *optimize* is now True by default. + .. versionchanged:: 2.9 + *dynamo* is now True by default. + """ + if dynamo is True or isinstance(model, torch.export.ExportedProgram): + from torch.onnx._internal.exporter import _compat + + if isinstance(args, torch.Tensor): + args = (args,) + # Prepare legacy export parameters for potential fallback + legacy_export_kwargs = { + "training": training, + "operator_export_type": operator_export_type, + "do_constant_folding": do_constant_folding, + "custom_opsets": custom_opsets, + "export_modules_as_functions": export_modules_as_functions, + "autograd_inlining": autograd_inlining, + } + + return _compat.export_compat( + model, + args, + f, + kwargs=kwargs, + export_params=export_params, + verbose=verbose, + input_names=input_names, + output_names=output_names, + opset_version=opset_version, + custom_translation_table=custom_translation_table, + dynamic_axes=dynamic_axes, + keep_initializers_as_inputs=keep_initializers_as_inputs, + external_data=external_data, + dynamic_shapes=dynamic_shapes, + report=report, + optimize=optimize, + verify=verify, + profile=profile, + dump_exported_program=dump_exported_program, + artifacts_dir=artifacts_dir, + fallback=fallback, + legacy_export_kwargs=legacy_export_kwargs, + ) + else: + import warnings + + from ._internal.torchscript_exporter.utils import export + + warnings.warn( + "You are using the legacy TorchScript-based ONNX export. Starting in PyTorch 2.9, " + "the new torch.export-based ONNX exporter has become the default. " + "Learn more about the new export logic: https://docs.pytorch.org/docs/stable/onnx_export.html. " + "For exporting control flow: " + "https://pytorch.org/tutorials/beginner/onnx/export_control_flow_model_to_onnx_tutorial.html", + category=DeprecationWarning, + stacklevel=2, + ) + + if dynamic_shapes: + raise ValueError( + "The exporter only supports dynamic shapes " + "through parameter dynamic_axes when dynamo=False." + ) + + export( + model, + args, + f, # type: ignore[arg-type] + kwargs=kwargs, + export_params=export_params, + verbose=verbose is True, + input_names=input_names, + output_names=output_names, + opset_version=opset_version, + dynamic_axes=dynamic_axes, + keep_initializers_as_inputs=keep_initializers_as_inputs, + training=training, + operator_export_type=operator_export_type, + do_constant_folding=do_constant_folding, + custom_opsets=custom_opsets, + export_modules_as_functions=export_modules_as_functions, + autograd_inlining=autograd_inlining, + ) + return None + + +def is_in_onnx_export() -> bool: + """Returns whether it is in the middle of ONNX export.""" + from torch.onnx._internal.exporter import _flags + from torch.onnx._internal.torchscript_exporter._globals import GLOBALS + + return GLOBALS.in_onnx_export or _flags._is_onnx_exporting diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_constants.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..36cbd2f491f268e5b13eb1411910e81e92f3004f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_constants.py @@ -0,0 +1,24 @@ +"""Constant values used in ONNX.""" + +ONNX_ARCHIVE_MODEL_PROTO_NAME = "__MODEL_PROTO" + +ONNX_BASE_OPSET = 9 +ONNX_MIN_OPSET = 7 +ONNX_MAX_OPSET = 23 +ONNX_TORCHSCRIPT_EXPORTER_MAX_OPSET = 20 +ONNX_DEFAULT_OPSET = 20 +ONNX_CONSTANT_FOLDING_MIN_OPSET = 9 + +PYTORCH_GITHUB_ISSUES_URL = "https://github.com/pytorch/pytorch/issues" + +INT64_MAX = 9223372036854775807 +INT32_MAX = 2147483647 +INT16_MAX = 32767 +INT8_MAX = 127 +UINT8_MAX = 255 + +INT64_MIN = -9223372036854775808 +INT32_MIN = -2147483648 +INT16_MIN = -32768 +INT8_MIN = -128 +UINT8_MIN = 0 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_flags.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..6b554ad725539533c985319fe7e68613e3c81312 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/_flags.py @@ -0,0 +1,55 @@ +"""Internal feature flags for torch.onnx. + +NOTE: These flags are experimental only. Any flag here can be removed at any +time without notice. +""" + +import logging +import os + + +logger = logging.getLogger(__name__) + + +def _load_boolean_flag( + name: str, + *, + this_will: str, + deprecated: bool = False, + default: bool = False, +) -> bool: + """Load a boolean flag from environment variable. + + Args: + name: The name of the environment variable. + this_will: A string that describes what this flag will do. + deprecated: Whether this flag is deprecated. + default: The default value if envvar not defined. + """ + undefined = os.getenv(name) is None + state = os.getenv(name) == "1" + if state: + if deprecated: + logger.error( + "Experimental flag %s is deprecated. Please remove it from your environment.", + name, + ) + else: + logger.warning( + "Experimental flag %s is enabled. This will %s.", name, this_will + ) + if undefined: + state = default + return state + + +ENABLE_DRAFT_EXPORT: bool = _load_boolean_flag( + "TORCH_ONNX_ENABLE_DRAFT_EXPORT", + this_will="enable torch.export.draft_export as a strategy for capturing models", + default=False, +) +PREFER_DEFERRED_RUNTIME_ASSERTS_OVER_GUARDS: bool = _load_boolean_flag( + "TORCH_ONNX_PREFER_DEFERRED_RUNTIME_ASSERTS_OVER_GUARDS", + this_will="set prefer_deferred_runtime_asserts_over_guards when calling torch.export", + default=True, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/errors.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..978c9d7aacb30739c70ee1dc2b5fe0b4b936dac5 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/errors.py @@ -0,0 +1,101 @@ +"""ONNX exporter exceptions.""" + +from __future__ import annotations + + +__all__ = [ + "OnnxExporterWarning", + "SymbolicValueError", + "UnsupportedOperatorError", +] + +import textwrap +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from torch import _C + + +class OnnxExporterWarning(UserWarning): + """Warnings in the ONNX exporter.""" + + +class OnnxExporterError(RuntimeError): + """Errors raised by the ONNX exporter. This is the base class for all exporter errors.""" + + +class UnsupportedOperatorError(OnnxExporterError): + """Raised when an operator is unsupported by the exporter.""" + + # NOTE: This is legacy and is only used by the torchscript exporter + # Clean up when the torchscript exporter is removed + def __init__(self, name: str, version: int, supported_version: int | None) -> None: + if supported_version is not None: + msg = ( + f"Exporting the operator '{name}' to ONNX opset version {version} " + "is not supported. Support for this operator was added in version " + f"{supported_version}, try exporting with this version" + ) + elif name.startswith(("aten::", "prim::", "quantized::")): + msg = ( + f"Exporting the operator '{name}' to ONNX opset version {version} " + "is not supported" + ) + else: + msg = ( + f"ONNX export failed on an operator with unrecognized namespace {name}. " + "If you are trying to export a custom operator, make sure you registered it with " + "the right domain and version." + ) + + super().__init__(msg) + + +class SymbolicValueError(OnnxExporterError): + """Errors around TorchScript values and nodes.""" + + # NOTE: This is legacy and is only used by the torchscript exporter + # Clean up when the torchscript exporter is removed + def __init__(self, msg: str, value: _C.Value) -> None: + message = ( + f"{msg} [Caused by the value '{value}' (type '{value.type()}') in the " + f"TorchScript graph. The containing node has kind '{value.node().kind()}'.] " + ) + + code_location = value.node().sourceRange() + if code_location: + message += f"\n (node defined in {code_location})" + + try: + # Add its input and output to the message. + message += "\n\n" + message += textwrap.indent( + ( + "Inputs:\n" + + ( + "\n".join( + f" #{i}: {input_} (type '{input_.type()}')" + for i, input_ in enumerate(value.node().inputs()) + ) + or " Empty" + ) + + "\n" + + "Outputs:\n" + + ( + "\n".join( + f" #{i}: {output} (type '{output.type()}')" + for i, output in enumerate(value.node().outputs()) + ) + or " Empty" + ) + ), + " ", + ) + except AttributeError: + message += ( + " Failed to obtain its input and output for debugging. " + "Please refer to the TorchScript graph for debugging information." + ) + + super().__init__(message) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/operators.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/operators.py new file mode 100644 index 0000000000000000000000000000000000000000..16e3f6d365fa6a427614412fe22488d7a0442623 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/operators.py @@ -0,0 +1,47 @@ +"""This file provides a location for operators that help exporting models via onnx. + +E.g. `shape_as_tensor` and `reshape_from_tensor_shape` +are to make all dynamic sizes operations traceable. + +NOTE: at one point these functions were implemented differently. +Since then we have implemented these directly in ATen, so this +file is kept purely for backward-compatibility. +""" + +from __future__ import annotations + + +__all__: list[str] = [] + +import torch + + +"""Get the shape of a tensor as a tensor. + +Args: + x (Tensor): The input tensor. + +Returns: + Tensor: A tensor of shape [len(x.shape)] containing the size of each dimension of x. + +Example: + >>> x = torch.randn(2, 3) + >>> shape_as_tensor(x) + tensor([2, 3]) + +""" +shape_as_tensor = torch._shape_as_tensor + +"""Reshape a tensor to the given shape. + +This function is used to make dynamic size operations traceable when exporting models via ONNX. +This function is kept for backward-compatibility. It is implemented directly in ATen. + +Parameters: + x (Tensor): the tensor to be reshaped. + shape (Tensor): the target shape. + +Returns: + Tensor: the reshaped tensor. +""" +reshape_from_tensor_shape = torch._reshape_from_tensor diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_helper.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..0f273d897e3776a2ef5795e9d5a96bfc8499fa61 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_helper.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_helper.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_helper import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset10.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset10.py new file mode 100644 index 0000000000000000000000000000000000000000..5884d928146c13f7e53187f63bb46876715b38c4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset10.py @@ -0,0 +1,11 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset10.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset10 import * # noqa: F401,F403 +from torch.onnx._internal.torchscript_exporter.symbolic_opset10 import ( # noqa: F401 + _slice, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset11.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset11.py new file mode 100644 index 0000000000000000000000000000000000000000..1bba5e4b7b5667d1200e03ef39f6db29fb77905e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset11.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset11.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset11 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset12.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset12.py new file mode 100644 index 0000000000000000000000000000000000000000..63a6bea25d61c3a990b23755453c8a31ae16bb0a --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset12.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset12.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset12 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset13.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset13.py new file mode 100644 index 0000000000000000000000000000000000000000..b4f9174c6fa61c38b4c1bfe4dffd255744a3e7fd --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset13.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset13.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset13 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset14.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset14.py new file mode 100644 index 0000000000000000000000000000000000000000..419fa7568fa5a8e308f808a5dc17e0c8557a148b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset14.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset14.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset14 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset15.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset15.py new file mode 100644 index 0000000000000000000000000000000000000000..f550b45ad61fed5bb5e67f1772039aa1d7f62820 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset15.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset15.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset15 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset16.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset16.py new file mode 100644 index 0000000000000000000000000000000000000000..43a478b39429a581c3ae5b100695ce47f8e4d6d3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset16.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset16.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset16 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset17.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset17.py new file mode 100644 index 0000000000000000000000000000000000000000..818fb6d58678840265f58c16ecdd787b01637ac9 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset17.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset17.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset17 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset18.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset18.py new file mode 100644 index 0000000000000000000000000000000000000000..61422d8430f8703e2d69bc924ac5f7aa1e1521fc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset18.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset18.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset18 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset19.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset19.py new file mode 100644 index 0000000000000000000000000000000000000000..5b7d3d8375b23067059943276a61ae17280d9665 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset19.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset19.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset19 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset20.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset20.py new file mode 100644 index 0000000000000000000000000000000000000000..cff751d89e8e0af9b4a1d44d4ec55a4c79f437d0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset20.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset20.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset20 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset7.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset7.py new file mode 100644 index 0000000000000000000000000000000000000000..22f08faf01c8dd0ee6d6789b2effb5458b580bba --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset7.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset7.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset7 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset8.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset8.py new file mode 100644 index 0000000000000000000000000000000000000000..7bbc004f53ccdb9bcba4734ad313f91e5a5e022e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset8.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset8.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset8 import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset9.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset9.py new file mode 100644 index 0000000000000000000000000000000000000000..8ea8c16be4f9ceb588c36f83c142dad32ed1a7c1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/symbolic_opset9.py @@ -0,0 +1,14 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset9.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset9 import * # noqa: F401,F403 +from torch.onnx._internal.torchscript_exporter.symbolic_opset9 import ( # noqa: F401 + _prepare_onnx_paddings, + _reshape_from_tensor, + _slice, + _var_mean, +) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/testing.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..cbf37c0756bf935ff440e6de8669d6195f5ed79b --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/testing.py @@ -0,0 +1,8 @@ +"""Utilities to aid in testing exported ONNX models.""" + +__all__ = ["assert_onnx_program"] + +from torch.onnx._internal.exporter._testing import assert_onnx_program + + +assert_onnx_program.__module__ = "torch.onnx.testing" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..658d5563c0a3a76e8f60033d578eec4b7f4dd327 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/utils.py @@ -0,0 +1,9 @@ +"""Backward compatibility module for torch.onnx.utils.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +# pyrefly: ignore [deprecated] +from torch.onnx._internal.torchscript_exporter.utils import * # noqa: F401,F403 diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/verification.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/verification.py new file mode 100644 index 0000000000000000000000000000000000000000..b01f2c39a4cdd47a1a7c94675d8842829ece9f62 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/onnx/verification.py @@ -0,0 +1,12 @@ +"""A set of tools to verify the correctness of ONNX models.""" + +__all__ = ["VerificationInfo", "verify_onnx_program"] + +from torch.onnx._internal.exporter._verification import ( + VerificationInfo, + verify_onnx_program, +) + + +VerificationInfo.__module__ = "torch.onnx.verification" +verify_onnx_program.__module__ = "torch.onnx.verification" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/__init__.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..59a8cac52e1a26cdbd83556791452e06b75bcb3c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/__init__.py @@ -0,0 +1,66 @@ +""" +:mod:`torch.optim` is a package implementing various optimization algorithms. + +Most commonly used methods are already supported, and the interface is general +enough, so that more sophisticated ones can also be easily integrated in the +future. +""" + +from torch.optim import lr_scheduler as lr_scheduler, swa_utils as swa_utils +from torch.optim._adafactor import Adafactor as Adafactor +from torch.optim._muon import Muon as Muon +from torch.optim.adadelta import Adadelta as Adadelta +from torch.optim.adagrad import Adagrad as Adagrad +from torch.optim.adam import Adam as Adam +from torch.optim.adamax import Adamax as Adamax +from torch.optim.adamw import AdamW as AdamW +from torch.optim.asgd import ASGD as ASGD +from torch.optim.lbfgs import LBFGS as LBFGS +from torch.optim.nadam import NAdam as NAdam +from torch.optim.optimizer import Optimizer as Optimizer +from torch.optim.radam import RAdam as RAdam +from torch.optim.rmsprop import RMSprop as RMSprop +from torch.optim.rprop import Rprop as Rprop +from torch.optim.sgd import SGD as SGD +from torch.optim.sparse_adam import SparseAdam as SparseAdam + + +Adafactor.__module__ = "torch.optim" +Muon.__module__ = "torch.optim" + + +del adadelta # type: ignore[name-defined] # noqa: F821 +del adagrad # type: ignore[name-defined] # noqa: F821 +del adam # type: ignore[name-defined] # noqa: F821 +del adamw # type: ignore[name-defined] # noqa: F821 +del sparse_adam # type: ignore[name-defined] # noqa: F821 +del adamax # type: ignore[name-defined] # noqa: F821 +del asgd # type: ignore[name-defined] # noqa: F821 +del sgd # type: ignore[name-defined] # noqa: F821 +del radam # type: ignore[name-defined] # noqa: F821 +del rprop # type: ignore[name-defined] # noqa: F821 +del rmsprop # type: ignore[name-defined] # noqa: F821 +del optimizer # type: ignore[name-defined] # noqa: F821 +del nadam # type: ignore[name-defined] # noqa: F821 +del lbfgs # type: ignore[name-defined] # noqa: F821 + +__all__ = [ + "Adafactor", + "Adadelta", + "Adagrad", + "Adam", + "Adamax", + "AdamW", + "ASGD", + "LBFGS", + "lr_scheduler", + "Muon", + "NAdam", + "Optimizer", + "RAdam", + "RMSprop", + "Rprop", + "SGD", + "SparseAdam", + "swa_utils", +] diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_adafactor.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_adafactor.py new file mode 100644 index 0000000000000000000000000000000000000000..c0b098a998c54fa0b33d8f2b6ed0aef4f488b2f3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_adafactor.py @@ -0,0 +1,661 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +from typing import cast, TYPE_CHECKING + +import torch +from torch import Tensor + +from .optimizer import ( + _disable_dynamo_if_unsupported, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + Optimizer, + ParamsT, + TensorListList, +) + + +__all__ = ["Adafactor", "adafactor"] + + +class Adafactor(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + beta2_decay: float = -0.8, + eps: tuple[float | None, float] = (None, 1e-3), + d: float = 1.0, + weight_decay: float = 0.0, + *, + foreach: bool | None = None, + maximize: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Learning rate should be >= 0 but is: {lr}") + if not 0.0 >= beta2_decay: + raise ValueError(f"beta2_decay should be <= 0 but is: {beta2_decay}") + if eps[0] is not None and not 0.0 <= eps[0]: + raise ValueError(f"epsilon1 should be >= 0 but is: {eps[0]}") + if not 0.0 <= eps[1]: + raise ValueError(f"epsilon2 should be >= 0 but is: {eps[1]}") + if not 1.0 <= d: + raise ValueError(f"Clipping threshold d should be >= 1 but is: {d}") + if not 0.0 <= weight_decay: + raise ValueError(f"weight_decay should be >= 0 but is: {weight_decay}") + defaults = { + "lr": lr, + "beta2_decay": beta2_decay, + "eps": eps, + "d": d, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = torch.tensor(step_val, dtype=_get_scalar_dtype()) + + def _init_group( + self, + group, + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + ) -> bool: + for p in group["params"]: + if p.grad is None: + continue + if torch.is_complex(p): + raise RuntimeError("Adafactor does not support complex parameters") + if p.grad.is_sparse: + raise RuntimeError("Adafactor does not support sparse gradients") + + params_with_grad.append(p) + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + # note(crcrpar): Deliberately host `step` on CPU if both capturable and fused are off. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = torch.tensor(0.0, dtype=_get_scalar_dtype()) + + if p.grad.dim() > 1: + row_shape = list(p.grad.shape) + row_shape[-1] = 1 + # Row factor of variance, NOT the same shape as grads (will be reduced along last dim) + state["row_var"] = p.grad.new_zeros(row_shape) + + col_shape = list(p.grad.shape) + col_shape[-2] = 1 + # Col factor of variance, NOT the same shape as grads (will be reduced along penultimate dim) + state["col_var"] = p.grad.new_zeros(col_shape) + else: + state["variance"] = torch.zeros_like( + p.grad, memory_format=torch.preserve_format + ) + + row_vars.append(state.get("row_var", None)) + col_vars.append(state.get("col_var", None)) + variances.append(state.get("variance", None)) + state_steps.append(state["step"]) + return False # has_complex + + @torch.no_grad() + def step(self, closure=None): + r"""Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + row_vars: list[Tensor | None] = [] + col_vars: list[Tensor | None] = [] + variances: list[Tensor | None] = [] + state_steps: list[Tensor] = [] + eps1, eps2 = group["eps"] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + ) + + adafactor( + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + d=group["d"], + lr=group["lr"], + beta2_decay=group["beta2_decay"], + weight_decay=group["weight_decay"], + eps1=eps1, + eps2=eps2, + foreach=group["foreach"], + maximize=group["maximize"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + has_complex=has_complex, + ) + + return loss + + +Adafactor.__doc__ = ( + r"""Implements Adafactor algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{(lr)}, \: \tau + \text{(}\beta_2\text{ decay)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, \\ + &\hspace{15mm} \: \epsilon_1, \epsilon_2 \text{ (epsilons)}, \: d \text{(clipping threshold)}, \\ + &\hspace{15mm} \: \lambda \text{(weight decay)}, + \: \textit{maximize} \\ + &\textbf{initialize} : \: R_0 \leftarrow 0 \text{ (second moment row factor)}, \\ + &\hspace{23mm} \: C_0 \leftarrow 0 \text{ (second moment col factor)}, \\ + &\hspace{23mm} \: \widehat{V}_0 \leftarrow 0 \text{ (second moment for vectors)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}G_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}G_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\widehat{\beta}_{2_t} \leftarrow 1 - t^{\tau} \\ + &\hspace{5mm}\rho_t \leftarrow min(lr, \frac{1}{\sqrt{t}}) \\ + &\hspace{5mm}\alpha_t \leftarrow max(\epsilon_2, + \text{RMS}(\theta_{t-1}))\rho_t \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{5mm}\textbf{if} \: \text{dim}(G_t) > 1: \\ + &\hspace{10mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t) \cdot 1_m \\ + &\hspace{10mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t) \\ + &\hspace{10mm}\widehat{V}_t \leftarrow + \frac{R_t \cdot C_t}{max(1^\top_n \cdot R_t, \epsilon_1)} \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{V}_t \leftarrow \widehat{\beta}_{2_t}\widehat{V}_{t-1}+ + (1-\widehat{\beta}_{2_t}) \cdot (G_t \odot G_t) \\ + &\hspace{5mm}U_t \leftarrow + \frac{G_t}{max(\sqrt{\widehat{V}_t}, \epsilon_1)} \\ + &\hspace{5mm}\widehat{U}_t \leftarrow \frac{U_t}{max(1, \frac{\text{RMS}(U_t)}{d})} \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \alpha_t \widehat{U}_t \\ + + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): unlike other optimizers, Adafactor does not require a + learning rate, and Noam Shazeer and Mitchell Stern do not use lr at all. + Deviating from the paper, this implementation uses lr for applying weight + decay and as the maximum value for relative step size rho_t. Note that in + the paper, a constant of 0.01 is used as the maximum value for relative + step size, and so we set 0.01 as the default value. (default: 1e-2) + beta2_decay (float, optional): the decay rate of beta2. beta2 standardly refers + to the coefficient used for computing the running average of the gradient + squared. (default: -0.8) + eps (Tuple[float, float], optional): epsilon1 is the term added to the denominator + of the update calculation to improve numerical stability. This use of epsilon1 + deviates from the algorithm written in the paper! See note below for more details. + epsilon2 is the term used to avoid having too small a weight update when applying + parameter scaling. (default: (None, 1e-3)) + d (float, optional): the clipping threshold, used to avoid larger-than-desired + updates. + weight_decay (float, optional): weight decay coefficient (default: 1e-2) + foreach (bool, optional): whether foreach implementation of optimizer is used. Note + that the foreach implementation uses ~ sizeof(params) more peak memory than the + for-loop version due to the intermediates being a tensorlist vs just one tensor. + As Adafactor is commonly used when memory is prohibitive, Adafactor will default + to the slower single tensor for-loop implementation unless this flag is explicitly + True. This behavior is contrary to other optimizers, which will attempt defaulting + to foreach on CUDA for faster runtime. (default: None) + {_maximize_doc}""" + + r""" + .. Note:: + The implementation of Adafactor subtly differs from Noam Shazeer and Mitchell Stern + and implementations in some other frameworks with its use of learning rate and + :math:`\epsilon_1`. + + Regarding the learning rate hyperparameter: Noam Shazeer and Mitchell Stern do not + use lr at all, as the stated algorithm uses :math:`\rho_t` and update clipping to + affect the step size. + + This implementation allows `lr` to influence the maximum value for :math:`\rho_t`: + + .. math:: + \begin{aligned} + &\hspace{5mm}\rho_t \leftarrow min(lr, \frac{1}{\sqrt{t}}) + \end{aligned} + + This differs from Noam Shazeer and Mitchell Stern, who use a constant of 0.01 as + the maximum value of :math:`\rho_t` + + .. math:: + \begin{aligned} + &\hspace{5mm}\rho_t \leftarrow min(0.01, \frac{1}{\sqrt{t}}) + \end{aligned} + + Noam Shazeer and Mitchell Stern do not enforce an opinion on how weight decay should + be computed, and so we use the learning rate as a coefficient for decoupled weight + decay, similar to what is suggested in `Decoupled Weight Decay Regularization`_. + + Regarding the use of :math:`\epsilon_1`: The implementation attempts to replicate the + presumed intention of Noam Shazeer and Mitchell Stern to use :math:`\epsilon_1` as + a stabilizing term when the squared gradient becomes small. + + This stabilization can be written as + + .. math:: + \begin{aligned} + &\hspace{5mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t + 1_n \cdot 1^\top_m) \cdot 1_m \\ + &\hspace{5mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t + 1_n \cdot 1^\top_m) \\ + &\hspace{5mm}\widehat{V}_t \leftarrow + \frac{R_t \cdot C_t}{max(1^\top_n \cdot R_t, \epsilon_1)} \\ + &\hspace{5mm}U_t \leftarrow \frac{G_t}{max(\sqrt{\widehat{V}_t}, \epsilon_1)} \\ + \end{aligned} + + where the row and column factors of gradient squared :math:`R_t` and :math:`C_t` + are left alone, and we apply :math:`\epsilon_1` at the final calculation of + the variance estimate :math:`\widehat{V}_t` and for the update :math:`U_t`. + + This is in contrast to Noam Shazeer and Mitchell Stern and other frameworks which + apply :math:`\epsilon_1` to both row and column factors of the squared gradient, but + not in the calculations after: + + .. math:: + \begin{aligned} + &\hspace{5mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t + \epsilon_1 1_n \cdot 1^\top_m) \cdot 1_m \\ + &\hspace{5mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t + \epsilon_1 1_n \cdot 1^\top_m) \\ + &\hspace{5mm}\widehat{V}_t \leftarrow \frac{R_t \cdot C_t}{1^\top_n \cdot R_t} \\ + &\hspace{5mm}U_t \leftarrow \frac{G_t}{\sqrt{\widehat{V}_t}} \\ + \end{aligned} + + You may note that Noam Shazeer and Mitchell Stern describe using the sum of squared gradients, + while this implementation uses the mean instead. This choice is mathematically equivalent and + allows for greater numerical stability for large sums. + + .. _Adafactor\: Adaptive Learning Rates with Sublinear Memory Cost: + https://arxiv.org/pdf/1804.04235 + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + """ +) + + +def _single_tensor_adafactor( + params: list[Tensor], + grads: list[Tensor], + # If grad is 1-dimensional (aka a vector), there is no factorization necessary + # so row_var and col_var will be None while variance will be filled. + # Contrarily, for a grad with multiple dimensions, we will factor along the last + # 2 dimensions, and so row_var and col_var will be filled and variance will be None. + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + d: float, + lr: Tensor | float, + beta2_decay: float, + weight_decay: float, + eps1: float | None, + eps2: float, + maximize: bool, + has_complex: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Grad scaling should occur outside of optimizer.step()") + + if torch.jit.is_scripting(): + # this assert is due to JIT being dumb and not realizing that the ops below + # have overloads to handle both float and Tensor lrs, so we just assert it's + # a float since most people using JIT are using floats + if not isinstance(lr, float): + raise AssertionError(f"Expected lr to be a float, but got {type(lr)}") + + else: + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + step_t = state_steps[i] + row_var = row_vars[i] + col_var = col_vars[i] + variance = variances[i] + if eps1 is None: + eps1 = torch.finfo(param.dtype).eps + + # update step + step_t += 1 + step_float = step_t.item() + + one_minus_beta2_t = step_float**beta2_decay + rho_t = min(lr, 1 / (step_float**0.5)) + alpha = max(eps2, param.norm(2).item() / (param.numel() ** 0.5)) * rho_t + + # Perform stepweight decay + if weight_decay != 0: + param.mul_(1 - lr * weight_decay) + + if grad.dim() > 1: + if row_var is None or col_var is None: + raise AssertionError( + "row_var and col_var should be defined when grad is multidimensional" + ) + # same as (g * g).mean(dim=-1) w/o materializing an intermediate size g + row_mean = ( + torch.norm(grad, dim=-1, keepdim=True).square_().div_(grad.size(-1)) + ) + row_var.lerp_(row_mean, one_minus_beta2_t) + # same as (g * g).mean(dim=-2) w/o materializing an intermediate size g + col_mean = ( + torch.norm(grad, dim=-2, keepdim=True).square_().div_(grad.size(-2)) + ) + col_var.lerp_(col_mean, one_minus_beta2_t) + var_estimate = row_var @ col_var + var_estimate.div_(row_var.mean(dim=-2, keepdim=True).clamp_(min=eps1)) + else: + if variance is None: + raise AssertionError("variance should be defined when grad is a vector") + grad_squared = grad * grad + variance.lerp_(grad_squared, one_minus_beta2_t) + # avoid writing into variance during update + var_estimate = variance.clone() + + # square the eps1 as we sqrt after to keep eps1's magnitude + update = var_estimate.clamp_(min=eps1 * eps1).rsqrt_() + update.mul_(grad) + denom = max(1.0, update.norm(2).item() / ((update.numel() ** 0.5) * d)) + param.add_(update, alpha=-alpha / denom) + + +def _group_tensors_by_device_dtype_and_is_multidim( + tensorlists: TensorListList, +) -> dict[ + tuple[torch.device | None, torch.dtype | None, bool], + list[list[Tensor | None]], +]: + """Groups tensors by device, dtype, AND multidimensionality -- whether the tensor + has multiple dims or just one dim (is a vector). This allows the foreach impl of + Adafactor to assume that every group of params will either be factored or not.""" + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype(tensorlists) + ultra_grouped_tensors: dict[ + tuple[torch.device | None, torch.dtype | None, bool], + list[list[Tensor | None]], + ] = {} + for (device, dtype), (tensorlists, _) in grouped_tensors.items(): + matrix_key = (device, dtype, True) + vector_key = (device, dtype, False) + + # assumes grad is the second tensorlist + for j, tensor in enumerate(tensorlists[1]): + if tensor is None: + raise AssertionError("grad should not be None") + if tensor.dim() > 1: + if matrix_key not in ultra_grouped_tensors: + ultra_grouped_tensors[matrix_key] = [[] for _ in tensorlists] + for i in range(len(tensorlists)): + ultra_grouped_tensors[matrix_key][i].append(tensorlists[i][j]) + else: + if vector_key not in ultra_grouped_tensors: + ultra_grouped_tensors[vector_key] = [[] for _ in tensorlists] + for i in range(len(tensorlists)): + ultra_grouped_tensors[vector_key][i].append(tensorlists[i][j]) + return ultra_grouped_tensors + + +def _multi_tensor_adafactor( + params: list[Tensor], + grads: list[Tensor], + # If grad is 1-dimensional (aka a vector), there is no factorization necessary + # so row_var and col_var will be None while variance will be filled. + # Contrarily, for a grad with multiple dimensions, we will factor along the last + # 2 dimensions, and so row_var and col_var will be filled and variance will be None. + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + d: float, + lr: Tensor | float, + beta2_decay: float, + weight_decay: float, + eps1: float | None, + eps2: float, + maximize: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if grad_scale is not None or found_inf is not None: + raise AssertionError("Grad scaling should occur outside of optimizer.step()") + + lr = _to_scalar(lr) + + grouped_tensors = _group_tensors_by_device_dtype_and_is_multidim( + [params, grads, row_vars, col_vars, variances, state_steps] # type: ignore[list-item] + ) + for (_, dtype, is_multidim), ( + ( + device_params_, + device_grads_, + device_row_vars_, + device_col_vars_, + device_variances_, + device_state_steps_, + ) + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_steps = cast(list[Tensor], device_state_steps_) + if eps1 is None: + if dtype is None: + raise AssertionError( + "dtype is needed to compute eps1 when eps1 is unset" + ) + eps1 = torch.finfo(dtype).eps + + if TYPE_CHECKING: + assert device_state_steps[0] is not None + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1.0) + + one_minus_beta2_ts = [] + beta2_ts = [] + rho_ts = [] + for s in device_state_steps: + one_minus_beta2_ts.append(s.item() ** beta2_decay) + beta2_ts.append(1 - s.item() ** beta2_decay) + rho_ts.append(min(lr, 1 / (s.item() ** 0.5))) + + alphas = [ + max(eps2, p.norm(2).item() / (p.numel() ** 0.5)) * r + for p, r in zip(device_params, rho_ts, strict=True) + ] + + # Perform stepweight decay + if weight_decay != 0: + torch._foreach_mul_(device_params, 1 - lr * weight_decay) + + if is_multidim: + device_row_vars = cast(list[Tensor], device_row_vars_) + device_col_vars = cast(list[Tensor], device_col_vars_) + if device_row_vars[0] is None or device_col_vars[0] is None: + raise AssertionError( + "row_var and col_var should be defined when grad is multidimensional" + ) + # same as (g * g).mean(dim=-1) w/o materializing an intermediate size g + row_means = [ + torch.norm(grad, dim=-1, keepdim=True) for grad in device_grads + ] + torch._foreach_mul_(row_means, row_means) + torch._foreach_div_(row_means, [grad.size(-1) for grad in device_grads]) + torch._foreach_lerp_(device_row_vars, row_means, one_minus_beta2_ts) + del row_means + + # same as (g * g).mean(dim=-2) w/o materializing an intermediate size g + col_means = [ + torch.norm(grad, dim=-2, keepdim=True) for grad in device_grads + ] + torch._foreach_mul_(col_means, col_means) + torch._foreach_div_(col_means, [grad.size(-2) for grad in device_grads]) + torch._foreach_lerp_(device_col_vars, col_means, one_minus_beta2_ts) + del col_means + + var_estimates = [ + row_var @ col_var + for row_var, col_var in zip( + device_row_vars, device_col_vars, strict=True + ) + ] + row_var_means = [ + row_var.mean(dim=-2, keepdim=True) for row_var in device_row_vars + ] + torch._foreach_clamp_min_(row_var_means, eps1) + torch._foreach_div_(var_estimates, row_var_means) + del row_var_means + else: + device_variances = cast(list[Tensor], device_variances_) + if device_variances[0] is None: + raise AssertionError("variance should be defined when grad is a vector") + + grads_squared = torch._foreach_mul(device_grads, device_grads) + torch._foreach_lerp_(device_variances, grads_squared, one_minus_beta2_ts) + del grads_squared + + # avoid writing into variance during update + var_estimates = [v.clone() for v in device_variances] + + # square the eps1 as we sqrt after to keep eps1's magnitude + torch._foreach_clamp_min_(var_estimates, eps1 * eps1) + torch._foreach_rsqrt_(var_estimates) + torch._foreach_mul_(var_estimates, device_grads) + updates = var_estimates + + alphas = [ + -a / (max(1.0, update.norm(2).item() / ((update.numel() ** 0.5) * d))) + for a, update in zip(alphas, updates, strict=True) + ] + torch._foreach_mul_(updates, alphas) + torch._foreach_add_(device_params, updates) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adafactor) +def adafactor( + params: list[Tensor], + grads: list[Tensor], + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + *, + d: float, + lr: float | Tensor, + beta2_decay: float, + weight_decay: float, + eps1: float, + eps2: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adafactor algorithm computation. + + See :class:`~torch.optim.Adafactor` for details. + """ + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "`state_steps` argument must contain a list of singleton tensors" + ) + + if foreach: + func = _multi_tensor_adafactor + else: + func = _single_tensor_adafactor + + func( + params, + grads, + row_vars, + col_vars, + variances, + state_steps, + d=d, + lr=lr, + beta2_decay=beta2_decay, + weight_decay=weight_decay, + eps1=eps1, + eps2=eps2, + maximize=maximize, + grad_scale=grad_scale, + found_inf=found_inf, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_functional.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..b6720e94f9ce9aa6122af9c7c2907a2e16dbff35 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_functional.py @@ -0,0 +1,84 @@ +# mypy: allow-untyped-defs +r"""Functional interface.""" + +import math + +from torch import Tensor + +from .adadelta import adadelta # type: ignore[attr-defined] # noqa: F401 +from .adagrad import _make_sparse, adagrad # type: ignore[attr-defined] # noqa: F401 +from .adam import adam # type: ignore[attr-defined] # noqa: F401 +from .adamax import adamax # type: ignore[attr-defined] # noqa: F401 +from .adamw import adamw # type: ignore[attr-defined] # noqa: F401 +from .asgd import asgd # type: ignore[attr-defined] # noqa: F401 +from .nadam import nadam # type: ignore[attr-defined] # noqa: F401 +from .radam import radam # type: ignore[attr-defined] # noqa: F401 +from .rmsprop import rmsprop # type: ignore[attr-defined] # noqa: F401 +from .rprop import rprop # type: ignore[attr-defined] # noqa: F401 +from .sgd import sgd # type: ignore[attr-defined] # noqa: F401 + + +# TODO: use foreach API in optim._functional to do all the computation + + +def sparse_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[int], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + maximize: bool, +) -> None: + r"""Functional API that performs Sparse Adam algorithm computation. + + See :class:`~torch.optim.SparseAdam` for details. + """ + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + grad = grad.coalesce() # the update is non-linear so indices must be unique + grad_indices = grad._indices() + grad_values = grad._values() + if grad_values.numel() == 0: + # Skip update for empty grad + continue + size = grad.size() + + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step = state_steps[i] + + def make_sparse(values): + constructor = grad.new + if grad_indices.dim() == 0 or values.dim() == 0: + return constructor().resize_as_(grad) + return constructor(grad_indices, values, size) + + # Decay the first and second moment running average coefficient + # old <- b * old + (1 - b) * new + # <==> old += (1 - b) * (new - old) + old_exp_avg_values = exp_avg.sparse_mask(grad)._values() + exp_avg_update_values = grad_values.sub(old_exp_avg_values).mul_(1 - beta1) + exp_avg.add_(make_sparse(exp_avg_update_values)) + old_exp_avg_sq_values = exp_avg_sq.sparse_mask(grad)._values() + exp_avg_sq_update_values = ( + grad_values.pow(2).sub_(old_exp_avg_sq_values).mul_(1 - beta2) + ) + exp_avg_sq.add_(make_sparse(exp_avg_sq_update_values)) + + # Dense addition again is intended, avoiding another sparse_mask + numer = exp_avg_update_values.add_(old_exp_avg_values) + exp_avg_sq_update_values.add_(old_exp_avg_sq_values) + denom = exp_avg_sq_update_values.sqrt_().add_(eps) + del exp_avg_update_values, exp_avg_sq_update_values + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + step_size = lr * math.sqrt(bias_correction2) / bias_correction1 + + param.add_(make_sparse(-step_size * numer.div_(denom))) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_muon.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_muon.py new file mode 100644 index 0000000000000000000000000000000000000000..d216cd02415879d0ba58ce241227725b4e9fb7b1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/_muon.py @@ -0,0 +1,360 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code=arg-type +"""Implementation of the Muon optimizer.""" + +import math +from collections.abc import MutableMapping + +import torch +from torch import Tensor + +from .optimizer import ( + _disable_dynamo_if_unsupported, + _params_doc, + _to_scalar, + Optimizer, + ParamsT, +) + + +__all__ = ["Muon"] + +# Constants from Keller Jordan's Muon post: https://kellerjordan.github.io/posts/muon/ +# github permlink: https://github.com/KellerJordan/Muon/blob/f90a42b28e00b8d9d2d05865fe90d9f39abcbcbd/muon.py#L16 +EPS = 1e-7 +DEFAULT_A = 3.4445 +DEFAULT_B = -4.7750 +DEFAULT_C = 2.0315 +DEFAULT_NS_STEPS = 5 + + +def _zeropower_via_newtonschulz( + grad: Tensor, ns_coefficients: tuple[float, float, float], ns_steps: int, eps: float +) -> Tensor: + """ + Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a + quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose + of minimizing steps, it turns out to be empirically effective to keep increasing the slope at + zero even beyond the point where the iteration no longer converges all the way to one everywhere + on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T + where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model + performance at all relative to UV^T, where USV^T = G is the SVD. + + Implementation reference: https://github.com/KellerJordan/Muon/blob/master/muon.py + with suggestions by @jxbz, @leloykun, and @YouJiacheng. + """ + if ns_steps >= 100: + raise ValueError( + "Number of steps must be less than 100 for computational efficiency" + ) + if len(grad.shape) != 2: + raise ValueError("Input tensor gradient must be a 2D matrix") + if len(ns_coefficients) != 3: + raise ValueError("Coefficients must be a tuple of exactly 3 values") + a, b, c = ns_coefficients + ortho_grad = grad.bfloat16() + if grad.size(0) > grad.size(1): + ortho_grad = ortho_grad.T + # Ensure spectral norm is at most 1 + ortho_grad.div_(ortho_grad.norm().clamp(min=eps)) + # Perform the NS iterations + for _ in range(ns_steps): + gram_matrix = ortho_grad @ ortho_grad.T + gram_update = torch.addmm( + gram_matrix, gram_matrix, gram_matrix, beta=b, alpha=c + ) + ortho_grad = torch.addmm(ortho_grad, gram_update, ortho_grad, beta=a) + + if grad.size(0) > grad.size(1): + ortho_grad = ortho_grad.T + return ortho_grad + + +def _adjust_lr(lr: float, adjust_lr_fn: str | None, param_shape: torch.Size) -> float: + """Default learning rate adjustment used by Muon.""" + A, B = param_shape[:2] + + if adjust_lr_fn is None or adjust_lr_fn == "original": + # pyrefly: ignore [no-matching-overload] + adjusted_ratio = math.sqrt(max(1, A / B)) + elif adjust_lr_fn == "match_rms_adamw": + adjusted_ratio = 0.2 * math.sqrt(max(A, B)) + else: + adjusted_ratio = 1.0 + return lr * adjusted_ratio + + +class Muon(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float = 1e-3, + weight_decay: float = 0.1, + momentum: float = 0.95, + nesterov: bool = True, + ns_coefficients: tuple[float, float, float] = (DEFAULT_A, DEFAULT_B, DEFAULT_C), + eps: float = EPS, + ns_steps: int = DEFAULT_NS_STEPS, + adjust_lr_fn: str | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Learning rate should be >= 0 but is: {lr}") + if not 0.0 <= momentum: + raise ValueError(f"momentum should be >= 0 but is: {momentum}") + if not 0.0 <= weight_decay: + raise ValueError(f"weight decay should be >= 0 but is: {weight_decay}") + if adjust_lr_fn is not None and adjust_lr_fn not in [ + "original", + "match_rms_adamw", + ]: + raise ValueError( + f"Adjust learning rate function {adjust_lr_fn} is not supported" + ) + + defaults = { + "lr": lr, + "weight_decay": weight_decay, + "momentum": momentum, + "nesterov": nesterov, + "ns_coefficients": ns_coefficients, + "eps": eps, + "ns_steps": ns_steps, + "adjust_lr_fn": adjust_lr_fn, + } + super().__init__(params, defaults) + + for group in self.param_groups: + for p in group["params"]: + if p.ndim != 2: + raise ValueError( + f"Muon only supports 2D parameters whereas we found a parameter with size: {p.size()}" + ) + + def _init_group( + self, + group: MutableMapping, + params_with_grad: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + ) -> bool: + for p in group["params"]: + if p.grad is None: + continue + + if torch.is_complex(p): + raise RuntimeError("Muon does not support complex parameters") + if p.grad.is_sparse: + raise RuntimeError("Muon does not support sparse gradients") + + params_with_grad.append(p) + grads.append(p.grad) + + state = self.state[p] + + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like( + p.grad, memory_format=torch.preserve_format + ) + muon_momentum_bufs.append(state["momentum_buffer"]) + + return False # has_complex + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step.""" + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + lr = group["lr"] + weight_decay = group["weight_decay"] + momentum = group["momentum"] + + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + muon_momentum_bufs: list[Tensor] = [] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + muon_momentum_bufs, + ) + + muon( + params_with_grad, + grads, + muon_momentum_bufs, + lr=lr, + weight_decay=weight_decay, + momentum=momentum, + nesterov=group["nesterov"], + ns_coefficients=group["ns_coefficients"], + eps=group["eps"], + ns_steps=group["ns_steps"], + adjust_lr_fn=group["adjust_lr_fn"], + has_complex=has_complex, + ) + return loss + + +Muon.__doc__ = ( + r"""Implements Muon algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)},\ \lambda \text{ (weight decay)},\ + \mu \text{ (momentum)},\ \textit{nesterov}\in\{True,False\},\\ + &\hspace{13mm}(a,b,c)\ \text{ (NS coefficients)},\ + \varepsilon \text{ (epsilon)},\ k \text{ (NS steps)},\ + \theta_0 \text{ (params)},\ f(\theta) \text{ (objective)} \\ + &\textbf{initialize} : B_0 \leftarrow 0 \text{ (momentum buffer)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for}\ t=1\ \textbf{to}\ \ldots\ \textbf{do} \\[0.25ex] + &\hspace{5mm} g_t \leftarrow \nabla_{\theta} f_t(\theta_{t-1}) \\[0.25ex] + &\hspace{5mm} B_t \leftarrow \mu B_{t-1} + g_t \\[0.25ex] + &\hspace{5mm} \widetilde{B}_t \leftarrow + \begin{cases} + g_t + \mu B_t, & \text{if nesterov}=True \\ + B_t, & \text{if nesterov}=False + \end{cases} \\[1.0ex] + &\hspace{5mm} O_t \leftarrow \mathrm{NS}^{(a,b,c)}_{k}\!\big(\widetilde{B}_t;\ \varepsilon\big) \\[0.5ex] + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma\,\lambda\,\theta_{t-1} + \quad\text{(decoupled weight decay)} \\[0.25ex] + + &\hspace{5mm} \gamma \leftarrow \mathrm{AdjustLR}\!\big(\gamma;\ \mathrm{shape}\!\big(\theta_t \big) \big) \\[0.25ex] + &\hspace{5mm} \theta_t \leftarrow \theta_t - \gamma\, O_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\mathbf{return}\ \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt}s + \end{aligned} + + Here, :math:`\mathrm{NS}^{(a,b,c)}_{k}(\cdot;\varepsilon)` denotes :math:`k` iterations of the + Newton–Schulz orthogonalization operator parameterized by coefficients :math:`(a,b,c)` + with numerical stabilization :math:`\varepsilon`. + + The purpose for :math:`\mathrm{AdjustLR}\!\big(\gamma;\ \mathrm{shape}\!\big(\theta_t \big) \big)` + is to make the orthogonalized update have a consistent :math:`RMS` across rectangular matrices. + + Keller's original implementation scales the update by :math:`\sqrt{\max\!\left(1, \frac{A}{B}\right)}`, + where :math:`A` and :math:`B` are dimension of the matrix being optimized. + + Moonshot's implementation also focuses on matching :math:`RMS` of AdamW. The adjustment is computed as: + :math:`\gamma \leftarrow {0.2}\gamma\,\sqrt{\max\!\left({A}, {B}\right)}` + The method is adopted from `Muon is Scalable for LLM Training`_. Research + results show that with this adjustment Muon can directly reuse the learning rate + and weight decay tuned for AdamW. + + We provide two options for the learning rate adjustment: "original", which follows Keller's + implementation, and "match_rms_adamw", which refers to Moonshot's implementation. This gives users the + flexibility to choose between the two. If `adjust_lr_fn` is not specified, the default is "original". + + For further details regarding the algorithm we refer to `Muon: An optimizer for hidden layers in neural networks`_ + and `Muon is Scalable for LLM Training`_. + """ + + rf""" + Args: + {_params_doc}. Note that Muon is an optimizer for 2D parameters of neural network hidden layers. Other + parameters, such as bias, and embedding, should be optimized by a standard method such as AdamW. + lr (float, Tensor, optional): learning rate (default: 1e-3). + weight_decay (float, optional): weight decay (L2 penalty). (default: 0.1) + momentum (float, optional): momentum factor (default: 0.95) + nesterov (bool, optional): enables Nesterov momentum. Only applicable + when momentum is non-zero + ns_coefficients (tuple of three floats, optional): coefficients \(a,b,c\) for the + Newton–Schulz orthogonalization polynomial (default: ({DEFAULT_A}, {DEFAULT_B}, {DEFAULT_C})) + eps (float, optional): term added to the denominator for numerical stability. (default: {EPS}) + ns_steps (int, optional): number of Newton–Schulz iteration steps. (default: {DEFAULT_NS_STEPS}) + adjust_lr_fn (str, optional): function to adjust learning rate. One of "original" and "match_rms_adamw". + If not specified, we will default to use "original". (default: None) + + .. _Muon\: An optimizer for hidden layers in neural networks: + https://kellerjordan.github.io/posts/muon/ + .. _Muon is Scalable for LLM Training: + https://arxiv.org/pdf/2502.16982 + + """ +) + + +def _single_tensor_muon( + params: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + *, + lr: float, + weight_decay: float, + momentum: float, + nesterov: bool, + ns_coefficients: tuple[float, float, float], + ns_steps: int, + eps: float, + adjust_lr_fn: str | None, + has_complex: bool, +) -> None: + lr = _to_scalar(lr) + if has_complex: + raise ValueError("Complex parameters are not supported") + + for i, param in enumerate(params): + grad = grads[i] + if grad.ndim != 2: + raise ValueError("Param gradient must be a 2D matrix") + + buf = muon_momentum_bufs[i] + buf.lerp_(grad, 1 - momentum) + update = grad.lerp(buf, momentum) if nesterov else buf + + update = _zeropower_via_newtonschulz(update, ns_coefficients, ns_steps, eps) + + adjusted_lr = _adjust_lr(lr, adjust_lr_fn, param.shape) + + param.mul_(1 - lr * weight_decay) + param.add_(update, alpha=-adjusted_lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_muon) +def muon( + params: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + *, + foreach: bool | None = None, + lr: float, + weight_decay: float, + momentum: float, + nesterov: bool, + ns_coefficients: tuple[float, float, float], + ns_steps: int, + eps: float, + adjust_lr_fn: str | None, + has_complex: bool, +) -> None: + r"""Functional API that performs Muon algorithm computation. + + See :class:`~torch.optim.Muon` for details. + """ + if foreach is not None and foreach: + raise RuntimeError("Foreach is not supported for Muon yet") + + func = _single_tensor_muon + + func( + params, + grads, + muon_momentum_bufs, + lr=lr, + weight_decay=weight_decay, + momentum=momentum, + nesterov=nesterov, + ns_coefficients=ns_coefficients, + ns_steps=ns_steps, + eps=eps, + adjust_lr_fn=adjust_lr_fn, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adadelta.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adadelta.py new file mode 100644 index 0000000000000000000000000000000000000000..36e58aa59e5320dbd79750bd12ae2e07d090d7e1 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adadelta.py @@ -0,0 +1,473 @@ +# mypy: allow-untyped-defs +from typing import Any, cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adadelta", "adadelta"] + + +class Adadelta(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1.0, + rho: float = 0.9, + eps: float = 1e-6, + weight_decay: float = 0, + foreach: bool | None = None, + *, + capturable: bool = False, + maximize: bool = False, + differentiable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= rho <= 1.0: + raise ValueError(f"Invalid rho value: {rho}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "rho": rho, + "eps": eps, + "weight_decay": weight_decay, + "maximize": maximize, + "capturable": capturable, + "foreach": foreach, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group: dict[str, Any], + params_with_grad: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + ): + has_complex = False + p: Tensor + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("Adadelta does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # Lazy state initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + + state["square_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + state["acc_delta"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + square_avgs.append(state["square_avg"]) + acc_deltas.append(state["acc_delta"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + square_avgs: list[Tensor] = [] + acc_deltas: list[Tensor] = [] + state_steps: list[Tensor] = [] + ( + lr, + rho, + eps, + weight_decay, + foreach, + maximize, + differentiable, + capturable, + ) = ( + group["lr"], + group["rho"], + group["eps"], + group["weight_decay"], + group["foreach"], + group["maximize"], + group["differentiable"], + group["capturable"], + ) + + has_complex = self._init_group( + group, params_with_grad, grads, square_avgs, acc_deltas, state_steps + ) + + adadelta( + params_with_grad, + grads, + square_avgs, + acc_deltas, + state_steps, + lr=lr, + rho=rho, + eps=eps, + weight_decay=weight_decay, + foreach=foreach, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) + + return loss + + +Adadelta.__doc__ = ( + r"""Implements Adadelta algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, + \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)}, + \: \lambda \text{ (weight decay)} \\ + &\textbf{initialize} : v_0 \leftarrow 0 \: \text{ (square avg)}, + \: u_0 \leftarrow 0 \: \text{ (accumulate variables)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm} v_t \leftarrow v_{t-1} \rho + g^2_t (1 - \rho) \\ + &\hspace{5mm}\Delta x_t \leftarrow \frac{\sqrt{u_{t-1} + + \epsilon }}{ \sqrt{v_t + \epsilon} }g_t \hspace{21mm} \\ + &\hspace{5mm} u_t \leftarrow u_{t-1} \rho + + \Delta x^2_t (1 - \rho) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \Delta x_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `ADADELTA: An Adaptive Learning Rate Method`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): coefficient that scale delta before it is applied + to the parameters (default: 1.0) + rho (float, optional): coefficient used for computing a running average + of squared gradients (default: 0.9). A higher value of `rho` will + result in a slower average, which can be helpful for preventing + oscillations in the learning process. + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-6). + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_capturable_doc} + {_maximize_doc} + {_differentiable_doc} + + .. _ADADELTA\: An Adaptive Learning Rate Method: + https://arxiv.org/abs/1212.5701 + + """ +) + + +def _single_tensor_adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for param, grad, square_avg, acc_delta, step in zip( + params, grads, square_avgs, acc_deltas, state_steps, strict=True + ): + step += 1 + grad = grad if not maximize else -grad + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + square_avg = torch.view_as_real(square_avg) + acc_delta = torch.view_as_real(acc_delta) + grad = torch.view_as_real(grad) + + square_avg.mul_(rho).addcmul_(grad, grad, value=1 - rho) + std = square_avg.add(eps).sqrt_() + delta = acc_delta.add(eps).sqrt_() + if differentiable: + delta = delta.clone() + delta.div_(std).mul_(grad) + acc_delta.mul_(rho).addcmul_(delta, delta, value=1 - rho) + + if torch.is_complex(param): + delta = torch.view_as_complex(delta) + param.add_(delta, alpha=-lr) + + +def _multi_tensor_adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, square_avgs, acc_deltas, state_steps] # type: ignore[list-item] + ) + for ( + device_params_, + device_grads_, + device_square_avgs_, + device_acc_deltas_, + device_state_steps_, + ), _ in grouped_tensors.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_square_avgs = cast(list[Tensor], device_square_avgs_) + device_acc_deltas = cast(list[Tensor], device_acc_deltas_) + device_state_steps = cast(list[Tensor], device_state_steps_) + if has_complex: + _view_as_real( + device_params, device_grads, device_square_avgs, device_acc_deltas + ) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + torch._foreach_mul_(device_square_avgs, rho) + torch._foreach_addcmul_( + device_square_avgs, device_grads, device_grads, value=1 - rho + ) + + std = torch._foreach_add(device_square_avgs, eps) + torch._foreach_sqrt_(std) + + deltas = torch._foreach_add(device_acc_deltas, eps) + torch._foreach_sqrt_(deltas) + torch._foreach_div_(deltas, std) + torch._foreach_mul_(deltas, device_grads) + + torch._foreach_mul_(device_acc_deltas, rho) + torch._foreach_addcmul_(device_acc_deltas, deltas, deltas, value=1 - rho) + + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + torch._foreach_mul_(deltas, -lr) + torch._foreach_add_(device_params, deltas) + else: + torch._foreach_add_(device_params, deltas, alpha=-lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adadelta) +def adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + capturable: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + has_complex: bool = False, + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adadelta algorithm computation. + + See :class:`~torch.optim.Adadelta` for details. + """ + + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + # We still respect when the user inputs False for foreach. + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adadelta + else: + func = _single_tensor_adadelta + + func( + params, + grads, + square_avgs, + acc_deltas, + state_steps, + lr=lr, + rho=rho, + eps=eps, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adagrad.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adagrad.py new file mode 100644 index 0000000000000000000000000000000000000000..f851f9558b6e57f11156547adb15a91a44eaffcc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adagrad.py @@ -0,0 +1,579 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _foreach_doc, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adagrad", "adagrad"] + + +class Adagrad(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + lr_decay: float = 0, + weight_decay: float = 0, + initial_accumulator_value: float = 0, + eps: float = 1e-10, + foreach: bool | None = None, + *, + maximize: bool = False, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= lr_decay: + raise ValueError(f"Invalid lr_decay value: {lr_decay}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= initial_accumulator_value: + raise ValueError( + f"Invalid initial_accumulator_value value: {initial_accumulator_value}" + ) + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + + defaults = { + "lr": lr, + "lr_decay": lr_decay, + "eps": eps, + "weight_decay": weight_decay, + "initial_accumulator_value": initial_accumulator_value, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "fused": fused, + } + super().__init__(params, defaults) + + if fused: + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + self._need_device_dtype_check_for_fused = True + + for group in self.param_groups: + for p in group["params"]: + state = self.state[p] + state["step"] = ( + torch.zeros( + (), + dtype=_get_scalar_dtype(is_fused=group["fused"]), + device=p.device, + ) + if group["fused"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + init_value = ( + complex(initial_accumulator_value, initial_accumulator_value) + if torch.is_complex(p) + else initial_accumulator_value + ) + state["sum"] = torch.full_like( + p, init_value, memory_format=torch.preserve_format + ) + + def __setstate__(self, state): + super().__setstate__(state) + # define "fused" for + # MYPY error: Name "fused" may be undefined + fused = None + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + fused = group.setdefault("fused", None) + + state_values = list(self.state.values()) + step_is_tensor = (len(state_values) != 0) and torch.is_tensor( + state_values[0]["step"] + ) + if not step_is_tensor: + for s in state_values: + s["step"] = torch.tensor( + float(s["step"]), dtype=_get_scalar_dtype(is_fused=fused) + ) + + def share_memory(self) -> None: + """Calls tensor.share_memory_() on the state sum tensors.""" + for group in self.param_groups: + for p in group["params"]: + state = self.state[p] + state["sum"].share_memory_() + + def _init_group(self, group, params_with_grad, grads, state_sums, state_steps): + has_sparse_grad, has_complex = False, False + for p in group["params"]: + if p.grad is not None: + if group["fused"] and getattr( + self, + "_need_device_dtype_check_for_fused", + True, + ): + _device_dtype_check_for_fused(p, cuda_unsupported=True) + self._need_device_dtype_check_for_fused = False + has_sparse_grad |= p.grad.is_sparse + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + grads.append(p.grad) + state = self.state[p] + state_sums.append(state["sum"]) + state_steps.append(state["step"]) + + return has_sparse_grad, has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + state_sums: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_sparse_grad, has_complex = self._init_group( + group, params_with_grad, grads, state_sums, state_steps + ) + + adagrad( + params_with_grad, + grads, + state_sums, + state_steps, + lr=group["lr"], + weight_decay=group["weight_decay"], + lr_decay=group["lr_decay"], + eps=group["eps"], + has_sparse_grad=has_sparse_grad, + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + has_complex=has_complex, + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + ) + + return loss + + +Adagrad.__doc__ = ( + r"""Implements Adagrad algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) + \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ + &\hspace{12mm} \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\ + &\textbf{initialize} : state\_sum_0 \leftarrow \tau \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \tilde{\gamma} \leftarrow \gamma / (1 +(t-1) \eta) \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ + &\hspace{5mm}\theta_t \leftarrow + \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning + and Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + lr_decay (float, optional): learning rate decay (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + initial_accumulator_value (float, optional): initial value of the + sum of squares of gradients (default: 0) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-10) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + fused (bool, optional): whether the fused implementation (CPU only) is used. + Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16` + are supported. (default: None). Please note that the fused implementations does not + support sparse or complex gradients. + .. _Adaptive Subgradient Methods for Online Learning and Stochastic + Optimization: http://jmlr.org/papers/v12/duchi11a.html + + """ +) + + +def adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting these as kwargs for now as functional API is compiled by torch/distributed/optim + has_sparse_grad: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + has_complex: bool = False, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adagrad algorithm computation. + + See :class:`~torch.optim.Adagrad` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if fused is None and foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if fused is None: + fused = False + if foreach is None: + foreach = False + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if fused and not torch.jit.is_scripting(): + func = _fused_adagrad + elif foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adagrad + else: + func = _single_tensor_adagrad + + func( + params, + grads, + state_sums, + state_steps, + lr=lr, + weight_decay=weight_decay, + lr_decay=lr_decay, + eps=eps, + has_sparse_grad=has_sparse_grad, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + grad_scale=grad_scale, + found_inf=found_inf, + ) + + +def _make_sparse(grad, grad_indices, values): + size = grad.size() + return torch.sparse_coo_tensor(grad_indices, values, size) + + +def _single_tensor_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for param, grad, state_sum, step_t in zip( + params, grads, state_sums, state_steps, strict=True + ): + # update step + step_t += 1 + step = _get_value(step_t) + grad = grad if not maximize else -grad + + if weight_decay != 0: + if grad.is_sparse: + raise RuntimeError( + "weight_decay option is not compatible with sparse gradients" + ) + grad = grad.add(param, alpha=weight_decay) + + clr = lr / (1 + (step - 1) * lr_decay) + + if grad.is_sparse: + grad = grad.coalesce() # the update is non-linear so indices must be unique + grad_indices = grad._indices() + grad_values = grad._values() + + state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2))) + std = state_sum.sparse_mask(grad) + std_values = std._values().sqrt_().add_(eps) + param.add_( + _make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr + ) + else: + is_complex = torch.is_complex(param) + if is_complex: + grad = torch.view_as_real(grad) + state_sum = torch.view_as_real(state_sum) + param = torch.view_as_real(param) + state_sum.addcmul_(grad, grad, value=1) + if differentiable: + std = state_sum.sqrt() + eps + else: + std = state_sum.sqrt().add_(eps) + param.addcdiv_(grad, std, value=-clr) + if is_complex: + param = torch.view_as_complex(param) + state_sum = torch.view_as_complex(state_sum) + + +def _multi_tensor_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + # Foreach functions will throw errors if given empty lists + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensorlists = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, state_sums, state_steps] # type: ignore[list-item] + ) + for ( + device_params_, + device_grads_, + device_state_sums_, + device_state_steps_, + ), _ in grouped_tensorlists.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_sums = cast(list[Tensor], device_state_sums_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_has_sparse_grad = has_sparse_grad and any( + grad.is_sparse for grad in device_grads + ) + + if device_has_sparse_grad: + _single_tensor_adagrad( + device_params, + device_grads, + device_state_sums, + device_state_steps, + lr=lr, + weight_decay=weight_decay, + lr_decay=lr_decay, + eps=eps, + has_sparse_grad=True, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + grad_scale=grad_scale, + found_inf=found_inf, + ) + continue + + # Handle complex parameters + if has_complex: + _view_as_real(device_params, device_grads, device_state_sums) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + minus_clr = [ + -lr / (1 + (_get_value(step) - 1) * lr_decay) for step in device_state_steps + ] + + torch._foreach_addcmul_(device_state_sums, device_grads, device_grads, value=1) + + std = torch._foreach_sqrt(device_state_sums) + torch._foreach_add_(std, eps) + + if weight_decay != 0 or maximize: + # Again, reuse the intermediate memory (device_grads) already allocated + torch._foreach_mul_(device_grads, minus_clr) + numerator = device_grads + else: + numerator = torch._foreach_mul(device_grads, minus_clr) # type: ignore[assignment] + + torch._foreach_addcdiv_(device_params, numerator, std) + + +def _fused_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if not params: + return + if has_sparse_grad or has_complex: + raise RuntimeError("`fused` does not support sparse grad or complex param") + + if differentiable: + raise RuntimeError( + "adagrad with fused=True does not support differentiable=True" + ) + + lr = _to_scalar(lr) + + grad_scale_dict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else None + ) + found_inf_dict = {found_inf.device: found_inf} if found_inf is not None else None + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, state_sums, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + device_params_, + device_grads_, + device_state_sums_, + device_state_steps_, + ), + _, + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_sums = cast(list[Tensor], device_state_sums_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_grad_scale, device_found_inf = None, None + if grad_scale is not None and grad_scale_dict is not None: + if device not in grad_scale_dict: + grad_scale_dict[device] = grad_scale.to(device, non_blocking=True) # type: ignore[index] + device_grad_scale = grad_scale_dict[device] # type: ignore[index] + if found_inf is not None and found_inf_dict is not None: + if found_inf not in found_inf_dict: + found_inf_dict[device] = found_inf.to(device, non_blocking=True) # type: ignore[index] + device_found_inf = found_inf_dict[device] # type: ignore[index] + torch._foreach_add_(device_state_steps, 1) + torch._fused_adagrad_( + device_params, + device_grads, + device_state_sums, + device_state_steps, + lr=lr, + lr_decay=lr_decay, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) + if device_found_inf is not None: + torch._foreach_sub_( + device_state_steps, [device_found_inf] * len(device_state_steps) + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adam.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adam.py new file mode 100644 index 0000000000000000000000000000000000000000..935bbcde8434f0efbab3a5ab7148649746684064 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adam.py @@ -0,0 +1,990 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _fused_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _stack_if_compiling, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + DeviceDict, + DeviceDtypeDict, + Optimizer, + ParamsT, +) + + +__all__ = ["Adam", "adam"] + + +class Adam(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float | Tensor, float | Tensor] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + amsgrad: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + decoupled_weight_decay: bool = False, + ) -> None: + if isinstance(lr, Tensor): + if foreach and not capturable: + raise ValueError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + if lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not ( + (isinstance(betas[0], float) and isinstance(betas[1], float)) + or (isinstance(betas[0], Tensor) and isinstance(betas[1], Tensor)) + ): + raise ValueError("betas must be either both floats or both Tensors") + if isinstance(betas[0], Tensor): + if not capturable and foreach: + raise ValueError( + "betas[0] as a Tensor is not supported for capturable=False and foreach=True" + ) + if betas[0].numel() != 1: + raise ValueError("Tensor betas[0] must be 1-element") + if isinstance(betas[1], Tensor): + if not capturable and foreach: + raise ValueError( + "betas[1] as a Tensor is not supported for capturable=False and foreach=True" + ) + if betas[1].numel() != 1: + raise ValueError("Tensor betas[1] must be 1-element") + betas = tuple(map(_to_scalar, betas)) + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "amsgrad": amsgrad, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "differentiable": differentiable, + "fused": fused, + "decoupled_weight_decay": decoupled_weight_decay, + } + super().__init__(params, defaults) + + if fused: + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + self._step_supports_amp_scaling = True + # TODO(crcrpar): [low prec params & their higher prec copy] + # Support AMP with FP16/BF16 model params which would need + # higher prec copy of params to do update math in higher prec to + # alleviate the loss of information. + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("amsgrad", False) + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("capturable", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + fused = group.setdefault("fused", None) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, + dtype=_get_scalar_dtype(is_fused=fused), + device=p.device, + ) + if group["capturable"] or group["fused"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError( + "Adam does not support sparse gradients, please consider SparseAdam instead" + ) + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + if group["fused"]: + _device_dtype_check_for_fused(p) + # note(crcrpar): [special device hosting for step] + # Deliberately host `step` on CPU if both capturable and fused are off. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = ( + torch.zeros( + (), + dtype=_get_scalar_dtype(is_fused=group["fused"]), + device=p.device, + ) + if group["capturable"] or group["fused"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["amsgrad"]: + # Maintains max of all exp. moving avg. of sq. grad. values + state["max_exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + + if group["amsgrad"]: + max_exp_avg_sqs.append(state["max_exp_avg_sq"]) + if group["differentiable"] and state["step"].requires_grad: + raise RuntimeError( + "`requires_grad` is not supported for `step` in differentiable mode" + ) + + # Foreach without capturable does not support a tensor lr + if ( + group["foreach"] + and torch.is_tensor(group["lr"]) + and not group["capturable"] + ): + raise RuntimeError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + max_exp_avg_sqs: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = group["betas"] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + ) + + adam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + amsgrad=group["amsgrad"], + has_complex=has_complex, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + eps=group["eps"], + maximize=group["maximize"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + decoupled_weight_decay=group["decoupled_weight_decay"], + ) + + return loss + + +Adam.__doc__ = ( + r"""Implements Adam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 + \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \lambda \text{ (weight decay)}, \: \textit{amsgrad}, + \:\textit{maximize}, \: \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{5mm}\textbf{if} \: amsgrad \\ + &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t) \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR + is not yet supported for all our implementations. Please use a float + LR if you are not also specifying fused=True or capturable=True. + betas (tuple[Union[float, Tensor], Union[float, Tensor]], optional): + coefficients used for computing running averages of gradient and + its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_weight_decay (bool, optional): if True, this optimizer is + equivalent to AdamW and the algorithm will not accumulate weight + decay in the momentum nor variance. (default: False) + amsgrad (bool, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + {_fused_doc} + .. Note:: + A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`. + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + + """ +) + + +def _single_tensor_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if torch.jit.is_scripting(): + # this assert is due to JIT being dumb and not realizing that the ops below + # have overloads to handle both float and Tensor lrs, so we just assert it's + # a float since most people using JIT are using floats + if not isinstance(lr, float): + raise AssertionError(f"Expected lr to be a float, but got {type(lr)}") + if not isinstance(beta1, float): + raise AssertionError(f"Expected beta1 to be a float, but got {type(beta1)}") + if not isinstance(beta2, float): + raise AssertionError(f"Expected beta2 to be a float, but got {type(beta2)}") + else: + lr = _to_scalar(lr) + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + # We only shuffle around the beta when it is a Tensor, otherwise, we prefer + # treating it as a scalar. + # Note: ensure type declaration is under conditional check for isinstance + # or else torchscript will get cranky about the DeviceDict type. + if isinstance(beta1, Tensor): + beta1_dict: DeviceDtypeDict | None = {(beta1.device, beta1.dtype): beta1} + else: + beta1_dict = None + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + param.mul_(1 - lr * weight_decay) + else: + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(weight_decay, Tensor): + if weight_decay.requires_grad: + grad = grad.addcmul_(param.clone(), weight_decay) + else: + # pyrefly: ignore [bad-argument-type] + grad = grad.add(param, alpha=weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + if amsgrad: + max_exp_avg_sqs[i] = torch.view_as_real(max_exp_avg_sqs[i]) + param = torch.view_as_real(param) + + device = param.device + + if beta1_dict is not None: + dtype = param.dtype # type: ignore[union-attr] + + # cast to workaround https://github.com/pytorch/pytorch/issues/140601 + key = (device, dtype) + if key not in beta1_dict: + beta1_dict[key] = beta1.to( # type: ignore[union-attr] + device=device, dtype=dtype, non_blocking=True + ) + + device_beta1: float | Tensor = beta1_dict[key] + else: + device_beta1 = beta1 + + # Decay the first and second moment running average coefficient + + exp_avg.lerp_(grad, 1 - device_beta1) + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta2, Tensor): + if beta2.requires_grad: + # Using lerp to only use 2 operations bc addcmul's value cannot be a tensor + # Showing equivalence of differentiable path and nondifferentiable path + # expavg * b2 + grad^2 * (1-b2) + # add expavg * (1-b2) - expavg * (1-b2) = 0 + # expavg * b2 + expavg * (1-b2) - expavg * (1-b2) + grad^2 * (1-b2) + # expavg - expavg * (1-b2) + grad^2 * (1-b2) + # expavg + (grad^2 - expavg) * (1-b2) + # expavg.lerp(grad^2, 1-beta2) + exp_avg_sq.lerp_(torch.square(grad), weight=1 - beta2) + else: + exp_avg_sq.mul_(beta2).addcmul_( + grad, grad, value=cast(float, 1 - beta2) + ) + else: + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) # type: ignore[arg-type] + + if capturable or differentiable: + step = step_t + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta1, Tensor): + if beta1.requires_grad: + bias_correction1 = 1 - beta1 ** step.clone() + else: + bias_correction1 = 1 - beta1**step + else: + bias_correction1 = 1 - beta1**step + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta2, Tensor): + if beta2.requires_grad: + bias_correction2 = 1 - beta2 ** step.clone() + else: + bias_correction2 = 1 - beta2**step + else: + bias_correction2 = 1 - beta2**step + + step_size = lr / bias_correction1 + step_size_neg = step_size.neg() + + bias_correction2_sqrt = bias_correction2.sqrt() + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + if differentiable: + max_exp_avg_sq = max_exp_avg_sqs[i].clone() + else: + max_exp_avg_sq = max_exp_avg_sqs[i] + + max_exp_avg_sqs[i].copy_(torch.maximum(max_exp_avg_sq, exp_avg_sq)) + + # Uses the max. for normalizing running avg. of gradient + # Folds in (admittedly ugly) 1-elem step_size math here to avoid extra param-set-sized read+write + # (can't fold it into addcdiv_ below because addcdiv_ requires value is a Number, not a Tensor) + denom = ( + max_exp_avg_sqs[i].sqrt() / (bias_correction2_sqrt * step_size_neg) + ).add_(eps / step_size_neg) + else: + denom = ( + exp_avg_sq.sqrt() / (bias_correction2_sqrt * step_size_neg) + ).add_(eps / step_size_neg) + + if differentiable: + param.addcdiv_(exp_avg.clone(), denom) + else: + param.addcdiv_(exp_avg, denom) + else: + step = _get_value(step_t) + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + + step_size = lr / bias_correction1 + + bias_correction2_sqrt = bias_correction2**0.5 + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + torch.maximum(max_exp_avg_sqs[i], exp_avg_sq, out=max_exp_avg_sqs[i]) + + # Use the max. for normalizing running avg. of gradient + denom = (max_exp_avg_sqs[i].sqrt() / bias_correction2_sqrt).add_(eps) + else: + denom = (exp_avg_sq.sqrt() / bias_correction2_sqrt).add_(eps) + + param.addcdiv_(exp_avg, denom, value=-step_size) # type: ignore[arg-type] + + # Lastly, switch back to complex view + if amsgrad and torch.is_complex(params[i]): + max_exp_avg_sqs[i] = torch.view_as_complex(max_exp_avg_sqs[i]) + + +def _multi_tensor_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if len(params) == 0: + return + + if isinstance(lr, Tensor): + if not capturable: + raise RuntimeError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + if lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + + if isinstance(beta1, Tensor): + if not capturable: + raise ValueError( + "beta1 as a Tensor is not supported for capturable=False and foreach=True" + ) + if beta1.numel() != 1: + raise ValueError("Tensor beta1 must be 1-element") + + if isinstance(beta2, Tensor): + if not capturable: + raise ValueError( + "beta2 as a Tensor is not supported for capturable=False and foreach=True" + ) + if beta2.numel() != 1: + raise ValueError("Tensor beta2 must be 1-element") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + lr = _to_scalar(lr) + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + + # We only shuffle around the beta when it is a Tensor and on CUDA, otherwise, we prefer + # treating it as a scalar. + beta1_dict: DeviceDict | None = ( # type: ignore[attr-defined] + {beta1.device: beta1} + if isinstance(beta1, Tensor) and str(beta1.device) != "cpu" + else None + ) + + for ( + device_params_, + device_grads_, + device_exp_avgs_, + device_exp_avg_sqs_, + device_max_exp_avg_sqs_, + device_state_steps_, + ), _ in grouped_tensors.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_exp_avgs = cast(list[Tensor], device_exp_avgs_) + device_exp_avg_sqs = cast(list[Tensor], device_exp_avg_sqs_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device = device_params[0].device + if beta1_dict is not None and device not in beta1_dict: + beta1_dict[device] = beta1.to(device=device, non_blocking=True) # type: ignore[union-attr, attr-defined] + + device_beta1 = beta1_dict[device] if beta1_dict else beta1 + + # Handle complex parameters + if has_complex: + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + _view_as_real( + device_params, + device_grads, + device_exp_avgs, + device_exp_avg_sqs, + device_max_exp_avg_sqs, + ) + else: + _view_as_real( + device_params, device_grads, device_exp_avgs, device_exp_avg_sqs + ) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + torch._foreach_mul_(device_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + # Use device beta1 if beta1 is a tensor to ensure all + # tensors are on the same device + torch._foreach_lerp_( + device_exp_avgs, device_grads, cast(float, 1 - device_beta1) + ) + + torch._foreach_mul_(device_exp_avg_sqs, beta2) + + # Due to the strictness of the _foreach_addcmul API, we can't have a single + # tensor scalar as the scalar arg (only python number is supported there) + # as a result, separate out the value mul + # Filed https://github.com/pytorch/pytorch/issues/139795 + if isinstance(beta2, torch.Tensor): + scaled_device_grads = torch._foreach_mul(device_grads, 1 - beta2) # type: ignore[assignment] + value = 1.0 + else: + scaled_device_grads = device_grads # type: ignore[assignment] + value = 1 - beta2 + + torch._foreach_addcmul_( + device_exp_avg_sqs, scaled_device_grads, device_grads, value + ) + + # Delete the local intermediate(s) since they won't be used anymore to save on peak memory + del device_grads + del scaled_device_grads + + bias_correction1: tuple[Tensor, ...] | list[Tensor] + bias_correction2: tuple[Tensor, ...] | list[Tensor] + bias_correction2_sqrt: tuple[Tensor, ...] | list[Tensor] + + if capturable: + bias_correction1 = torch._foreach_pow(beta1, device_state_steps) # type: ignore[arg-type] + bias_correction2 = torch._foreach_pow(beta2, device_state_steps) # type: ignore[arg-type] + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_correction1, 1) + torch._foreach_sub_(bias_correction2, 1) + # we do not negate bias_correction1 as it'll need to be negated later anyway + torch._foreach_neg_(bias_correction2) + + # foreach_div doesn't allow a scalar as the first arg + torch._foreach_div_(bias_correction1, lr) + torch._foreach_reciprocal_(bias_correction1) + + torch._foreach_sqrt_(bias_correction2) + + # Re-assign for clarity as we maintain minimal intermediates: we'll have + # step_size = - lr / (1 - beta1 ^ t) where t = num_steps + # bias_correction2_sqrt = sqrt(1 - beta2 ^ t) + step_size = bias_correction1 + bias_correction2_sqrt = bias_correction2 + + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + # Maintains the maximum of all 2nd moment running avg. till now + torch._foreach_maximum_(device_max_exp_avg_sqs, device_exp_avg_sqs) # type: ignore[assignment] + + # Set intermediate to the max. for normalizing running avg. of gradient when amsgrad + exp_avg_sq_sqrt = torch._foreach_sqrt(device_max_exp_avg_sqs) + else: + exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction2_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + torch._foreach_div_(exp_avg_sq_sqrt, step_size) + + # at this point, exp_avg_sq_sqrt = - (1 - beta^t) * [sqrt(exp_avg_sq / (1 - beta2^t)) + eps] / lr + torch._foreach_addcdiv_(device_params, device_exp_avgs, exp_avg_sq_sqrt) + else: + bias_correction1 = [ + 1 - beta1 ** _get_value(step) for step in device_state_steps + ] + bias_correction2 = [ + 1 - beta2 ** _get_value(step) for step in device_state_steps + ] + + step_size = _stack_if_compiling([(lr / bc) * -1 for bc in bias_correction1]) + + bias_correction2_sqrt = [bc**0.5 for bc in bias_correction2] # type: ignore[arg-type] + + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + # Maintains the maximum of all 2nd moment running avg. till now + torch._foreach_maximum_(device_max_exp_avg_sqs, device_exp_avg_sqs) + + # Use the max. for normalizing running avg. of gradient + exp_avg_sq_sqrt = torch._foreach_sqrt(device_max_exp_avg_sqs) + else: + exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction2_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + torch._foreach_addcdiv_( + device_params, + device_exp_avgs, + exp_avg_sq_sqrt, + step_size, # type: ignore[arg-type] + ) + + +def _fused_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, # Needed for consistency. + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, # Needed for consistency. + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if not params: + return + if differentiable: + raise RuntimeError("Adam with fused=True does not support differentiable=True") + + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + grad_scale_dict: DeviceDict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else {} + ) + found_inf_dict: DeviceDict = ( + {found_inf.device: found_inf} if found_inf is not None else {} + ) + + # We only shuffle around the lr when it is a Tensor and on CUDA, otherwise, we prefer + # treating it as a scalar. + lr_dict: DeviceDict | None = ( + {lr.device: lr} if isinstance(lr, Tensor) and str(lr.device) != "cpu" else None + ) + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + device_params_, + device_grads_, + device_exp_avgs_, + device_exp_avg_sqs_, + device_max_exp_avg_sqs, + device_state_steps_, + ), + _, + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_exp_avgs = cast(list[Tensor], device_exp_avgs_) + device_exp_avg_sqs = cast(list[Tensor], device_exp_avg_sqs_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_grad_scale, device_found_inf = None, None + if grad_scale is not None: + device_grad_scale = grad_scale_dict.setdefault( + device, grad_scale.to(device, non_blocking=True) + ) + if found_inf is not None: + device_found_inf = found_inf_dict.setdefault( + device, found_inf.to(device, non_blocking=True) + ) + if lr_dict is not None and device not in lr_dict: + lr_dict[device] = lr.to(device=device, non_blocking=True) # type: ignore[union-attr] + lr = lr_dict[device] + torch._foreach_add_(device_state_steps, 1) + func = torch._fused_adam_ if not decoupled_weight_decay else torch._fused_adamw_ + func( + device_params, + device_grads, + device_exp_avgs, + device_exp_avg_sqs, + device_max_exp_avg_sqs, # type: ignore[arg-type] + device_state_steps, + amsgrad=amsgrad, + lr=lr, # type: ignore[arg-type] + beta1=beta1, + beta2=beta2, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) + if device_found_inf is not None: + torch._foreach_sub_( + device_state_steps, [device_found_inf] * len(device_state_steps) + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adam) +def adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + decoupled_weight_decay: bool = False, + *, + amsgrad: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adam algorithm computation. + + See :class:`~torch.optim.Adam` for details. + """ + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if fused is None and foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + # Do not flip on foreach for the unsupported case where lr is a Tensor and capturable=False. + if foreach and isinstance(lr, Tensor) and not capturable: + foreach = False + if fused is None: + fused = False + if foreach is None: + foreach = False + + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if fused and not torch.jit.is_scripting(): + func = _fused_adam + elif foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adam + else: + func = _single_tensor_adam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + amsgrad=amsgrad, + has_complex=has_complex, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + grad_scale=grad_scale, + found_inf=found_inf, + decoupled_weight_decay=decoupled_weight_decay, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamax.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamax.py new file mode 100644 index 0000000000000000000000000000000000000000..3e24ae140cef6ffcc615edc594d355acd959b663 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamax.py @@ -0,0 +1,485 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adamax", "adamax"] + + +class Adamax(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 2e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + foreach: bool | None = None, + *, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, group, params_with_grad, grads, exp_avgs, exp_infs, state_steps + ): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("Adamax does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + state["exp_inf"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_infs.append(state["exp_inf"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Performs a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_infs: list[Tensor] = [] + state_steps: list[Tensor] = [] + + beta1, beta2 = group["betas"] + eps = group["eps"] + lr = group["lr"] + weight_decay = group["weight_decay"] + foreach = group["foreach"] + maximize = group["maximize"] + differentiable = group["differentiable"] + capturable = group["capturable"] + + has_complex = self._init_group( + group, params_with_grad, grads, exp_avgs, exp_infs, state_steps + ) + + adamax( + params_with_grad, + grads, + exp_avgs, + exp_infs, + state_steps, + eps=eps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + foreach=foreach, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) + + return loss + + +Adamax.__doc__ = ( + r"""Implements Adamax algorithm (a variant of Adam based on infinity norm). + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 + \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}, + \: \lambda \text{ (weight decay)}, \\ + &\hspace{13mm} \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + u_0 \leftarrow 0 \text{ ( infinity norm)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}u_t \leftarrow \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + {_capturable_doc} + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + + """ +) + + +def _single_tensor_adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + exp_avg = exp_avgs[i] + exp_inf = exp_infs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_inf = torch.view_as_real(exp_inf) + + # Update biased first moment estimate. + exp_avg.lerp_(grad, 1 - beta1) + # Update the exponentially weighted infinity norm. + if not differentiable: + torch.maximum( + exp_inf.mul_(beta2), + grad.abs().add_(eps), + out=exp_inf, + ) + else: + norm_buf = torch.cat( + [exp_inf.mul_(beta2).unsqueeze(0), grad.abs().add_(eps).unsqueeze_(0)], + 0, + ) + exp_inf.copy_(torch.amax(norm_buf, 0, keepdim=False)) + + if capturable: + # why jump through extra hoops and negate bias_correction? check out #121238 + # once fixed, we should use bias_correction with addcdiv value=-1 for readability + neg_bias_correction = beta1**step_t - 1 + neg_bias_correction.div_(lr) + denom = exp_inf * neg_bias_correction + param.addcdiv_(exp_avg, denom) + else: + bias_correction = 1 - beta1 ** _get_value(step_t) + clr = lr / bias_correction + + param.addcdiv_(exp_avg, exp_inf, value=-clr) + + +def _multi_tensor_adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + if len(params) == 0: + return + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_infs, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_infs_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_infs = cast(list[Tensor], grouped_exp_infs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_infs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + if maximize: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Update biased first moment estimate. + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + # Update the exponentially weighted infinity norm. + torch._foreach_mul_(grouped_exp_infs, beta2) + + # in this case, we need to introduce a copy of the grads + # since one has not been introduced previously + if not maximize and weight_decay == 0: + grouped_grads = torch._foreach_abs(grouped_grads) # type: ignore[assignment] + else: + torch._foreach_abs_(grouped_grads) + + torch._foreach_add_(grouped_grads, eps) + torch._foreach_maximum_(grouped_exp_infs, grouped_grads) + + bias_corrections: tuple[Tensor, ...] | list[Tensor] + if capturable: + bias_corrections = torch._foreach_pow(beta1, grouped_state_steps) + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_corrections, 1) + torch._foreach_div_(bias_corrections, lr) + + denom = torch._foreach_mul(grouped_exp_infs, bias_corrections) + torch._foreach_addcdiv_(grouped_params, grouped_exp_avgs, denom) + else: + bias_corrections = [ + 1 - beta1 ** _get_value(step) for step in grouped_state_steps + ] + step_size = [(_get_value(lr) / bc) * -1 for bc in bias_corrections] + torch._foreach_addcdiv_( + grouped_params, grouped_exp_avgs, grouped_exp_infs, step_size + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adamax) +def adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, +) -> None: + r"""Functional API that performs adamax algorithm computation. + + See :class:`~torch.optim.Adamax` for details. + """ + + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adamax + else: + func = _single_tensor_adamax + + func( + params, + grads, + exp_avgs, + exp_infs, + state_steps, + eps=eps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + capturable=capturable, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamw.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamw.py new file mode 100644 index 0000000000000000000000000000000000000000..f00b45dd4886899f2522930c9b35ae97c716d5d2 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/adamw.py @@ -0,0 +1,181 @@ +# mypy: allow-untyped-defs + +from torch import Tensor + +from .adam import Adam, adam +from .optimizer import ( + _capturable_doc, + _differentiable_doc, + _foreach_doc, + _fused_doc, + _maximize_doc, + _params_doc, + ParamsT, +) + + +__all__ = ["AdamW", "adamw"] + + +class AdamW(Adam): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float | Tensor, float | Tensor] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 1e-2, + amsgrad: bool = False, + *, + maximize: bool = False, + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: + super().__init__( + params, + lr, + betas, + eps, + weight_decay, + amsgrad, + foreach=foreach, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + fused=fused, + decoupled_weight_decay=True, + ) + + # Preserve decoupled_weight_decay from AdamW for backwards compatibility. The following + # guarantees that decoupled_weight_decay will always be True for loading any state into + # AdamW + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group["decoupled_weight_decay"] = True + + +AdamW.__doc__ = ( + r"""Implements AdamW algorithm, where weight decay does not accumulate in the momentum nor variance. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{(lr)}, \: \beta_1, \beta_2 + \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, + \: \epsilon \text{ (epsilon)} \\ + &\hspace{13mm} \lambda \text{(weight decay)}, \: \textit{amsgrad}, + \: \textit{maximize} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0 + \text{ ( second moment)}, \: v_0^{max}\leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{5mm}\textbf{if} \: amsgrad \\ + &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t) \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR + is not yet supported for all our implementations. Please use a float + LR if you are not also specifying fused=True or capturable=True. + betas (tuple[Union[float, Tensor], Union[float, Tensor]], optional): + coefficients used for computing running averages of gradient and + its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay coefficient (default: 1e-2) + amsgrad (bool, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + {_maximize_doc} + {_foreach_doc} + {_capturable_doc} + {_differentiable_doc} + {_fused_doc} + .. Note:: + A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`. + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + + """ +) + + +# @_disable_dynamo_if_unsupported logic occurs in the decorator that's applied to F.adam +def adamw( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + *, + amsgrad: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs AdamW algorithm computation. + + See :class:`~torch.optim.AdamW` for details. + """ + adam( + params, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + foreach=foreach, + capturable=capturable, + differentiable=differentiable, + fused=fused, + grad_scale=grad_scale, + found_inf=found_inf, + has_complex=has_complex, + amsgrad=amsgrad, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + decoupled_weight_decay=True, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/asgd.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/asgd.py new file mode 100644 index 0000000000000000000000000000000000000000..51f67a286cea9323771f5fa51125aa174c7754c3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/asgd.py @@ -0,0 +1,479 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["ASGD", "asgd"] + + +class ASGD(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + lambd: float = 1e-4, + alpha: float = 0.75, + t0: float = 1e6, + weight_decay: float = 0, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "lambd": lambd, + "alpha": alpha, + "t0": t0, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0: + if not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if not torch.is_tensor(p_state["eta"]): + p_state["eta"] = torch.tensor( + p_state["eta"], dtype=_get_scalar_dtype(), device=p.device + ) + if not torch.is_tensor(p_state["mu"]): + p_state["mu"] = torch.tensor( + p_state["mu"], dtype=_get_scalar_dtype(), device=p.device + ) + + def _init_group(self, group, params_with_grad, grads, mus, axs, etas, state_steps): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("ASGD does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # State initialization + if len(state) == 0: + state["step"] = torch.zeros( + (), device=p.device, dtype=_get_scalar_dtype() + ) + state["eta"] = ( + torch.as_tensor( + _to_scalar(group["lr"]), + device=p.device, + dtype=_get_scalar_dtype(), + ) + .clone() + .detach() + ) + state["mu"] = torch.ones( + (), device=p.device, dtype=_get_scalar_dtype() + ) + state["ax"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + mus.append(state["mu"]) + axs.append(state["ax"]) + etas.append(state["eta"]) + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + mus: list[Tensor] = [] + axs: list[Tensor] = [] + etas: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_complex = self._init_group( + group, params_with_grad, grads, mus, axs, etas, state_steps + ) + + asgd( + params_with_grad, + grads, + axs, + mus, + etas, + state_steps, + lambd=group["lambd"], + lr=group["lr"], + t0=group["t0"], + alpha=group["alpha"], + weight_decay=group["weight_decay"], + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +ASGD.__doc__ = rf"""Implements Averaged Stochastic Gradient Descent. + + It has been proposed in `Acceleration of stochastic approximation by + averaging`_. + + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + lambd (float, optional): decay term (default: 1e-4) + alpha (float, optional): power for eta update (default: 0.75) + t0 (float, optional): point at which to start averaging (default: 1e6) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + {_capturable_doc} + + .. _Acceleration of stochastic approximation by averaging: + https://meyn.ece.ufl.edu/wp-content/uploads/sites/77/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf + + """ + + +def _single_tensor_asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + mu = mus[i] + ax = axs[i] + eta = etas[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type + == mu.device.type + == eta.device.type + == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params, mus, etas, and state_steps must be " + f"on supported devices: {capturable_supported_devices}." + ) + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + param = torch.view_as_real(param) + ax = torch.view_as_real(ax) + + # update step + step_t += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if capturable: + param.mul_(1 - lambd * eta) + param.addcmul_(grad, eta, value=-1) # update parameter + else: + eta_value = _get_value(eta) + param.mul_(1 - lambd * eta_value) # decay term + param.add_(grad, alpha=-eta_value) # update parameter + + # averaging + if capturable or mu.item() != 1: + ax.add_(param.sub(ax).mul_(mu)) + else: + ax.copy_(param) + + if capturable: + # pyrefly: ignore [unsupported-operation] + eta.copy_(lr / ((1 + lambd * lr * step_t) ** alpha)) + mu.copy_(1 / torch.maximum(step_t - t0, torch.ones_like(step_t))) + else: + step = _get_value(step_t) + new_eta = torch.as_tensor(lr / ((1 + lambd * lr * step) ** alpha)) + eta.copy_(new_eta) + new_mu = torch.as_tensor(1 / max(1, step - t0)) + mu.copy_(new_mu) + + +def _multi_tensor_asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == mu.device.type == eta.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, mu, eta, step in zip(params, mus, etas, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params, mus, etas, and state_steps must be on " + f"supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, axs, mus, etas, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + grouped_params_, + grouped_grads_, + grouped_axs_, + grouped_mus_, + grouped_etas_, + grouped_state_steps_, + ), + _, + ) in grouped_tensors.items(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_axs = cast(list[Tensor], grouped_axs_) + grouped_mus = cast(list[Tensor], grouped_mus_) + grouped_etas = cast(list[Tensor], grouped_etas_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + _view_as_real(grouped_params, grouped_grads, grouped_axs) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + # intermediate = grad + param * lambd + intermediate: tuple[Tensor, ...] | list[Tensor] + if weight_decay != 0: + if maximize: + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + intermediate = grouped_grads + else: + intermediate = torch._foreach_add( + grouped_grads, grouped_params, alpha=weight_decay + ) + + torch._foreach_add_(intermediate, grouped_params, alpha=lambd) + else: + intermediate = torch._foreach_add( + grouped_grads, grouped_params, alpha=lambd + ) + + # update param + # param * (1 - lambd * eta) - eta * grad + # => param - param * lambd * eta - eta * grad + # => param - eta * intermediate + torch._foreach_addcmul_(grouped_params, intermediate, grouped_etas, value=-1) + del intermediate + + # update grouped_axs + # averaging: ax = ax + mu * (param - ax) + # Note (mlazos): We can't use lerp here since it requires weight to be float64 + # and our grouping code requires dtypes to match for all tensors in a group (and it should, since + # we use the mus in other places) + # all dtypes need to match, so we could introduce a cast in a loop + # but since this only adds one additional kernel launch, this looks like the cleaner + # and faster solution + intermediate = torch._foreach_sub(grouped_params, grouped_axs) + torch._foreach_addcmul_(grouped_axs, intermediate, grouped_mus) + del intermediate + + new_etas: tuple[Tensor, ...] | list[Tensor] + new_mus: tuple[Tensor, ...] | list[Tensor] + if capturable: + # update grouped_mus + new_mus = torch._foreach_sub(grouped_state_steps, t0) + torch._foreach_maximum_(new_mus, 1.0) + torch._foreach_reciprocal_(new_mus) + torch._foreach_copy_(grouped_mus, new_mus) + del new_mus + + # update eta = lr / ((1 + lambd * lr * step)^alpha) + new_etas = torch._foreach_mul(grouped_state_steps, lambd) + torch._foreach_mul_(new_etas, lr) + torch._foreach_add_(new_etas, 1) + torch._foreach_pow_(new_etas, alpha) + torch._foreach_reciprocal_(new_etas) + torch._foreach_mul_(new_etas, lr) + torch._foreach_copy_(grouped_etas, new_etas) + else: + new_etas = [ + torch.as_tensor(lr / ((1 + lambd * lr * step) ** alpha), device=device) + for step in grouped_state_steps + ] + new_mus = [ + torch.as_tensor(1 / max(1, _get_value(step) - t0), device=device) + for step in grouped_state_steps + ] + torch._foreach_copy_(grouped_etas, new_etas) + torch._foreach_copy_(grouped_mus, new_mus) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_asgd) +def asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, +) -> None: + r"""Functional API that performs asgd algorithm computation. + + See :class:`~torch.optim.ASGD` for details. + """ + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_asgd + else: + func = _single_tensor_asgd + + func( + params, + grads, + axs, + mus, + etas, + state_steps, + lambd=lambd, + lr=lr, + t0=t0, + alpha=alpha, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lbfgs.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lbfgs.py new file mode 100644 index 0000000000000000000000000000000000000000..d6789d585a4b9bedc94fc2d400b280e251830b00 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lbfgs.py @@ -0,0 +1,537 @@ +# mypy: allow-untyped-defs + +import torch +from torch import Tensor + +from .optimizer import _to_scalar, Optimizer, ParamsT + + +__all__ = ["LBFGS"] + + +def _cubic_interpolate(x1, f1, g1, x2, f2, g2, bounds=None): + # ported from https://github.com/torch/optim/blob/master/polyinterp.lua + # Compute bounds of interpolation area + if bounds is not None: + xmin_bound, xmax_bound = bounds + else: + xmin_bound, xmax_bound = (x1, x2) if x1 <= x2 else (x2, x1) + + # Code for most common case: cubic interpolation of 2 points + # w/ function and derivative values for both + # Solution in this case (where x2 is the farthest point): + # d1 = g1 + g2 - 3*(f1-f2)/(x1-x2); + # d2 = sqrt(d1^2 - g1*g2); + # min_pos = x2 - (x2 - x1)*((g2 + d2 - d1)/(g2 - g1 + 2*d2)); + # t_new = min(max(min_pos,xmin_bound),xmax_bound); + d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2) + d2_square = d1**2 - g1 * g2 + if d2_square >= 0: + d2 = d2_square.sqrt() + if x1 <= x2: + min_pos = x2 - (x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2 * d2)) + else: + min_pos = x1 - (x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2 * d2)) + return min(max(min_pos, xmin_bound), xmax_bound) + else: + return (xmin_bound + xmax_bound) / 2.0 + + +def _strong_wolfe( + obj_func, x, t, d, f, g, gtd, c1=1e-4, c2=0.9, tolerance_change=1e-9, max_ls=25 +): + # ported from https://github.com/torch/optim/blob/master/lswolfe.lua + d_norm = d.abs().max() + g = g.clone(memory_format=torch.contiguous_format) + # evaluate objective and gradient using initial step + f_new, g_new = obj_func(x, t, d) + ls_func_evals = 1 + gtd_new = g_new.dot(d) + + # bracket an interval containing a point satisfying the Wolfe criteria + t_prev, f_prev, g_prev, gtd_prev = 0, f, g, gtd + done = False + ls_iter = 0 + while ls_iter < max_ls: + # check conditions + if f_new > (f + c1 * t * gtd) or (ls_iter > 1 and f_new >= f_prev): + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + if abs(gtd_new) <= -c2 * gtd: + bracket = [t] + bracket_f = [f_new] + bracket_g = [g_new] + done = True + break + + if gtd_new >= 0: + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + # interpolate + min_step = t + 0.01 * (t - t_prev) + max_step = t * 10 + tmp = t + t = _cubic_interpolate( + t_prev, f_prev, gtd_prev, t, f_new, gtd_new, bounds=(min_step, max_step) + ) + + # next step + t_prev = tmp + f_prev = f_new + g_prev = g_new.clone(memory_format=torch.contiguous_format) + gtd_prev = gtd_new + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + # reached max number of iterations? + if ls_iter == max_ls: + bracket = [0, t] + bracket_f = [f, f_new] + bracket_g = [g, g_new] + + # zoom phase: we now have a point satisfying the criteria, or + # a bracket around it. We refine the bracket until we find the + # exact point satisfying the criteria + insuf_progress = False + # find high and low points in bracket + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[-1] else (1, 0) # type: ignore[possibly-undefined] + while not done and ls_iter < max_ls: + # line-search bracket is so small + if abs(bracket[1] - bracket[0]) * d_norm < tolerance_change: # type: ignore[possibly-undefined] + break + + # compute new trial value + t = _cubic_interpolate( + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + bracket[0], + # pyrefly: ignore [unbound-name] + bracket_f[0], + bracket_gtd[0], # type: ignore[possibly-undefined] + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + bracket[1], + # pyrefly: ignore [unbound-name] + bracket_f[1], + # pyrefly: ignore [unbound-name] + bracket_gtd[1], + ) + + # test that we are making sufficient progress: + # in case `t` is so close to boundary, we mark that we are making + # insufficient progress, and if + # + we have made insufficient progress in the last step, or + # + `t` is at one of the boundary, + # we will move `t` to a position which is `0.1 * len(bracket)` + # away from the nearest boundary point. + # pyrefly: ignore [unbound-name] + eps = 0.1 * (max(bracket) - min(bracket)) + # pyrefly: ignore [unbound-name] + if min(max(bracket) - t, t - min(bracket)) < eps: + # interpolation close to boundary + # pyrefly: ignore [unbound-name] + if insuf_progress or t >= max(bracket) or t <= min(bracket): + # evaluate at 0.1 away from boundary + # pyrefly: ignore [unbound-name] + if abs(t - max(bracket)) < abs(t - min(bracket)): + # pyrefly: ignore [unbound-name] + t = max(bracket) - eps + else: + # pyrefly: ignore [unbound-name] + t = min(bracket) + eps + insuf_progress = False + else: + insuf_progress = True + else: + insuf_progress = False + + # Evaluate new point + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + # pyrefly: ignore [unbound-name] + if f_new > (f + c1 * t * gtd) or f_new >= bracket_f[low_pos]: + # Armijo condition not satisfied or not lower than lowest point + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[high_pos] = t + # pyrefly: ignore [unbound-name] + bracket_f[high_pos] = f_new + bracket_g[high_pos] = g_new.clone(memory_format=torch.contiguous_format) # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[high_pos] = gtd_new + # pyrefly: ignore [unbound-name] + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[1] else (1, 0) + else: + if abs(gtd_new) <= -c2 * gtd: + # Wolfe conditions satisfied + done = True + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + elif gtd_new * (bracket[high_pos] - bracket[low_pos]) >= 0: + # old high becomes new low + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[high_pos] = bracket[low_pos] + # pyrefly: ignore [unbound-name] + bracket_f[high_pos] = bracket_f[low_pos] + bracket_g[high_pos] = bracket_g[low_pos] # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[high_pos] = bracket_gtd[low_pos] + + # new point becomes new low + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[low_pos] = t + # pyrefly: ignore [unbound-name] + bracket_f[low_pos] = f_new + bracket_g[low_pos] = g_new.clone(memory_format=torch.contiguous_format) # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[low_pos] = gtd_new + + # return stuff + t = bracket[low_pos] # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + f_new = bracket_f[low_pos] + g_new = bracket_g[low_pos] # type: ignore[possibly-undefined] + return f_new, g_new, t, ls_func_evals + + +class LBFGS(Optimizer): + """Implements L-BFGS algorithm. + + Heavily inspired by `minFunc + `_. + + .. warning:: + This optimizer doesn't support per-parameter options and parameter + groups (there can be only one). + + .. warning:: + Right now all parameters have to be on a single device. This will be + improved in the future. + + .. note:: + This is a very memory intensive optimizer (it requires additional + ``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory + try reducing the history size, or use a different algorithm. + + Args: + params (iterable): iterable of parameters to optimize. Parameters must be real. + lr (float, optional): learning rate (default: 1) + max_iter (int, optional): maximal number of iterations per optimization step + (default: 20) + max_eval (int, optional): maximal number of function evaluations per optimization + step (default: max_iter * 1.25). + tolerance_grad (float, optional): termination tolerance on first order optimality + (default: 1e-7). + tolerance_change (float, optional): termination tolerance on function + value/parameter changes (default: 1e-9). + history_size (int, optional): update history size (default: 100). + line_search_fn (str, optional): either 'strong_wolfe' or None (default: None). + """ + + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1, + max_iter: int = 20, + max_eval: int | None = None, + tolerance_grad: float = 1e-7, + tolerance_change: float = 1e-9, + history_size: int = 100, + line_search_fn: str | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if max_eval is None: + max_eval = max_iter * 5 // 4 + defaults = { + "lr": lr, + "max_iter": max_iter, + "max_eval": max_eval, + "tolerance_grad": tolerance_grad, + "tolerance_change": tolerance_change, + "history_size": history_size, + "line_search_fn": line_search_fn, + } + super().__init__(params, defaults) + + if len(self.param_groups) != 1: + raise ValueError( + "LBFGS doesn't support per-parameter options (parameter groups)" + ) + + self._params = self.param_groups[0]["params"] + self._numel_cache = None + + def _numel(self): + if self._numel_cache is None: + # pyrefly: ignore [bad-assignment] + self._numel_cache = sum( + 2 * p.numel() if torch.is_complex(p) else p.numel() + for p in self._params + ) + + return self._numel_cache + + def _gather_flat_grad(self): + views = [] + for p in self._params: + if p.grad is None: + view = p.new(p.numel()).zero_() + elif p.grad.is_sparse: + view = p.grad.to_dense().view(-1) + else: + view = p.grad.view(-1) + if torch.is_complex(view): + view = torch.view_as_real(view).view(-1) + views.append(view) + return torch.cat(views, 0) + + def _add_grad(self, step_size, update) -> None: + offset = 0 + for p in self._params: + if torch.is_complex(p): + p = torch.view_as_real(p) + numel = p.numel() + # view as to avoid deprecated pointwise semantics + p.add_(update[offset : offset + numel].view_as(p), alpha=step_size) + offset += numel + if offset != self._numel(): + raise AssertionError(f"Expected offset {offset} to equal {self._numel()}") + + def _clone_param(self): + return [p.clone(memory_format=torch.contiguous_format) for p in self._params] + + def _set_param(self, params_data) -> None: + for p, pdata in zip(self._params, params_data, strict=True): + p.copy_(pdata) + + def _directional_evaluate(self, closure, x, t, d): + self._add_grad(t, d) + loss = float(closure()) + flat_grad = self._gather_flat_grad() + self._set_param(x) + return loss, flat_grad + + @torch.no_grad() + def step(self, closure): # type: ignore[override] + """Perform a single optimization step. + + Args: + closure (Callable): A closure that reevaluates the model + and returns the loss. + """ + if len(self.param_groups) != 1: + raise AssertionError( + f"Expected exactly one param_group, but got {len(self.param_groups)}" + ) + + # Make sure the closure is always called with grad enabled + closure = torch.enable_grad()(closure) + + group = self.param_groups[0] + lr = _to_scalar(group["lr"]) + max_iter = group["max_iter"] + max_eval = group["max_eval"] + tolerance_grad = group["tolerance_grad"] + tolerance_change = group["tolerance_change"] + line_search_fn = group["line_search_fn"] + history_size = group["history_size"] + + # NOTE: LBFGS has only global state, but we register it as state for + # the first param, because this helps with casting in load_state_dict + state = self.state[self._params[0]] + state.setdefault("func_evals", 0) + state.setdefault("n_iter", 0) + + # evaluate initial f(x) and df/dx + orig_loss = closure() + loss = float(orig_loss) + current_evals = 1 + state["func_evals"] += 1 + + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + + # optimal condition + if opt_cond: + return orig_loss + + # tensors cached in state (for tracing) + d = state.get("d") + t = state.get("t") + old_dirs = state.get("old_dirs") + old_stps = state.get("old_stps") + ro = state.get("ro") + H_diag = state.get("H_diag") + prev_flat_grad = state.get("prev_flat_grad") + prev_loss = state.get("prev_loss") + + n_iter = 0 + # optimize for a max of max_iter iterations + while n_iter < max_iter: + # keep track of nb of iterations + n_iter += 1 + state["n_iter"] += 1 + + ############################################################ + # compute gradient descent direction + ############################################################ + if state["n_iter"] == 1: + d = flat_grad.neg() + old_dirs = [] + old_stps = [] + ro = [] + H_diag = 1 + else: + # do lbfgs update (update memory) + y = flat_grad.sub(prev_flat_grad) + s = d.mul(t) + ys = y.dot(s) # y*s + if ys > 1e-10: + # updating memory + if len(old_dirs) == history_size: + # shift history by one (limited-memory) + old_dirs.pop(0) + old_stps.pop(0) + ro.pop(0) + + # store new direction/step + old_dirs.append(y) + old_stps.append(s) + ro.append(1.0 / ys) + + # update scale of initial Hessian approximation + H_diag = ys / y.dot(y) # (y*y) + + # compute the approximate (L-BFGS) inverse Hessian + # multiplied by the gradient + num_old = len(old_dirs) + + if "al" not in state: + state["al"] = [None] * history_size + al = state["al"] + + # iteration in L-BFGS loop collapsed to use just one buffer + q = flat_grad.neg() + for i in range(num_old - 1, -1, -1): + al[i] = old_stps[i].dot(q) * ro[i] + q.add_(old_dirs[i], alpha=-al[i]) + + # multiply by initial Hessian + # r/d is the final direction + d = r = torch.mul(q, H_diag) + for i in range(num_old): + be_i = old_dirs[i].dot(r) * ro[i] + r.add_(old_stps[i], alpha=al[i] - be_i) + + if prev_flat_grad is None: + prev_flat_grad = flat_grad.clone(memory_format=torch.contiguous_format) + else: + prev_flat_grad.copy_(flat_grad) + prev_loss = loss + + ############################################################ + # compute step length + ############################################################ + # reset initial guess for step size + if state["n_iter"] == 1: + t = min(1.0, 1.0 / flat_grad.abs().sum()) * lr + else: + t = lr + + # directional derivative + gtd = flat_grad.dot(d) # g * d + + # directional derivative is below tolerance + if gtd > -tolerance_change: + break + + # optional line search: user function + ls_func_evals = 0 + if line_search_fn is not None: + # perform line search, using user function + if line_search_fn != "strong_wolfe": + raise RuntimeError("only 'strong_wolfe' is supported") + else: + x_init = self._clone_param() + + def obj_func(x, t, d): + return self._directional_evaluate(closure, x, t, d) + + loss, flat_grad, t, ls_func_evals = _strong_wolfe( + obj_func, + x_init, + t, + d, + loss, + flat_grad, + gtd, + max_ls=max_eval - current_evals, + ) + self._add_grad(t, d) + opt_cond = flat_grad.abs().max() <= tolerance_grad + else: + # no line search, simply move with fixed-step + self._add_grad(t, d) + if n_iter != max_iter: + # re-evaluate function only if not in last iteration + # the reason we do this: in a stochastic setting, + # no use to re-evaluate that function here + with torch.enable_grad(): + loss = closure() + loss = float(loss) + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + ls_func_evals = 1 + + # update func eval + current_evals += ls_func_evals + state["func_evals"] += ls_func_evals + + ############################################################ + # check conditions + ############################################################ + if n_iter == max_iter: + break + + if current_evals >= max_eval: + break + + # optimal condition + if opt_cond: + break + + # lack of progress + if d.mul(t).abs().max() <= tolerance_change: + break + + if abs(loss - prev_loss) < tolerance_change: + break + + state["d"] = d + state["t"] = t + state["old_dirs"] = old_dirs + state["old_stps"] = old_stps + state["ro"] = ro + state["H_diag"] = H_diag + state["prev_flat_grad"] = prev_flat_grad + state["prev_loss"] = prev_loss + + return orig_loss diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lr_scheduler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..90988e8f337b446baf43eaa661d28332c32bc774 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/lr_scheduler.py @@ -0,0 +1,2602 @@ +# mypy: allow-untyped-defs +r"""Learning Rate Scheduler.""" + +from __future__ import annotations + +import math +import types +import warnings +from bisect import bisect_right +from collections import Counter +from functools import partial, wraps +from typing import Any, cast, Literal, SupportsFloat, TYPE_CHECKING, TypedDict +from typing_extensions import override, Self +from weakref import ref + +from torch import inf, Tensor + +from .optimizer import _to_scalar, Optimizer + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Sequence + + +__all__ = [ + "LambdaLR", + "MultiplicativeLR", + "StepLR", + "MultiStepLR", + "ConstantLR", + "LinearLR", + "ExponentialLR", + "SequentialLR", + "CosineAnnealingLR", + "ChainedScheduler", + "ReduceLROnPlateau", + "CyclicLR", + "CosineAnnealingWarmRestarts", + "OneCycleLR", + "PolynomialLR", + "LRScheduler", +] + +EPOCH_DEPRECATION_WARNING = ( + "The epoch parameter in `scheduler.step()` was not necessary and is being " + "deprecated where possible. Please use `scheduler.step()` to step the " + "scheduler. During the deprecation, if epoch is different from None, the " + "closed form is used instead of the new chainable form, where available. " + "Please open an issue if you are unable to replicate your use case: " + "https://github.com/pytorch/pytorch/issues/new/choose." +) + + +def _format_param(name: str, optimizer: Optimizer, param): + """Return correctly formatted lr/momentum for each param group.""" + + def _copy(_param): + return _param.clone() if isinstance(_param, Tensor) else _param + + if isinstance(param, (list, tuple)): + if len(param) != len(optimizer.param_groups): + raise ValueError( + f"{name} must have the same length as optimizer.param_groups. " + f"{name} has {len(param)} values, param_groups has {len(optimizer.param_groups)}." + ) + else: + param = [param] * len(optimizer.param_groups) + + return list(map(_copy, param)) + + +def _param_groups_val_list(optimizer: Optimizer, key: str) -> list[Any]: + """Create a list containing group[key] for each optimizer param_group. + Prevents aliasing when group[key] could be a Tensor. + Raises a KeyError when group[key] does not exist. + """ + return [ + group[key].clone() if isinstance(group[key], Tensor) else group[key] + for group in optimizer.param_groups + ] + + +def _update_param_group_val( + param_group: dict[str, Any], key: str, val: float | Tensor +) -> None: + """Set param_group[key] to val without aliasing or assignment when they're + both tensors. Raises a KeyError if param_group[key] does not exist. + """ + if isinstance(param_group[key], Tensor): + param_group[key].fill_(_to_scalar(val)) + else: + param_group[key] = val + + +class LRScheduler: + r"""Base class for all learning rate schedulers. + + Subclasses implement :meth:`get_lr` and optionally override :meth:`step` to + define scheduling behavior. + + Args: + optimizer (Optimizer): The optimizer this scheduler will adjust the + learning rates of. + last_epoch (int): Index of the last epoch seen by the scheduler. Use + ``-1`` (default) to initialize the scheduler. Only use a non-default + value when restoring this scheduler from a saved checkpoint. + + .. warning:: + Initializing a scheduler overwrites its optimizer's + ``param_group["lr"]``\s. When restoring a checkpoint, initialize the + scheduler **before** calling your optimizer's + :meth:`~torch.optim.Optimizer.load_state_dict` to avoid overwriting the + loaded learning rates. + """ + + _get_lr_called_within_step: bool = False + _is_initial: bool = False + + def __init__( + self, + optimizer: Optimizer, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + # Initialize epoch and base learning rates + if last_epoch == -1: + for group in optimizer.param_groups: + initial_lr = group["lr"] + if isinstance(initial_lr, Tensor): + initial_lr = initial_lr.clone() + group.setdefault("initial_lr", initial_lr) + else: + for i, group in enumerate(optimizer.param_groups): + if "initial_lr" not in group: + raise KeyError( + f"param 'initial_lr' is not specified in param_groups[{i}] when resuming scheduler with last_epoch >= 0.\n" + "This typically happens when:\n" + "1. You're trying to resume training from a checkpoint but haven't properly loaded the optimizer state\n" + "2. You're using last_epoch >= 0 for a fresh training run (not recommended)" + ) + self.base_lrs: list[float | Tensor] = _param_groups_val_list( + optimizer, "initial_lr" + ) + self.last_epoch = last_epoch + + # Following https://github.com/pytorch/pytorch/issues/20124 + # We would like to ensure that `lr_scheduler.step()` is called after + # `optimizer.step()` + def patch_track_step_called(opt: Optimizer): + if hasattr(opt.step, "_wrapped_by_lr_sched"): + # we've already patched + return opt.step + + def wrap_step(step_fn): + opt_ref = ref(self.optimizer) + func = step_fn.__func__ + + @wraps(func) + def wrapper(*args, **kwargs): + opt = opt_ref() + opt._opt_called = True # type: ignore[union-attr] + return func.__get__(opt, opt.__class__)(*args, **kwargs) + + wrapper._wrapped_by_lr_sched = True # type: ignore[attr-defined] + return wrapper + + opt.step = wrap_step(opt.step) # type: ignore[method-assign] + + patch_track_step_called(self.optimizer) + self._initial_step() + + def _initial_step(self) -> None: + """Initialize step counts and perform a step.""" + self._step_count = 0 + with _initial_mode(self): + self.step() + + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + """ + return { + key: value for key, value in self.__dict__.items() if key != "optimizer" + } + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + + def get_last_lr(self) -> list[float | Tensor]: + r"""Get the most recent learning rates computed by this scheduler. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates with entries + for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`, with the same types as + their ``group["lr"]``\s. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + # We always update self._last_lr with _param_groups_val_list, so it's a + # .clone() of the group["lr"]s. If we didn't do this, the user could + # corrupt their learning rates by modifying the outputs in place. + return self._last_lr + + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + raise NotImplementedError + + def step(self, epoch: int | None = None) -> None: + """Step the scheduler. + + Args: + epoch (int, optional): + .. deprecated:: 1.4 + If provided, sets :attr:`last_epoch` to ``epoch`` and uses + :meth:`_get_closed_form_lr` if it is available. This is not + universally supported. Use :meth:`step` without arguments + instead. + + .. note:: + Call this method after calling the optimizer's + :meth:`~torch.optim.Optimizer.step`. + """ + # Raise a warning if old pattern is detected + # https://github.com/pytorch/pytorch/issues/20124 + if self._step_count == 1: + if not hasattr(self.optimizer.step, "_wrapped_by_lr_sched"): + warnings.warn( + "Seems like `optimizer.step()` has been overridden after learning rate scheduler " + "initialization. Please, make sure to call `optimizer.step()` before " + "`lr_scheduler.step()`. See more details at " + "https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate", + UserWarning, + stacklevel=2, + ) + + # Just check if there were two first lr_scheduler.step() calls before optimizer.step() + elif not getattr(self.optimizer, "_opt_called", False): + warnings.warn( + "Detected call of `lr_scheduler.step()` before `optimizer.step()`. " + "In PyTorch 1.1.0 and later, you should call them in the opposite order: " + "`optimizer.step()` before `lr_scheduler.step()`. Failure to do this " + "will result in PyTorch skipping the first value of the learning rate schedule. " + "See more details at " + "https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate", + UserWarning, + stacklevel=2, + ) + + self._step_count += 1 + if epoch is not None: + warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning, stacklevel=2) + self._update_lr(epoch) + + def _update_lr(self, epoch: int | None = None) -> None: + with _enable_get_lr_call(self): + if epoch is None: + self.last_epoch += 1 + values = self.get_lr() + else: + self.last_epoch = epoch + if hasattr(self, "_get_closed_form_lr"): + values = cast(list[float | Tensor], self._get_closed_form_lr()) + else: + values = self.get_lr() + + for param_group, lr in zip(self.optimizer.param_groups, values, strict=True): + _update_param_group_val(param_group, "lr", lr) + + self._last_lr: list[float | Tensor] = _param_groups_val_list( + self.optimizer, "lr" + ) + + +def _warn_get_lr_called_within_step(lr_scheduler: LRScheduler) -> None: + if not lr_scheduler._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, " + "please use `get_last_lr()`.", + UserWarning, + stacklevel=2, + ) + + +# Including _LRScheduler for backwards compatibility +# Subclass instead of assign because we want __name__ of _LRScheduler to be _LRScheduler (assigning would make it LRScheduler). +class _LRScheduler(LRScheduler): + pass + + +class _enable_get_lr_call: + def __init__(self, o: LRScheduler) -> None: + self.o = o + + def __enter__(self) -> Self: + self.o._get_lr_called_within_step = True + return self + + def __exit__(self, type, value, traceback) -> None: + self.o._get_lr_called_within_step = False + + +class _initial_mode: + def __init__(self, o: LRScheduler) -> None: + self.o = o + + def __enter__(self): + self.o._is_initial = True + + def __exit__(self, type, value, traceback): + self.o._is_initial = False + + +class LambdaLR(LRScheduler): + """Sets the initial learning rate. + + The learning rate of each parameter group is set to the initial lr + times a given function. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_lambda (function or list): A function which computes a multiplicative + factor given an integer parameter epoch, or a list of such + functions, one for each group in optimizer.param_groups. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer has two groups. + >>> num_epochs = 100 + >>> lambda1 = lambda epoch: epoch // 30 + >>> lambda2 = lambda epoch: 0.95**epoch + >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2]) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + >>> + >>> # Alternatively, you can use a single lambda function for all groups. + >>> scheduler = LambdaLR(opt, lr_lambda=lambda epoch: epoch // 30) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/LambdaLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + lr_lambda: Callable[[int], float] | list[Callable[[int], float]], + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.optimizer = optimizer + + self.lr_lambdas: list[Callable[[int], float]] + if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple): + self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups) + else: + if len(lr_lambda) != len(optimizer.param_groups): + raise ValueError( + f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}" + ) + self.lr_lambdas = list(lr_lambda) + super().__init__(optimizer, last_epoch) + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "lr_lambdas") + } + state_dict["lr_lambdas"] = [None] * len(self.lr_lambdas) + + for idx, fn in enumerate(self.lr_lambdas): + if not isinstance(fn, types.FunctionType): + # pyrefly: ignore [unsupported-operation] + state_dict["lr_lambdas"][idx] = fn.__dict__.copy() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + lr_lambdas = state_dict.pop("lr_lambdas") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["lr_lambdas"] = lr_lambdas + + for idx, fn in enumerate(lr_lambdas): + if fn is not None: + self.lr_lambdas[idx].__dict__.update(fn) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the :attr:`base_lrs` by the outputs of the :attr:`lr_lambdas` at + :attr:`last_epoch`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + return [ + base_lr * lmbda(self.last_epoch) + for lmbda, base_lr in zip(self.lr_lambdas, self.base_lrs, strict=True) + ] + + +class MultiplicativeLR(LRScheduler): + """Multiply the learning rate of each parameter group by the factor given in the specified function. + + When last_epoch=-1, set initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_lambda (function or list): A function which computes a multiplicative + factor given an integer parameter epoch, or a list of such + functions, one for each group in optimizer.param_groups. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> lmbda = lambda epoch: 0.95 + >>> scheduler = MultiplicativeLR(optimizer, lr_lambda=lmbda) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/MultiplicativeLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + lr_lambda: Callable[[int], float] | list[Callable[[int], float]], + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.optimizer = optimizer + + self.lr_lambdas: list[Callable[[int], float]] + if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple): + self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups) + else: + if len(lr_lambda) != len(optimizer.param_groups): + raise ValueError( + f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}" + ) + self.lr_lambdas = list(lr_lambda) + for lr_lambda in self.lr_lambdas: + if not callable(lr_lambda): + raise TypeError( + f"lr_lambda should be a function, but got {type(lr_lambda).__name__}" + ) + super().__init__(optimizer, last_epoch) + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "lr_lambdas") + } + state_dict["lr_lambdas"] = [None] * len(self.lr_lambdas) + + for idx, fn in enumerate(self.lr_lambdas): + if not isinstance(fn, types.FunctionType): + # pyrefly: ignore [unsupported-operation] + state_dict["lr_lambdas"][idx] = fn.__dict__.copy() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + lr_lambdas = state_dict.pop("lr_lambdas") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["lr_lambdas"] = lr_lambdas + + for idx, fn in enumerate(lr_lambdas): + if fn is not None: + self.lr_lambdas[idx].__dict__.update(fn) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the current ``group["lr"]``\s in each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by the outputs of the + :attr:`lr_lambdas` at :attr:`last_epoch`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if not self._is_initial: + return [ + group["lr"] * lmbda(self.last_epoch) + for lmbda, group in zip( + self.lr_lambdas, self.optimizer.param_groups, strict=True + ) + ] + else: + return _param_groups_val_list(self.optimizer, "lr") + + +class StepLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma every step_size epochs. + + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + step_size (int): Period of learning rate decay. + gamma (float): Multiplicative factor of learning rate decay. + Default: 0.1. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch < 30 + >>> # lr = 0.005 if 30 <= epoch < 60 + >>> # lr = 0.0005 if 60 <= epoch < 90 + >>> # ... + >>> scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/StepLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + step_size: int, + gamma: float = 0.1, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.step_size = step_size + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + If the current epoch is a non-zero multiple of :attr:`step_size`, we + scale the current ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if (self.last_epoch == 0) or (self.last_epoch % self.step_size != 0): + return _param_groups_val_list(self.optimizer, "lr") + return [group["lr"] * self.gamma for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr * self.gamma ** (self.last_epoch // self.step_size) + for base_lr in self.base_lrs + ] + + +class MultiStepLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma once the number of epoch reaches one of the milestones. + + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + milestones (list): List of epoch indices. Must be increasing. + gamma (float): Multiplicative factor of learning rate decay. + Default: 0.1. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch < 30 + >>> # lr = 0.005 if 30 <= epoch < 80 + >>> # lr = 0.0005 if epoch >= 80 + >>> scheduler = MultiStepLR(optimizer, milestones=[30, 80], gamma=0.1) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/MultiStepLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + milestones: Iterable[int], + gamma: float = 0.1, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.milestones = Counter(milestones) + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + If the current epoch is in :attr:`milestones`, decays the + ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + If the current epoch appears in :attr:`milestones` ``n`` times, we + scale by :attr:`gamma` to the power of ``n`` + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch not in self.milestones: + return _param_groups_val_list(self.optimizer, "lr") + return [ + group["lr"] * self.gamma ** self.milestones[self.last_epoch] + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + milestones = sorted(self.milestones.elements()) + return [ + base_lr * self.gamma ** bisect_right(milestones, self.last_epoch) + for base_lr in self.base_lrs + ] + + +class ConstantLR(LRScheduler): + """Multiply the learning rate of each parameter group by a small constant factor. + + The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters. + Notice that such multiplication of the small constant factor can + happen simultaneously with other changes to the learning rate from outside this scheduler. + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + factor (float): The number we multiply learning rate until the milestone. Default: 1./3. + total_iters (int): The number of steps that the scheduler multiplies the learning rate by the factor. + Default: 5. + last_epoch (int): The index of the last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.025 if epoch == 0 + >>> # lr = 0.025 if epoch == 1 + >>> # lr = 0.025 if epoch == 2 + >>> # lr = 0.025 if epoch == 3 + >>> # ... + >>> # lr = 0.05 if epoch >= 40 + >>> scheduler = ConstantLR(optimizer, factor=0.5, total_iters=40) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ConstantLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + factor: float = 1.0 / 3, + total_iters: int = 5, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if factor > 1.0 or factor < 0: + raise ValueError( + "Constant multiplicative factor expected to be between 0 and 1." + ) + + self.factor = factor + self.total_iters = total_iters + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + When :attr:`last_epoch` is 0, this method scales the ``group["lr"]``\s + in each of the optimizer's :attr:`~torch.optim.Optimizer.param_groups` + by :attr:`factor`. Once :attr:`total_iters` is reached, it undoes this, + scaling by ``1 / factor``. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch == 0: + return [group["lr"] * self.factor for group in self.optimizer.param_groups] + + if self.last_epoch != self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + return [ + group["lr"] * (1.0 / self.factor) for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr + * (self.factor + (self.last_epoch >= self.total_iters) * (1 - self.factor)) + for base_lr in self.base_lrs + ] + + +class LinearLR(LRScheduler): + """Decays the learning rate of each parameter group by linearly changing small multiplicative factor. + + The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters. + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + start_factor (float): The number we multiply learning rate in the first epoch. + The multiplication factor changes towards end_factor in the following epochs. + Default: 1./3. + end_factor (float): The number we multiply learning rate at the end of linear changing + process. Default: 1.0. + total_iters (int): The number of iterations that multiplicative factor reaches to 1. + Default: 5. + last_epoch (int): The index of the last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.003687 if epoch == 0 + >>> # lr = 0.004875 if epoch == 1 + >>> # lr = 0.006062 if epoch == 2 + >>> # lr = 0.00725 if epoch == 3 + >>> # ... + >>> # lr = 0.05 if epoch >= 40 + >>> scheduler = LinearLR(optimizer, start_factor=0.05, total_iters=40) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/LinearLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + start_factor: float = 1.0 / 3, + end_factor: float = 1.0, + total_iters: int = 5, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if start_factor > 1.0 or start_factor <= 0: + raise ValueError( + "Starting multiplicative factor expected to be greater than 0 and less or equal to 1." + ) + + if end_factor > 1.0 or end_factor < 0: + raise ValueError( + "Ending multiplicative factor expected to be between 0 and 1." + ) + + self.start_factor = start_factor + self.end_factor = end_factor + self.total_iters = total_iters + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that successive steps + interpolate linearly from :attr:`start_factor` up to :attr:`end_factor` + across :attr:`total_iters` steps. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch == 0: + return [ + group["lr"] * self.start_factor for group in self.optimizer.param_groups + ] + + if self._is_initial or self.last_epoch > self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + return [ + group["lr"] + * ( + 1.0 + + (self.end_factor - self.start_factor) + / ( + self.total_iters * self.start_factor + + (self.last_epoch - 1) * (self.end_factor - self.start_factor) + ) + ) + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr + * ( + self.start_factor + + (self.end_factor - self.start_factor) + * min(self.total_iters, self.last_epoch) + / self.total_iters + ) + for base_lr in self.base_lrs + ] + + +class ExponentialLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma every epoch. + + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + gamma (float): Multiplicative factor of learning rate decay. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> scheduler = ExponentialLR(optimizer, gamma=0.95) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ExponentialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + gamma: float, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Multiplies the current ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + # when loading from a checkpoint, we don't want _initial_step (called from the constructor) + # to update the lr one more step ahead of itself. + if self._is_initial: + return _param_groups_val_list(self.optimizer, "lr") + return [group["lr"] * self.gamma for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [base_lr * self.gamma**self.last_epoch for base_lr in self.base_lrs] + + +class SequentialLR(LRScheduler): + """Contains a list of schedulers expected to be called sequentially during the optimization process. + + Specifically, the schedulers will be called according to the milestone points, which should provide exact + intervals by which each scheduler should be called at a given epoch. + + Args: + optimizer (Optimizer): Wrapped optimizer. + schedulers (list): List of chained schedulers. + milestones (list): List of integers that reflects milestone points. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.005 if epoch == 0 + >>> # lr = 0.005 if epoch == 1 + >>> # lr = 0.005 if epoch == 2 + >>> # ... + >>> # lr = 0.05 if epoch == 20 + >>> # lr = 0.045 if epoch == 21 + >>> # lr = 0.0405 if epoch == 22 + >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=20) + >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9) + >>> scheduler = SequentialLR( + ... optimizer, + ... schedulers=[scheduler1, scheduler2], + ... milestones=[20], + ... ) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/SequentialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + schedulers: list[LRScheduler], + milestones: list[int], + last_epoch: int = -1, + ) -> None: # noqa: D107 + if len(schedulers) < 1: + raise ValueError( + f"{self.__class__.__name__} expects at least one scheduler, but got no scheduler." + ) + + for scheduler_idx, scheduler in enumerate(schedulers): + if not hasattr(scheduler, "optimizer"): + raise TypeError( + f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute." + ) + if isinstance(scheduler, ReduceLROnPlateau): + raise ValueError( + f"{self.__class__.__name__} does not support `ReduceLROnPlateau` scheduler as it " + "requires additional kwargs to be specified when calling `step`, " + f"but got one at index {scheduler_idx} in the given schedulers sequence." + ) + if optimizer != scheduler.optimizer: + raise ValueError( + f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but " + f"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, " + f"which is different from {optimizer.__class__.__name__}." + ) + + if len(milestones) != len(schedulers) - 1: + raise ValueError( + "Sequential Schedulers expects number of schedulers provided to be one more " + f"than the number of milestone points, but got number of schedulers {len(schedulers)} and the " + f"number of milestones to be equal to {len(milestones)}" + ) + self._schedulers = schedulers + self._milestones = milestones + self.last_epoch = last_epoch + 1 + self.optimizer = optimizer + + # Reset learning rates back to initial values + for group in self.optimizer.param_groups: + _update_param_group_val(group, "lr", group["initial_lr"]) + + # "Undo" the step performed by other schedulers + self.recursive_undo() + + # Perform the initial step for only the first scheduler + self._schedulers[0]._initial_step() + + self._last_lr = schedulers[0].get_last_lr() + + def recursive_undo(self, sched=None) -> None: + """ + Recursively undo any step performed by the initialisation of + schedulers. + """ + scheds = self if sched is None else sched + + if hasattr(scheds, "_schedulers"): + for s in scheds._schedulers: + self.recursive_undo(s) + elif hasattr(scheds, "last_epoch"): + scheds.last_epoch -= 1 + + def step(self) -> None: # type: ignore[override] + """Perform a step.""" + self.last_epoch += 1 + idx = bisect_right(self._milestones, self.last_epoch) + scheduler = self._schedulers[idx] + if idx > 0 and self._milestones[idx - 1] == self.last_epoch: + scheduler._update_lr(0) + else: + scheduler.step() + + self._last_lr = scheduler.get_last_lr() + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The wrapped scheduler states will also be saved. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "_schedulers") + } + state_dict["_schedulers"] = [None] * len(self._schedulers) + + for idx, s in enumerate(self._schedulers): + # pyrefly: ignore [unsupported-operation] + state_dict["_schedulers"][idx] = s.state_dict() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + _schedulers = state_dict.pop("_schedulers") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["_schedulers"] = _schedulers + + for idx, s in enumerate(_schedulers): + self._schedulers[idx].load_state_dict(s) + + +class PolynomialLR(LRScheduler): + """Decays the learning rate of each parameter group using a polynomial function in the given total_iters. + + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + total_iters (int): The number of steps that the scheduler decays the learning rate. Default: 5. + power (float): The power of the polynomial. Default: 1.0. + + Example: + >>> # xdoctest: +SKIP("undefined vars") + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.0490 if epoch == 0 + >>> # lr = 0.0481 if epoch == 1 + >>> # lr = 0.0472 if epoch == 2 + >>> # ... + >>> # lr = 0.0 if epoch >= 50 + >>> scheduler = PolynomialLR(optimizer, total_iters=50, power=0.9) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/PolynomialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + total_iters: int = 5, + power: float = 1.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.total_iters = total_iters + self.power = power + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that the learning rates + follow + + .. math:: + \texttt{base\_lr} \cdot \left(1 - \frac{\texttt{last\_epoch}} + {\texttt{total\_iters}} \right)^\texttt{power} + + Returns the current learning rates unchanged after :attr:`total_iters` + is reached. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self._is_initial or self.last_epoch > self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + decay_factor = ( + (1.0 - self.last_epoch / self.total_iters) + / (1.0 - (self.last_epoch - 1) / self.total_iters) + ) ** self.power + return [group["lr"] * decay_factor for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + ( + base_lr + * (1.0 - min(self.total_iters, self.last_epoch) / self.total_iters) + ** self.power + ) + for base_lr in self.base_lrs + ] + + +class CosineAnnealingLR(LRScheduler): + r""" + Set the learning rate of each parameter group using a cosine annealing schedule. + + The learning rate is updated recursively using: + + .. math:: + \eta_{t+1} = \eta_{\min} + (\eta_t - \eta_{\min}) \cdot + \frac{1 + \cos\left(\frac{(T_{cur}+1) \pi}{T_{max}}\right)} + {1 + \cos\left(\frac{T_{cur} \pi}{T_{max}}\right)} + + This implements a recursive approximation of the closed-form schedule proposed in + `SGDR: Stochastic Gradient Descent with Warm Restarts`_: + + .. math:: + \eta_t = \eta_{\min} + \frac{1}{2}(\eta_{\max} - \eta_{\min}) \left( + 1 + \cos\left(\frac{T_{cur} \pi}{T_{max}}\right) \right) + + where: + + - :math:`\eta_t` is the learning rate at step :math:`t` + - :math:`T_{cur}` is the number of epochs since the last restart + - :math:`T_{max}` is the maximum number of epochs in a cycle + + Note: + Although SGDR includes periodic restarts, this implementation performs cosine annealing + **without restarts**, so :math:`T_{cur} = t` and increases monotonically with each call + to :meth:`step`. + + Args: + optimizer (Optimizer): Wrapped optimizer. + T_max (int): Maximum number of iterations. + eta_min (float): Minimum learning rate. Default: 0. + last_epoch (int): The index of the last epoch. Default: -1. + + .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: + https://arxiv.org/abs/1608.03983 + + Example: + >>> # xdoctest: +SKIP + >>> num_epochs = 100 + >>> scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CosineAnnealingLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + T_max: int, + eta_min: float = 0.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.T_max = T_max + self.eta_min = eta_min + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that their learning + rates approximate + + .. math:: + \texttt{eta\_min} + \frac{1}{2} (\texttt{base\_lr} - + \texttt{eta\_min}) \left(1 + \cos\left(\pi \cdot + \frac{\texttt{last\_epoch}}{\texttt{T\_max}}\right) \right) + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self._is_initial: + return _param_groups_val_list(self.optimizer, "lr") + elif self._step_count == 1 and self.last_epoch > 0: + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos((self.last_epoch) * math.pi / self.T_max)) + / 2 + for base_lr, group in zip( + self.base_lrs, self.optimizer.param_groups, strict=True + ) + ] + elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: + return [ + group["lr"] + + (base_lr - self.eta_min) * (1 - math.cos(math.pi / self.T_max)) / 2 + for base_lr, group in zip( + self.base_lrs, self.optimizer.param_groups, strict=True + ) + ] + return [ + (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / (1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) + * (group["lr"] - self.eta_min) + + self.eta_min + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / 2 + for base_lr in self.base_lrs + ] + + +class ChainedScheduler(LRScheduler): + """Chains a list of learning rate schedulers. + + Takes in a sequence of chainable learning rate schedulers and calls their + step() functions consecutively in just one call to step(). + + Args: + schedulers (sequence): sequence of chained schedulers. + optimizer (Optimizer, optional): Wrapped optimizer. Default: None. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch == 0 + >>> # lr = 0.0450 if epoch == 1 + >>> # lr = 0.0405 if epoch == 2 + >>> # ... + >>> # lr = 0.00675 if epoch == 19 + >>> # lr = 0.06078 if epoch == 20 + >>> # lr = 0.05470 if epoch == 21 + >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=20) + >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9) + >>> scheduler = ChainedScheduler([scheduler1, scheduler2], optimizer=optimizer) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ChainedScheduler.png + """ + + def __init__( + self, schedulers: Sequence[LRScheduler], optimizer: Optimizer | None = None + ) -> None: # noqa: D107 + if len(schedulers) < 1: + raise ValueError( + f"{self.__class__.__name__} expects at least one scheduler to be chained, but got no scheduler." + ) + + optimizer = optimizer or schedulers[0].optimizer + for scheduler_idx, scheduler in enumerate(schedulers): + if not hasattr(scheduler, "optimizer"): + raise TypeError( + f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute." + ) + if isinstance(scheduler, ReduceLROnPlateau): + raise ValueError( + f"{self.__class__.__name__} does not support `ReduceLROnPlateau` scheduler as it " + "requires additional kwargs to be specified when calling `step`, " + f"but got one at index {scheduler_idx} in the given schedulers sequence." + ) + if optimizer != scheduler.optimizer: + raise ValueError( + f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but " + f"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, " + f"which is different from {optimizer.__class__.__name__}." + ) + self._schedulers = schedulers + self.optimizer = optimizer + self._last_lr = _param_groups_val_list(self._schedulers[-1].optimizer, "lr") + + def step(self) -> None: # type: ignore[override] + """Perform a step.""" + for scheduler in self._schedulers: + scheduler.step() + self._last_lr = _param_groups_val_list(self._schedulers[-1].optimizer, "lr") + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The wrapped scheduler states will also be saved. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "_schedulers") + } + state_dict["_schedulers"] = [None] * len(self._schedulers) + + for idx, s in enumerate(self._schedulers): + # pyrefly: ignore [unsupported-operation] + state_dict["_schedulers"][idx] = s.state_dict() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + _schedulers = state_dict.pop("_schedulers") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["_schedulers"] = _schedulers + + for idx, s in enumerate(_schedulers): + self._schedulers[idx].load_state_dict(s) + + +class ReduceLROnPlateau(LRScheduler): + """Reduce learning rate when a metric has stopped improving. + + Models often benefit from reducing the learning rate by a factor + of 2-10 once learning stagnates. This scheduler reads a metrics + quantity and if no improvement is seen for a 'patience' number + of epochs, the learning rate is reduced. + + Args: + optimizer (Optimizer): Wrapped optimizer. + mode (str): One of `min`, `max`. In `min` mode, lr will + be reduced when the quantity monitored has stopped + decreasing; in `max` mode it will be reduced when the + quantity monitored has stopped increasing. Default: 'min'. + factor (float): Factor by which the learning rate will be + reduced. new_lr = lr * factor. Default: 0.1. + patience (int): The number of allowed epochs with no improvement after + which the learning rate will be reduced. + For example, consider the case of having no patience (`patience = 0`). + In the first epoch, a baseline is established and is always considered good as there's no previous baseline. + In the second epoch, if the performance is worse than the baseline, + we have what is considered an intolerable epoch. + Since the count of intolerable epochs (1) is greater than the patience level (0), + the learning rate is reduced at the end of this epoch. + From the third epoch onwards, the learning rate continues to be reduced at the end of each epoch + if the performance is worse than the baseline. If the performance improves or remains the same, + the learning rate is not adjusted. + Default: 10. + threshold (float): Threshold for measuring the new optimum, + to only focus on significant changes. Default: 1e-4. + threshold_mode (str): One of `rel`, `abs`. In `rel` mode, + dynamic_threshold = best * ( 1 + threshold ) in 'max' + mode or best * ( 1 - threshold ) in `min` mode. + In `abs` mode, dynamic_threshold = best + threshold in + `max` mode or best - threshold in `min` mode. Default: 'rel'. + cooldown (int): Number of epochs to wait before resuming + normal operation after lr has been reduced. Default: 0. + min_lr (float or list): A scalar or a list of scalars. A + lower bound on the learning rate of all param groups + or each group respectively. Default: 0. + eps (float): Minimal decay applied to lr. If the difference + between new and old lr is smaller than eps, the update is + ignored. Default: 1e-8. + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = ReduceLROnPlateau(optimizer, "min") + >>> for epoch in range(10): + >>> train(...) + >>> val_loss = validate(...) + >>> # Note that step should be called after validate() + >>> scheduler.step(val_loss) + + .. image:: ../scripts/lr_scheduler_images/ReduceLROnPlateau.png + """ + + def __init__( + self, + optimizer: Optimizer, + mode: Literal["min", "max"] = "min", + factor: float = 0.1, + patience: int = 10, + threshold: float = 1e-4, + threshold_mode: Literal["rel", "abs"] = "rel", + cooldown: int = 0, + min_lr: list[float] | float = 0, + eps: float = 1e-8, + ) -> None: # noqa: D107 + if factor >= 1.0: + raise ValueError("Factor should be < 1.0.") + self.factor = factor + + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + if isinstance(min_lr, (list, tuple)): + if len(min_lr) != len(optimizer.param_groups): + raise ValueError( + f"expected {len(optimizer.param_groups)} min_lrs, got {len(min_lr)}" + ) + self.default_min_lr = None + self.min_lrs = list(min_lr) + else: + # pyrefly: ignore [bad-assignment] + self.default_min_lr = min_lr + self.min_lrs = [min_lr] * len(optimizer.param_groups) + + self.patience = patience + self.cooldown = cooldown + self.eps = eps + self.last_epoch = 0 + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + self._init_is_better( + mode=mode, threshold=threshold, threshold_mode=threshold_mode + ) + self._reset() + + def _reset(self) -> None: + """Reset num_bad_epochs counter and cooldown counter.""" + self.best = self.mode_worse + self.cooldown_counter = 0 + self.num_bad_epochs = 0 + + def step(self, metrics: SupportsFloat, epoch=None) -> None: # type: ignore[override] + """Perform a step.""" + # convert `metrics` to float, in case it's a zero-dim Tensor + current = float(metrics) + if epoch is None: + epoch = self.last_epoch + 1 + else: + warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning, stacklevel=2) + self.last_epoch = epoch + + if self._is_better(current, self.best): + self.best = current + self.num_bad_epochs = 0 + else: + self.num_bad_epochs += 1 + + if self.in_cooldown: + self.cooldown_counter -= 1 + self.num_bad_epochs = 0 # ignore any bad epochs in cooldown + + if self.num_bad_epochs > self.patience: + self._reduce_lr(epoch) + self.cooldown_counter = self.cooldown + self.num_bad_epochs = 0 + + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + + def _reduce_lr(self, epoch) -> None: + if len(self.optimizer.param_groups) != len(self.min_lrs): + if self.default_min_lr is None: + raise RuntimeError( + "The number of param groups in the `optimizer` " + f"({len(self.optimizer.param_groups)}) differs " + f"from when `ReduceLROnPlateau` was initialized " + f"({len(self.min_lrs)}), usually due to a new " + "param group being added to the optimizer. Please " + "modify the `min_lrs` field to match the length " + "of the `optimizer` param groups." + ) + else: + # pyrefly: ignore [bad-assignment] + self.min_lrs = [self.default_min_lr] * len(self.optimizer.param_groups) + + for i, param_group in enumerate(self.optimizer.param_groups): + old_lr = float(param_group["lr"]) + new_lr = max(old_lr * self.factor, self.min_lrs[i]) + if old_lr - new_lr > self.eps: + _update_param_group_val(param_group, "lr", new_lr) + + @property + def in_cooldown(self): # noqa: D102 + return self.cooldown_counter > 0 + + def _is_better(self, a, best): # noqa: D102 + if self.mode == "min" and self.threshold_mode == "rel": + rel_epsilon = 1.0 - self.threshold + return a < best * rel_epsilon + + elif self.mode == "min" and self.threshold_mode == "abs": + return a < best - self.threshold + + elif self.mode == "max" and self.threshold_mode == "rel": + rel_epsilon = self.threshold + 1.0 + return a > best * rel_epsilon + + else: # mode == 'max' and epsilon_mode == 'abs': + return a > best + self.threshold + + def _init_is_better(self, mode, threshold, threshold_mode) -> None: + if mode not in {"min", "max"}: + raise ValueError("mode " + mode + " is unknown!") + if threshold_mode not in {"rel", "abs"}: + raise ValueError("threshold mode " + threshold_mode + " is unknown!") + + # the worse value for the chosen mode + if mode == "min": + self.mode_worse = inf + else: # mode == 'max': + self.mode_worse = -inf + + self.mode = mode + self.threshold = threshold + self.threshold_mode = threshold_mode + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state.""" + self.__dict__.update(state_dict) + self._init_is_better( + mode=self.mode, threshold=self.threshold, threshold_mode=self.threshold_mode + ) + + +class CyclicLR(LRScheduler): + r"""Sets the learning rate of each parameter group according to cyclical learning rate policy (CLR). + + The policy cycles the learning rate between two boundaries with a constant frequency, + as detailed in the paper `Cyclical Learning Rates for Training Neural Networks`_. + The distance between the two boundaries can be scaled on a per-iteration + or per-cycle basis. + + Cyclical learning rate policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This class has three built-in policies, as put forth in the paper: + + * "triangular": A basic triangular cycle without amplitude scaling. + * "triangular2": A basic triangular cycle that scales initial amplitude by half each cycle. + * "exp_range": A cycle that scales initial amplitude by :math:`\text{gamma}^{\text{cycle iterations}}` + at each cycle iteration. + + This implementation was adapted from the github repo: `bckenstler/CLR`_ + + Args: + optimizer (Optimizer): Wrapped optimizer. + base_lr (float or list): Initial learning rate which is the + lower boundary in the cycle for each parameter group. + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_lr - base_lr). + The lr at any cycle is the sum of base_lr + and some scaling of the amplitude; therefore + max_lr may not actually be reached depending on + scaling function. + step_size_up (int): Number of training iterations in the + increasing half of a cycle. Default: 2000 + step_size_down (int): Number of training iterations in the + decreasing half of a cycle. If step_size_down is None, + it is set to step_size_up. Default: None + mode (str): One of {triangular, triangular2, exp_range}. + Values correspond to policies detailed above. + If scale_fn is not None, this argument is ignored. + Default: 'triangular' + gamma (float): Constant in 'exp_range' scaling function: + gamma**(cycle iterations) + Default: 1.0 + scale_fn (function): Custom scaling policy defined by a single + argument lambda function, where + 0 <= scale_fn(x) <= 1 for all x >= 0. + If specified, then 'mode' is ignored. + Default: None + scale_mode (str): {'cycle', 'iterations'}. + Defines whether scale_fn is evaluated on + cycle number or cycle iterations (training + iterations since start of cycle). + Default: 'cycle' + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'base_momentum' and 'max_momentum'. + Default: True + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.8 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + The momentum at any cycle is the difference of max_momentum + and some scaling of the amplitude; therefore + base_momentum may not actually be reached depending on + scaling function. Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is 'max_momentum' + and learning rate is 'base_lr' + Default: 0.9 + last_epoch (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_epoch=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = torch.optim.lr_scheduler.CyclicLR( + ... optimizer, + ... base_lr=0.01, + ... max_lr=0.1, + ... step_size_up=10, + ... ) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CyclicLR.png + + .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186 + .. _bckenstler/CLR: https://github.com/bckenstler/CLR + """ + + def __init__( + self, + optimizer: Optimizer, + base_lr: float | list[float], + max_lr: float | list[float], + step_size_up: int = 2000, + step_size_down: int | None = None, + mode: Literal["triangular", "triangular2", "exp_range"] = "triangular", + gamma: float = 1.0, + scale_fn: Callable[[float], float] | None = None, + scale_mode: Literal["cycle", "iterations"] = "cycle", + cycle_momentum: bool = True, + base_momentum: float = 0.8, + max_momentum: float = 0.9, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + base_lrs = _format_param("base_lr", optimizer, base_lr) + if last_epoch == -1: + for lr, group in zip(base_lrs, optimizer.param_groups, strict=True): + _update_param_group_val(group, "lr", lr) + + self.max_lrs = _format_param("max_lr", optimizer, max_lr) + + # pyrefly: ignore [bad-assignment] + step_size_up = float(step_size_up) + step_size_down = ( + # pyrefly: ignore [bad-assignment] + float(step_size_down) if step_size_down is not None else step_size_up + ) + # pyrefly: ignore [unsupported-operation] + self.total_size = step_size_up + step_size_down + self.step_ratio = step_size_up / self.total_size + + if mode not in ["triangular", "triangular2", "exp_range"] and scale_fn is None: + raise ValueError("mode is invalid and scale_fn is None") + + self.mode = mode + self.gamma = gamma + + self._scale_fn_ref: Callable[[float], float] + self._scale_fn_custom = scale_fn + self.scale_mode = scale_mode + self._init_scale_fn() + + self.cycle_momentum = cycle_momentum + if cycle_momentum: + if ( + "momentum" not in optimizer.defaults + and "betas" not in optimizer.defaults + ): + raise ValueError( + "optimizer must support momentum or beta1 with `cycle_momentum` option enabled" + ) + + self.use_beta1 = "betas" in self.optimizer.defaults + self.base_momentums = _format_param( + "base_momentum", optimizer, base_momentum + ) + self.max_momentums = _format_param("max_momentum", optimizer, max_momentum) + if last_epoch == -1: + for m_momentum, b_momentum, group in zip( + self.max_momentums, + self.base_momentums, + optimizer.param_groups, + strict=True, + ): + if self.use_beta1: + group["betas"] = (m_momentum, *group["betas"][1:]) + else: + group["momentum"] = m_momentum + group["max_momentum"] = m_momentum + group["base_momentum"] = b_momentum + + super().__init__(optimizer, last_epoch) + self.base_lrs = base_lrs + + def _init_scale_fn(self) -> None: + if self._scale_fn_custom is not None: + return + if self.mode == "triangular": + self._scale_fn_ref = self._triangular_scale_fn + self.scale_mode = "cycle" + elif self.mode == "triangular2": + self._scale_fn_ref = self._triangular2_scale_fn + self.scale_mode = "cycle" + elif self.mode == "exp_range": + self._scale_fn_ref = partial(self._exp_range_scale_fn, self.gamma) + self.scale_mode = "iterations" + + def scale_fn(self, x) -> float: + """Get the scaling policy.""" + if self._scale_fn_custom is not None: + return self._scale_fn_custom(x) + else: + return self._scale_fn_ref(x) # static method + + @staticmethod + def _triangular_scale_fn(x: float) -> float: + return 1.0 + + @staticmethod + def _triangular2_scale_fn(x: float) -> float: + return 1 / (2.0 ** (x - 1)) + + @staticmethod + def _exp_range_scale_fn(gamma: float, x: float) -> float: + return gamma**x + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Advances each ``group["lr"]`` in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` along a cycle between the + group's ``base_lr`` and ``max_lr`` using :meth:`scale_fn`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + This method treats :attr:`last_epoch` as the index of the previous + batch. + + .. note:: + When :attr:`cycle_momentum` is ``True``, this method has a side + effect of updating the optimizer's momentum. + """ + _warn_get_lr_called_within_step(self) + + cycle = math.floor(1 + self.last_epoch / self.total_size) + x = 1.0 + self.last_epoch / self.total_size - cycle + if x <= self.step_ratio: + scale_factor = x / self.step_ratio + else: + scale_factor = (x - 1) / (self.step_ratio - 1) + + lrs = [] + for base_lr, max_lr in zip(self.base_lrs, self.max_lrs, strict=True): + base_height = (max_lr - base_lr) * scale_factor + if self.scale_mode == "cycle": + lr = base_lr + base_height * self.scale_fn(cycle) + else: + lr = base_lr + base_height * self.scale_fn(self.last_epoch) + lrs.append(lr) + + if self.cycle_momentum: + momentums = [] + for base_momentum, max_momentum in zip( + self.base_momentums, self.max_momentums, strict=True + ): + base_height = (max_momentum - base_momentum) * scale_factor + if self.scale_mode == "cycle": + momentum = max_momentum - base_height * self.scale_fn(cycle) + else: + momentum = max_momentum - base_height * self.scale_fn( + self.last_epoch + ) + momentums.append(momentum) + for param_group, momentum in zip( + self.optimizer.param_groups, momentums, strict=True + ): + if self.use_beta1: + param_group["betas"] = (momentum, *param_group["betas"][1:]) + else: + param_group["momentum"] = momentum + + return lrs + + @override + def state_dict(self) -> dict[str, Any]: # noqa: D102 + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + """ + state = super().state_dict() + # We are dropping the `_scale_fn_ref` attribute because it is a + # `weakref.WeakMethod` and can't be pickled. + state.pop("_scale_fn_ref", None) + fn = state.pop("_scale_fn_custom") + state["_scale_fn_custom"] = None + if fn is not None and not isinstance(fn, types.FunctionType): + # The _scale_fn_custom will only be saved if it is a callable object + # and not if it is a function or lambda. + state["_scale_fn_custom"] = fn.__dict__.copy() + + return state + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state.""" + fn = state_dict.pop("_scale_fn_custom") + super().load_state_dict(state_dict) + if fn is not None: + self._scale_fn_custom.__dict__.update(fn) + self._init_scale_fn() + + +class CosineAnnealingWarmRestarts(LRScheduler): + r"""Set the learning rate of each parameter group using a cosine annealing schedule. + + The :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}` + is the number of epochs since the last restart and :math:`T_{i}` is the number + of epochs between two warm restarts in SGDR: + + .. math:: + \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 + + \cos\left(\frac{T_{cur}}{T_{i}}\pi\right)\right) + + When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`. + When :math:`T_{cur}=0` after restart, set :math:`\eta_t=\eta_{max}`. + + It has been proposed in + `SGDR: Stochastic Gradient Descent with Warm Restarts`_. + + Args: + optimizer (Optimizer): Wrapped optimizer. + T_0 (int): Number of iterations until the first restart. + T_mult (int, optional): A factor by which :math:`T_{i}` increases after a restart. Default: 1. + eta_min (float, optional): Minimum learning rate. Default: 0. + last_epoch (int, optional): The index of the last epoch. Default: -1. + + .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: + https://arxiv.org/abs/1608.03983 + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.05) + >>> scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( + ... optimizer, T_0=20 + ... ) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CosineAnnealingWarmRestarts.png + """ + + def __init__( + self, + optimizer: Optimizer, + T_0: int, + T_mult: int = 1, + eta_min: float = 0.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if T_0 <= 0 or not isinstance(T_0, int): + raise ValueError(f"Expected positive integer T_0, but got {T_0}") + if T_mult < 1 or not isinstance(T_mult, int): + raise ValueError(f"Expected integer T_mult >= 1, but got {T_mult}") + if not isinstance(eta_min, (float, int)): + raise ValueError( + f"Expected float or int eta_min, but got {eta_min} of type {type(eta_min)}" + ) + self.T_0 = T_0 + self.T_i = T_0 + self.T_mult = T_mult + self.eta_min = eta_min + self.T_cur = last_epoch + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Computes learning rates for the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` following: + + .. math:: + \texttt{eta\_min} + \frac{1}{2}(\texttt{base\_lr} - + \texttt{eta\_min})\left(1 + \cos\left(\pi \cdot + \frac{\texttt{T\_cur}}{\texttt{T\_i}}\right)\right) + + Where :attr:`T_cur` is the number of epochs since the last restart and + :attr:`T_i` is the number of epochs between two restarts. Both + :attr:`T_cur` and :attr:`T_i` are updated in :meth:`step`, and + :attr:`T_i` becomes :attr:`T_mult` times larger after each restart. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.T_cur / self.T_i)) + / 2 + for base_lr in self.base_lrs + ] + + @override + def step(self, epoch=None) -> None: + """Step could be called after every batch update. + + Example: + >>> # xdoctest: +SKIP("Undefined vars") + >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult) + >>> iters = len(dataloader) + >>> for epoch in range(20): + >>> for i, sample in enumerate(dataloader): + >>> inputs, labels = sample['inputs'], sample['labels'] + >>> optimizer.zero_grad() + >>> outputs = net(inputs) + >>> loss = criterion(outputs, labels) + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.step(epoch + i / iters) + + This function can be called in an interleaved way. + + Example: + >>> # xdoctest: +SKIP("Undefined vars") + >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult) + >>> for epoch in range(20): + >>> scheduler.step() + >>> scheduler.step(26) + >>> scheduler.step() # scheduler.step(27), instead of scheduler(20) + """ + if epoch is None and self.last_epoch < 0: + epoch = 0 + + if epoch is None: + epoch = self.last_epoch + 1 + self.T_cur = self.T_cur + 1 + if self.T_cur >= self.T_i: + self.T_cur = self.T_cur % self.T_i + self.T_i = self.T_i * self.T_mult + else: + if epoch < 0: + raise ValueError(f"Expected non-negative epoch, but got {epoch}") + if epoch >= self.T_0: + if self.T_mult == 1: + self.T_cur = epoch % self.T_0 + else: + n = int( + math.log( + (epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult + ) + ) + self.T_cur = epoch - self.T_0 * (self.T_mult**n - 1) / ( + self.T_mult - 1 + ) + self.T_i = self.T_0 * self.T_mult ** (n) + else: + self.T_i = self.T_0 + self.T_cur = epoch + self.last_epoch = math.floor(epoch) + + with _enable_get_lr_call(self): + for param_group, lr in zip( + self.optimizer.param_groups, self.get_lr(), strict=True + ): + _update_param_group_val(param_group, "lr", lr) + + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + + +class _SchedulePhase(TypedDict): + end_step: float + start_lr: str + end_lr: str + start_momentum: str + end_momentum: str + + +class OneCycleLR(LRScheduler): + r"""Sets the learning rate of each parameter group according to the 1cycle learning rate policy. + + The 1cycle policy anneals the learning rate from an initial learning rate to some maximum + learning rate and then from that maximum learning rate to some minimum learning rate much + lower than the initial learning rate. + This policy was initially described in the paper `Super-Convergence: + Very Fast Training of Neural Networks Using Large Learning Rates`_. + + The 1cycle learning rate policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This scheduler is not chainable. + + Note also that the total number of steps in the cycle can be determined in one + of two ways (listed in order of precedence): + + #. A value for total_steps is explicitly provided. + #. A number of epochs (epochs) and a number of steps per epoch + (steps_per_epoch) are provided. + In this case, the number of total steps is inferred by + total_steps = epochs * steps_per_epoch + + You must either provide a value for total_steps or provide a value for both + epochs and steps_per_epoch. + + The default behaviour of this scheduler follows the fastai implementation of 1cycle, which + claims that "unpublished work has shown even better results by using only two phases". To + mimic the behaviour of the original paper instead, set ``three_phase=True``. + + Args: + optimizer (Optimizer): Wrapped optimizer. + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. + total_steps (int): The total number of steps in the cycle. Note that + if a value is not provided here, then it must be inferred by providing + a value for epochs and steps_per_epoch. + Default: None + epochs (int): The number of epochs to train for. This is used along + with steps_per_epoch in order to infer the total number of steps in the cycle + if a value for total_steps is not provided. + Default: None + steps_per_epoch (int): The number of steps per epoch to train for. This is + used along with epochs in order to infer the total number of steps in the + cycle if a value for total_steps is not provided. + Default: None + pct_start (float): The percentage of the cycle (in number of steps) spent + increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: "cos" for cosine annealing, "linear" for + linear annealing. + Default: 'cos' + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'base_momentum' and 'max_momentum'. + Default: True + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.85 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is 'max_momentum' + and learning rate is 'base_lr' + Default: 0.95 + div_factor (float): Determines the initial learning rate via + initial_lr = max_lr/div_factor + Default: 25 + final_div_factor (float): Determines the minimum learning rate via + min_lr = initial_lr/final_div_factor + Default: 1e4 + three_phase (bool): If ``True``, use a third phase of the schedule to annihilate the + learning rate according to 'final_div_factor' instead of modifying the second + phase (the first two phases will be symmetrical about the step indicated by + 'pct_start'). + last_epoch (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_epoch=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> # xdoctest: +SKIP + >>> data_loader = torch.utils.data.DataLoader(...) + >>> optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9) + >>> scheduler = torch.optim.lr_scheduler.OneCycleLR( + ... optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10 + ... ) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> optimizer.step() + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/OneCycleLR.png + + .. _Super-Convergence\: Very Fast Training of Neural Networks Using Large Learning Rates: + https://arxiv.org/abs/1708.07120 + """ + + def __init__( + self, + optimizer: Optimizer, + max_lr: float | list[float], + total_steps: int | None = None, + epochs: int | None = None, + steps_per_epoch: int | None = None, + pct_start: float = 0.3, + anneal_strategy: Literal["cos", "linear"] = "cos", + cycle_momentum: bool = True, + base_momentum: float | list[float] = 0.85, + max_momentum: float | list[float] = 0.95, + div_factor: float = 25.0, + final_div_factor: float = 1e4, + three_phase: bool = False, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Validate optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + # Validate total_steps + if total_steps is not None: + if total_steps <= 0 or not isinstance(total_steps, int): + raise ValueError( + f"Expected positive integer total_steps, but got {total_steps}" + ) + self.total_steps = total_steps + elif epochs is not None and steps_per_epoch is not None: + if not isinstance(epochs, int) or epochs <= 0: + raise ValueError(f"Expected positive integer epochs, but got {epochs}") + if not isinstance(steps_per_epoch, int) or steps_per_epoch <= 0: + raise ValueError( + f"Expected positive integer steps_per_epoch, but got {steps_per_epoch}" + ) + self.total_steps = epochs * steps_per_epoch + else: + raise ValueError( + "You must define either total_steps OR (epochs AND steps_per_epoch)" + ) + + self._schedule_phases: list[_SchedulePhase] + if three_phase: + self._schedule_phases = [ + { + "end_step": float(pct_start * self.total_steps) - 1, + "start_lr": "initial_lr", + "end_lr": "max_lr", + "start_momentum": "max_momentum", + "end_momentum": "base_momentum", + }, + { + "end_step": float(2 * pct_start * self.total_steps) - 2, + "start_lr": "max_lr", + "end_lr": "initial_lr", + "start_momentum": "base_momentum", + "end_momentum": "max_momentum", + }, + { + "end_step": self.total_steps - 1, + "start_lr": "initial_lr", + "end_lr": "min_lr", + "start_momentum": "max_momentum", + "end_momentum": "max_momentum", + }, + ] + else: + self._schedule_phases = [ + { + "end_step": float(pct_start * self.total_steps) - 1, + "start_lr": "initial_lr", + "end_lr": "max_lr", + "start_momentum": "max_momentum", + "end_momentum": "base_momentum", + }, + { + "end_step": self.total_steps - 1, + "start_lr": "max_lr", + "end_lr": "min_lr", + "start_momentum": "base_momentum", + "end_momentum": "max_momentum", + }, + ] + + # Validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError( + f"Expected float between 0 and 1 pct_start, but got {pct_start}" + ) + + # Validate anneal_strategy + if anneal_strategy not in ["cos", "linear"]: + raise ValueError( + f"anneal_strategy must be one of 'cos' or 'linear', instead got {anneal_strategy}" + ) + else: + self._anneal_func_type = anneal_strategy + + # Initialize learning rate variables + max_lrs = _format_param("max_lr", self.optimizer, max_lr) + if last_epoch == -1: + for idx, group in enumerate(self.optimizer.param_groups): + group["initial_lr"] = max_lrs[idx] / div_factor + group["max_lr"] = max_lrs[idx] + group["min_lr"] = group["initial_lr"] / final_div_factor + + # Initialize momentum variables + self.cycle_momentum = cycle_momentum + if self.cycle_momentum: + if ( + "momentum" not in self.optimizer.defaults + and "betas" not in self.optimizer.defaults + ): + raise ValueError( + "optimizer must support momentum or beta1 with `cycle_momentum` option enabled" + ) + self.use_beta1 = "betas" in self.optimizer.defaults + max_momentums = _format_param("max_momentum", optimizer, max_momentum) + base_momentums = _format_param("base_momentum", optimizer, base_momentum) + if last_epoch == -1: + for m_momentum, b_momentum, group in zip( + max_momentums, base_momentums, optimizer.param_groups, strict=True + ): + if self.use_beta1: + group["betas"] = (m_momentum, *group["betas"][1:]) + else: + group["momentum"] = m_momentum + group["max_momentum"] = m_momentum + group["base_momentum"] = b_momentum + + super().__init__(optimizer, last_epoch) + + def _anneal_func(self, *args, **kwargs): + if hasattr(self, "_anneal_func_type"): + if self._anneal_func_type == "cos": + return self._annealing_cos(*args, **kwargs) + elif self._anneal_func_type == "linear": + return self._annealing_linear(*args, **kwargs) + else: + raise ValueError(f"Unknown _anneal_func_type: {self._anneal_func_type}") + else: + # For BC + return self.anneal_func(*args, **kwargs) # type: ignore[attr-defined] + + @staticmethod + def _annealing_cos(start, end, pct): + """Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.""" + cos_out = math.cos(math.pi * pct) + 1 + return end + (start - end) / 2.0 * cos_out + + @staticmethod + def _annealing_linear(start, end, pct): + """Linearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.""" + return (end - start) * pct + start + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Finds the appropriate :attr:`_schedule_phases` entry for the current + step and interpolates between its ``start_lr`` and ``end_lr`` using + :meth:`_anneal_func`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + When :attr:`cycle_momentum` is ``True``, this method has a side + effect of updating the optimizer's momentum. + """ + _warn_get_lr_called_within_step(self) + + lrs = [] + step_num = self.last_epoch + + if step_num > self.total_steps: + raise ValueError( + f"Tried to step {step_num} times. The specified number of total steps is {self.total_steps}" + ) + + for group in self.optimizer.param_groups: + start_step = 0.0 + for i, phase in enumerate(self._schedule_phases): + end_step = phase["end_step"] + if step_num <= end_step or i == len(self._schedule_phases) - 1: + pct = (step_num - start_step) / (end_step - start_step) + computed_lr = self._anneal_func( + group[phase["start_lr"]], group[phase["end_lr"]], pct + ) + if self.cycle_momentum: + computed_momentum = self._anneal_func( + group[phase["start_momentum"]], + group[phase["end_momentum"]], + pct, + ) + break + start_step = phase["end_step"] + + lrs.append(computed_lr) # type: ignore[possibly-undefined] + if self.cycle_momentum: + if self.use_beta1: + group["betas"] = (computed_momentum, *group["betas"][1:]) # type: ignore[possibly-undefined] + else: + group["momentum"] = computed_momentum # type: ignore[possibly-undefined] + + return lrs diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/nadam.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/nadam.py new file mode 100644 index 0000000000000000000000000000000000000000..b2341adc11a7150d0e23324921b6ee6fad26bc33 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/nadam.py @@ -0,0 +1,673 @@ +# mypy: allow-untyped-defs +r"""Implementation for the NAdam algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _stack_if_compiling, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["NAdam", "nadam"] + + +class NAdam(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 2e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + momentum_decay: float = 4e-3, + decoupled_weight_decay: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= momentum_decay: + raise ValueError(f"Invalid momentum_decay value: {momentum_decay}") + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "momentum_decay": momentum_decay, + "decoupled_weight_decay": decoupled_weight_decay, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("capturable", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0: + if not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + if not torch.is_tensor(p_state["mu_product"]): + mu_prod_val = p_state["mu_product"] + p_state["mu_product"] = ( + torch.tensor( + mu_prod_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(mu_prod_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("NAdam does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + # note(crcrpar): [special device hosting for step] + # Deliberately host `step` and `mu_product` on CPU if capturable is False. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + state["mu_product"] = ( + torch.ones((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(1.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + mu_products.append(state["mu_product"]) + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + mu_products: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = cast(tuple[float, float], group["betas"]) + + has_complex = self._init_group( + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + ) + + nadam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + momentum_decay=group["momentum_decay"], + eps=group["eps"], + maximize=group["maximize"], + decoupled_weight_decay=group["decoupled_weight_decay"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + has_complex=has_complex, + ) + + return loss + + +NAdam.__doc__ = ( + r"""Implements NAdam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)}, + \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)} \\ + &\hspace{13mm} \: \textit{decoupled\_weight\_decay}, \:\textit{maximize} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0 \leftarrow 0 \text{ ( second moment)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay} \\ + &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{10mm}\textbf{else} \\ + &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{t \psi} \big) \\ + &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex] + & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i}) \\ + &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + momentum_decay (float, optional): momentum momentum_decay (default: 4e-3) + decoupled_weight_decay (bool, optional): whether to decouple the weight + decay as in AdamW to obtain NAdamW. If True, the algorithm does not + accumulate weight decay in the momentum nor variance. (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + + .. _Incorporating Nesterov Momentum into Adam: + https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + + """ +) + + +def _single_tensor_nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, + decoupled_weight_decay: bool, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + mu_product = mu_products[i] + step_t = state_steps[i] + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == mu_product.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params, mu_products and state_steps must be " + f"on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if capturable: + step = step_t + else: + step = _get_value(step_t) + + bias_correction2 = 1 - beta2**step + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + param.mul_(1 - lr * weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + # calculate the momentum cache \mu^{t} and \mu^{t+1} + mu = beta1 * (1.0 - 0.5 * (0.96 ** (step * momentum_decay))) + mu_next = beta1 * (1.0 - 0.5 * (0.96 ** ((step + 1) * momentum_decay))) + + # update mu_product + mu_product *= mu + + # decay the first and second moment running average coefficient + exp_avg.lerp_(grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + denom = exp_avg_sq.div(bias_correction2).sqrt() + + if differentiable or capturable: + denom = denom.add(eps) + # Make autograd track the operations + # by updating the grad and exp_avg directly and not using the + # scalar "value" argument of addcdiv. + mu_product_next = mu_product * mu_next + grad = grad * (-lr * (1.0 - mu) / (1.0 - mu_product)) + exp_avg = exp_avg * (-lr * mu_next / (1.0 - mu_product_next)) + param.addcdiv_(grad, denom) + param.addcdiv_(exp_avg, denom) + else: + mu_product_next = _get_value(mu_product) * mu_next + denom.add_(eps) + param.addcdiv_( + grad, denom, value=(-lr * (1.0 - mu) / (1.0 - _get_value(mu_product))) + ) + param.addcdiv_( + exp_avg, + denom, + value=cast(float, (-lr * mu_next) / (1.0 - mu_product_next)), + ) + + +def _multi_tensor_nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, + decoupled_weight_decay: bool, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == mp.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, mp, step in zip(params, mu_products, state_steps, strict=True) + ): + raise AssertionError( + "If capturable=True, " + "params, mu_products, and state_steps must be on supported devices: " + f"{capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, mu_products, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_avg_sqs_, + grouped_mu_products_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_avg_sqs = cast(list[Tensor], grouped_exp_avg_sqs_) + grouped_mu_products = cast(list[Tensor], grouped_mu_products_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # handle complex + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_avg_sqs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + torch._foreach_mul_(grouped_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_( + grouped_grads, grouped_params, alpha=weight_decay + ) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + torch._foreach_mul_(grouped_exp_avg_sqs, beta2) + torch._foreach_addcmul_( + grouped_exp_avg_sqs, grouped_grads, grouped_grads, 1 - beta2 + ) + + exp_avg_sq_sqrt = torch._foreach_sqrt(grouped_exp_avg_sqs) + + bias_correction_sqrt: tuple[Tensor, ...] | list[Tensor] + mus: tuple[Tensor, ...] | list[Tensor] + mu_nexts: tuple[Tensor, ...] | list[Tensor] + if capturable: + # mus will be beta1 * (1 - 0.5 * 0.96 ** (step * momentum_decay)) + exponent = torch._foreach_mul(grouped_state_steps, momentum_decay) + mus = torch._foreach_pow(0.96, exponent) + torch._foreach_mul_(mus, -0.5) + torch._foreach_add_(mus, 1.0) + torch._foreach_mul_(mus, beta1) + + # mu_nexts will be beta1 * (1 - 0.5 * 0.96 ** ((step + 1) * momentum_decay)) + torch._foreach_add_(exponent, momentum_decay) + mu_nexts = torch._foreach_pow(0.96, exponent) + torch._foreach_mul_(mu_nexts, -0.5) + torch._foreach_add_(mu_nexts, 1.0) + torch._foreach_mul_(mu_nexts, beta1) + + # save peak memory as we don't need exponent anymore + del exponent + + bias_correction_sqrt = torch._foreach_pow(beta2, grouped_state_steps) + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_correction_sqrt, 1.0) + torch._foreach_neg_(bias_correction_sqrt) + torch._foreach_sqrt_(bias_correction_sqrt) + else: + bias_correction_sqrt = [ + (1 - beta2 ** _get_value(step)) ** 0.5 for step in grouped_state_steps + ] + mus = [ + beta1 * (1.0 - 0.5 * (0.96 ** (_get_value(step) * momentum_decay))) + for step in grouped_state_steps + ] + mu_nexts = [ + beta1 + * (1.0 - 0.5 * (0.96 ** ((_get_value(step) + 1) * momentum_decay))) + for step in grouped_state_steps + ] + + # update mu_products + torch._foreach_mul_(grouped_mu_products, mus) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + + # explicitly delete bias_correction refs to save memory + del bias_correction_sqrt + + if capturable: + # Build up the step_size multiplier for grad, reusing mus' memory + torch._foreach_sub_(mus, 1.0) + torch._foreach_mul_(mus, lr) + # foreach_sub doesn't allow a scalar as the first arg + denom = torch._foreach_sub(grouped_mu_products, 1.0) + torch._foreach_neg_(denom) + torch._foreach_div_(mus, denom) + # - lr * (1 - mu) / (1 - mu_product) + step_size_grads = mus + # explicitly delete denom to save memory + del denom + + # Build up the step_size multiplier for exp_avg, reusing mu_nexts' memory + denom = torch._foreach_mul(grouped_mu_products, mu_nexts) + torch._foreach_mul_(mu_nexts, lr) + # foreach_sub doesn't allow a scalar as the first arg, but it's okay because + # we need a negative here anyway + torch._foreach_sub_(denom, 1.0) + torch._foreach_div_(mu_nexts, denom) + # - lr * mu_next / (1 - mu_product * mu_next) + step_size_expavg = mu_nexts + # explicitly delete denom to save memory + del denom + + # we cannot inplace into step_size_grads cuz it is a list of ScalarTensors + # and mul'ing with grouped_grads will result in a list of bigger Tensors + numerator = torch._foreach_mul(step_size_grads, grouped_grads) + torch._foreach_addcmul_(numerator, step_size_expavg, grouped_exp_avgs) + + # finally, update params + torch._foreach_addcdiv_(grouped_params, numerator, exp_avg_sq_sqrt) + else: + step_size_grads = _stack_if_compiling( + [ + (_get_value(lr) * (1.0 - mu) / (1.0 - _get_value(mu_product))) * -1 + for mu_product, mu in zip(grouped_mu_products, mus, strict=True) + ] + ) + step_size_expavg = _stack_if_compiling( + [ + ( + _get_value(lr) + * mu_next + / (1.0 - _get_value(mu_product) * mu_next) + ) + * -1 + for mu_product, mu_next in zip( + grouped_mu_products, mu_nexts, strict=True + ) + ] + ) + + torch._foreach_addcdiv_( + grouped_params, + grouped_grads, + exp_avg_sq_sqrt, + step_size_grads, # type: ignore[arg-type] + ) + torch._foreach_addcdiv_( + grouped_params, + grouped_exp_avgs, + exp_avg_sq_sqrt, + step_size_expavg, # type: ignore[arg-type] + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_nadam) +def nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + decoupled_weight_decay: bool = False, + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + has_complex: bool = False, + maximize: bool = False, + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, +) -> None: + r"""Functional API that performs NAdam algorithm computation. + + See :class:`~torch.optim.NAdam` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if not all(isinstance(t, torch.Tensor) for t in mu_products): + raise RuntimeError( + "API has changed, `mu_products` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_nadam + else: + func = _single_tensor_nadam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + momentum_decay=momentum_decay, + maximize=maximize, + decoupled_weight_decay=decoupled_weight_decay, + eps=eps, + capturable=capturable, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/optimizer.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..d008faf4e6e02403a41920a29b9088b56ff94708 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/optimizer.py @@ -0,0 +1,1185 @@ +# mypy: allow-untyped-defs +"""Base optimizer.""" + +import functools +import warnings +from collections import defaultdict, OrderedDict +from collections.abc import Callable, Hashable, Iterable, Sequence +from copy import deepcopy +from itertools import chain +from typing import Any, cast, overload, TypeAlias, TypeVar +from typing_extensions import ParamSpec, Self + +import torch +import torch.utils.hooks as hooks +from torch.utils._foreach_utils import ( + _get_foreach_kernels_supported_devices, + _get_fused_kernels_supported_devices, + _group_tensors_by_device_and_dtype, + Indices, + TensorListList, +) +from torch.utils.hooks import RemovableHandle + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +Args: TypeAlias = tuple[Any, ...] +Kwargs: TypeAlias = dict[str, Any] +StateDict: TypeAlias = dict[str, Any] +DeviceDict: TypeAlias = dict[torch.device | None, torch.Tensor] +DeviceDtypeDict: TypeAlias = dict[tuple[torch.device, torch.dtype] | None, torch.Tensor] + +GlobalOptimizerPreHook: TypeAlias = Callable[ + ["Optimizer", Args, Kwargs], tuple[Args, Kwargs] | None +] +GlobalOptimizerPostHook: TypeAlias = Callable[["Optimizer", Args, Kwargs], None] + +__all__ = [ + "Optimizer", + "register_optimizer_step_pre_hook", + "register_optimizer_step_post_hook", +] +_global_optimizer_pre_hooks: dict[int, GlobalOptimizerPreHook] = OrderedDict() +_global_optimizer_post_hooks: dict[int, GlobalOptimizerPostHook] = OrderedDict() +_foreach_supported_types = [torch.Tensor, torch.nn.parameter.Parameter] + + +class _RequiredParameter: + """Singleton class representing a required parameter for an Optimizer.""" + + def __repr__(self) -> str: + return "" + + +required = _RequiredParameter() + + +def _use_grad_for_differentiable(func: Callable[_P, _T]) -> Callable[_P, _T]: + def _use_grad(*args: _P.args, **kwargs: _P.kwargs) -> _T: + import torch._dynamo + + # pyrefly: ignore [unsupported-operation] + self = cast(Optimizer, args[0]) # assume first positional arg is `self` + prev_grad = torch.is_grad_enabled() + try: + # Note on graph break below: + # we need to graph break to ensure that aot respects the no_grad annotation. + # This is important for perf because without this, functionalization will generate an epilogue + # which updates the mutated parameters of the optimizer which is *not* visible to inductor, as a result, + # inductor will allocate for every parameter in the model, which is horrible. + # With this, aot correctly sees that this is an inference graph, and functionalization will generate + # an epilogue which is appended to the graph, which *is* visible to inductor, as a result, inductor sees that + # step is in place and is able to avoid the extra allocation. + # In the future, we will either 1) continue to graph break on backward, so this graph break does not matter + # or 2) have a fully fused forward and backward graph, which will have no_grad by default, and we can remove this + # graph break to allow the fully fused fwd-bwd-optimizer graph to be compiled. + # see https://github.com/pytorch/pytorch/issues/104053 + torch.set_grad_enabled(self.defaults["differentiable"]) + torch._dynamo.graph_break() + ret = func(*args, **kwargs) + finally: + torch._dynamo.graph_break() + torch.set_grad_enabled(prev_grad) + return ret + + functools.update_wrapper(_use_grad, func) + return _use_grad + + +def _get_value(x): + # item is significantly faster than a cpu tensor in eager mode + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + return x + else: + return x.item() if isinstance(x, torch.Tensor) else x + + +def _stack_if_compiling(x): + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + return torch.stack(x) + else: + return x + + +def _disable_dynamo_if_unsupported( + single_tensor_fn: Callable[..., object] | None = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + # workaround for torchscript BC + # it requires all called functions to be in the + # global environment at the site at which the + # maybe_fallback closure is created + if single_tensor_fn: + globals()[single_tensor_fn.__name__] = single_tensor_fn + + def wrapper(func: Callable[_P, _T]) -> Callable[_P, _T]: + import inspect + + disabled_func = torch._disable_dynamo(func) + ps = inspect.signature(func).parameters + has_state_steps = True + try: + state_steps_ind = list(ps.keys()).index("state_steps") + except ValueError: + has_state_steps = False + + # Today, there are cases where we stack state steps + # and pass them as the value arg of foreach ops. + # Having state steps on cuda as the value arg is not supported in eager, + # but this only occurs in the rare case that the user explicitly deletes + # the capturable flag. If capturable=True, this is not a problem. + @functools.wraps(func) + def maybe_fallback(*args: _P.args, **kwargs: _P.kwargs): + if torch.compiler.is_compiling() and ( + not kwargs.get("capturable", False) + and has_state_steps + # pyrefly: ignore [unsupported-operation] + and (arg := args[state_steps_ind]) + and isinstance(arg, Sequence) + and arg[0].is_cuda + or ( + "state_steps" in kwargs + # pyrefly: ignore [unsupported-operation] + and (kwarg := kwargs["state_steps"]) + and isinstance(kwarg, Sequence) + and kwarg[0].is_cuda + ) + ): + return disabled_func(*args, **kwargs) + else: + return func(*args, **kwargs) + + return maybe_fallback + + return wrapper + + +# For any optimizer with a faster implementation, we attempt to default to the +# fastest + stablest whenever possible. For foreach, the requirements are to have +# native params all on CUDA. For fused, there's currently the additional requirement +# that the tensors' dtypes must be floating point. Neither alternative supports +# torch.jit.script nor differentiable, so we fall back to the single tensor +# implementation in those cases. +def _default_to_fused_or_foreach( + params: list[torch.Tensor], differentiable: bool, use_fused: bool = False +) -> tuple[bool, bool]: + if torch.jit.is_scripting() or differentiable: + return False, False + + fused_supported_devices = _get_fused_kernels_supported_devices() + foreach_supported_devices = _get_foreach_kernels_supported_devices() + fused = use_fused and all( + p is None + or ( + type(p) in _foreach_supported_types + and p.device.type in fused_supported_devices + and torch.is_floating_point(p) + ) + for p in params + ) + foreach = not fused and all( + p is None + or ( + type(p) in _foreach_supported_types + and p.device.type in foreach_supported_devices + ) + for p in params + ) + return fused, foreach + + +def _device_dtype_check_for_fused( + p: torch.Tensor, cuda_unsupported: bool = False +) -> None: + fused_supported_devices = _get_fused_kernels_supported_devices() + if cuda_unsupported: + fused_supported_devices.remove("cuda") + if not (p.device.type in fused_supported_devices and torch.is_floating_point(p)): + raise RuntimeError( + "`fused=True` requires all the params to be floating point Tensors of " + f"supported devices: {fused_supported_devices} but {p.dtype} and {p.device.type}" + ) + + +def _view_as_real(params, *state_and_grads) -> None: + for i, p in enumerate(params): + if torch.is_complex(p): + params[i] = torch.view_as_real(params[i]) + for s in state_and_grads: + s[i] = torch.view_as_real(s[i]) + + +def _get_scalar_dtype(is_fused=None): + if is_fused: + return torch.float32 + return ( + torch.float64 if torch.get_default_dtype() == torch.float64 else torch.float32 + ) + + +def _get_capturable_supported_devices(supports_xla: bool = True) -> list[str]: + r"""Return the device type list that supports capturable optimizer.""" + capturable_supported_devices = ["cuda", "xpu", "hpu"] + if not torch.jit.is_scripting(): + capturable_supported_devices.append(torch._C._get_privateuse1_backend_name()) + if supports_xla: + capturable_supported_devices.append("xla") + return capturable_supported_devices + + +def _to_scalar(x: float | torch.Tensor): + r"""This function converts a hyperparameter to a 0-dimension (scalar) tensor + if it is a nonzero-dimensions 1-element tensor. If it is not a tensor, it is + kept as is. + + Args: + x (float or Tensor): A hyperparameter of the optimizer. + If it is Tensor, it is needed to be 1-element. + + Returns: + float or Tensor: + a scalar tensor if x is Tensor otherwise Python scalar (float) value. + """ + if isinstance(x, torch.Tensor) and x.dim() != 0: + return x.squeeze() + else: + return x + + +# Common doc strings among optimizers +_params_doc = r"""params (iterable): iterable of parameters or named_parameters to optimize + or iterable of dicts defining parameter groups. When using named_parameters, + all parameters in all groups should be named""" + +_foreach_doc = r"""foreach (bool, optional): whether foreach implementation of optimizer + is used. If unspecified by the user (so foreach is None), we will try to use + foreach over the for-loop implementation on CUDA, since it is usually + significantly more performant. Note that the foreach implementation uses + ~ sizeof(params) more peak memory than the for-loop version due to the intermediates + being a tensorlist vs just one tensor. If memory is prohibitive, batch fewer + parameters through the optimizer at a time or switch this flag to False (default: None)""" + +_fused_doc = r"""fused (bool, optional): whether the fused implementation is used. + Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16` + are supported. (default: None) + + .. note:: The foreach and fused implementations are typically faster than the for-loop, + single-tensor implementation, with fused being theoretically fastest with both + vertical and horizontal fusion. As such, if the user has not specified either + flag (i.e., when foreach = fused = None), we will attempt defaulting to the foreach + implementation when the tensors are all on CUDA. Why not fused? Since the fused + implementation is relatively new, we want to give it sufficient bake-in time. + To specify fused, pass True for fused. To force running the for-loop + implementation, pass False for either foreach or fused. """ + +_capturable_doc = r"""capturable (bool, optional): whether this instance is safe to + capture in a graph, whether for CUDA graphs or for torch.compile support. + Tensors are only capturable when on supported :ref:`accelerators`. + Passing True can impair ungraphed performance, so if you don't intend to graph + capture this instance, leave it False (default: False)""" + +_differentiable_doc = r"""differentiable (bool, optional): whether autograd should + occur through the optimizer step in training. Otherwise, the step() + function runs in a torch.no_grad() context. Setting to True can impair + performance, so leave it False if you don't intend to run autograd + through this instance (default: False)""" + +_maximize_doc = r"""maximize (bool, optional): maximize the objective with respect to the + params, instead of minimizing (default: False)""" + + +def register_optimizer_step_pre_hook(hook: GlobalOptimizerPreHook) -> RemovableHandle: + r"""Register a pre hook common to all optimizers. + + The hook should have the following signature:: + + hook(optimizer, args, kwargs) -> None or modified args and kwargs + + Args: + hook (Callable): A user defined hook which is registered on all optimizers. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(_global_optimizer_pre_hooks) + _global_optimizer_pre_hooks[handle.id] = hook + return handle + + +def register_optimizer_step_post_hook(hook: GlobalOptimizerPostHook) -> RemovableHandle: + r"""Register a post hook common to all optimizers. + + The hook should have the following signature:: + + hook(optimizer, args, kwargs) -> None + + Args: + hook (Callable): A user defined hook which is registered on all optimizers. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(_global_optimizer_post_hooks) + _global_optimizer_post_hooks[handle.id] = hook + return handle + + +ParamsT: TypeAlias = ( + Iterable[torch.Tensor] + | Iterable[dict[str, Any]] + | Iterable[tuple[str, torch.Tensor]] +) + +R = TypeVar("R") +T = TypeVar("T") + + +class Optimizer: + r"""Base class for all optimizers. + + .. warning:: + Parameters need to be specified as collections that have a deterministic + ordering that is consistent between runs. Examples of objects that don't + satisfy those properties are sets and iterators over values of dictionaries. + + Args: + params (iterable): an iterable of :class:`torch.Tensor` s or + :class:`dict` s. Specifies what Tensors should be optimized. + defaults: (dict): a dict containing default values of optimization + options (used when a parameter group doesn't specify them). + """ + + OptimizerPreHook: TypeAlias = Callable[ + [Self, Args, Kwargs], # type: ignore[misc] + tuple[Args, Kwargs] | None, + ] + OptimizerPostHook: TypeAlias = Callable[[Self, Args, Kwargs], None] # type: ignore[misc] + + _optimizer_step_pre_hooks: dict[int, OptimizerPreHook] + _optimizer_step_post_hooks: dict[int, OptimizerPostHook] + # pyrefly: ignore [not-a-type] + _optimizer_state_dict_pre_hooks: 'OrderedDict[int, Callable[["Optimizer"], None]]' + _optimizer_state_dict_post_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer", StateDict], StateDict | None]]' + ) + _optimizer_load_state_dict_pre_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer", StateDict], StateDict | None]]' + ) + _optimizer_load_state_dict_post_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer"], None]]' + ) + + def __init__(self, params: ParamsT, defaults: dict[str, Any]) -> None: # noqa: D107 + torch._C._log_api_usage_once("python.optimizer") + self.defaults = defaults + self._optimizer_step_pre_hooks = OrderedDict() + self._optimizer_step_post_hooks = OrderedDict() + self._optimizer_state_dict_pre_hooks = OrderedDict() + self._optimizer_state_dict_post_hooks = OrderedDict() + self._optimizer_load_state_dict_pre_hooks = OrderedDict() + self._optimizer_load_state_dict_post_hooks = OrderedDict() + + self._patch_step_function() + + if isinstance(params, torch.Tensor): + raise TypeError( + "params argument given to the optimizer should be " + "an iterable of Tensors or dicts, but got " + torch.typename(params) + ) + + self.state: defaultdict[torch.Tensor, Any] = defaultdict(dict) + self.param_groups: list[dict[str, Any]] = [] + + param_groups = list(params) + if len(param_groups) == 0: + raise ValueError("optimizer got an empty parameter list") + if not isinstance(param_groups[0], dict): + param_groups = [{"params": param_groups}] + + for param_group in param_groups: + self.add_param_group(cast(dict, param_group)) + + # Allows _cuda_graph_capture_health_check to rig a poor man's TORCH_WARN_ONCE in python, + # which I don't think exists + # https://github.com/pytorch/pytorch/issues/72948 + self._warned_capturable_if_run_uncaptured = True + + def __getstate__(self) -> dict[str, Any]: # noqa: D105 + return { + "defaults": self.defaults, + "state": self.state, + "param_groups": self.param_groups, + } + + def __setstate__(self, state: dict[str, Any]) -> None: # noqa: D105 + self.__dict__.update(state) + if "_optimizer_step_pre_hooks" not in self.__dict__: + self._optimizer_step_pre_hooks = OrderedDict() + if "_optimizer_step_post_hooks" not in self.__dict__: + self._optimizer_step_post_hooks = OrderedDict() + if "_optimizer_state_dict_pre_hooks" not in self.__dict__: + self._optimizer_state_dict_pre_hooks = OrderedDict() + if "_optimizer_state_dict_post_hooks" not in self.__dict__: + self._optimizer_state_dict_post_hooks = OrderedDict() + if "_optimizer_load_state_dict_pre_hooks" not in self.__dict__: + self._optimizer_load_state_dict_pre_hooks = OrderedDict() + if "_optimizer_load_state_dict_post_hooks" not in self.__dict__: + self._optimizer_load_state_dict_post_hooks = OrderedDict() + self._patch_step_function() # To support multiprocessing pickle/unpickle + self.defaults.setdefault("differentiable", False) + + def __repr__(self) -> str: # noqa: D105 + format_string = self.__class__.__name__ + " (" + for i, group in enumerate(self.param_groups): + format_string += "\n" + format_string += f"Parameter Group {i}\n" + for key in sorted(group.keys()): + if key != "params": + format_string += f" {key}: {group[key]}\n" + format_string += ")" + return format_string + + # Currently needed by Adam and AdamW + def _cuda_graph_capture_health_check(self) -> None: + # Note [torch.compile x capturable] + # If we are compiling, we try to take the capturable path automatically by + # setting the flag to True during tracing. Due to this, we skip all the checks + # normally required for determining whether we can use CUDA graphs and + # shunt the responsibility to torch.inductor. This saves time during tracing + # since the checks are slow without sacrificing UX since inductor will warn + # later if CUDA graphs cannot be enabled, e.g., + # https://github.com/pytorch/pytorch/blob/d3ba8901d8640eb16f88b2bfef9df7fa383d4b47/torch/_inductor/compile_fx.py#L390. + # Thus, when compiling, inductor will determine if cudagraphs + # can be enabled based on whether there is input mutation or CPU tensors. + if ( + not torch.compiler.is_compiling() + and torch.backends.cuda.is_built() + and torch.cuda.is_available() + ): + capturing = torch.cuda.is_current_stream_capturing() + + if capturing and not all( + group["capturable"] for group in self.param_groups + ): + raise RuntimeError( + "Attempting CUDA graph capture of step() for an instance of " + + self.__class__.__name__ + + " but param_groups' capturable is False." + ) + + if ( + (not getattr(self, "_warned_capturable_if_run_uncaptured", False)) + and all(group["capturable"] for group in self.param_groups) + and (not capturing) + ): + warnings.warn( + "This instance was constructed with capturable=True or some of all the param_groups came with capturable=True, " + "but step() is running without CUDA graph capture. If you never intend to graph-capture this " + "instance, capturable=True can impair performance, and you should set capturable=False.", + stacklevel=2, + ) + self._warned_capturable_if_run_uncaptured = True + + def _optimizer_step_code(self) -> None: + """Entry point for `torch.profile.profiler`. + + When python tracing is enabled the profiler will hook into this + function at the CPython level to inspect the optimizer's parameters and + param groups. It is called it after `step()` since many optimizers + lazily initialize state. + + This is a workaround due to lack of a proper step hook on the optimizer, + and will be removed if it exists. + """ + + @staticmethod + def profile_hook_step(func: Callable[_P, R]) -> Callable[_P, R]: # noqa: D102 + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> R: + self, *_ = args + self = cast(Optimizer, self) + profile_name = f"Optimizer.step#{self.__class__.__name__}.step" + with torch.autograd.profiler.record_function(profile_name): + # call optimizer step pre hooks + for pre_hook in chain( + _global_optimizer_pre_hooks.values(), + self._optimizer_step_pre_hooks.values(), + ): + result = pre_hook(self, args, kwargs) + if result is not None: + if isinstance(result, tuple) and len(result) == 2: + args, kwargs = result # type: ignore[assignment] + else: + raise RuntimeError( + f"{func} must return None or a tuple of (new_args, new_kwargs), but got {result}." + ) + + # pyrefly: ignore [invalid-param-spec] + out = func(*args, **kwargs) + self._optimizer_step_code() + + # call optimizer step post hooks + for post_hook in chain( + self._optimizer_step_post_hooks.values(), + _global_optimizer_post_hooks.values(), + ): + post_hook(self, args, kwargs) + + return out + + return wrapper + + @staticmethod + def _group_tensors_by_device_and_dtype( + tensorlistlist: TensorListList, + with_indices: bool = False, + ) -> ( + dict[tuple[None, None], tuple[TensorListList, Indices]] + | dict[tuple[torch.device, torch.dtype], tuple[TensorListList, Indices]] + ): + """Group a list of lists of tensors by device and dtype. + + Skips this step if we are compiling since this will occur during inductor lowering. + """ + if torch.compiler.is_compiling(): + return {(None, None): (tensorlistlist, list(range(len(tensorlistlist[0]))))} + else: + return _group_tensors_by_device_and_dtype(tensorlistlist, with_indices) # type: ignore[return-value, arg-type] + + def _patch_step_function(self) -> None: + self._zero_grad_profile_name = ( + f"Optimizer.zero_grad#{self.__class__.__name__}.zero_grad" + ) + hooked = getattr(self.__class__.step, "hooked", None) + if not hooked: + self.__class__.step = self.profile_hook_step(self.__class__.step) # type: ignore[assignment] + self.__class__.step.hooked = True # type: ignore[attr-defined] + + def register_step_pre_hook(self, hook: OptimizerPreHook) -> RemovableHandle: + r"""Register an optimizer step pre hook which will be called before optimizer step. + + It should have the following signature:: + + hook(optimizer, args, kwargs) -> None or modified args and kwargs + + The ``optimizer`` argument is the optimizer instance being used. If + args and kwargs are modified by the pre-hook, then the transformed + values are returned as a tuple containing the new_args and new_kwargs. + + Args: + hook (Callable): The user defined hook to be registered. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_step_pre_hooks) + self._optimizer_step_pre_hooks[handle.id] = hook + return handle + + def register_step_post_hook(self, hook: OptimizerPostHook) -> RemovableHandle: + r"""Register an optimizer step post hook which will be called after optimizer step. + + It should have the following signature:: + + hook(optimizer, args, kwargs) -> None + + The ``optimizer`` argument is the optimizer instance being used. + + Args: + hook (Callable): The user defined hook to be registered. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_step_post_hooks) + self._optimizer_step_post_hooks[handle.id] = hook + return handle + + def register_state_dict_pre_hook( + self, hook: Callable[["Optimizer"], None], prepend: bool = False + ) -> RemovableHandle: # noqa: D101 + r"""Register a state dict pre-hook which will be called before :meth:`~torch.optim.Optimizer.state_dict` is called. + + It should have the following signature:: + + hook(optimizer) -> None + + The ``optimizer`` argument is the optimizer instance being used. + The hook will be called with argument ``self`` before calling ``state_dict`` on ``self``. + The registered hook can be used to perform pre-processing before the ``state_dict`` + call is made. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided pre ``hook`` will be fired before + all the already registered pre-hooks on ``state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + pre-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_state_dict_pre_hooks) + self._optimizer_state_dict_pre_hooks[handle.id] = hook + if prepend: + self._optimizer_state_dict_pre_hooks.move_to_end(handle.id, last=False) + return handle + + def register_state_dict_post_hook( + self, + hook: Callable[["Optimizer", StateDict], StateDict | None], + prepend: bool = False, + ) -> RemovableHandle: + r"""Register a state dict post-hook which will be called after :meth:`~torch.optim.Optimizer.state_dict` is called. + + It should have the following signature:: + + hook(optimizer, state_dict) -> state_dict or None + + The hook will be called with arguments ``self`` and ``state_dict`` after generating + a ``state_dict`` on ``self``. The hook may modify the state_dict inplace or optionally + return a new one. The registered hook can be used to perform post-processing + on the ``state_dict`` before it is returned. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided post ``hook`` will be fired before + all the already registered post-hooks on ``state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + post-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_state_dict_post_hooks) + self._optimizer_state_dict_post_hooks[handle.id] = hook + if prepend: + self._optimizer_state_dict_post_hooks.move_to_end(handle.id, last=False) + return handle + + @torch._disable_dynamo + def state_dict(self) -> StateDict: + r"""Return the state of the optimizer as a :class:`dict`. + + It contains two entries: + + * ``state``: a Dict holding current optimization state. Its content + differs between optimizer classes, but some common characteristics + hold. For example, state is saved per parameter, and the parameter + itself is NOT saved. ``state`` is a Dictionary mapping parameter ids + to a Dict with state corresponding to each parameter. + * ``param_groups``: a List containing all parameter groups where each + parameter group is a Dict. Each parameter group contains metadata + specific to the optimizer, such as learning rate and weight decay, + as well as a List of parameter IDs of the parameters in the group. + If a param group was initialized with ``named_parameters()`` the names + content will also be saved in the state dict. + + NOTE: The parameter IDs may look like indices but they are just IDs + associating state with param_group. When loading from a state_dict, + the optimizer will zip the param_group ``params`` (int IDs) and the + optimizer ``param_groups`` (actual ``nn.Parameter`` s) in order to + match state WITHOUT additional verification. + + A returned state dict might look something like: + + .. code-block:: text + + { + 'state': { + 0: {'momentum_buffer': tensor(...), ...}, + 1: {'momentum_buffer': tensor(...), ...}, + 2: {'momentum_buffer': tensor(...), ...}, + 3: {'momentum_buffer': tensor(...), ...} + }, + 'param_groups': [ + { + 'lr': 0.01, + 'weight_decay': 0, + ... + 'params': [0] + 'param_names' ['param0'] (optional) + }, + { + 'lr': 0.001, + 'weight_decay': 0.5, + ... + 'params': [1, 2, 3] + 'param_names': ['param1', 'layer.weight', 'layer.bias'] (optional) + } + ] + } + + """ + for pre_hook in self._optimizer_state_dict_pre_hooks.values(): + pre_hook(self) + + # Save order indices instead of Tensors + param_mappings: dict[int, int] = {} + start_index = 0 + + def pack_group(group: dict[str, Any]) -> dict[str, Any]: + nonlocal start_index + packed = {k: v for k, v in group.items() if k != "params"} + param_mappings.update( + { + id(p): i + for i, p in enumerate(group["params"], start_index) + if id(p) not in param_mappings + } + ) + packed["params"] = [param_mappings[id(p)] for p in group["params"]] + start_index += len(packed["params"]) + return packed + + param_groups = [pack_group(g) for g in self.param_groups] + # Remap state to use order indices as keys + packed_state = { + (param_mappings[id(k)] if isinstance(k, torch.Tensor) else k): v + for k, v in self.state.items() + } + + state_dict = { + "state": packed_state, + "param_groups": param_groups, + } + + for post_hook in self._optimizer_state_dict_post_hooks.values(): + hook_result = post_hook(self, state_dict) + if hook_result is not None: + state_dict = hook_result + return state_dict + + @staticmethod + def _process_value_according_to_param_policy( + param: torch.Tensor, + value: torch.Tensor, + param_id: int, + param_groups: list[dict[Any, Any]], + key: Hashable = None, + ) -> torch.Tensor: + # Floating-point types are a bit special here. They are the only ones + # that are assumed to always match the type of params. + # Make sure state['step'] is not casted https://github.com/pytorch/pytorch/issues/74424 + # UNLESS fused or capturable, see note [special device hosting for step] + fused = False + capturable = False + if param_groups is None: + raise AssertionError("Expected param_groups to be set") + for pg in param_groups: + if param_id in pg["params"]: + fused = pg.get("fused", False) + capturable = pg.get("capturable", False) + break + if key == "step": + if capturable or fused: + return value.to(dtype=torch.float32, device=param.device) + else: + return value + else: + if param.is_floating_point(): + return value.to(dtype=param.dtype, device=param.device) + else: + return value.to(device=param.device) + + def register_load_state_dict_pre_hook( + self, + hook: Callable[["Optimizer", StateDict], StateDict | None], + prepend: bool = False, + ) -> RemovableHandle: # noqa: D205 D400 + r"""Register a load_state_dict pre-hook which will be called before + :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the + following signature:: + + hook(optimizer, state_dict) -> state_dict or None + + The ``optimizer`` argument is the optimizer instance being used and the + ``state_dict`` argument is a shallow copy of the ``state_dict`` the user + passed in to ``load_state_dict``. The hook may modify the state_dict inplace + or optionally return a new one. If a state_dict is returned, it will be used + to be loaded into the optimizer. + + The hook will be called with argument ``self`` and ``state_dict`` before + calling ``load_state_dict`` on ``self``. The registered hook can be used to + perform pre-processing before the ``load_state_dict`` call is made. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided pre ``hook`` will be fired before + all the already registered pre-hooks on ``load_state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + pre-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_load_state_dict_pre_hooks) + self._optimizer_load_state_dict_pre_hooks[handle.id] = hook + if prepend: + self._optimizer_load_state_dict_pre_hooks.move_to_end(handle.id, last=False) + return handle + + def register_load_state_dict_post_hook( + self, hook: Callable[["Optimizer"], None], prepend: bool = False + ) -> RemovableHandle: # noqa: D205 D400 + r"""Register a load_state_dict post-hook which will be called after + :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the + following signature:: + + hook(optimizer) -> None + + The ``optimizer`` argument is the optimizer instance being used. + + The hook will be called with argument ``self`` after calling + ``load_state_dict`` on ``self``. The registered hook can be used to + perform post-processing after ``load_state_dict`` has loaded the + ``state_dict``. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided post ``hook`` will be fired before + all the already registered post-hooks on ``load_state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + post-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_load_state_dict_post_hooks) + self._optimizer_load_state_dict_post_hooks[handle.id] = hook + if prepend: + self._optimizer_load_state_dict_post_hooks.move_to_end( + handle.id, last=False + ) # type: ignore[attr-defined] + return handle + + @torch._disable_dynamo + def load_state_dict(self, state_dict: StateDict) -> None: + r"""Load the optimizer state. + + Args: + state_dict (dict): optimizer state. Should be an object returned + from a call to :meth:`state_dict`. + + .. warning:: + Make sure this method is called after initializing :class:`torch.optim.lr_scheduler.LRScheduler`, + as calling it beforehand will overwrite the loaded learning rates. + + .. note:: + The names of the parameters (if they exist under the "param_names" key of each param group + in :meth:`state_dict`) will not affect the loading process. + To use the parameters' names for custom cases (such as when the parameters in the loaded state dict + differ from those initialized in the optimizer), + a custom ``register_load_state_dict_pre_hook`` should be implemented to adapt the loaded dict + accordingly. + If ``param_names`` exist in loaded state dict ``param_groups`` they will be saved and override + the current names, if present, in the optimizer state. If they do not exist in loaded state dict, + the optimizer ``param_names`` will remain unchanged. + + Example: + >>> # xdoctest: +SKIP + >>> model = torch.nn.Linear(10, 10) + >>> optim = torch.optim.SGD(model.parameters(), lr=3e-4) + >>> scheduler1 = torch.optim.lr_scheduler.LinearLR( + ... optim, + ... start_factor=0.1, + ... end_factor=1, + ... total_iters=20, + ... ) + >>> scheduler2 = torch.optim.lr_scheduler.CosineAnnealingLR( + ... optim, + ... T_max=80, + ... eta_min=3e-5, + ... ) + >>> lr = torch.optim.lr_scheduler.SequentialLR( + ... optim, + ... schedulers=[scheduler1, scheduler2], + ... milestones=[20], + ... ) + >>> lr.load_state_dict(torch.load("./save_seq.pt")) + >>> # now load the optimizer checkpoint after loading the LRScheduler + >>> optim.load_state_dict(torch.load("./save_optim.pt")) + + """ + # shallow copy, to be consistent with module API + state_dict = state_dict.copy() + + for pre_hook in self._optimizer_load_state_dict_pre_hooks.values(): + hook_result = pre_hook(self, state_dict) + if hook_result is not None: + state_dict = hook_result + + # Validate the state_dict + groups = self.param_groups + + # Deepcopy as we write into saved_groups later to update state + saved_groups = deepcopy(state_dict["param_groups"]) + + if len(groups) != len(saved_groups): + raise ValueError( + "loaded state dict has a different number of parameter groups" + ) + param_lens = (len(g["params"]) for g in groups) + saved_lens = (len(g["params"]) for g in saved_groups) + if any( + p_len != s_len for p_len, s_len in zip(param_lens, saved_lens, strict=True) + ): + raise ValueError( + "loaded state dict contains a parameter group " + "that doesn't match the size of optimizer's group" + ) + + # Update the state + id_map = dict( + zip( + chain.from_iterable(g["params"] for g in saved_groups), + chain.from_iterable(g["params"] for g in groups), + strict=True, + ) + ) + + def _cast(param, value, param_id=None, param_groups=None, key=None): + r"""Make a deep copy of value, casting all tensors to device of param.""" + if isinstance(value, torch.Tensor): + return Optimizer._process_value_according_to_param_policy( + param, + value, + # pyrefly: ignore [bad-argument-type] + param_id, + # pyrefly: ignore [bad-argument-type] + param_groups, + key, + ) + elif isinstance(value, dict): + return { + k: _cast( + param, v, param_id=param_id, param_groups=param_groups, key=k + ) + for k, v in value.items() + } + elif isinstance(value, Iterable): + return type(value)( + # pyrefly: ignore [bad-argument-count] + _cast(param, v, param_id=param_id, param_groups=param_groups) + for v in value + ) # type: ignore[call-arg] + else: + return value + + # Copy state assigned to params (and cast tensors to appropriate types). + # State that is not assigned to params is copied as is (needed for + # backward compatibility). + state: defaultdict[torch.Tensor, dict[Any, Any]] = defaultdict(dict) + for k, v in state_dict["state"].items(): + if k in id_map: + param = id_map[k] + state[param] = _cast( + param, v, param_id=k, param_groups=state_dict["param_groups"] + ) + else: + state[k] = v + + # Update parameter groups, setting their 'params' value + def update_group( + group: dict[str, Any], new_group: dict[str, Any] + ) -> dict[str, Any]: + new_group["params"] = group["params"] + if "param_names" in group and "param_names" not in new_group: + new_group["param_names"] = group["param_names"] + return new_group + + param_groups = [ + update_group(g, ng) for g, ng in zip(groups, saved_groups, strict=True) + ] + self.__setstate__({"state": state, "param_groups": param_groups}) + + for post_hook in self._optimizer_load_state_dict_post_hooks.values(): + post_hook(self) + + @torch._disable_dynamo + def zero_grad(self, set_to_none: bool = True) -> None: + r"""Reset the gradients of all optimized :class:`torch.Tensor` s. + + Args: + set_to_none (bool, optional): Instead of setting to zero, set the grads to None. Default: ``True`` + + This will in general have lower memory footprint, and can modestly improve performance. + However, it changes certain behaviors. For example: + + 1. When the user tries to access a gradient and perform manual ops on it, + a None attribute or a Tensor full of 0s will behave differently. + 2. If the user requests ``zero_grad(set_to_none=True)`` followed by a backward pass, ``.grad``\ s + are guaranteed to be None for params that did not receive a gradient. + 3. ``torch.optim`` optimizers have a different behavior if the gradient is 0 or None + (in one case it does the step with a gradient of 0 and in the other it skips + the step altogether). + """ + foreach = self.defaults.get("foreach", False) or self.defaults.get( + "fused", False + ) + + if not hasattr(self, "_zero_grad_profile_name"): + self._patch_step_function() + + per_device_and_dtype_grads: ( + defaultdict[torch.device, defaultdict[torch.dtype, list[torch.Tensor]]] + | None + ) + if foreach: + per_device_and_dtype_grads = defaultdict(lambda: defaultdict(list)) + else: + per_device_and_dtype_grads = None + + with torch.autograd.profiler.record_function(self._zero_grad_profile_name): + for group in self.param_groups: + for p in group["params"]: + if p.grad is not None: + if set_to_none: + p.grad = None + else: + if p.grad.grad_fn is not None: + p.grad.detach_() + else: + p.grad.requires_grad_(False) + if not foreach or p.grad.is_sparse: + p.grad.zero_() + else: + if per_device_and_dtype_grads is None: + raise AssertionError( + "Expected per_device_and_dtype_grads to be set" + ) + per_device_and_dtype_grads[p.grad.device][ + p.grad.dtype + ].append(p.grad) + if foreach: + if per_device_and_dtype_grads is None: + raise AssertionError( + "Expected per_device_and_dtype_grads to be set" + ) + for per_dtype_grads in per_device_and_dtype_grads.values(): + for grads in per_dtype_grads.values(): + torch._foreach_zero_(grads) + + @overload + def step(self, closure: None = None) -> None: ... + + @overload + def step(self, closure: Callable[[], float]) -> float: ... + + def step(self, closure: Callable[[], float] | None = None) -> float | None: + r"""Perform a single optimization step to update parameter. + + Args: + closure (Callable): A closure that reevaluates the model and + returns the loss. Optional for most optimizers. + """ + raise NotImplementedError + + @torch._disable_dynamo + def add_param_group(self, param_group: dict[str, Any]) -> None: + r"""Add a param group to the :class:`Optimizer` s `param_groups`. + + This can be useful when fine tuning a pre-trained network as frozen layers can be made + trainable and added to the :class:`Optimizer` as training progresses. + + Args: + param_group (dict): Specifies what Tensors should be optimized along with group + specific optimization options. + """ + if not isinstance(param_group, dict): + raise TypeError(f"param_group must be a dict, but got {type(param_group)}") + + params = param_group["params"] + if isinstance(params, torch.Tensor): + param_group["params"] = [params] + elif isinstance(params, set): + raise TypeError( + "optimizer parameters need to be organized in ordered collections, but " + "the ordering of tensors in sets will change between runs. Please use a list instead." + ) + else: + param_group["params"] = list(params) + + extracted_param_tensors = [] + extracted_param_names = [] + for param in param_group["params"]: + if isinstance(param, tuple): + param_name = param[0] + extracted_param_names.append(param_name) + extracted_param_tensors.append(param[1]) + else: + extracted_param_tensors.append(param) + + param_group["params"] = extracted_param_tensors + if len(extracted_param_names) != 0: + if len(extracted_param_names) == len(extracted_param_tensors): + param_group["param_names"] = extracted_param_names + else: + raise ValueError( + "all optimizer params should be with/without names. Some param names are missing" + ) + + for param in param_group["params"]: + if not isinstance(param, torch.Tensor): + raise TypeError( + "optimizer can only optimize Tensors, " + "but one of the params is " + torch.typename(param) + ) + if not self.defaults.get("differentiable", None) and not ( + param.is_leaf or param.retains_grad + ): + raise ValueError("can't optimize a non-leaf Tensor") + + for name, default in self.defaults.items(): + if default is required and name not in param_group: + raise ValueError( + f"parameter group didn't specify a value of required optimization parameter {name}" + ) + else: + param_group.setdefault(name, default) + + params = param_group["params"] + if len(params) != len(set(params)): + warnings.warn( + "optimizer contains a parameter group with duplicate parameters; " + "in future, this will cause an error; " + "see github.com/pytorch/pytorch/issues/40967 for more information", + stacklevel=3, + ) + + param_set: set[torch.Tensor] = set() + for group in self.param_groups: + param_set.update(set(group["params"])) + if ("param_names" in param_group) != ("param_names" in group): + current_group_txt = ( + "with names" if "param_names" in param_group else "without names" + ) + raise ValueError( + "all optimizer param groups should be with/without names. " + f"cannot add param group {current_group_txt} to the optimizer" + ) + + if not param_set.isdisjoint(set(param_group["params"])): + raise ValueError("some parameters appear in more than one parameter group") + + self.param_groups.append(param_group) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/radam.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/radam.py new file mode 100644 index 0000000000000000000000000000000000000000..b5feee1f6d72a30d4a06b10407f13d91291c72b3 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/radam.py @@ -0,0 +1,628 @@ +# mypy: allow-untyped-defs +r"""Implementation for the RAdam algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["RAdam", "radam"] + + +class RAdam(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + decoupled_weight_decay: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "decoupled_weight_decay": decoupled_weight_decay, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, group, params_with_grad, grads, exp_avgs, exp_avg_sqs, state_steps + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("RAdam does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = cast(tuple[float, float], group["betas"]) + + has_complex = self._init_group( + group, params_with_grad, grads, exp_avgs, exp_avg_sqs, state_steps + ) + + radam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + eps=group["eps"], + maximize=group["maximize"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + decoupled_weight_decay=group["decoupled_weight_decay"], + has_complex=has_complex, + ) + + return loss + + +RAdam.__doc__ = ( + r"""Implements RAdam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \beta_1, \beta_2 + \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \: + \lambda \text{ (weightdecay)}, \:\textit{maximize} \\ + &\hspace{13mm} \epsilon \text{ (epsilon)}, \textit{decoupled\_weight\_decay} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0 \leftarrow 0 \text{ ( second moment)}, \\ + &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{6mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{12mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{6mm}\textbf{else} \\ + &\hspace{12mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{6mm} \theta_t \leftarrow \theta_{t-1} \\ + &\hspace{6mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{12mm}\textbf{if} \: \textit{decoupled\_weight\_decay} \\ + &\hspace{18mm} \theta_t \leftarrow \theta_{t} - \gamma \lambda \theta_{t} \\ + &\hspace{12mm}\textbf{else} \\ + &\hspace{18mm} g_t \leftarrow g_t + \lambda \theta_{t} \\ + &\hspace{6mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{6mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{6mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} - + 2 t \beta^t_2 /\big(1-\beta_2^t \big) \\[0.1.ex] + &\hspace{6mm}\textbf{if} \: \rho_t > 5 \\ + &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon } \\ + &\hspace{12mm} r_t \leftarrow + \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\ + &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t \\ + &\hspace{6mm}\textbf{else} \\ + &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `On the variance of the adaptive learning rate and beyond`_. + + This implementation provides an option to use either the original weight_decay implementation as in Adam + (where the weight_decay is applied to the gradient) or the one from AdamW (where weight_decay is applied + to the weight) through the decoupled_weight_decay option. When decoupled_weight_decay is set to False + (default), it uses the original Adam style weight decay, otherwise, it uses the AdamW style which + corresponds more closely to the `author's implementation`_ in the RAdam paper. Further information + about decoupled weight decay can be found in `Decoupled Weight Decay Regularization`_. + + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_weight_decay (bool, optional): whether to decouple the weight + decay as in AdamW to obtain RAdamW. If True, the algorithm does not + accumulate weight decay in the momentum nor variance. (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + + .. _On the variance of the adaptive learning rate and beyond: + https://arxiv.org/abs/1908.03265 + .. _author's implementation: + https://github.com/LiyuanLucasLiu/RAdam + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + + """ +) + + +def _single_tensor_radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, + decoupled_weight_decay: bool, + differentiable: bool, + maximize: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + + # update step + step_t += 1 + step = step_t if capturable else _get_value(step_t) + + if weight_decay != 0: + if decoupled_weight_decay: + param.mul_(1 - lr * weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + # Decay the first and second moment running average coefficient + exp_avg.lerp_(grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + + # correcting bias for the first moving moment + bias_corrected_exp_avg = exp_avg / bias_correction1 + + # maximum length of the approximated SMA + rho_inf = 2 / (1 - beta2) - 1 + # compute the length of the approximated SMA + rho_t = rho_inf - 2 * step * (beta2**step) / bias_correction2 + + def _compute_rect(): + # pyrefly: ignore [unsupported-operation] + return ( + (rho_t - 4) + * (rho_t - 2) + * rho_inf + / ((rho_inf - 4) * (rho_inf - 2) * rho_t) + ) ** 0.5 + + def _compute_adaptive_lr(): + exp_avg_sq_sqrt = exp_avg_sq.sqrt() + if differentiable: + exp_avg_sq_sqrt = exp_avg_sq_sqrt.add(eps) + else: + exp_avg_sq_sqrt = exp_avg_sq_sqrt.add_(eps) + + # pyrefly: ignore [unsupported-operation] + return (bias_correction2**0.5) / exp_avg_sq_sqrt + + # Compute the variance rectification term and update parameters accordingly + if capturable: + update = torch.where( + rho_t > 5.0, _compute_rect() * _compute_adaptive_lr(), 1.0 + ) + param.add_(bias_corrected_exp_avg * lr * update, alpha=-1.0) + else: + if rho_t > 5.0: + param.add_( + bias_corrected_exp_avg + * lr + * _compute_adaptive_lr() + * _compute_rect(), + alpha=-1.0, + ) + else: + param.add_(bias_corrected_exp_avg * lr, alpha=-1.0) + + +def _multi_tensor_radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, + decoupled_weight_decay: bool, + differentiable: bool, + maximize: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_avg_sqs_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_avg_sqs = cast(list[Tensor], grouped_exp_avg_sqs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_avg_sqs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # maximum length of the approximated SMA + rho_inf = 2 / (1 - beta2) - 1 + # compute the length of the approximated SMA + bias_correction1: tuple[Tensor, ...] | list[Tensor] + bias_correction2: tuple[Tensor, ...] | list[Tensor] + rho_t_list: tuple[Tensor, ...] | list[Tensor] + if capturable: + bias_correction1 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_neg_(bias_correction1) + torch._foreach_add_(bias_correction1, 1) + bias_correction2 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_mul_(bias_correction2, grouped_state_steps) + torch._foreach_mul_(bias_correction2, 2) + torch._foreach_div_(bias_correction2, bias_correction1) + torch._foreach_neg_(bias_correction2) + torch._foreach_add_(bias_correction2, rho_inf) + rho_t_list = bias_correction2 + else: + rho_t_list = [ + rho_inf + - 2 + * _get_value(step) + * (beta2 ** _get_value(step)) + / (1 - beta2 ** _get_value(step)) + for step in grouped_state_steps + ] + + if weight_decay != 0: + if decoupled_weight_decay: + torch._foreach_mul_(grouped_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_( + grouped_grads, grouped_params, alpha=weight_decay + ) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + torch._foreach_mul_(grouped_exp_avg_sqs, beta2) + torch._foreach_addcmul_( + grouped_exp_avg_sqs, grouped_grads, grouped_grads, 1 - beta2 + ) + + # Delete the local intermediate since it won't be used anymore to save on peak memory + del grouped_grads + + if capturable: + num = torch._foreach_sub(rho_t_list, 4) + sub2 = torch._foreach_sub(rho_t_list, 2) + torch._foreach_mul_(num, sub2) + del sub2 + torch._foreach_mul_(num, rho_inf) + rho_inf = (rho_inf - 4) * (rho_inf - 2) + denom = torch._foreach_mul(rho_t_list, rho_inf) + torch._foreach_div_(num, denom) + del denom + torch._foreach_sqrt_(num) + + # TODO(mlazos): we should try and get a foreach_where op https://github.com/pytorch/pytorch/issues/117884 + rect = [ + torch.where(rho_t > 5.0, n, 0.0) + for n, rho_t in zip(num, rho_t_list, strict=True) + ] + del num + del rho_t_list + unrect_step_size = [torch.where(rect > 0, 0.0, 1.0) for rect in rect] + torch._foreach_mul_(unrect_step_size, lr) + + bias_correction1 = torch._foreach_pow(beta1, grouped_state_steps) + torch._foreach_neg_(bias_correction1) + torch._foreach_add_(bias_correction1, 1) + + torch._foreach_div_(unrect_step_size, bias_correction1) + torch._foreach_neg_(unrect_step_size) + + bias_correction2 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_neg_(bias_correction2) + torch._foreach_add_(bias_correction2, 1) + torch._foreach_sqrt_(bias_correction2) + torch._foreach_mul_(bias_correction2, lr) + torch._foreach_mul_(bias_correction2, rect) + del rect + torch._foreach_neg_(bias_correction2) + torch._foreach_div_(bias_correction2, bias_correction1) + del bias_correction1 + else: + rect = [ + ( # type: ignore[misc] + (rho_t - 4) # type: ignore[arg-type] + * (rho_t - 2) + * rho_inf + / ((rho_inf - 4) * (rho_inf - 2) * rho_t) + ) + ** 0.5 + if rho_t > 5 + else 0 + for rho_t in rho_t_list + ] + unrectified = [0 if rect > 0 else 1.0 for rect in rect] + + bias_correction1 = [ + 1 - beta1 ** _get_value(step) for step in grouped_state_steps + ] + unrect_step_size = [ + (lr * rect / bc) * -1 + for rect, bc in zip(unrectified, bias_correction1, strict=True) + ] + bias_correction2 = [ + ((1 - beta2 ** _get_value(step)) ** 0.5) * (lr * rect / bc) * -1 + for step, rect, bc in zip( + grouped_state_steps, rect, bias_correction1, strict=True + ) + ] + + buffer = torch._foreach_sqrt(grouped_exp_avg_sqs) + torch._foreach_add_(buffer, eps) + torch._foreach_div_(buffer, bias_correction2) + torch._foreach_reciprocal_(buffer) + torch._foreach_add_(buffer, unrect_step_size) + + # Here, buffer = sqrt(1 - beta2^t) * rect_step_size / (sqrt(v) + eps) + unrect_step_size + torch._foreach_addcmul_(grouped_params, grouped_exp_avgs, buffer) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_radam) +def radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + decoupled_weight_decay: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + maximize: bool = False, + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, +) -> None: + r"""Functional API that performs RAdam algorithm computation. + + See :class:`~torch.optim.RAdam` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_radam + else: + func = _single_tensor_radam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + decoupled_weight_decay=decoupled_weight_decay, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rmsprop.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rmsprop.py new file mode 100644 index 0000000000000000000000000000000000000000..e3878b8e04d9ae4407d036b2d094f890498f5151 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rmsprop.py @@ -0,0 +1,542 @@ +# mypy: allow-untyped-defs +r"""Implementation for the RMSprop algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["RMSprop", "rmsprop"] + + +class RMSprop(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + alpha: float = 0.99, + eps: float = 1e-8, + weight_decay: float = 0, + momentum: float = 0, + centered: bool = False, + capturable: bool = False, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= momentum: + raise ValueError(f"Invalid momentum value: {momentum}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= alpha: + raise ValueError(f"Invalid alpha value: {alpha}") + + defaults = { + "lr": lr, + "momentum": momentum, + "alpha": alpha, + "eps": eps, + "centered": centered, + "weight_decay": weight_decay, + "capturable": capturable, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("momentum", 0) + group.setdefault("centered", False) + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + square_avgs, + momentum_buffer_list, + grad_avgs, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + + if p.grad.is_sparse: + raise RuntimeError("RMSprop does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + state["square_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["momentum"] > 0: + state["momentum_buffer"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["centered"]: + state["grad_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + square_avgs.append(state["square_avg"]) + state_steps.append(state["step"]) + + if group["momentum"] > 0: + momentum_buffer_list.append(state["momentum_buffer"]) + if group["centered"]: + grad_avgs.append(state["grad_avg"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + square_avgs: list[Tensor] = [] + grad_avgs: list[Tensor] = [] + momentum_buffer_list: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + square_avgs, + momentum_buffer_list, + grad_avgs, + state_steps, + ) + + rmsprop( + params_with_grad, + grads, + square_avgs, + grad_avgs, + momentum_buffer_list, + state_steps, + lr=group["lr"], + alpha=group["alpha"], + eps=group["eps"], + weight_decay=group["weight_decay"], + momentum=group["momentum"], + centered=group["centered"], + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +RMSprop.__doc__ = ( + r"""Implements RMSprop algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \alpha \text{ (alpha)}, \: \gamma \text{ (lr)}, + \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \lambda \text{ (weight decay)},\: \mu \text{ (momentum)}, + \: centered, \: \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : v_0 \leftarrow 0 \text{ (square average)}, \: + \textbf{b}_0 \leftarrow 0 \text{ (buffer)}, \: g^{ave}_0 \leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}v_t \leftarrow \alpha v_{t-1} + (1 - \alpha) g^2_t + \hspace{8mm} \\ + &\hspace{5mm} \tilde{v_t} \leftarrow v_t \\ + &\hspace{5mm}if \: centered \\ + &\hspace{10mm} g^{ave}_t \leftarrow g^{ave}_{t-1} \alpha + (1-\alpha) g_t \\ + &\hspace{10mm} \tilde{v_t} \leftarrow \tilde{v_t} - \big(g^{ave}_{t} \big)^2 \\ + &\hspace{5mm}if \: \mu > 0 \\ + &\hspace{10mm} \textbf{b}_t\leftarrow \mu \textbf{b}_{t-1} + + g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big) \\ + &\hspace{10mm} \theta_t \leftarrow \theta_{t-1} - \gamma \textbf{b}_t \\ + &\hspace{5mm} else \\ + &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - + \gamma g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big) \hspace{3mm} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to + `lecture notes `_ by G. Hinton. + and centered version `Generating Sequences + With Recurrent Neural Networks `_. + The implementation here takes the square root of the gradient average before + adding epsilon (note that TensorFlow interchanges these two operations). The effective + learning rate is thus :math:`\gamma/(\sqrt{v} + \epsilon)` where :math:`\gamma` + is the scheduled learning rate and :math:`v` is the weighted moving average + of the squared gradient. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + alpha (float, optional): smoothing constant (default: 0.99) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + momentum (float, optional): momentum factor (default: 0) + centered (bool, optional) : if ``True``, compute the centered RMSProp, + the gradient is normalized by an estimation of its variance + {_capturable_doc} + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + + """ +) + + +def _single_tensor_rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + step = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + grad = grads[i] + grad = grad if not maximize else -grad + square_avg = square_avgs[i] + + step += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + is_complex_param = torch.is_complex(param) + if is_complex_param: + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + square_avg = torch.view_as_real(square_avg) + + square_avg.mul_(alpha).addcmul_(grad, grad, value=1 - alpha) + + if centered: + grad_avg = grad_avgs[i] + if is_complex_param: + grad_avg = torch.view_as_real(grad_avg) + grad_avg.lerp_(grad, 1 - alpha) + avg = square_avg.addcmul(grad_avg, grad_avg, value=-1).sqrt_() + else: + avg = square_avg.sqrt() + + if differentiable: + avg = avg.add(eps) + else: + avg = avg.add_(eps) + + if momentum > 0: + buf = momentum_buffer_list[i] + if is_complex_param: + buf = torch.view_as_real(buf) + buf.mul_(momentum).addcdiv_(grad, avg) + param.add_(buf, alpha=-lr) + else: + param.addcdiv_(grad, avg, value=-lr) + + +def _multi_tensor_rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, square_avgs, grad_avgs, momentum_buffer_list, state_steps] # type: ignore[list-item] + ) + for ( + ( + grouped_params_, + grouped_grads_, + grouped_square_avgs_, + grouped_grad_avgs_, + grouped_momentum_buffer_list_, + grouped_state_steps_, + ) + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_square_avgs = cast(list[Tensor], grouped_square_avgs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + state_and_grads = [grouped_grads, grouped_square_avgs] + if momentum > 0: + grouped_momentum_buffer_list = cast( + list[Tensor], grouped_momentum_buffer_list_ + ) + state_and_grads.append(grouped_momentum_buffer_list) + if centered: + grouped_grad_avgs = cast(list[Tensor], grouped_grad_avgs_) + state_and_grads.append(grouped_grad_avgs) + _view_as_real(grouped_params, *state_and_grads) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + torch._foreach_mul_(grouped_square_avgs, alpha) + torch._foreach_addcmul_( + grouped_square_avgs, grouped_grads, grouped_grads, value=1 - alpha + ) + + if centered: + grouped_grad_avgs = cast(list[Tensor], grouped_grad_avgs_) + torch._foreach_lerp_(grouped_grad_avgs, grouped_grads, 1 - alpha) + avg = torch._foreach_addcmul( + grouped_square_avgs, grouped_grad_avgs, grouped_grad_avgs, value=-1 + ) + torch._foreach_sqrt_(avg) + torch._foreach_add_(avg, eps) + else: + avg = torch._foreach_sqrt(grouped_square_avgs) + torch._foreach_add_(avg, eps) + + if momentum > 0: + grouped_momentum_buffer_list = cast( + list[Tensor], grouped_momentum_buffer_list_ + ) + torch._foreach_mul_(grouped_momentum_buffer_list, momentum) + torch._foreach_addcdiv_(grouped_momentum_buffer_list, grouped_grads, avg) + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + momentum_lr = torch._foreach_mul(grouped_momentum_buffer_list, -lr) + torch._foreach_add_(grouped_params, momentum_lr) + else: + torch._foreach_add_( + grouped_params, grouped_momentum_buffer_list, alpha=-lr + ) + else: + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + torch._foreach_div_(avg, -lr) + torch._foreach_addcdiv_(grouped_params, grouped_grads, avg) + else: + torch._foreach_addcdiv_(grouped_params, grouped_grads, avg, value=-lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_rmsprop) +def rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, +) -> None: + r"""Functional API that performs rmsprop algorithm computation. + + See :class:`~torch.optim.RMSProp` for details. + """ + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_rmsprop + else: + func = _single_tensor_rmsprop + + func( + params, + grads, + square_avgs, + grad_avgs, + momentum_buffer_list, + state_steps, + lr=lr, + alpha=alpha, + eps=eps, + weight_decay=weight_decay, + momentum=momentum, + centered=centered, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rprop.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rprop.py new file mode 100644 index 0000000000000000000000000000000000000000..ad5aa3fabf21fae0ca077e3d8dc305db9021f71e --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/rprop.py @@ -0,0 +1,472 @@ +# mypy: allow-untyped-defs +r"""Implementation for the Resilient backpropagation.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Rprop", "rprop"] + + +class Rprop(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + etas: tuple[float, float] = (0.5, 1.2), + step_sizes: tuple[float, float] = (1e-6, 50), + *, + capturable: bool = False, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 < etas[0] < 1.0 < etas[1]: + raise ValueError(f"Invalid eta values: {etas[0]}, {etas[1]}") + + defaults = { + "lr": lr, + "etas": etas, + "step_sizes": step_sizes, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group(self, group, params, grads, prevs, step_sizes, state_steps): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params.append(p) + grad = p.grad + if grad.is_sparse: + raise RuntimeError("Rprop does not support sparse gradients") + + grads.append(grad) + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + + state["prev"] = torch.zeros_like(p, memory_format=torch.preserve_format) + if p.dtype.is_complex: + # Complex Number should be as if they are two independent real numbers. + # Hence the step_size shouldn't be zero for imaginary part. + state["step_size"] = torch.full_like( + grad, complex(group["lr"], group["lr"]) + ) + else: + state["step_size"] = torch.full_like(grad, _to_scalar(group["lr"])) + + prevs.append(state["prev"]) + step_sizes.append(state["step_size"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params: list[Tensor] = [] + grads: list[Tensor] = [] + prevs: list[Tensor] = [] + step_sizes: list[Tensor] = [] + state_steps: list[Tensor] = [] + + etaminus, etaplus = group["etas"] + step_size_min, step_size_max = group["step_sizes"] + foreach = group["foreach"] + maximize = group["maximize"] + + has_complex = self._init_group( + group, params, grads, prevs, step_sizes, state_steps + ) + + rprop( + params, + grads, + prevs, + step_sizes, + state_steps, + step_size_min=step_size_min, + step_size_max=step_size_max, + etaminus=etaminus, + etaplus=etaplus, + foreach=foreach, + maximize=maximize, + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +Rprop.__doc__ = ( + r"""Implements the resilient backpropagation algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta) + \text{ (objective)}, \\ + &\hspace{13mm} \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min} + \text{ (step sizes)} \\ + &\textbf{initialize} : g^0_{prev} \leftarrow 0, + \: \eta_0 \leftarrow \text{lr (learning rate)} \\ + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \textbf{for} \text{ } i = 0, 1, \ldots, d-1 \: \mathbf{do} \\ + &\hspace{10mm} \textbf{if} \: g^i_{prev} g^i_t > 0 \\ + &\hspace{15mm} \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+}, + \Gamma_{max}) \\ + &\hspace{10mm} \textbf{else if} \: g^i_{prev} g^i_t < 0 \\ + &\hspace{15mm} \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-}, + \Gamma_{min}) \\ + &\hspace{15mm} g^i_t \leftarrow 0 \\ + &\hspace{10mm} \textbf{else} \: \\ + &\hspace{15mm} \eta^i_t \leftarrow \eta^i_{t-1} \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t) \\ + &\hspace{5mm}g_{prev} \leftarrow g_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to the paper + `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm + `_.""" # codespell:ignore + + rf""" + + Args: + {_params_doc} + lr (float, optional): learning rate (default: 1e-2) + etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that + are multiplicative increase and decrease factors + (default: (0.5, 1.2)) + step_sizes (Tuple[float, float], optional): a pair of minimal and + maximal allowed step sizes (default: (1e-6, 50)) + {_capturable_doc} + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + + """ +) + + +def _single_tensor_rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + prev = prevs[i] + step_size = step_sizes[i] + step = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + step += 1 + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + prev = torch.view_as_real(prev) + param = torch.view_as_real(param) + step_size = torch.view_as_real(step_size) + if differentiable: + sign = grad.mul(prev.clone()).sign() + else: + sign = grad.mul(prev).sign() + + if capturable: + sign.copy_(torch.where(sign.gt(0), etaplus, sign)) + sign.copy_(torch.where(sign.lt(0), etaminus, sign)) + sign.copy_(torch.where(sign.eq(0), 1, sign)) + else: + sign[sign.gt(0)] = etaplus + sign[sign.lt(0)] = etaminus + sign[sign.eq(0)] = 1 + + # update stepsizes with step size updates + step_size.mul_(sign).clamp_(step_size_min, step_size_max) + + # for dir<0, dfdx=0 + # for dir>=0 dfdx=dfdx + grad = grad.clone(memory_format=torch.preserve_format) + if capturable: + grad.copy_(torch.where(sign.eq(etaminus), 0, grad)) + else: + grad[sign.eq(etaminus)] = 0 + + # update parameters + param.addcmul_(grad.sign(), step_size, value=-1) + prev.copy_(grad) + + +def _multi_tensor_rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, prevs, step_sizes, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_prevs_, + grouped_step_sizes_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_prevs = cast(list[Tensor], grouped_prevs_) + grouped_step_sizes = cast(list[Tensor], grouped_step_sizes_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + # Handle complex params + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_prevs, grouped_step_sizes + ) + + signs = torch._foreach_mul(grouped_grads, grouped_prevs) + if maximize: + torch._foreach_neg_(signs) + + # At the end of the step, grouped_prevs will contain the current grads, so we reuse + # grouped_prevs memory instead of creating a new buffer, but, for clarity, we reassign + # to keep referring to the buffer as grouped_grads. + torch._foreach_copy_(grouped_prevs, grouped_grads) + if maximize: + torch._foreach_neg_(grouped_prevs) + grouped_grads = grouped_prevs + + torch._foreach_sign_(signs) + if capturable: + for sign in signs: + sign.copy_(torch.where(sign.gt(0), etaplus, sign)) + sign.copy_(torch.where(sign.lt(0), etaminus, sign)) + sign.copy_(torch.where(sign.eq(0), 1, sign)) + else: + for sign in signs: + sign[sign.gt(0)] = etaplus + sign[sign.lt(0)] = etaminus + sign[sign.eq(0)] = 1 + + # update stepsizes with step size updates + torch._foreach_mul_(grouped_step_sizes, signs) + for step_size in grouped_step_sizes: + step_size.clamp_(step_size_min, step_size_max) + + # for dir<0, dfdx=0 + # for dir>=0 dfdx=dfdx + grouped_grads = list(grouped_grads) + for i in range(len(grouped_grads)): + grouped_grads[i].copy_( + torch.where(signs[i].eq(etaminus), 0, grouped_grads[i]) + ) + + # explicitly del signs as it's not used after here to save memory + del signs + + # update parameters + grad_signs = [grad.sign() for grad in grouped_grads] + torch._foreach_addcmul_( + grouped_params, grad_signs, grouped_step_sizes, value=-1 + ) + + # Logically, you may expect grouped_prevs to get updated to grouped_grads, but that's + # basically already happened since we've been using grouped_prevs' memory to store + # updated grouped_grads! + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_rprop) +def rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + maximize: bool = False, + differentiable: bool = False, + has_complex: bool = False, + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, +) -> None: + r"""Functional API that performs rprop algorithm computation. + + See :class:`~torch.optim.Rprop` for details. + """ + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_rprop + else: + func = _single_tensor_rprop + + func( + params, + grads, + prevs, + step_sizes, + state_steps, + step_size_min=step_size_min, + step_size_max=step_size_max, + etaminus=etaminus, + etaplus=etaplus, + capturable=capturable, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sgd.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sgd.py new file mode 100644 index 0000000000000000000000000000000000000000..bb43ae3f6ce4509f5ce76e9311290af95ff692cc --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sgd.py @@ -0,0 +1,545 @@ +# mypy: allow-untyped-defs +r"""Implementation for Stochastic Gradient Descent optimizer.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _foreach_doc, + _fused_doc, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + DeviceDict, + Optimizer, + ParamsT, +) + + +__all__ = ["SGD", "sgd"] + + +class SGD(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + momentum: float = 0, + dampening: float = 0, + weight_decay: float | Tensor = 0, + nesterov: bool = False, + *, + maximize: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if momentum < 0.0: + raise ValueError(f"Invalid momentum value: {momentum}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "momentum": momentum, + "dampening": dampening, + "weight_decay": weight_decay, + "nesterov": nesterov, + "maximize": maximize, + "foreach": foreach, + "differentiable": differentiable, + "fused": fused, + } + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super().__init__(params, defaults) + + if fused: + self._step_supports_amp_scaling = True + self._need_device_dtype_check_for_fused = True + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("nesterov", False) + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("differentiable", False) + group.setdefault("fused", False) + + def _init_group(self, group, params, grads, momentum_buffer_list): + has_sparse_grad = False + + for p in group["params"]: + if p.grad is not None: + if group["fused"] and getattr( + self, "_need_device_dtype_check_for_fused", True + ): + _device_dtype_check_for_fused(p) + self._need_device_dtype_check_for_fused = False + params.append(p) + grads.append(p.grad) + if p.grad.is_sparse: + has_sparse_grad = True + + if group["momentum"] != 0: + state = self.state[p] + momentum_buffer_list.append(state.get("momentum_buffer")) + + return has_sparse_grad + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params: list[Tensor] = [] + grads: list[Tensor] = [] + momentum_buffer_list: list[Tensor | None] = [] + + has_sparse_grad = self._init_group( + group, params, grads, momentum_buffer_list + ) + + sgd( + params, + grads, + momentum_buffer_list, + weight_decay=group["weight_decay"], + momentum=group["momentum"], + lr=group["lr"], + dampening=group["dampening"], + nesterov=group["nesterov"], + maximize=group["maximize"], + has_sparse_grad=has_sparse_grad, + foreach=group["foreach"], + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + ) + + if group["momentum"] != 0: + # update momentum_buffers in state + for p, momentum_buffer in zip( + params, momentum_buffer_list, strict=True + ): + state = self.state[p] + state["momentum_buffer"] = momentum_buffer + + return loss + + +SGD.__doc__ = ( + r"""Implements stochastic gradient descent (optionally with momentum). + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) + \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ + &\hspace{13mm} \:\mu \text{ (momentum)}, \:\tau \text{ (dampening)}, + \:\textit{ nesterov,}\:\textit{ maximize} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}\textbf{if} \: \mu \neq 0 \\ + &\hspace{10mm}\textbf{if} \: t > 1 \\ + &\hspace{15mm} \textbf{b}_t \leftarrow \mu \textbf{b}_{t-1} + (1-\tau) g_t \\ + &\hspace{10mm}\textbf{else} \\ + &\hspace{15mm} \textbf{b}_t \leftarrow g_t \\ + &\hspace{10mm}\textbf{if} \: \textit{nesterov} \\ + &\hspace{15mm} g_t \leftarrow g_{t} + \mu \textbf{b}_t \\ + &\hspace{10mm}\textbf{else} \\[-1.ex] + &\hspace{15mm} g_t \leftarrow \textbf{b}_t \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma g_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + Nesterov momentum is based on the formula from + `On the importance of initialization and momentum in deep learning`__. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + momentum (float, optional): momentum factor (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + nesterov (bool, optional): enables Nesterov momentum. Only applicable + when momentum is non-zero. (default: False) + {_maximize_doc} + {_foreach_doc} + {_differentiable_doc} + {_fused_doc} + """ + + r""" + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + + __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + + .. note:: + The implementation of SGD with Momentum/Nesterov subtly differs from + Sutskever et al. and implementations in some other frameworks. + + Considering the specific case of Momentum, the update can be written as + + .. math:: + \begin{aligned} + v_{t+1} & = \mu * v_{t} + g_{t+1}, \\ + p_{t+1} & = p_{t} - \text{lr} * v_{t+1}, + \end{aligned} + + where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the + parameters, gradient, velocity, and momentum respectively. + + This is in contrast to Sutskever et al. and + other frameworks which employ an update of the form + + .. math:: + \begin{aligned} + v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\ + p_{t+1} & = p_{t} - v_{t+1}. + \end{aligned} + + The Nesterov version is analogously modified. + + Moreover, the initial value of the momentum buffer is set to the + gradient value at the first step. This is in contrast to some other + frameworks that initialize it to all zeros. One notable side effect + of this decision is that the first momentum value will not be scaled + by dampening. Dampening will be applied starting at the second step. + + """ +) + + +def sgd( + params: list[Tensor], + d_p_list: list[Tensor], + momentum_buffer_list: list[Tensor | None], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + has_sparse_grad: bool = False, + foreach: bool | None = None, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, +) -> None: + r"""Functional API that performs SGD algorithm computation. + + See :class:`~torch.optim.SGD` for details. + """ + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if foreach is None and fused is None: + # why must we be explicit about an if statement for torch.jit.is_scripting here? + # because JIT can't handle Optionals nor fancy conditionals when scripting + if not torch.jit.is_scripting(): + fused, foreach = _default_to_fused_or_foreach( + params, differentiable=False, use_fused=False + ) + else: + foreach = False + fused = False + if foreach is None: + foreach = False + if fused is None: + fused = False + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_sgd + elif fused and not torch.jit.is_scripting(): + func = _fused_sgd + else: + func = _single_tensor_sgd + + func( + params, + d_p_list, + momentum_buffer_list, + weight_decay=weight_decay, + momentum=momentum, + lr=lr, + dampening=dampening, + nesterov=nesterov, + has_sparse_grad=has_sparse_grad, + maximize=maximize, + grad_scale=grad_scale, + found_inf=found_inf, + ) + + +def _single_tensor_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + + if weight_decay != 0: + # Nested if is necessary to bypass jitscript rules + if isinstance(weight_decay, Tensor): + if weight_decay.requires_grad: + # usually this is the differentiable path, which is why the param.clone() is needed + grad = grad.addcmul_(param.clone(), weight_decay) + else: + # pyrefly: ignore [bad-argument-type] + grad = grad.add(param, alpha=weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + if momentum != 0: + buf = momentum_buffer_list[i] + + if buf is None: + buf = grad.detach().clone() + momentum_buffer_list[i] = buf + else: + buf.mul_(momentum).add_(grad, alpha=1 - dampening) + + if nesterov: + grad = grad.add(buf, alpha=momentum) + else: + grad = buf + + # Nested if is necessary to bypass jitscript rules + if isinstance(lr, Tensor): + if lr.requires_grad: + param.addcmul_(grad, lr, value=-1) + else: + # pyrefly: ignore [bad-argument-type] + param.add_(grad, alpha=-lr) + else: + param.add_(grad, alpha=-lr) + + +def _multi_tensor_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, momentum_buffer_list], # type: ignore[list-item] + with_indices=True, + ) + for ( + device_params_, + device_grads_, + device_momentum_buffer_list, + ), indices in grouped_tensors.values(): + device_params: list[Tensor] = cast(list[Tensor], device_params_) + device_grads: list[Tensor] = cast(list[Tensor], device_grads_) + + device_has_sparse_grad = has_sparse_grad and any( + grad.is_sparse for grad in device_grads + ) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + if momentum != 0: + bufs: list[Tensor] = [] + + all_states_with_momentum_buffer = True + for i in range(len(device_momentum_buffer_list)): + if device_momentum_buffer_list[i] is None: + all_states_with_momentum_buffer = False + break + else: + bufs.append(cast(Tensor, device_momentum_buffer_list[i])) + + if all_states_with_momentum_buffer: + torch._foreach_mul_(bufs, momentum) + torch._foreach_add_(bufs, device_grads, alpha=1 - dampening) + else: + bufs = [] + + for i in range(len(device_momentum_buffer_list)): + if device_momentum_buffer_list[i] is None: + buf = device_momentum_buffer_list[i] = momentum_buffer_list[ + indices[i] + ] = device_grads[i].detach().clone() + else: + buf = cast(Tensor, device_momentum_buffer_list[i]) + buf.mul_(momentum).add_(device_grads[i], alpha=1 - dampening) + + bufs.append(buf) + + if nesterov: + torch._foreach_add_(device_grads, bufs, alpha=momentum) + else: + device_grads = bufs + + if not device_has_sparse_grad: + # handle internal item() call if lr is a tensor + if isinstance(lr, torch.Tensor) and torch.compiler.is_compiling(): + grads_x_lr = torch._foreach_mul(device_grads, -lr) + torch._foreach_add_(device_params, grads_x_lr) + else: + torch._foreach_add_(device_params, device_grads, alpha=-lr) + else: + # foreach APIs don't support sparse + for i in range(len(device_params)): + device_params[i].add_(device_grads[i], alpha=-lr) + + +def _fused_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if not params: + return + if has_sparse_grad: + raise RuntimeError("`_fused_sgd` does not support sparse gradients") + grad_scale_dict: DeviceDict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else {} + ) + found_inf_dict: DeviceDict = ( + {found_inf.device: found_inf} if found_inf is not None else {} + ) + + no_momentum_buffer = momentum == 0 + is_first_step = ( + all(t is None for t in momentum_buffer_list) and not no_momentum_buffer + ) + if is_first_step: + for i, g in enumerate(grads): + momentum_buffer_list[i] = torch.empty_like(g) + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, momentum_buffer_list], # type: ignore[list-item] + with_indices=False, + ) + for (device, _), ( + (device_params_, device_grads_, device_momentum_buffer_list), + _, + ) in grouped_tensors.items(): + device_params: list[Tensor] = cast(list[Tensor], device_params_) + device_grads: list[Tensor] = cast(list[Tensor], device_grads_) + device_grad_scale, device_found_inf = None, None + if grad_scale is not None: + device_grad_scale = grad_scale_dict.setdefault( + device, grad_scale.to(device) + ) + if found_inf_dict is not None and found_inf is not None: + device_found_inf = found_inf_dict.setdefault(device, found_inf.to(device)) + torch._fused_sgd_( + device_params, + device_grads, + [] + if no_momentum_buffer + else cast(list[Tensor], device_momentum_buffer_list), + weight_decay=weight_decay, + momentum=momentum, + lr=lr, + dampening=dampening, + nesterov=nesterov, + maximize=maximize, + is_first_step=is_first_step, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sparse_adam.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sparse_adam.py new file mode 100644 index 0000000000000000000000000000000000000000..e3724b0468f50a9411e47868564f2463840d0116 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/sparse_adam.py @@ -0,0 +1,189 @@ +# mypy: allow-untyped-defs + +import torch +from torch import Tensor + +from . import _functional as F +from .optimizer import _maximize_doc, _params_doc, _to_scalar, Optimizer, ParamsT + + +__all__ = ["SparseAdam"] + + +class SparseAdam(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + maximize: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 < lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 < eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "maximize": maximize, + } + super().__init__(params, defaults) + + sparse_params = [] + complex_params = [] + for index, param_group in enumerate(self.param_groups): + if not isinstance(param_group, dict): + raise AssertionError( + f"param_groups must be a list of dicts, but got {type(param_group)}" + ) + # given param group, convert given params to a list first before iterating + for d_index, d_param in enumerate(param_group["params"]): + if d_param.is_sparse: + sparse_params.append([index, d_index]) + if d_param.is_complex(): + complex_params.append([index, d_index]) + if sparse_params: + raise ValueError( + f"Sparse params at indices {sparse_params}: SparseAdam requires dense parameter tensors" + ) + if complex_params: + raise ValueError( + f"Complex params at indices {complex_params}: SparseAdam does not support complex parameters" + ) + + @torch.no_grad() + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + state_steps: list[int] = [] + beta1, beta2 = group["betas"] + maximize = group.get("maximize", False) + + for p in group["params"]: + if p.grad is not None: + params_with_grad.append(p) + if not p.grad.is_sparse: + raise RuntimeError( + "SparseAdam does not support dense gradients, please consider Adam instead" + ) + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = 0 + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + + # update the steps for each param group update + state["step"] += 1 + # record the step after step update + state_steps.append(state["step"]) + + F.sparse_adam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + eps=group["eps"], + beta1=beta1, + beta2=beta2, + lr=_to_scalar(group["lr"]), + maximize=maximize, + ) + + return loss + + +SparseAdam.__doc__ = rf"""SparseAdam implements a masked version of the Adam algorithm + suitable for sparse gradients. Currently, due to implementation constraints (explained + below), SparseAdam is only intended for a narrow subset of use cases, specifically + parameters of a dense layout with gradients of a sparse layout. This occurs in a + special case where the module backwards produces grads already in a sparse layout. + One example NN module that behaves as such is ``nn.Embedding(sparse=True)``. + + SparseAdam approximates the Adam algorithm by masking out the parameter and moment + updates corresponding to the zero values in the gradients. Whereas the Adam algorithm + will update the first moment, the second moment, and the parameters based on all values + of the gradients, SparseAdam only updates the moments and parameters corresponding + to the non-zero values of the gradients. + + A simplified way of thinking about the `intended` implementation is as such: + + 1. Create a mask of the non-zero values in the sparse gradients. For example, + if your gradient looks like [0, 5, 0, 0, 9], the mask would be [0, 1, 0, 0, 1]. + 2. Apply this mask over the running moments and do computation on only the + non-zero values. + 3. Apply this mask over the parameters and only apply an update on non-zero values. + + In actuality, we use sparse layout Tensors to optimize this approximation, which means the + more gradients that are masked by not being materialized, the more performant the optimization. + Since we rely on using sparse layout tensors, we infer that any materialized value in the + sparse layout is non-zero and we do NOT actually verify that all values are not zero! + It is important to not conflate a semantically sparse tensor (a tensor where many + of its values are zeros) with a sparse layout tensor (a tensor where ``.is_sparse`` + returns ``True``). The SparseAdam approximation is intended for `semantically` sparse + tensors and the sparse layout is only a implementation detail. A clearer implementation + would be to use MaskedTensors, but those are experimental. + + + .. note:: + + If you suspect your gradients are semantically sparse (but do not have sparse + layout), this variant may not be the best for you. Ideally, you want to avoid + materializing anything that is suspected to be sparse in the first place, since + needing to convert all your grads from dense layout to sparse layout may outweigh + the performance gain. Here, using Adam may be the best alternative, unless you + can easily rig up your module to output sparse grads similar to + ``nn.Embedding(sparse=True)``. If you insist on converting your grads, you can do + so by manually overriding your parameters' ``.grad`` fields with their sparse + equivalents before calling ``.step()``. + + + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + {_maximize_doc} + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + + """ diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/swa_utils.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/swa_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cbac02a853be9347e0d1e84078ac6d4c915773db --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/optim/swa_utils.py @@ -0,0 +1,549 @@ +# mypy: allow-untyped-defs +r"""Implementation for Stochastic Weight Averaging implementation.""" + +import itertools +import math +import warnings +from collections.abc import Callable, Iterable +from copy import deepcopy +from typing import Any, cast, Literal, Union +from typing_extensions import override + +import torch +from torch import Tensor +from torch.nn import Module +from torch.optim.lr_scheduler import _format_param, LRScheduler +from torch.utils._foreach_utils import _get_foreach_kernels_supported_devices + +from .optimizer import Optimizer + + +__all__ = [ + "AveragedModel", + "update_bn", + "SWALR", + "get_ema_multi_avg_fn", + "get_swa_multi_avg_fn", + "get_ema_avg_fn", + "get_swa_avg_fn", +] + +from torch.utils._foreach_utils import _group_tensors_by_device_and_dtype + + +PARAM_LIST = Union[tuple[Tensor, ...], list[Tensor]] + + +def get_ema_multi_avg_fn(decay=0.999): + """Get the function applying exponential moving average (EMA) across multiple params.""" + + if decay < 0.0 or decay > 1.0: + raise ValueError( + f"Invalid decay value {decay} provided. Please provide a value in [0,1] range." + ) + + @torch.no_grad() + def ema_update( + ema_param_list: PARAM_LIST, current_param_list: PARAM_LIST, _ + ) -> None: + # foreach lerp only handles float and complex + if torch.is_floating_point(ema_param_list[0]) or torch.is_complex( + ema_param_list[0] + ): + torch._foreach_lerp_(ema_param_list, current_param_list, 1 - decay) + else: + for p_ema, p_model in zip(ema_param_list, current_param_list, strict=True): + p_ema.copy_(p_ema * decay + p_model * (1 - decay)) + + return ema_update + + +def get_swa_multi_avg_fn(): + """Get the function applying stochastic weight average (SWA) across multiple params.""" + + @torch.no_grad() + def swa_update( + averaged_param_list: PARAM_LIST, + current_param_list: PARAM_LIST, + num_averaged: Tensor | int, + ) -> None: + # foreach lerp only handles float and complex + if torch.is_floating_point(averaged_param_list[0]) or torch.is_complex( + averaged_param_list[0] + ): + torch._foreach_lerp_( + averaged_param_list, + current_param_list, + cast(float, 1 / (num_averaged + 1)), + ) + else: + diffs = torch._foreach_sub(current_param_list, averaged_param_list) + if isinstance(num_averaged, Tensor): + torch._foreach_addcdiv_( + averaged_param_list, + diffs, + [num_averaged + 1] * len(averaged_param_list), + ) + else: + torch._foreach_add_( + averaged_param_list, diffs, alpha=1.0 / (num_averaged + 1) + ) + + return swa_update + + +def get_ema_avg_fn(decay=0.999): + """Get the function applying exponential moving average (EMA) across a single param.""" + + if decay < 0.0 or decay > 1.0: + raise ValueError( + f"Invalid decay value {decay} provided. Please provide a value in [0,1] range." + ) + + @torch.no_grad() + def ema_update(ema_param: Tensor, current_param: Tensor, num_averaged): + return decay * ema_param + (1 - decay) * current_param + + return ema_update + + +def get_swa_avg_fn(): + """Get the function applying stochastic weight average (SWA) across a single param.""" + + @torch.no_grad() + def swa_update( + averaged_param: Tensor, current_param: Tensor, num_averaged: Tensor | int + ): + return averaged_param + (current_param - averaged_param) / (num_averaged + 1) + + return swa_update + + +class AveragedModel(Module): + r"""Implements averaged model for Stochastic Weight Averaging (SWA) and Exponential Moving Average (EMA). + + Stochastic Weight Averaging was proposed in `Averaging Weights Leads to + Wider Optima and Better Generalization`_ by Pavel Izmailov, Dmitrii + Podoprikhin, Timur Garipov, Dmitry Vetrov and Andrew Gordon Wilson + (UAI 2018). + + Exponential Moving Average is a variation of `Polyak averaging`_, + but using exponential weights instead of equal weights across iterations. + + AveragedModel class creates a copy of the provided module :attr:`model` + on the device :attr:`device` and allows to compute running averages of the + parameters of the :attr:`model`. + + Args: + model (torch.nn.Module): model to use with SWA/EMA + device (torch.device, optional): if provided, the averaged model will be + stored on the :attr:`device` + avg_fn (function, optional): the averaging function used to update + parameters; the function must take in the current value of the + :class:`AveragedModel` parameter, the current value of :attr:`model` + parameter, and the number of models already averaged; if None, + an equally weighted average is used (default: None) + multi_avg_fn (function, optional): the averaging function used to update + parameters inplace; the function must take in the current values of the + :class:`AveragedModel` parameters as a list, the current values of :attr:`model` + parameters as a list, and the number of models already averaged; if None, + an equally weighted average is used (default: None) + use_buffers (bool): if ``True``, it will compute running averages for + both the parameters and the buffers of the model. (default: ``False``) + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> loader, optimizer, model, loss_fn = ... + >>> swa_model = torch.optim.swa_utils.AveragedModel(model) + >>> scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, + >>> T_max=300) + >>> swa_start = 160 + >>> swa_scheduler = SWALR(optimizer, swa_lr=0.05) + >>> for i in range(300): + >>> for input, target in loader: + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + >>> if i > swa_start: + >>> swa_model.update_parameters(model) + >>> swa_scheduler.step() + >>> else: + >>> scheduler.step() + >>> + >>> # Update bn statistics for the swa_model at the end + >>> torch.optim.swa_utils.update_bn(loader, swa_model) + + You can also use custom averaging functions with the `avg_fn` or `multi_avg_fn` parameters. + If no averaging function is provided, the default is to compute + equally-weighted average of the weights (SWA). + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> # Compute exponential moving averages of the weights and buffers + >>> ema_model = torch.optim.swa_utils.AveragedModel(model, + >>> torch.optim.swa_utils.get_ema_multi_avg_fn(0.9), use_buffers=True) + + .. note:: + When using SWA/EMA with models containing Batch Normalization you may + need to update the activation statistics for Batch Normalization. + This can be done either by using the :meth:`torch.optim.swa_utils.update_bn` + or by setting :attr:`use_buffers` to `True`. The first approach updates the + statistics in a post-training step by passing data through the model. The + second does it during the parameter update phase by averaging all buffers. + Empirical evidence has shown that updating the statistics in normalization + layers increases accuracy, but you may wish to empirically test which + approach yields the best results in your problem. + + .. note:: + :attr:`avg_fn` and `multi_avg_fn` are not saved in the :meth:`state_dict` of the model. + + .. note:: + When :meth:`update_parameters` is called for the first time (i.e. + :attr:`n_averaged` is `0`) the parameters of `model` are copied + to the parameters of :class:`AveragedModel`. For every subsequent + call of :meth:`update_parameters` the function `avg_fn` is used + to update the parameters. + + .. _Averaging Weights Leads to Wider Optima and Better Generalization: + https://arxiv.org/abs/1803.05407 + .. _There Are Many Consistent Explanations of Unlabeled Data: Why You Should + Average: + https://arxiv.org/abs/1806.05594 + .. _SWALP: Stochastic Weight Averaging in Low-Precision Training: + https://arxiv.org/abs/1904.11943 + .. _Stochastic Weight Averaging in Parallel: Large-Batch Training That + Generalizes Well: + https://arxiv.org/abs/2001.02312 + .. _Polyak averaging: + https://paperswithcode.com/method/polyak-averaging + """ + + n_averaged: Tensor + + def __init__( + self, + model: Module, + device: int | torch.device | None = None, + avg_fn: Callable[[Tensor, Tensor, Tensor | int], Tensor] | None = None, + multi_avg_fn: Callable[[PARAM_LIST, PARAM_LIST, Tensor | int], None] + | None = None, + use_buffers=False, + ) -> None: # noqa: D107 + super().__init__() + if avg_fn is not None and multi_avg_fn is not None: + raise AssertionError( + "Only one of avg_fn and multi_avg_fn should be provided" + ) + self.module = deepcopy(model) + if device is not None: + self.module = self.module.to(device) + self.register_buffer( + "n_averaged", torch.tensor(0, dtype=torch.long, device=device) + ) + self.avg_fn = avg_fn + self.multi_avg_fn = multi_avg_fn + self.use_buffers = use_buffers + + def forward(self, *args, **kwargs): + """Forward pass.""" + return self.module(*args, **kwargs) + + def update_parameters(self, model: Module) -> None: + """Update model parameters.""" + self_param = ( + # pyrefly: ignore [bad-argument-type] + itertools.chain(self.module.parameters(), self.module.buffers()) + if self.use_buffers + else self.parameters() + ) + model_param = ( + # pyrefly: ignore [bad-argument-type] + itertools.chain(model.parameters(), model.buffers()) + if self.use_buffers + else model.parameters() + ) + self_param_detached: list[Tensor | None] = [] + model_param_detached: list[Tensor | None] = [] + copy_param = bool(self.n_averaged == 0) + for p_averaged, p_model in zip(self_param, model_param, strict=False): + p_model_ = p_model.detach().to(p_averaged.device) + self_param_detached.append(p_averaged.detach()) + model_param_detached.append(p_model_) + if copy_param: + p_averaged.detach().copy_(p_model_) + + if self.n_averaged > 0: + if self.multi_avg_fn is not None or self.avg_fn is None: + grouped_tensors = _group_tensors_by_device_and_dtype( + [self_param_detached, model_param_detached] + ) + for (device, _), ( + [self_params, model_params], + _, + ) in grouped_tensors.items(): + if self.multi_avg_fn: + self.multi_avg_fn( + self_params, # type: ignore[arg-type] + model_params, # type: ignore[arg-type] + self.n_averaged.to(device), + ) + elif ( + device is not None + and device.type in _get_foreach_kernels_supported_devices() + ): + multi_avg_fn = get_swa_multi_avg_fn() + multi_avg_fn( + self_params, model_params, self.n_averaged.to(device) + ) + else: + avg_fn = get_swa_avg_fn() + n_averaged = self.n_averaged.to(device) + for p_averaged, p_model in zip( # type: ignore[assignment] + self_params, model_params, strict=True + ): + # pyrefly: ignore [missing-attribute] + p_averaged.copy_(avg_fn(p_averaged, p_model, n_averaged)) + else: + for p_averaged, p_model in zip( # type: ignore[assignment] + self_param_detached, model_param_detached, strict=True + ): + # pyrefly: ignore [missing-attribute] + n_averaged = self.n_averaged.to(p_averaged.device) + # pyrefly: ignore [missing-attribute] + p_averaged.detach().copy_( + # pyrefly: ignore [missing-attribute, bad-argument-type] + self.avg_fn(p_averaged.detach(), p_model, n_averaged) + ) + + if not self.use_buffers: + # If not apply running averages to the buffers, + # keep the buffers in sync with the source model. + for b_swa, b_model in zip( + self.module.buffers(), model.buffers(), strict=True + ): + b_swa.detach().copy_(b_model.detach().to(b_swa.device)) + self.n_averaged += 1 + + +@torch.no_grad() +def update_bn( + loader: Iterable[Any], + model: Module, + device: int | torch.device | None = None, +) -> None: + r"""Update BatchNorm running_mean, running_var buffers in the model. + + It performs one pass over data in `loader` to estimate the activation + statistics for BatchNorm layers in the model. + + Args: + loader (torch.utils.data.DataLoader): dataset loader to compute the + activation statistics on. Each data batch should be either a + tensor, or a list/tuple whose first element is a tensor + containing data. + model (torch.nn.Module): model for which we seek to update BatchNorm + statistics. + device (torch.device, optional): If set, data will be transferred to + :attr:`device` before being passed into :attr:`model`. + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> loader, model = ... + >>> torch.optim.swa_utils.update_bn(loader, model) + + .. note:: + The `update_bn` utility assumes that each data batch in :attr:`loader` + is either a tensor or a list or tuple of tensors; in the latter case it + is assumed that :meth:`model.forward()` should be called on the first + element of the list or tuple corresponding to the data batch. + """ + momenta = {} + for module in model.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm): + module.reset_running_stats() + momenta[module] = module.momentum + + if not momenta: + return + + was_training = model.training + model.train() + for module in momenta: + module.momentum = None + + for input in loader: + if isinstance(input, (list, tuple)): + input = input[0] + if device is not None: + input = input.to(device) + + model(input) + + for bn_module in momenta: + bn_module.momentum = momenta[bn_module] + model.train(was_training) + + +class SWALR(LRScheduler): + r"""Anneals the learning rate in each parameter group to a fixed value. + + This learning rate scheduler is meant to be used with Stochastic Weight + Averaging (SWA) method (see `torch.optim.swa_utils.AveragedModel`). + + Args: + optimizer (torch.optim.Optimizer): wrapped optimizer + swa_lrs (float or list): the learning rate value for all param groups + together or separately for each group. + annealing_epochs (int): number of epochs in the annealing phase + (default: 10) + annealing_strategy (str): "cos" or "linear"; specifies the annealing + strategy: "cos" for cosine annealing, "linear" for linear annealing + (default: "cos") + last_epoch (int): the index of the last epoch (default: -1) + + The :class:`SWALR` scheduler can be used together with other + schedulers to switch to a constant learning rate late in the training + as in the example below. + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> loader, optimizer, model = ... + >>> lr_lambda = lambda epoch: 0.9 + >>> scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, + >>> lr_lambda=lr_lambda) + >>> swa_scheduler = torch.optim.swa_utils.SWALR(optimizer, + >>> anneal_strategy="linear", anneal_epochs=20, swa_lr=0.05) + >>> swa_start = 160 + >>> for i in range(300): + >>> for input, target in loader: + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + >>> if i > swa_start: + >>> swa_scheduler.step() + >>> else: + >>> scheduler.step() + + .. _Averaging Weights Leads to Wider Optima and Better Generalization: + https://arxiv.org/abs/1803.05407 + """ + + def __init__( + self, + optimizer: Optimizer, + swa_lr: float, + anneal_epochs=10, + anneal_strategy: Literal["cos", "linear"] = "cos", + last_epoch=-1, + ) -> None: # noqa: D107 + swa_lrs = _format_param("swa_lr", optimizer, swa_lr) + for swa_lr, group in zip(swa_lrs, optimizer.param_groups, strict=True): + group["swa_lr"] = swa_lr + if anneal_strategy not in ["cos", "linear"]: + raise ValueError( + "anneal_strategy must by one of 'cos' or 'linear', " + f"instead got {anneal_strategy}" + ) + self._set_anneal_func(anneal_strategy) + if not isinstance(anneal_epochs, int) or anneal_epochs < 0: + raise ValueError( + f"anneal_epochs must be equal or greater than 0, got {anneal_epochs}" + ) + self.anneal_epochs = anneal_epochs + super().__init__(optimizer, last_epoch) + + @staticmethod + def _linear_anneal(t): + return t + + @staticmethod + def _cosine_anneal(t): + return (1 - math.cos(math.pi * t)) / 2 + + @staticmethod + def _get_initial_lr(lr, swa_lr, alpha): + if alpha == 1: + return swa_lr + return (lr - alpha * swa_lr) / (1 - alpha) + + @override + def get_lr(self): + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Uses :attr:`anneal_func` to interpolate between each group's + ``group["lr"]`` and ``group["swa_lr"]`` over :attr:`anneal_epochs` + epochs. Once :attr:`anneal_epochs` is reached, keeps the learning rate + fixed at ``group["swa_lr"]``. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + # `_get_lr_called_within_step` is only available `_enable_get_lr_call`, + # so we ignore the type error here. See `LRScheduler.step()` for more details. + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, " + "please use `get_last_lr()`.", + UserWarning, + stacklevel=2, + ) + # Set in `LRScheduler._initial_step()` + step = self._step_count - 1 + if self.anneal_epochs == 0: + step = max(1, step) + # pyrefly: ignore [no-matching-overload] + prev_t = max(0, min(1, (step - 1) / max(1, self.anneal_epochs))) + prev_alpha = self.anneal_func(prev_t) + prev_lrs = [ + self._get_initial_lr(group["lr"], group["swa_lr"], prev_alpha) + for group in self.optimizer.param_groups + ] + # pyrefly: ignore [no-matching-overload] + t = max(0, min(1, step / max(1, self.anneal_epochs))) + alpha = self.anneal_func(t) + return [ + group["swa_lr"] * alpha + lr * (1 - alpha) + for group, lr in zip(self.optimizer.param_groups, prev_lrs, strict=True) + ] + + def _set_anneal_func(self, anneal_strategy: Literal["cos", "linear"]) -> None: + self._anneal_strategy = anneal_strategy + if anneal_strategy == "cos": + self.anneal_func = self._cosine_anneal + else: + self.anneal_func = self._linear_anneal + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in self.__dict__ which + is not the optimizer or anneal_func. + """ + return { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "anneal_func") + } + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + self._set_anneal_func(self._anneal_strategy) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_digraph.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_digraph.py new file mode 100644 index 0000000000000000000000000000000000000000..891693ec1b96fbf0ad85415b8f30b1795f5adfc0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_digraph.py @@ -0,0 +1,173 @@ +# mypy: allow-untyped-defs +from collections import deque + + +class DiGraph: + """Really simple unweighted directed graph data structure to track dependencies. + + The API is pretty much the same as networkx so if you add something just + copy their API. + """ + + def __init__(self): + # Dict of node -> dict of arbitrary attributes + self._node = {} + # Nested dict of node -> successor node -> nothing. + # (didn't implement edge data) + self._succ = {} + # Nested dict of node -> predecessor node -> nothing. + self._pred = {} + + # Keep track of the order in which nodes are added to + # the graph. + self._node_order = {} + self._insertion_idx = 0 + + def add_node(self, n, **kwargs): + """Add a node to the graph. + + Args: + n: the node. Can we any object that is a valid dict key. + **kwargs: any attributes you want to attach to the node. + """ + if n not in self._node: + self._node[n] = kwargs + self._succ[n] = {} + self._pred[n] = {} + self._node_order[n] = self._insertion_idx + self._insertion_idx += 1 + else: + self._node[n].update(kwargs) + + def add_edge(self, u, v): + """Add an edge to graph between nodes ``u`` and ``v`` + + ``u`` and ``v`` will be created if they do not already exist. + """ + # add nodes + self.add_node(u) + self.add_node(v) + + # add the edge + self._succ[u][v] = True + self._pred[v][u] = True + + def successors(self, n): + """Returns an iterator over successor nodes of n.""" + try: + return iter(self._succ[n]) + except KeyError as e: + raise ValueError(f"The node {n} is not in the digraph.") from e + + def predecessors(self, n): + """Returns an iterator over predecessors nodes of n.""" + try: + return iter(self._pred[n]) + except KeyError as e: + raise ValueError(f"The node {n} is not in the digraph.") from e + + @property + def edges(self): + """Returns an iterator over all edges (u, v) in the graph""" + for n, successors in self._succ.items(): + for succ in successors: + yield n, succ + + @property + def nodes(self): + """Returns a dictionary of all nodes to their attributes.""" + return self._node + + def __iter__(self): + """Iterate over the nodes.""" + return iter(self._node) + + def __contains__(self, n): + """Returns True if ``n`` is a node in the graph, False otherwise.""" + try: + return n in self._node + except TypeError: + return False + + def forward_transitive_closure(self, src: str) -> set[str]: + """Returns a set of nodes that are reachable from src""" + + result = set(src) + working_set = deque(src) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.successors(cur): + if n not in result: + result.add(n) + working_set.append(n) + return result + + def backward_transitive_closure(self, src: str) -> set[str]: + """Returns a set of nodes that are reachable from src in reverse direction""" + + result = set(src) + working_set = deque(src) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.predecessors(cur): + if n not in result: + result.add(n) + working_set.append(n) + return result + + def all_paths(self, src: str, dst: str): + """Returns a subgraph rooted at src that shows all the paths to dst.""" + + result_graph = DiGraph() + # First compute forward transitive closure of src (all things reachable from src). + forward_reachable_from_src = self.forward_transitive_closure(src) + + if dst not in forward_reachable_from_src: + return result_graph + + # Second walk the reverse dependencies of dst, adding each node to + # the output graph iff it is also present in forward_reachable_from_src. + # we don't use backward_transitive_closures for optimization purposes + working_set = deque(dst) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.predecessors(cur): + if n in forward_reachable_from_src: + result_graph.add_edge(n, cur) + # only explore further if its reachable from src + working_set.append(n) + + return result_graph.to_dot() + + def first_path(self, dst: str) -> list[str]: + """Returns a list of nodes that show the first path that resulted in dst being added to the graph.""" + path = [] + + while dst: + path.append(dst) + candidates = self._pred[dst].keys() + dst, min_idx = "", None + for candidate in candidates: + idx = self._node_order.get(candidate, None) + if idx is None: + break + if min_idx is None or idx < min_idx: + min_idx = idx + dst = candidate + + return list(reversed(path)) + + def to_dot(self) -> str: + """Returns the dot representation of the graph. + + Returns: + A dot representation of the graph. + """ + edges = "\n".join(f'"{f}" -> "{t}";' for f, t in self.edges) + return f"""\ +digraph G {{ +rankdir = LR; +node [shape=box]; +{edges} +}} +""" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_directory_reader.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_directory_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..227dafc365e01c31ea3d61782edbba2c88ac0d05 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_directory_reader.py @@ -0,0 +1,66 @@ +# mypy: allow-untyped-defs +import os.path +from glob import glob +from typing import cast + +import torch +from torch.types import Storage + + +__serialization_id_record_name__ = ".data/serialization_id" + + +# because get_storage_from_record returns a tensor!? +class _HasStorage: + def __init__(self, storage): + self._storage = storage + + def storage(self): + return self._storage + + +class DirectoryReader: + """ + Class to allow PackageImporter to operate on unzipped packages. Methods + copy the behavior of the internal PyTorchFileReader class (which is used for + accessing packages in all other cases). + + N.B.: ScriptObjects are not depickleable or accessible via this DirectoryReader + class due to ScriptObjects requiring an actual PyTorchFileReader instance. + """ + + def __init__(self, directory): + self.directory = directory + + def get_record(self, name): + filename = f"{self.directory}/{name}" + with open(filename, "rb") as f: + return f.read() + + def get_storage_from_record(self, name, numel, dtype): + filename = f"{self.directory}/{name}" + nbytes = torch._utils._element_size(dtype) * numel + storage = cast(Storage, torch.UntypedStorage) + return _HasStorage(storage.from_file(filename=filename, nbytes=nbytes)) + + def has_record(self, path): + full_path = os.path.join(self.directory, path) + return os.path.isfile(full_path) + + def get_all_records( + self, + ): + files = [ + filename[len(self.directory) + 1 :] + for filename in glob(f"{self.directory}/**", recursive=True) + if not os.path.isdir(filename) + ] + return files + + def serialization_id( + self, + ): + if self.has_record(__serialization_id_record_name__): + return self.get_record(__serialization_id_record_name__) + else: + return "" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_importlib.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_importlib.py new file mode 100644 index 0000000000000000000000000000000000000000..5a67e2d4923e07dcd967cd3a3c170297c2329c77 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_importlib.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +import _warnings +import os.path + + +# note: implementations +# copied from cpython's import code + + +# _zip_searchorder defines how we search for a module in the Zip +# archive: we first search for a package __init__, then for +# non-package .pyc, and .py entries. The .pyc entries +# are swapped by initzipimport() if we run in optimized mode. Also, +# '/' is replaced by path_sep there. + +_zip_searchorder = ( + ("/__init__.py", True), + (".py", False), +) + + +# Replace any occurrences of '\r\n?' in the input string with '\n'. +# This converts DOS and Mac line endings to Unix line endings. +def _normalize_line_endings(source): + source = source.replace(b"\r\n", b"\n") + source = source.replace(b"\r", b"\n") + return source + + +def _resolve_name(name, package, level): + """Resolve a relative module name to an absolute one.""" + bits = package.rsplit(".", level - 1) + if len(bits) < level: + raise ValueError("attempted relative import beyond top-level package") + base = bits[0] + return f"{base}.{name}" if name else base + + +def _sanity_check(name, package, level): + """Verify arguments are "sane".""" + if not isinstance(name, str): + raise TypeError(f"module name must be str, not {type(name)}") + if level < 0: + raise ValueError("level must be >= 0") + if level > 0: + if not isinstance(package, str): + raise TypeError("__package__ not set to a string") + elif not package: + raise ImportError("attempted relative import with no known parent package") + if not name and level == 0: + raise ValueError("Empty module name") + + +def _calc___package__(globals): + """Calculate what __package__ should be. + + __package__ is not guaranteed to be defined or could be set to None + to represent that its proper value is unknown. + + """ + package = globals.get("__package__") + spec = globals.get("__spec__") + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn( # noqa: G010 + f"__package__ != __spec__.parent ({package!r} != {spec.parent!r})", # noqa: G004 + ImportWarning, + stacklevel=3, + ) + return package + elif spec is not None: + return spec.parent + else: + _warnings.warn( # noqa: G010 + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, + stacklevel=3, + ) + package = globals["__name__"] + if "__path__" not in globals: + package = package.rpartition(".")[0] + return package + + +def _normalize_path(path): + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + parent, file_name = os.path.split(path) + if parent: + raise ValueError(f"{path!r} must be only a file name") + else: + return file_name diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mangling.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mangling.py new file mode 100644 index 0000000000000000000000000000000000000000..2a50840adcc917c9e3bde842d614a59e3f5e1248 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mangling.py @@ -0,0 +1,65 @@ +# mypy: allow-untyped-defs +"""Import mangling. +See mangling.md for details. +""" + +import re + + +_mangle_index = 0 + + +class PackageMangler: + """ + Used on import, to ensure that all modules imported have a shared mangle parent. + """ + + def __init__(self) -> None: + global _mangle_index + self._mangle_index = _mangle_index + # Increment the global index + _mangle_index += 1 + # Angle brackets are used so that there is almost no chance of + # confusing this module for a real module. Plus, it is Python's + # preferred way of denoting special modules. + self._mangle_parent = f"" + + def mangle(self, name) -> str: + assert len(name) != 0 + return self._mangle_parent + "." + name + + def demangle(self, mangled: str) -> str: + """ + Note: This only demangles names that were mangled by this specific + PackageMangler. It will pass through names created by a different + PackageMangler instance. + """ + if mangled.startswith(self._mangle_parent + "."): + return mangled.partition(".")[2] + + # wasn't a mangled name + return mangled + + def parent_name(self): + return self._mangle_parent + + +def is_mangled(name: str) -> bool: + return bool(re.match(r"", name)) + + +def demangle(name: str) -> str: + """ + Note: Unlike PackageMangler.demangle, this version works on any + mangled name, irrespective of which PackageMangler created it. + """ + if is_mangled(name): + _first, sep, last = name.partition(".") + # If there is only a base mangle prefix, e.g. '', + # then return an empty string. + return last if len(sep) != 0 else "" + return name + + +def get_mangle_prefix(name: str) -> str: + return name.partition(".")[0] if is_mangled(name) else name diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mock.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mock.py new file mode 100644 index 0000000000000000000000000000000000000000..d34970e908461d005382e2d0fc47d6c9ac10dffa --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_mock.py @@ -0,0 +1,123 @@ +# mypy: allow-untyped-defs +_magic_methods = [ + "__subclasscheck__", + "__hex__", + "__rmul__", + "__float__", + "__idiv__", + "__setattr__", + "__div__", + "__invert__", + "__nonzero__", + "__rshift__", + "__eq__", + "__pos__", + "__round__", + "__rand__", + "__or__", + "__complex__", + "__divmod__", + "__len__", + "__reversed__", + "__copy__", + "__reduce__", + "__deepcopy__", + "__rdivmod__", + "__rrshift__", + "__ifloordiv__", + "__hash__", + "__iand__", + "__xor__", + "__isub__", + "__oct__", + "__ceil__", + "__imod__", + "__add__", + "__truediv__", + "__unicode__", + "__le__", + "__delitem__", + "__sizeof__", + "__sub__", + "__ne__", + "__pow__", + "__bytes__", + "__mul__", + "__itruediv__", + "__bool__", + "__iter__", + "__abs__", + "__gt__", + "__iadd__", + "__enter__", + "__floordiv__", + "__call__", + "__neg__", + "__and__", + "__ixor__", + "__getitem__", + "__exit__", + "__cmp__", + "__getstate__", + "__index__", + "__contains__", + "__floor__", + "__lt__", + "__getattr__", + "__mod__", + "__trunc__", + "__delattr__", + "__instancecheck__", + "__setitem__", + "__ipow__", + "__ilshift__", + "__long__", + "__irshift__", + "__imul__", + "__lshift__", + "__dir__", + "__ge__", + "__int__", + "__ior__", +] + + +class MockedObject: + _name: str + + def __new__(cls, *args, **kwargs): + # _suppress_err is set by us in the mocked module impl, so that we can + # construct instances of MockedObject to hand out to people looking up + # module attributes. + + # Any other attempt to construct a MockedObject instance (say, in the + # unpickling process) should give an error. + if not kwargs.get("_suppress_err"): + raise NotImplementedError( + f"Object '{cls._name}' was mocked out during packaging " + f"but it is being used in '__new__'. If this error is " + "happening during 'load_pickle', please ensure that your " + "pickled object doesn't contain any mocked objects." + ) + # Otherwise, this is just a regular object creation + # (e.g. `x = MockedObject("foo")`), so pass it through normally. + return super().__new__(cls) + + def __init__(self, name: str, _suppress_err: bool): + self.__dict__["_name"] = name + + def __repr__(self): + return f"MockedObject({self._name})" + + +def install_method(method_name): + def _not_implemented(self, *args, **kwargs): + raise NotImplementedError( + f"Object '{self._name}' was mocked out during packaging but it is being used in {method_name}" + ) + + setattr(MockedObject, method_name, _not_implemented) + + +for method_name in _magic_methods: + install_method(method_name) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_pickler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_pickler.py new file mode 100644 index 0000000000000000000000000000000000000000..00c9a12cb9aa9a4640978f1422e8871dccbf68cf --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_pickler.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +# pyrefly: ignore [missing-module-attribute] +import sys +from pickle import ( # type: ignore[attr-defined] + _compat_pickle, + _extension_registry, + _getattribute, + _Pickler, + EXT1, + EXT2, + EXT4, + GLOBAL, + PicklingError, + STACK_GLOBAL, +) +from struct import pack +from types import FunctionType + +from .importer import Importer, ObjMismatchError, ObjNotFoundError, sys_importer + + +class _PyTorchLegacyPickler(_Pickler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._persistent_id = None + + def persistent_id(self, obj): + if self._persistent_id is None: + return super().persistent_id(obj) + return self._persistent_id(obj) + + +class PackagePickler(_PyTorchLegacyPickler): + """Package-aware pickler. + + This behaves the same as a normal pickler, except it uses an `Importer` + to find objects and modules to save. + """ + + def __init__(self, importer: Importer, *args, **kwargs): + self.importer = importer + super().__init__(*args, **kwargs) + + # Make sure the dispatch table copied from _Pickler is up-to-date. + # Previous issues have been encountered where a library (e.g. dill) + # mutate _Pickler.dispatch, PackagePickler makes a copy when this lib + # is imported, then the offending library removes its dispatch entries, + # leaving PackagePickler with a stale dispatch table that may cause + # unwanted behavior. + self.dispatch = _Pickler.dispatch.copy() # type: ignore[misc] + self.dispatch[FunctionType] = PackagePickler.save_global # type: ignore[assignment] + + def save_global(self, obj, name=None): + # ruff: noqa: F841 + # unfortunately the pickler code is factored in a way that + # forces us to copy/paste this function. The only change is marked + # CHANGED below. + write = self.write # type: ignore[attr-defined] + memo = self.memo # type: ignore[attr-defined] + + # CHANGED: import module from module environment instead of __import__ + try: + module_name, name = self.importer.get_name(obj, name) + except (ObjNotFoundError, ObjMismatchError) as err: + raise PicklingError(f"Can't pickle {obj}: {str(err)}") from err + + module = self.importer.import_module(module_name) + if sys.version_info >= (3, 14): + # pickle._getattribute signature changes in 3.14 + # to take iterable and return just the object (not tuple) + # We need to get the parent object that contains the attribute + name_parts = name.split(".") + if "" in name_parts: + raise PicklingError(f"Can't pickle local object {obj!r}") + if len(name_parts) == 1: + parent = module + else: + parent = _getattribute(module, name_parts[:-1]) + else: + _, parent = _getattribute(module, name) + # END CHANGED + + if self.proto >= 2: # type: ignore[attr-defined] + code = _extension_registry.get((module_name, name)) + if code: + assert code > 0 + if code <= 0xFF: + write(EXT1 + pack("= 3. + if self.proto >= 4: # type: ignore[attr-defined] + self.save(module_name) # type: ignore[attr-defined] + self.save(name) # type: ignore[attr-defined] + write(STACK_GLOBAL) + elif parent is not module: + self.save_reduce(getattr, (parent, lastname)) # type: ignore[attr-defined] + elif self.proto >= 3: # type: ignore[attr-defined] + write( + GLOBAL + + bytes(module_name, "utf-8") + + b"\n" + + bytes(name, "utf-8") + + b"\n" + ) + else: + if self.fix_imports: # type: ignore[attr-defined] + r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING + r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING + if (module_name, name) in r_name_mapping: + module_name, name = r_name_mapping[(module_name, name)] + elif module_name in r_import_mapping: + module_name = r_import_mapping[module_name] + try: + write( + GLOBAL + + bytes(module_name, "ascii") + + b"\n" + + bytes(name, "ascii") + + b"\n" + ) + except UnicodeEncodeError as exc: + raise PicklingError( + f"can't pickle global identifier '{module}.{name}' using " + f"pickle protocol {self.proto:d}" # type: ignore[attr-defined] + ) from exc + + self.memoize(obj) # type: ignore[attr-defined] + + +def create_pickler(data_buf, importer, protocol=4): + if importer is sys_importer: + # if we are using the normal import library system, then + # we can use the C implementation of pickle which is faster + return _PyTorchLegacyPickler(data_buf, protocol=protocol) + else: + return PackagePickler(importer, data_buf, protocol=protocol) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_unpickler.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_unpickler.py new file mode 100644 index 0000000000000000000000000000000000000000..ac5dd35e527cd4b6f8be92525231fc61bdc2f7c0 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_package_unpickler.py @@ -0,0 +1,27 @@ +# mypy: allow-untyped-defs +import _compat_pickle +import pickle + +from .importer import Importer + + +class PackageUnpickler(pickle._Unpickler): # type: ignore[name-defined] + """Package-aware unpickler. + + This behaves the same as a normal unpickler, except it uses `importer` to + find any global names that it encounters while unpickling. + """ + + def __init__(self, importer: Importer, *args, **kwargs): + super().__init__(*args, **kwargs) + self._importer = importer + + def find_class(self, module, name): + # Subclasses may override this. + if self.proto < 3 and self.fix_imports: # type: ignore[attr-defined] + if (module, name) in _compat_pickle.NAME_MAPPING: + module, name = _compat_pickle.NAME_MAPPING[(module, name)] + elif module in _compat_pickle.IMPORT_MAPPING: + module = _compat_pickle.IMPORT_MAPPING[module] + mod = self._importer.import_module(module) + return getattr(mod, name) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_stdlib.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_stdlib.py new file mode 100644 index 0000000000000000000000000000000000000000..d853b43a7127273d67123e532fd1fb8c4e680444 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/_stdlib.py @@ -0,0 +1,21 @@ +# mypy: allow-untyped-defs +"""List of Python standard library modules. + +Sadly, there is no reliable way to tell whether a module is part of the +standard library except by comparing to a canonical list. + +This is taken from https://github.com/PyCQA/isort/tree/develop/isort/stdlibs, +which itself is sourced from the Python documentation. +""" + +import sys + + +def is_stdlib_module(module: str) -> bool: + base_module = module.partition(".")[0] + return base_module in _get_stdlib_modules() + + +def _get_stdlib_modules(): + assert sys.version_info >= (3, 10) + return sys.stdlib_module_names diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/file_structure_representation.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/file_structure_representation.py new file mode 100644 index 0000000000000000000000000000000000000000..170d3ae7701068e3a12ed0c995a11555f7d69f48 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/file_structure_representation.py @@ -0,0 +1,137 @@ +# mypy: allow-untyped-defs + +from .glob_group import GlobGroup, GlobPattern + + +__all__ = ["Directory"] + + +class Directory: + """A file structure representation. Organized as Directory nodes that have lists of + their Directory children. Directories for a package are created by calling + :meth:`PackageImporter.file_structure`.""" + + def __init__(self, name: str, is_dir: bool): + self.name = name + self.is_dir = is_dir + self.children: dict[str, Directory] = {} + + def _get_dir(self, dirs: list[str]) -> "Directory": + """Builds path of Directories if not yet built and returns last directory + in list. + + Args: + dirs (List[str]): List of directory names that are treated like a path. + + Returns: + :class:`Directory`: The last Directory specified in the dirs list. + """ + if len(dirs) == 0: + return self + dir_name = dirs[0] + if dir_name not in self.children: + self.children[dir_name] = Directory(dir_name, True) + return self.children[dir_name]._get_dir(dirs[1:]) + + def _add_file(self, file_path: str): + """Adds a file to a Directory. + + Args: + file_path (str): Path of file to add. Last element is added as a file while + other paths items are added as directories. + """ + *dirs, file = file_path.split("/") + dir = self._get_dir(dirs) + dir.children[file] = Directory(file, False) + + def has_file(self, filename: str) -> bool: + """Checks if a file is present in a :class:`Directory`. + + Args: + filename (str): Path of file to search for. + Returns: + bool: If a :class:`Directory` contains the specified file. + """ + lineage = filename.split("/", maxsplit=1) + child = lineage[0] + grandchildren = lineage[1] if len(lineage) > 1 else None + if child in self.children: + if grandchildren is None: + return True + else: + return self.children[child].has_file(grandchildren) + return False + + def __str__(self): + str_list: list[str] = [] + self._stringify_tree(str_list) + return "".join(str_list) + + def _stringify_tree( + self, + str_list: list[str], + preamble: str = "", + dir_ptr: str = "\u2500\u2500\u2500 ", + ): + """Recursive method to generate print-friendly version of a Directory.""" + space = " " + branch = "\u2502 " + tee = "\u251c\u2500\u2500 " + last = "\u2514\u2500\u2500 " + + # add this directory's representation + str_list.append(f"{preamble}{dir_ptr}{self.name}\n") + + # add directory's children representations + if dir_ptr == tee: + preamble = preamble + branch + else: + preamble = preamble + space + + file_keys: list[str] = [] + dir_keys: list[str] = [] + for key, val in self.children.items(): + if val.is_dir: + dir_keys.append(key) + else: + file_keys.append(key) + + for index, key in enumerate(sorted(dir_keys)): + if (index == len(dir_keys) - 1) and len(file_keys) == 0: + self.children[key]._stringify_tree(str_list, preamble, last) + else: + self.children[key]._stringify_tree(str_list, preamble, tee) + for index, file in enumerate(sorted(file_keys)): + pointer = last if (index == len(file_keys) - 1) else tee + str_list.append(f"{preamble}{pointer}{file}\n") + + +def _create_directory_from_file_list( + filename: str, + file_list: list[str], + include: "GlobPattern" = "**", + exclude: "GlobPattern" = (), +) -> Directory: + """Return a :class:`Directory` file structure representation created from a list of files. + + Args: + filename (str): The name given to the top-level directory that will be the + relative root for all file paths found in the file_list. + + file_list (List[str]): List of files to add to the top-level directory. + + include (Union[List[str], str]): An optional pattern that limits what is included from the file_list to + files whose name matches the pattern. + + exclude (Union[List[str], str]): An optional pattern that excludes files whose name match the pattern. + + Returns: + :class:`Directory`: a :class:`Directory` file structure representation created from a list of files. + """ + glob_pattern = GlobGroup(include, exclude=exclude, separator="/") + + top_dir = Directory(filename, True) + for file in file_list: + if glob_pattern.matches(file): + top_dir._add_file(file) + return top_dir diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/find_file_dependencies.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/find_file_dependencies.py new file mode 100644 index 0000000000000000000000000000000000000000..c94c800ea5e071d7c0b60ed55556fed1b4f89af4 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/find_file_dependencies.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +import ast + +from ._importlib import _resolve_name + + +class _ExtractModuleReferences(ast.NodeVisitor): + """ + Extract the list of global variables a block of code will read and write + """ + + @classmethod + def run(cls, src: str, package: str) -> list[tuple[str, str | None]]: + visitor = cls(package) + tree = ast.parse(src) + visitor.visit(tree) + return list(visitor.references.keys()) + + def __init__(self, package): + super().__init__() + self.package = package + self.references = {} + + def _absmodule(self, module_name: str, level: int) -> str: + if level > 0: + return _resolve_name(module_name, self.package, level) + return module_name + + def visit_Import(self, node): + for alias in node.names: + self.references[(alias.name, None)] = True + + def visit_ImportFrom(self, node): + name = self._absmodule(node.module, 0 if node.level is None else node.level) + for alias in node.names: + # from my_package import foo + # foo may be a module, so we have to add it to the list of + # potential references, if import of it fails, we will ignore it + if alias.name != "*": + self.references[(name, alias.name)] = True + else: + self.references[(name, None)] = True + + def _grab_node_int(self, node): + return node.value + + def _grab_node_str(self, node): + return node.value + + def visit_Call(self, node): + # __import__ calls aren't routed to the visit_Import/From nodes + if hasattr(node.func, "id") and node.func.id == "__import__": + try: + name = self._grab_node_str(node.args[0]) + fromlist: list[str] = [] + level = 0 + if len(node.args) > 3: + fromlist.extend(self._grab_node_str(v) for v in node.args[3].elts) + elif hasattr(node, "keywords"): + for keyword in node.keywords: + if keyword.arg == "fromlist": + fromlist.extend( + self._grab_node_str(v) for v in keyword.value.elts + ) + if len(node.args) > 4: + level = self._grab_node_int(node.args[4]) + elif hasattr(node, "keywords"): + for keyword in node.keywords: + if keyword.arg == "level": + level = self._grab_node_int(keyword.value) + if fromlist == []: + # the top-level package (the name up till the first dot) is returned + # when the fromlist argument is empty in normal import system, + # we need to include top level package to match this behavior and last + # level package to capture the intended dependency of user + self.references[(name, None)] = True + top_name = name.rsplit(".", maxsplit=1)[0] + if top_name != name: + top_name = self._absmodule(top_name, level) + self.references[(top_name, None)] = True + else: + name = self._absmodule(name, level) + for alias in fromlist: + # fromlist args may be submodules, so we have to add the fromlist args + # to the list of potential references. If import of an arg fails we + # will ignore it, similar to visit_ImportFrom + if alias != "*": + self.references[(name, alias)] = True + else: + self.references[(name, None)] = True + except Exception: + return + + +find_files_source_depends_on = _ExtractModuleReferences.run diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/glob_group.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/glob_group.py new file mode 100644 index 0000000000000000000000000000000000000000..e696f2b4eab77a044316ef4df68a09e66c56a2df --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/glob_group.py @@ -0,0 +1,85 @@ +# mypy: allow-untyped-defs +import re +from collections.abc import Iterable +from typing import Union + + +GlobPattern = Union[str, Iterable[str]] + + +class GlobGroup: + """A set of patterns that candidate strings will be matched against. + + A candidate is composed of a list of segments separated by ``separator``, e.g. "foo.bar.baz". + + A pattern contains one or more segments. Segments can be: + - A literal string (e.g. "foo"), which matches exactly. + - A string containing a wildcard (e.g. "torch*", or "foo*baz*"). The wildcard matches + any string, including the empty string. + - A double wildcard ("**"). This matches against zero or more complete segments. + + Examples: + ``torch.**``: matches ``torch`` and all its submodules, e.g. ``torch.nn`` and ``torch.nn.functional``. + ``torch.*``: matches ``torch.nn`` or ``torch.functional``, but not ``torch.nn.functional``. + ``torch*.**``: matches ``torch``, ``torchvision``, and all their submodules. + + A candidates will match the ``GlobGroup`` if it matches any of the ``include`` patterns and + none of the ``exclude`` patterns. + + Args: + include (Union[str, Iterable[str]]): A string or list of strings, + each representing a pattern to be matched against. A candidate + will match if it matches *any* include pattern + exclude (Union[str, Iterable[str]]): A string or list of strings, + each representing a pattern to be matched against. A candidate + will be excluded from matching if it matches *any* exclude pattern. + separator (str): A string that delimits segments in candidates and + patterns. By default this is "." which corresponds to how modules are + named in Python. Another common value for this is "/", which is + the Unix path separator. + """ + + def __init__( + self, include: GlobPattern, *, exclude: GlobPattern = (), separator: str = "." + ): + self._dbg = f"GlobGroup(include={include}, exclude={exclude})" + self.include = GlobGroup._glob_list(include, separator) + self.exclude = GlobGroup._glob_list(exclude, separator) + self.separator = separator + + def __str__(self): + return self._dbg + + def __repr__(self): + return self._dbg + + def matches(self, candidate: str) -> bool: + candidate = self.separator + candidate + return any(p.fullmatch(candidate) for p in self.include) and all( + not p.fullmatch(candidate) for p in self.exclude + ) + + @staticmethod + def _glob_list(elems: GlobPattern, separator: str = "."): + if isinstance(elems, str): + return [GlobGroup._glob_to_re(elems, separator)] + else: + return [GlobGroup._glob_to_re(e, separator) for e in elems] + + @staticmethod + def _glob_to_re(pattern: str, separator: str = "."): + # to avoid corner cases for the first component, we prefix the candidate string + # with '.' so `import torch` will regex against `.torch`, assuming '.' is the separator + def component_to_re(component): + if "**" in component: + if component == "**": + return "(" + re.escape(separator) + "[^" + separator + "]+)*" + else: + raise ValueError("** can only appear as an entire path segment") + else: + return re.escape(separator) + ("[^" + separator + "]*").join( + re.escape(x) for x in component.split("*") + ) + + result = "".join(component_to_re(c) for c in pattern.split(separator)) + return re.compile(result) diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/importer.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/importer.py new file mode 100644 index 0000000000000000000000000000000000000000..c5c5097196cdda62439d1e3ed2407ecccf71e165 --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/importer.py @@ -0,0 +1,264 @@ +# mypy: allow-untyped-defs +import importlib +import logging +import sys +from abc import ABC, abstractmethod + +# pyrefly: ignore [missing-module-attribute] +from pickle import ( # type: ignore[attr-defined] + _getattribute, + _Pickler, + whichmodule as _pickle_whichmodule, # pyrefly: ignore # missing-module-attribute +) +from types import ModuleType +from typing import Any + +from ._mangling import demangle, get_mangle_prefix, is_mangled + + +__all__ = ["ObjNotFoundError", "ObjMismatchError", "Importer", "OrderedImporter"] +log = logging.getLogger(__name__) + + +class ObjNotFoundError(Exception): + """Raised when an importer cannot find an object by searching for its name.""" + + +class ObjMismatchError(Exception): + """Raised when an importer found a different object with the same name as the user-provided one.""" + + +class Importer(ABC): + """Represents an environment to import modules from. + + By default, you can figure out what module an object belongs by checking + __module__ and importing the result using __import__ or importlib.import_module. + + torch.package introduces module importers other than the default one. + Each PackageImporter introduces a new namespace. Potentially a single + name (e.g. 'foo.bar') is present in multiple namespaces. + + It supports two main operations: + import_module: module_name -> module object + get_name: object -> (parent module name, name of obj within module) + + The guarantee is that following round-trip will succeed or throw an ObjNotFoundError/ObjMisMatchError. + module_name, obj_name = env.get_name(obj) + module = env.import_module(module_name) + obj2 = getattr(module, obj_name) + assert obj1 is obj2 + """ + + modules: dict[str, ModuleType] + + @abstractmethod + def import_module(self, module_name: str) -> ModuleType: + """Import `module_name` from this environment. + + The contract is the same as for importlib.import_module. + """ + + def get_name(self, obj: Any, name: str | None = None) -> tuple[str, str]: + """Given an object, return a name that can be used to retrieve the + object from this environment. + + Args: + obj: An object to get the module-environment-relative name for. + name: If set, use this name instead of looking up __name__ or __qualname__ on `obj`. + This is only here to match how Pickler handles __reduce__ functions that return a string, + don't use otherwise. + Returns: + A tuple (parent_module_name, attr_name) that can be used to retrieve `obj` from this environment. + Use it like: + mod = importer.import_module(parent_module_name) + obj = getattr(mod, attr_name) + + Raises: + ObjNotFoundError: we couldn't retrieve `obj by name. + ObjMisMatchError: we found a different object with the same name as `obj`. + """ + if name is None and obj and _Pickler.dispatch.get(type(obj)) is None: + # Honor the string return variant of __reduce__, which will give us + # a global name to search for in this environment. + # TODO: I guess we should do copyreg too? + reduce = getattr(obj, "__reduce__", None) + if reduce is not None: + try: + rv = reduce() + if isinstance(rv, str): + name = rv + except Exception: + pass + if name is None: + name = getattr(obj, "__qualname__", None) + if name is None: + name = obj.__name__ + + orig_module_name = self.whichmodule(obj, name) + # Demangle the module name before importing. If this obj came out of a + # PackageImporter, `__module__` will be mangled. See mangling.md for + # details. + module_name = demangle(orig_module_name) + + # Check that this name will indeed return the correct object + try: + module = self.import_module(module_name) + if sys.version_info >= (3, 14): + # pickle._getatribute signature changes in 3.14 + # to take iterable and return just one object + obj2 = _getattribute(module, name.split(".")) + else: + obj2, _ = _getattribute(module, name) + except (ImportError, KeyError, AttributeError): + raise ObjNotFoundError( + f"{obj} was not found as {module_name}.{name}" + ) from None + + if obj is obj2: + return module_name, name + + def get_obj_info(obj): + assert name is not None + module_name = self.whichmodule(obj, name) + is_mangled_ = is_mangled(module_name) + location = ( + get_mangle_prefix(module_name) + if is_mangled_ + else "the current Python environment" + ) + importer_name = ( + f"the importer for {get_mangle_prefix(module_name)}" + if is_mangled_ + else "'sys_importer'" + ) + return module_name, location, importer_name + + obj_module_name, obj_location, obj_importer_name = get_obj_info(obj) + obj2_module_name, obj2_location, obj2_importer_name = get_obj_info(obj2) + msg = ( + f"\n\nThe object provided is from '{obj_module_name}', " + f"which is coming from {obj_location}." + f"\nHowever, when we import '{obj2_module_name}', it's coming from {obj2_location}." + "\nTo fix this, make sure this 'PackageExporter's importer lists " + f"{obj_importer_name} before {obj2_importer_name}." + ) + raise ObjMismatchError(msg) + + def whichmodule(self, obj: Any, name: str) -> str: + """Find the module name an object belongs to. + + This should be considered internal for end-users, but developers of + an importer can override it to customize the behavior. + + Taken from pickle.py, but modified to exclude the search into sys.modules + """ + module_name = getattr(obj, "__module__", None) + if module_name is not None: + return module_name + + # Protect the iteration by using a list copy of self.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in self.modules.copy().items(): + if ( + module_name == "__main__" + or module_name == "__mp_main__" # bpo-42406 + or module is None + ): + continue + try: + if _getattribute(module, name)[0] is obj: + return module_name + except AttributeError: + pass + + return "__main__" + + +class _SysImporter(Importer): + """An importer that implements the default behavior of Python.""" + + def import_module(self, module_name: str): + return importlib.import_module(module_name) + + def whichmodule(self, obj: Any, name: str) -> str: + # In Python 3.14+, pickle.whichmodule tries to import the module, + # which fails for mangled package names like ''. + # Check __module__ first before calling pickle.whichmodule. + module_name = getattr(obj, "__module__", None) + if module_name is not None: + return module_name + return _pickle_whichmodule(obj, name) + + +sys_importer = _SysImporter() + + +class OrderedImporter(Importer): + """A compound importer that takes a list of importers and tries them one at a time. + + The first importer in the list that returns a result "wins". + """ + + def __init__(self, *args): + self._importers: list[Importer] = list(args) + + def _is_torchpackage_dummy(self, module): + """Returns true iff this module is an empty PackageNode in a torch.package. + + If you intern `a.b` but never use `a` in your code, then `a` will be an + empty module with no source. This can break cases where we are trying to + re-package an object after adding a real dependency on `a`, since + OrderedImportere will resolve `a` to the dummy package and stop there. + + See: https://github.com/pytorch/pytorch/pull/71520#issuecomment-1029603769 + """ + if not getattr(module, "__torch_package__", False): + return False + if not hasattr(module, "__path__"): + return False + if not hasattr(module, "__file__"): + return True + return module.__file__ is None + + def get_name(self, obj: Any, name: str | None = None) -> tuple[str, str]: + for importer in self._importers: + try: + return importer.get_name(obj, name) + except (ObjNotFoundError, ObjMismatchError) as e: + warning_message = ( + f"Tried to call get_name with obj {obj}, " + f"and name {name} on {importer} and got {e}" + ) + log.warning(warning_message) + raise ObjNotFoundError( + f"Could not find obj {obj} and name {name} in any of the importers {self._importers}" + ) + + def import_module(self, module_name: str) -> ModuleType: + last_err = None + for importer in self._importers: + if not isinstance(importer, Importer): + raise TypeError( + f"{importer} is not a Importer. " + "All importers in OrderedImporter must inherit from Importer." + ) + try: + module = importer.import_module(module_name) + if self._is_torchpackage_dummy(module): + continue + return module + except ModuleNotFoundError as err: + last_err = err + + if last_err is not None: + raise last_err + else: + raise ModuleNotFoundError(module_name) + + def whichmodule(self, obj: Any, name: str) -> str: + for importer in self._importers: + module_name = importer.whichmodule(obj, name) + if module_name != "__main__": + return module_name + + return "__main__" diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_exporter.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_exporter.py new file mode 100644 index 0000000000000000000000000000000000000000..c7402e0b74ca584470d60ec75cc9971376f23a2f --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_exporter.py @@ -0,0 +1,1190 @@ +# mypy: allow-untyped-defs +import collections +import importlib.machinery +import io +import linecache +import os +import pickletools +import platform +import types +from collections import defaultdict, OrderedDict +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from importlib.machinery import SourceFileLoader +from pathlib import Path +from typing import Any, cast, IO + +import torch +from torch.serialization import location_tag, normalize_storage_type +from torch.types import FileLike, Storage +from torch.utils.hooks import RemovableHandle + +from ._digraph import DiGraph +from ._importlib import _normalize_path +from ._mangling import demangle, is_mangled +from ._package_pickler import create_pickler +from ._stdlib import is_stdlib_module +from .find_file_dependencies import find_files_source_depends_on +from .glob_group import GlobGroup, GlobPattern +from .importer import Importer, OrderedImporter, sys_importer + + +__all__ = [ + "PackagingErrorReason", + "EmptyMatchError", + "PackagingError", + "PackageExporter", +] + +_gate_torchscript_serialization = True + +ActionHook = Callable[["PackageExporter", str], None] + + +class _ModuleProviderAction(Enum): + """Represents one of the actions that :class:`PackageExporter` can take on a module. + + See :meth:`PackageExporter.extern` and friends for a description of what the actions do. + """ + + INTERN = 1 + EXTERN = 2 + MOCK = 3 + DENY = 4 + # Special case: when a module is mocked, PackageExporter writes out a + # `_mock` module that implements our mocking stubs. If we re-package code, + # we may encounter a `_mock` module from the original package. If we do, + # just ignore it and write a `_mock` module once. + REPACKAGED_MOCK_MODULE = 5 + # Special case: PackageImporter adds a fake module + # (`torch_package_importer`) that allows packaged code to access it. Don't + # re-export this. + SKIP = 6 + + +class PackagingErrorReason(Enum): + """Listing of different reasons a dependency may fail to package. + + This enum is used to provide good error messages when + :class:`PackagingError` is raised. + """ + + def __repr__(self): + return f"<{self.__class__.__name__}.{self.name}>" + + IS_EXTENSION_MODULE = ( + "Module is a C extension module. torch.package supports Python modules only." + ) + NO_DUNDER_FILE = "Module had no __file__ defined." + SOURCE_FILE_NOT_FOUND = ( + "Module had a __file__, but we could not find it in your filesystem." + ) + DEPENDENCY_RESOLUTION_FAILED = "Dependency resolution failed." + NO_ACTION = ( + "Module did not match against any action pattern. Extern, mock, or intern it." + ) + DENIED = "Module was denied by a pattern." + MOCKED_BUT_STILL_USED = ( + "Module was mocked out, but is still being used in the package. " + "Please intern or extern the mocked modules if objects are supposed to be in " + "the package." + ) + + +@dataclass +class _PatternInfo: + """Holds :class:`PackageExporter`-specific info about how to execute matches against""" + + # What action to take on a module that matches this pattern. + action: _ModuleProviderAction + # The value of `allow_empty` the user gave when specifying the pattern. + allow_empty: bool + # Whether this pattern has been matched during packaging. + was_matched: bool + + def __init__(self, action, allow_empty): + self.action = action + self.allow_empty = allow_empty + self.was_matched = False + + +class EmptyMatchError(Exception): + """This is an exception that is thrown when a mock or extern is marked as + ``allow_empty=False``, and is not matched with any module during packaging. + """ + + +class PackagingError(Exception): + """This exception is raised when there is an issue with exporting a package. + ``PackageExporter`` will attempt to gather up all the errors and present + them to you at once. + """ + + def __init__(self, dependency_graph: DiGraph, debug=False): + # Group errors by reason. + broken: dict[PackagingErrorReason, list[str]] = defaultdict(list) + for module_name, attrs in dependency_graph.nodes.items(): + error = attrs.get("error") + if error is None: + continue + if error == PackagingErrorReason.NO_ACTION: + assert "action" not in attrs + broken[error].append(module_name) + + message = io.StringIO() + message.write("\n") + + for reason, module_names in broken.items(): + message.write(f"* {reason.value}\n") + for module_name in module_names: + message.write(f" {module_name}\n") + + # Print additional context if it's provided. + error_context = dependency_graph.nodes[module_name].get("error_context") + if error_context is not None: + message.write(f" Context: {error_context}\n") + if module_name in _DISALLOWED_MODULES: + message.write( + " Note: While we usually use modules in the python standard library " + f"from the local environment, `{module_name}` has a lot of system " + "level access and therefore can pose a security risk. We heavily " + f"recommend removing `{module_name}` from your packaged code. However, if that " + "is not possible, add it to the extern list by calling " + f'PackageExporter.extern("`{module_name}`")\n' + ) + if debug: + module_path = dependency_graph.first_path(module_name) + message.write( + f" A path to {module_name}: {' -> '.join(module_path)}\n" + ) + if not debug: + message.write("\n") + message.write( + "Set debug=True when invoking PackageExporter for a visualization of where " + "broken modules are coming from!\n" + ) + # Save the dependency graph so that tooling can get at it. + self.dependency_graph = dependency_graph + super().__init__(message.getvalue()) + + +class PackageExporter: + """Exporters allow you to write packages of code, pickled Python data, and + arbitrary binary and text resources into a self-contained package. + + Imports can load this code in a hermetic way, such that code is loaded + from the package rather than the normal Python import system. This allows + for the packaging of PyTorch model code and data so that it can be run + on a server or used in the future for transfer learning. + + The code contained in packages is copied file-by-file from the original + source when it is created, and the file format is a specially organized + zip file. Future users of the package can unzip the package, and edit the code + in order to perform custom modifications to it. + + The importer for packages ensures that code in the module can only be loaded from + within the package, except for modules explicitly listed as external using :meth:`extern`. + The file ``extern_modules`` in the zip archive lists all the modules that a package externally depends on. + This prevents "implicit" dependencies where the package runs locally because it is importing + a locally-installed package, but then fails when the package is copied to another machine. + + When source code is added to the package, the exporter can optionally scan it + for further code dependencies (``dependencies=True``). It looks for import statements, + resolves relative references to qualified module names, and performs an action specified by the user + (See: :meth:`extern`, :meth:`mock`, and :meth:`intern`). + """ + + """A importer that will be searched in order to find the modules referenced by other modules or by + pickled objects. The default module environment just uses sys_importer, which searches the Python environment. + """ + importer: Importer + + def __init__( + self, + f: FileLike, + importer: Importer | Sequence[Importer] = sys_importer, + debug: bool = False, + ) -> None: + """ + Create an exporter. + + Args: + f: The location to export to. Can be a ``string``/``Path`` object containing a filename + or a binary I/O object. + importer: If a single Importer is passed, use that to search for modules. + If a sequence of importers are passed, an ``OrderedImporter`` will be constructed out of them. + debug: If set to True, add path of broken modules to PackagingErrors. + """ + torch._C._log_api_usage_once("torch.package.PackageExporter") + self.debug = debug + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + self.buffer: IO[bytes] | None = None + else: # is a byte buffer + self.buffer = f + + self.zip_file = torch._C.PyTorchFileWriter(f) + self.zip_file.set_min_version(6) + self._written_files: set[str] = set() + + self.serialized_reduces: dict[int, Any] = {} + + # A graph tracking all the modules and pickle objects added to this + # package and the dependencies between them. + # - Each node is a module name (or a pickle name that looks like '') + # - Each directed edge (u, v) means u depends on v. + # - Nodes may contain metadata that describe how to write the thing to the zipfile. + self.dependency_graph = DiGraph() + self.script_module_serializer = torch._C.ScriptModuleSerializer(self.zip_file) + self.storage_context = self.script_module_serializer.storage_context() + + # These are OrderedDicts for compatibility with RemovableHandle. + # Generic OrderedDict type annotations are not present until 3.7. + # The real type signature is OrderedDict[int, Callable[[PackageExporter, str], None]] + self._extern_hooks: OrderedDict = OrderedDict() + self._mock_hooks: OrderedDict = OrderedDict() + self._intern_hooks: OrderedDict = OrderedDict() + + if isinstance(importer, Importer): + self.importer = importer + else: + if not isinstance(importer, collections.abc.Sequence): + raise TypeError( + "importer arg should be an Importer or a sequence of Importers, " + f"got {type(importer)} instead." + ) + self.importer = OrderedImporter(*importer) + + self.patterns: dict[GlobGroup, _PatternInfo] = {} + self._unique_id = 0 + + def save_source_file( + self, module_name: str, file_or_directory: str, dependencies=True + ): + """Adds the local file system ``file_or_directory`` to the source package to provide the code + for ``module_name``. + + Args: + module_name (str): e.g. ``"my_package.my_subpackage"``, code will be saved to provide code for this package. + file_or_directory (str): the path to a file or directory of code. When a directory, all python files in the directory + are recursively copied using :meth:`save_source_file`. If a file is named ``"/__init__.py"`` the code is treated + as a package. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + path = Path(file_or_directory) + if path.is_dir(): + to_save = [] # list of tuples with arguments to save_source_string + module_path = module_name.replace(".", "/") + for filename in path.glob("**/*.py"): + relative_path = filename.relative_to(path).as_posix() + archivename = module_path + "/" + relative_path + submodule_name = None + if filename.name == "__init__.py": + submodule_name = archivename[: -len("/__init__.py")].replace( + "/", "." + ) + is_package = True + else: + submodule_name = archivename[: -len(".py")].replace("/", ".") + is_package = False + + # we delay the call to save_source_string so that we record all the source files + # being provided by this directory structure _before_ attempting to resolve the dependencies + # on the source. This makes sure we don't try to copy over modules that will just get + # overwritten by this directory blob + to_save.append( + ( + submodule_name, + _read_file(str(filename)), + is_package, + dependencies, + ) + ) + + for item in to_save: + self.save_source_string(*item) + else: + is_package = path.name == "__init__.py" + self.save_source_string( + module_name, + _read_file(file_or_directory), + is_package, + dependencies, + ) + + def get_unique_id(self) -> str: + """Get an id. This id is guaranteed to only be handed out once for this package.""" + ret = str(self._unique_id) + self._unique_id += 1 + return ret + + def _get_dependencies( + self, src: str, module_name: str, is_package: bool + ) -> list[str]: + """Return all modules that this source code depends on. + + Dependencies are found by scanning the source code for import-like statements. + + Arguments: + src: The Python source code to analyze for dependencies. + module_name: The name of the module that ``src`` corresponds to. + is_package: Whether this module should be treated as a package. + See :py:meth:`save_source_string` for more info. + + Returns: + A list containing modules detected as direct dependencies in + ``src``. The items in the list are guaranteed to be unique. + """ + package_name = ( + module_name if is_package else module_name.rsplit(".", maxsplit=1)[0] + ) + try: + dep_pairs = find_files_source_depends_on(src, package_name) + except Exception as e: + self.dependency_graph.add_node( + module_name, + error=PackagingErrorReason.DEPENDENCY_RESOLUTION_FAILED, + error_context=str(e), + ) + return [] + + # Use a dict to get uniquing but also deterministic order + dependencies = {} + for dep_module_name, dep_module_obj in dep_pairs: + # handle the case where someone did something like `from pack import sub` + # where `sub` is a submodule. In this case we don't have to save pack, just sub. + # this ensures we don't pick up additional dependencies on pack. + # However, in the case where `sub` is not a submodule but an object, then we do have + # to save pack. + if dep_module_obj is not None: + possible_submodule = f"{dep_module_name}.{dep_module_obj}" + if self._module_exists(possible_submodule): + dependencies[possible_submodule] = True + # we don't need to save `pack` + continue + if self._module_exists(dep_module_name): + dependencies[dep_module_name] = True + + return list(dependencies.keys()) + + def save_source_string( + self, + module_name: str, + src: str, + is_package: bool = False, + dependencies: bool = True, + ): + """Adds ``src`` as the source code for ``module_name`` in the exported package. + + Args: + module_name (str): e.g. ``my_package.my_subpackage``, code will be saved to provide code for this package. + src (str): The Python source code to save for this package. + is_package (bool, optional): If ``True``, this module is treated as a package. Packages are allowed to have submodules + (e.g. ``my_package.my_subpackage.my_subsubpackage``), and resources can be saved inside them. Defaults to ``False``. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + self.dependency_graph.add_node( + module_name, + source=src, + is_package=is_package, + provided=True, + action=_ModuleProviderAction.INTERN, + ) + + if dependencies: + deps = self._get_dependencies(src, module_name, is_package) + + for dep in deps: + self.dependency_graph.add_edge(module_name, dep) + self.add_dependency(dep) + + def _write_source_string( + self, + module_name: str, + src: str, + is_package: bool = False, + ): + """Write ``src`` as the source code for ``module_name`` in the zip archive. + + Arguments are otherwise the same as for :meth:`save_source_string`. + """ + extension = "/__init__.py" if is_package else ".py" + filename = module_name.replace(".", "/") + extension + + self._write(filename, src) + + def _import_module(self, module_name: str): + try: + return self.importer.import_module(module_name) + except ModuleNotFoundError: + if not is_mangled(module_name): + raise + msg = ( + f"Module not found: '{module_name}'. Make sure the PackageImporter that " + "created this module is present in `self.importer`" + ) + raise ModuleNotFoundError(msg) from None + + def _module_exists(self, module_name: str) -> bool: + try: + self._import_module(module_name) + return True + except Exception: + return False + + def _get_source_of_module(self, module: types.ModuleType) -> str | None: + filename = None + spec = getattr(module, "__spec__", None) + if spec is not None: + loader = getattr(spec, "loader", None) + if loader is not None and isinstance(loader, SourceFileLoader): + try: + filename = loader.get_filename(module.__name__) + except ImportError: + pass + if filename is None: + filename = getattr(module, "__file__", None) + if isinstance(filename, str) and filename.endswith(".py"): + return "".join(linecache.getlines(filename, module.__dict__)) + return None + + def add_dependency(self, module_name: str, dependencies=True): + """Given a module, add it to the dependency graph according to patterns + specified by the user. + """ + if ( + module_name in self.dependency_graph + and self.dependency_graph.nodes[module_name].get("provided") is True + ): + return + + # Special case: PackageImporter provides a special module called + # `torch_package_importer` that allows packaged modules to reference + # their PackageImporter. We don't want to re-export this. + if module_name == "torch_package_importer": + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.SKIP, + provided=True, + ) + return + + if module_name == "_mock": + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.REPACKAGED_MOCK_MODULE, + provided=True, + ) + return + + if self._can_implicitly_extern(module_name): + self.dependency_graph.add_node( + module_name, action=_ModuleProviderAction.EXTERN, provided=True + ) + return + + for pattern, pattern_info in self.patterns.items(): + if pattern.matches(module_name): + pattern_info.was_matched = True + self.dependency_graph.add_node( + module_name, action=pattern_info.action, provided=True + ) + + if pattern_info.action == _ModuleProviderAction.DENY: + # Requiring a denied module just adds an error to the graph. + self.dependency_graph.add_node( + module_name, error=PackagingErrorReason.DENIED + ) + + # If we are interning this module, we need to retrieve its + # dependencies and package those as well. + if pattern_info.action == _ModuleProviderAction.INTERN: + self._intern_module(module_name, dependencies) + return + + # No patterns have matched. Explicitly add this as an error. + self.dependency_graph.add_node( + module_name, error=PackagingErrorReason.NO_ACTION + ) + + def save_module(self, module_name: str, dependencies=True): + """Save the code for ``module`` into the package. Code for the module is resolved using the ``importers`` path to find the + module object, and then using its ``__file__`` attribute to find the source code. + + Args: + module_name (str): e.g. ``my_package.my_subpackage``, code will be saved to provide code + for this package. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + if not isinstance(module_name, str): + raise TypeError( + "save_module() expects a string input, did you perhaps mean to pass `__name__`?" + ) + + self._intern_module(module_name, dependencies) + + def _intern_module( + self, + module_name: str, + dependencies: bool, + ): + """Adds the module to the dependency graph as an interned module, + along with any metadata needed to write it out to the zipfile at serialization time. + """ + module_obj = self._import_module(module_name) + # Subtle: if the import above succeeded, either: + # 1. The module name is not mangled, and this was just a regular import, or + # 2. The module name is mangled, but one of the importers was able to + # recognize the mangling and import it. + # Either way, it is now safe to demangle this name so that we don't + # serialize the mangled version to the package. + module_name = demangle(module_name) + + # Find dependencies of this module and require them as well. + is_package = hasattr(module_obj, "__path__") + source = self._get_source_of_module(module_obj) + if source is None: + # Couldn't find a source! Add it to our dependency graph as broken + # and continue. + filename = getattr(module_obj, "__file__", None) + error_context = None + if filename is None: + packaging_error = PackagingErrorReason.NO_DUNDER_FILE + elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)): + packaging_error = PackagingErrorReason.IS_EXTENSION_MODULE + else: + packaging_error = PackagingErrorReason.SOURCE_FILE_NOT_FOUND + error_context = f"filename: {filename}" + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.INTERN, + is_package=is_package, + error=packaging_error, + error_context=error_context, + provided=True, + ) + return + + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.INTERN, + is_package=is_package, + source=source, + provided=True, + ) + + if dependencies: + deps = self._get_dependencies(source, module_name, is_package) + for dep in deps: + self.dependency_graph.add_edge(module_name, dep) + self.add_dependency(dep) + + def save_pickle( + self, + package: str, + resource: str, + obj: Any, + dependencies: bool = True, + pickle_protocol: int = 3, + ): + """Save a python object to the archive using pickle. Equivalent to :func:`torch.save` but saving into + the archive rather than a stand-alone file. Standard pickle does not save the code, only the objects. + If ``dependencies`` is true, this method will also scan the pickled objects for which modules are required + to reconstruct them and save the relevant code. + + To be able to save an object where ``type(obj).__name__`` is ``my_module.MyObject``, + ``my_module.MyObject`` must resolve to the class of the object according to the ``importer`` order. When saving objects that + have previously been packaged, the importer's ``import_module`` method will need to be present in the ``importer`` list + for this to work. + + Args: + package (str): The name of module package this resource should go in (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + obj (Any): The object to save, must be picklable. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + + assert (pickle_protocol == 4) or (pickle_protocol == 3), ( + "torch.package only supports pickle protocols 3 and 4" + ) + + filename = self._filename(package, resource) + # Write the pickle data for `obj` + data_buf = io.BytesIO() + pickler = create_pickler(data_buf, self.importer, protocol=pickle_protocol) + pickler.persistent_id = self._persistent_id + pickler.dump(obj) + data_value = data_buf.getvalue() + mocked_modules = defaultdict(list) + name_in_dependency_graph = f"<{package}.{resource}>" + self.dependency_graph.add_node( + name_in_dependency_graph, + action=_ModuleProviderAction.INTERN, + provided=True, + is_pickle=True, + ) + + def _check_mocked_error(module: str | None, field: str | None): + """ + checks if an object (field) comes from a mocked module and then adds + the pair to mocked_modules which contains mocked modules paired with their + list of mocked objects present in the pickle. + + We also hold the invariant that the first user defined rule that applies + to the module is the one we use. + """ + + assert isinstance(module, str) + assert isinstance(field, str) + if self._can_implicitly_extern(module): + return + for pattern, pattern_info in self.patterns.items(): + if pattern.matches(module): + if pattern_info.action == _ModuleProviderAction.MOCK: + mocked_modules[module].append(field) + return + + if dependencies: + all_dependencies = [] + module = None + field = None + memo: defaultdict[int, str] = defaultdict(None) + memo_count = 0 + # pickletools.dis(data_value) + # pyrefly: ignore [bad-assignment] + for opcode, arg, _pos in pickletools.genops(data_value): + if pickle_protocol == 4: + if ( + opcode.name == "SHORT_BINUNICODE" + or opcode.name == "BINUNICODE" + or opcode.name == "BINUNICODE8" + ): + assert isinstance(arg, str) + module = field + field = arg + memo[memo_count] = arg + elif ( + opcode.name == "LONG_BINGET" + or opcode.name == "BINGET" + or opcode.name == "GET" + ): + assert isinstance(arg, int) + module = field + field = memo.get(arg, None) + elif opcode.name == "MEMOIZE": + memo_count += 1 + elif opcode.name == "STACK_GLOBAL": + if module is None: + # If not module was passed on in the entries preceding this one, continue. + continue + assert isinstance(module, str) + if module not in all_dependencies: + all_dependencies.append(module) + _check_mocked_error(module, field) + elif ( + pickle_protocol == 3 and opcode.name == "GLOBAL" + ): # a global reference + assert isinstance(arg, str) + module, field = arg.split(" ") + if module not in all_dependencies: + all_dependencies.append(module) + _check_mocked_error(module, field) + for module_name in all_dependencies: + self.dependency_graph.add_edge(name_in_dependency_graph, module_name) + + """ If an object happens to come from a mocked module, then we collect these errors and spit them + out with the other errors found by package exporter. + """ + if module_name in mocked_modules: + assert isinstance(module_name, str) + fields = mocked_modules[module_name] + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.MOCK, + error=PackagingErrorReason.MOCKED_BUT_STILL_USED, + error_context=f"Object(s) '{fields}' from module `{module_name}` was mocked out during packaging " + f"but is being used in resource - `{resource}` in package `{package}`. ", + provided=True, + ) + else: + self.add_dependency(module_name) + + self._write(filename, data_value) + + def save_text(self, package: str, resource: str, text: str): + """Save text data to the package. + + Args: + package (str): The name of module package this resource should go it (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + text (str): The contents to save. + """ + return self.save_binary(package, resource, text.encode("utf-8")) + + def save_binary(self, package, resource, binary: bytes): + """Save raw bytes to the package. + + Args: + package (str): The name of module package this resource should go it (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + binary (str): The data to save. + """ + filename = self._filename(package, resource) + self._write(filename, binary) + + def register_extern_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers an extern hook on the exporter. + + The hook will be called each time a module matches against an :meth:`extern` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._extern_hooks) + self._extern_hooks[handle.id] = hook + return handle + + def register_mock_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers a mock hook on the exporter. + + The hook will be called each time a module matches against a :meth:`mock` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._mock_hooks) + self._mock_hooks[handle.id] = hook + return handle + + def register_intern_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers an intern hook on the exporter. + + The hook will be called each time a module matches against an :meth:`intern` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._intern_hooks) + self._intern_hooks[handle.id] = hook + return handle + + def intern( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Specify modules that should be packaged. A module must match some ``intern`` pattern in order to be + included in the package and have its dependencies processed recursively. + + Args: + include (Union[List[str], str]): A string e.g. "my_package.my_subpackage", or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + + allow_empty (bool): An optional flag that specifies whether the intern modules specified by this call + to the ``intern`` method must be matched to some module during packaging. If an ``intern`` module glob + pattern is added with ``allow_empty=False``, and :meth:`close` is called (either explicitly or via ``__exit__``) + before any modules match that pattern, an exception is thrown. If ``allow_empty=True``, no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.INTERN, allow_empty + ) + + def mock( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Replace some required modules with a mock implementation. Mocked modules will return a fake + object for any attribute accessed from it. Because we copy file-by-file, the dependency resolution will sometimes + find files that are imported by model files but whose functionality is never used + (e.g. custom serialization code or training helpers). + Use this function to mock this functionality out without having to modify the original code. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be mocked out. Strings can also be a glob-style pattern + string that may match multiple modules. Any required dependencies that match this pattern + string will be mocked out automatically. + + Examples : + ``'torch.**'`` -- matches ``torch`` and all submodules of torch, e.g. ``'torch.nn'`` + and ``'torch.nn.functional'`` + + ``'torch.*'`` -- matches ``'torch.nn'`` or ``'torch.functional'``, but not + ``'torch.nn.functional'`` + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + e.g. ``include='torch.**', exclude='torch.foo'`` will mock all torch packages except ``'torch.foo'``, + Default: is ``[]``. + + allow_empty (bool): An optional flag that specifies whether the mock implementation(s) specified by this call + to the :meth:`mock` method must be matched to some module during packaging. If a mock is added with + ``allow_empty=False``, and :meth:`close` is called (either explicitly or via ``__exit__``) and the mock has + not been matched to a module used by the package being exported, an exception is thrown. + If ``allow_empty=True``, no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.MOCK, allow_empty + ) + + def extern( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Include ``module`` in the list of external modules the package can import. + This will prevent dependency discovery from saving + it in the package. The importer will load an external module directly from the standard import system. + Code for extern modules must also exist in the process loading the package. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as + described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the + include string. + + allow_empty (bool): An optional flag that specifies whether the extern modules specified by this call + to the ``extern`` method must be matched to some module during packaging. If an extern module glob + pattern is added with ``allow_empty=False``, and :meth:`close` is called (either explicitly or via + ``__exit__``) before any modules match that pattern, an exception is thrown. If ``allow_empty=True``, + no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.EXTERN, allow_empty + ) + + def deny(self, include: "GlobPattern", *, exclude: "GlobPattern" = ()): + """Blocklist modules who names match the given glob patterns from the list of modules the package can import. + If a dependency on any matching packages is found, a :class:`PackagingError` is raised. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.DENY, allow_empty=True + ) + + def _persistent_id(self, obj): + if torch.is_storage(obj) or isinstance(obj, torch.storage.TypedStorage): + storage: Storage + if isinstance(obj, torch.storage.TypedStorage): + # TODO: Once we decide to break serialization FC, we can + # remove this case + untyped_storage = obj._untyped_storage + storage_type_str = obj.pickle_storage_type() + storage_type = getattr(torch, storage_type_str) + storage = cast(Storage, untyped_storage) + storage_numel = obj.size() + + elif isinstance(obj, torch.UntypedStorage): + untyped_storage = obj + storage = cast(Storage, untyped_storage) + storage_type = normalize_storage_type(type(storage)) + storage_numel = storage.nbytes() + else: + raise RuntimeError(f"storage type not recognized: {type(obj)}") + + location = location_tag(storage) + + # serialize storage if not already written + storage_present = self.storage_context.has_storage(storage) + storage_id = self.storage_context.get_or_add_storage(storage) + if not storage_present: + if storage.device.type != "cpu": + storage = storage.cpu() + num_bytes = storage.nbytes() + self.zip_file.write_record( + f".data/{storage_id}.storage", storage, num_bytes + ) + return ("storage", storage_type, storage_id, location, storage_numel) + + if hasattr(obj, "__reduce_package__"): + if _gate_torchscript_serialization and isinstance( + obj, torch.jit.RecursiveScriptModule + ): + raise Exception( # noqa: TRY002 + "Serializing ScriptModules directly into a package is a beta feature. " + "To use, set global " + "`torch.package.package_exporter._gate_torchscript_serialization` to `False`." + ) + if self.serialized_reduces.get(id(obj)) is None: + self.serialized_reduces[id(obj)] = ( + "reduce_package", + id(obj), + *obj.__reduce_package__(self), + ) + + return self.serialized_reduces[id(obj)] + + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + # If __exit__ was called because an exception was raised, we do not + # attempt to finalize the package. Instead, control is returned to the + # caller to continue raising the exception. + if exc_type is not None: + # Do the bare minimum to leave the open buffer in a valid state. + self._finalize_zip() + return + + self.close() + + def _write(self, filename, str_or_bytes): + if filename in self._written_files: + raise AssertionError( + f"Tried to write file '{filename}', but it already exists in this archive. " + "Please file a bug." + ) + self._written_files.add(filename) + + if is_mangled(filename): + raise AssertionError( + f"Tried to save a torch.package'd module as '{filename}'. " + "Directly saving torch.package'd modules is not allowed." + ) + if isinstance(str_or_bytes, str): + str_or_bytes = str_or_bytes.encode("utf-8") + self.zip_file.write_record(filename, str_or_bytes, len(str_or_bytes)) + + def _validate_dependency_graph(self): + # 1. Check the graph for any errors inserted during dependency analysis. + for attrs in self.dependency_graph.nodes.values(): + if "error" in attrs: + raise PackagingError(self.dependency_graph, debug=self.debug) + + # 2. Check that all patterns for which allow_empty=False have been matched at least once. + for pattern, pattern_info in self.patterns.items(): + if not pattern_info.allow_empty and not pattern_info.was_matched: + raise EmptyMatchError( + f"Exporter did not match any modules to {pattern}, which was marked as allow_empty=False" + ) + + def _write_mock_file(self): + if "_mock.py" not in self._written_files: + mock_file = str(Path(__file__).parent / "_mock.py") + self._write_source_string("_mock", _read_file(mock_file), is_package=False) + + def _execute_dependency_graph(self): + """Takes a finalized dependency graph describing how to package all + modules and executes it, writing to the ZIP archive. + """ + self._validate_dependency_graph() + + extern_modules = [] + for module_name, attrs in self.dependency_graph.nodes.items(): + action = attrs["action"] + + if action == _ModuleProviderAction.EXTERN: + for hook in self._extern_hooks.values(): + hook(self, module_name) + + extern_modules.append(module_name) + + elif action == _ModuleProviderAction.MOCK: + for hook in self._mock_hooks.values(): + hook(self, module_name) + + self._write_mock_file() + + is_package = hasattr(self._import_module(module_name), "__path__") + self._write_source_string(module_name, _MOCK_IMPL, is_package) + + elif action == _ModuleProviderAction.INTERN: + for hook in self._intern_hooks.values(): + hook(self, module_name) + + # The node in the dependency graph contains metadata that tells us + # how to intern the module. + if "provided" not in attrs: + raise AssertionError( + f"Module was marked `intern` but not provided: {module_name}" + ) + + if attrs.get("is_pickle") is True: + # This node came from save_pickle, we don't need to write any source for it. + continue + + is_package = attrs["is_package"] + source = attrs["source"] + self._write_source_string(module_name, source, is_package) + + elif action == _ModuleProviderAction.REPACKAGED_MOCK_MODULE: + self._write_mock_file() + elif action == _ModuleProviderAction.SKIP: + continue + else: + raise AssertionError( + f"Invalid action: {module_name}, {action}. Please report a bug to PyTorch." + ) + + extern_file_contents = "\n".join(extern_modules) + "\n" + self._write(".data/extern_modules", extern_file_contents) + + def _write_python_version(self): + """Writes the python version that the package was created with to .data/python_version""" + self._write(".data/python_version", platform.python_version()) + + def close(self): + """Write the package to the filesystem. Any calls after :meth:`close` are now invalid. + It is preferable to use resource guard syntax instead:: + + with PackageExporter("file.zip") as e: + ... + """ + self._execute_dependency_graph() + self._write_python_version() + + self.script_module_serializer.write_files() + self._finalize_zip() + + def _finalize_zip(self): + """Called at the very end of packaging to leave the zipfile in a closed but valid state.""" + del self.zip_file + if self.buffer: + self.buffer.flush() + + def _filename(self, package, resource): + package_path = package.replace(".", "/") + resource = _normalize_path(resource) + return f"{package_path}/{resource}" + + def _can_implicitly_extern(self, module_name: str): + top_level_package_name = module_name.partition(".")[0] + return top_level_package_name == "torch" or ( + top_level_package_name not in _DISALLOWED_MODULES + and is_stdlib_module(top_level_package_name) + ) + + def dependency_graph_string(self) -> str: + """Returns digraph string representation of dependencies in package. + + Returns: + A string representation of dependencies in package. + """ + return self.dependency_graph.to_dot() + + def _nodes_with_action_type( + self, action: _ModuleProviderAction | None + ) -> list[str]: + result = [] + for name, node_dict in self.dependency_graph.nodes.items(): + node_action = node_dict.get("action", None) + if node_action == action and "is_pickle" not in node_dict: + result.append(name) + result.sort() + return result + + def externed_modules(self) -> list[str]: + """Return all modules that are currently externed. + + Returns: + A list containing the names of modules which will be + externed in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.EXTERN) + + def interned_modules(self) -> list[str]: + """Return all modules that are currently interned. + + Returns: + A list containing the names of modules which will be + interned in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.INTERN) + + def mocked_modules(self) -> list[str]: + """Return all modules that are currently mocked. + + Returns: + A list containing the names of modules which will be + mocked in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.MOCK) + + def denied_modules(self) -> list[str]: + """Return all modules that are currently denied. + + Returns: + A list containing the names of modules which will be + denied in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.DENY) + + def get_rdeps(self, module_name: str) -> list[str]: + """Return a list of all modules which depend on the module ``module_name``. + + Returns: + A list containing the names of modules which depend on ``module_name``. + """ + if module_name in self.dependency_graph._pred: + return list(self.dependency_graph._pred[module_name].keys()) + else: + return [] + + def all_paths(self, src: str, dst: str) -> str: + """Return a dot representation of the subgraph + that has all paths from src to dst. + + Returns: + A dot representation containing all paths from src to dst. + (https://graphviz.org/doc/info/lang.html) + """ + return self.dependency_graph.all_paths(src, dst) + + +# even though these are in the standard library, we do not allow them to be +# automatically externed since they offer a lot of system level access +_DISALLOWED_MODULES = ["sys", "io"] + +_MOCK_IMPL = """\ +from _mock import MockedObject +def __getattr__(attr: str): + return MockedObject(__name__ + '.' + attr, _suppress_err=True) +""" + + +def _read_file(filename: str) -> str: + with open(filename, "rb") as f: + b = f.read() + return b.decode("utf-8") diff --git a/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_importer.py b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_importer.py new file mode 100644 index 0000000000000000000000000000000000000000..1d858db71615bb56f2248192f78ee6117dcec36c --- /dev/null +++ b/TTS/Kokoro/.venv/Lib/site-packages/torch/package/package_importer.py @@ -0,0 +1,789 @@ +# mypy: allow-untyped-defs +import builtins +import importlib +import importlib.machinery +import inspect +import io +import linecache +import os +import sys +import types +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from typing import Any, cast, TYPE_CHECKING +from weakref import WeakValueDictionary + +import torch +from torch.serialization import _get_restore_location, _maybe_decode_ascii +from torch.types import FileLike + +from ._directory_reader import DirectoryReader +from ._importlib import ( + _calc___package__, + _normalize_line_endings, + _normalize_path, + _resolve_name, + _sanity_check, +) +from ._mangling import demangle, PackageMangler +from ._package_unpickler import PackageUnpickler +from .file_structure_representation import _create_directory_from_file_list, Directory +from .importer import Importer + + +if TYPE_CHECKING: + from .glob_group import GlobPattern + +__all__ = ["PackageImporter"] + + +# This is a list of imports that are implicitly allowed even if they haven't +# been marked as extern. This is to work around the fact that Torch implicitly +# depends on numpy and package can't track it. +# https://github.com/pytorch/multipy/issues/46 # codespell:ignore multipy +IMPLICIT_IMPORT_ALLOWLIST: Iterable[str] = [ + "numpy", + "numpy.core", + "numpy.core._multiarray_umath", + # FX GraphModule might depend on builtins module and users usually + # don't extern builtins. Here we import it here by default. + "builtins", +] + + +# Compatibility name mapping to facilitate upgrade of external modules. +# The primary motivation is to enable Numpy upgrade that many modules +# depend on. The latest release of Numpy removed `numpy.str` and +# `numpy.bool` breaking unpickling for many modules. +EXTERN_IMPORT_COMPAT_NAME_MAPPING: dict[str, dict[str, Any]] = { + "numpy": { + "str": str, + "bool": bool, + }, +} + + +class PackageImporter(Importer): + """Importers allow you to load code written to packages by :class:`PackageExporter`. + Code is loaded in a hermetic way, using files from the package + rather than the normal python import system. This allows + for the packaging of PyTorch model code and data so that it can be run + on a server or used in the future for transfer learning. + + The importer for packages ensures that code in the module can only be loaded from + within the package, except for modules explicitly listed as external during export. + The file ``extern_modules`` in the zip archive lists all the modules that a package externally depends on. + This prevents "implicit" dependencies where the package runs locally because it is importing + a locally-installed package, but then fails when the package is copied to another machine. + """ + + """The dictionary of already loaded modules from this package, equivalent to ``sys.modules`` but + local to this importer. + """ + + modules: dict[str, types.ModuleType] + + def __init__( + self, + file_or_buffer: FileLike | torch._C.PyTorchFileReader, + module_allowed: Callable[[str], bool] = lambda module_name: True, + ): + """Open ``file_or_buffer`` for importing. This checks that the imported package only requires modules + allowed by ``module_allowed`` + + Args: + file_or_buffer: a file-like object (has to implement :meth:`read`, :meth:`readline`, :meth:`tell`, and :meth:`seek`), + a string, or an ``os.PathLike`` object containing a filename. + module_allowed (Callable[[str], bool], optional): A method to determine if a externally provided module + should be allowed. Can be used to ensure packages loaded do not depend on modules that the server + does not support. Defaults to allowing anything. + + Raises: + ImportError: If the package will use a disallowed module. + """ + torch._C._log_api_usage_once("torch.package.PackageImporter") + + self.zip_reader: Any + if isinstance(file_or_buffer, torch._C.PyTorchFileReader): + self.filename = "" + self.zip_reader = file_or_buffer + elif isinstance(file_or_buffer, (os.PathLike, str)): + self.filename = os.fspath(file_or_buffer) + if not os.path.isdir(self.filename): + self.zip_reader = torch._C.PyTorchFileReader(self.filename) + else: + self.zip_reader = DirectoryReader(self.filename) + else: + self.filename = "" + self.zip_reader = torch._C.PyTorchFileReader(file_or_buffer) + + torch._C._log_api_usage_metadata( + "torch.package.PackageImporter.metadata", + { + "serialization_id": self.zip_reader.serialization_id(), + "file_name": self.filename, + }, + ) + + self.root = _PackageNode(None) + self.modules = {} + self.extern_modules = self._read_extern() + + for extern_module in self.extern_modules: + if not module_allowed(extern_module): + raise ImportError( + f"package '{file_or_buffer}' needs the external module '{extern_module}' " + f"but that module has been disallowed" + ) + self._add_extern(extern_module) + + for fname in self.zip_reader.get_all_records(): + self._add_file(fname) + + self.patched_builtins = builtins.__dict__.copy() + self.patched_builtins["__import__"] = self.__import__ + # Allow packaged modules to reference their PackageImporter + self.modules["torch_package_importer"] = self # type: ignore[assignment] + + self._mangler = PackageMangler() + + # used for reduce deserializaiton + self.storage_context: Any = None + self.last_map_location = None + + # used for torch.serialization._load + self.Unpickler = lambda *args, **kwargs: PackageUnpickler(self, *args, **kwargs) + + def import_module(self, name: str, package=None): + """Load a module from the package if it hasn't already been loaded, and then return + the module. Modules are loaded locally + to the importer and will appear in ``self.modules`` rather than ``sys.modules``. + + Args: + name (str): Fully qualified name of the module to load. + package ([type], optional): Unused, but present to match the signature of importlib.import_module. Defaults to ``None``. + + Returns: + types.ModuleType: The (possibly already) loaded module. + """ + # We should always be able to support importing modules from this package. + # This is to support something like: + # obj = importer.load_pickle(...) + # importer.import_module(obj.__module__) <- this string will be mangled + # + # Note that _mangler.demangle will not demangle any module names + # produced by a different PackageImporter instance. + name = self._mangler.demangle(name) + + return self._gcd_import(name) + + def load_binary(self, package: str, resource: str) -> bytes: + """Load raw bytes. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + + Returns: + bytes: The loaded data. + """ + + path = self._zipfile_path(package, resource) + return self.zip_reader.get_record(path) + + def load_text( + self, + package: str, + resource: str, + encoding: str = "utf-8", + errors: str = "strict", + ) -> str: + """Load a string. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + encoding (str, optional): Passed to ``decode``. Defaults to ``'utf-8'``. + errors (str, optional): Passed to ``decode``. Defaults to ``'strict'``. + + Returns: + str: The loaded text. + """ + data = self.load_binary(package, resource) + return data.decode(encoding, errors) + + def load_pickle(self, package: str, resource: str, map_location=None) -> Any: + """Unpickles the resource from the package, loading any modules that are needed to construct the objects + using :meth:`import_module`. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + map_location: Passed to `torch.load` to determine how tensors are mapped to devices. Defaults to ``None``. + + Returns: + Any: The unpickled object. + """ + pickle_file = self._zipfile_path(package, resource) + restore_location = _get_restore_location(map_location) + loaded_storages = {} + loaded_reduces = {} + storage_context = torch._C.DeserializationStorageContext() + + def load_tensor(dtype, size, key, location, restore_location): + name = f"{key}.storage" + + if storage_context.has_storage(name): + storage = storage_context.get_storage(name, dtype)._typed_storage() + else: + tensor = self.zip_reader.get_storage_from_record( + ".data/" + name, size, dtype + ) + if isinstance(self.zip_reader, torch._C.PyTorchFileReader): + storage_context.add_storage(name, tensor) + storage = tensor._typed_storage() + loaded_storages[key] = restore_location(storage, location) + + def persistent_load(saved_id): + assert isinstance(saved_id, tuple) + typename = _maybe_decode_ascii(saved_id[0]) + data = saved_id[1:] + + if typename == "storage": + storage_type, key, location, size = data + if storage_type is torch.UntypedStorage: + dtype = torch.uint8 + else: + dtype = storage_type.dtype + + if key not in loaded_storages: + load_tensor( + dtype, + size, + key, + _maybe_decode_ascii(location), + restore_location, + ) + storage = loaded_storages[key] + # TODO: Once we decide to break serialization FC, we can + # stop wrapping with TypedStorage + return torch.storage.TypedStorage( + wrap_storage=storage._untyped_storage, dtype=dtype, _internal=True + ) + elif typename == "reduce_package": + # to fix BC breaking change, objects on this load path + # will be loaded multiple times erroneously + if len(data) == 2: + func, args = data + return func(self, *args) + reduce_id, func, args = data + if reduce_id not in loaded_reduces: + loaded_reduces[reduce_id] = func(self, *args) + return loaded_reduces[reduce_id] + else: + f"Unknown typename for persistent_load, expected 'storage' or 'reduce_package' but got '{typename}'" + + # Load the data (which may in turn use `persistent_load` to load tensors) + data_file = io.BytesIO(self.zip_reader.get_record(pickle_file)) + unpickler = self.Unpickler(data_file) + unpickler.persistent_load = persistent_load # type: ignore[assignment] + + @contextmanager + def set_deserialization_context(): + # to let reduce_package access deserializaiton context + self.storage_context = storage_context + self.last_map_location = map_location + try: + yield + finally: + self.storage_context = None + self.last_map_location = None + + with set_deserialization_context(): + result = unpickler.load() + + # TODO from zdevito: + # This stateful weird function will need to be removed in our efforts + # to unify the format. It has a race condition if multiple python + # threads try to read independent files + torch._utils._validate_loaded_sparse_tensors() + + return result + + def id(self): + """ + Returns internal identifier that torch.package uses to distinguish :class:`PackageImporter` instances. + Looks like:: + + + """ + return self._mangler.parent_name() + + def file_structure( + self, *, include: "GlobPattern" = "**", exclude: "GlobPattern" = () + ) -> Directory: + """Returns a file structure representation of package's zipfile. + + Args: + include (Union[List[str], str]): An optional string e.g. ``"my_package.my_subpackage"``, or optional list of strings + for the names of the files to be included in the zipfile representation. This can also be + a glob-style pattern, as described in :meth:`PackageExporter.mock` + + exclude (Union[List[str], str]): An optional pattern that excludes files whose name match the pattern. + + Returns: + :class:`Directory` + """ + return _create_directory_from_file_list( + self.filename, self.zip_reader.get_all_records(), include, exclude + ) + + def python_version(self): + """Returns the version of python that was used to create this package. + + Note: this function is experimental and not Forward Compatible. The plan is to move this into a lock + file later on. + + Returns: + :class:`Optional[str]` a python version e.g. 3.8.9 or None if no version was stored with this package + """ + python_version_path = ".data/python_version" + return ( + self.zip_reader.get_record(python_version_path).decode("utf-8").strip() + if self.zip_reader.has_record(python_version_path) + else None + ) + + def _read_extern(self): + return ( + self.zip_reader.get_record(".data/extern_modules") + .decode("utf-8") + .splitlines(keepends=False) + ) + + def _make_module( + self, name: str, filename: str | None, is_package: bool, parent: str + ): + mangled_filename = self._mangler.mangle(filename) if filename else None + spec = importlib.machinery.ModuleSpec( + name, + self, # type: ignore[arg-type] + origin="", + is_package=is_package, + ) + module = importlib.util.module_from_spec(spec) + self.modules[name] = module + module.__name__ = self._mangler.mangle(name) + ns = module.__dict__ + ns["__spec__"] = spec + ns["__loader__"] = self + ns["__file__"] = mangled_filename + ns["__cached__"] = None + ns["__builtins__"] = self.patched_builtins + ns["__torch_package__"] = True + + # Add this module to our private global registry. It should be unique due to mangling. + assert module.__name__ not in _package_imported_modules + _package_imported_modules[module.__name__] = module + + # preemptively install on the parent to prevent IMPORT_FROM from trying to + # access sys.modules + self._install_on_parent(parent, name, module) + + if filename is not None: + assert mangled_filename is not None + # preemptively install the source in `linecache` so that stack traces, + # `inspect`, etc. work. + assert filename not in linecache.cache # type: ignore[attr-defined] + linecache.lazycache(mangled_filename, ns) + + code = self._compile_source(filename, mangled_filename) + exec(code, ns) + + return module + + def _load_module(self, name: str, parent: str): + cur: _PathNode = self.root + for atom in name.split("."): + if not isinstance(cur, _PackageNode) or atom not in cur.children: + if name in IMPLICIT_IMPORT_ALLOWLIST: + module = self.modules[name] = importlib.import_module(name) + return module + raise ModuleNotFoundError( + f'No module named "{name}" in self-contained archive "{self.filename}"' + f" and the module is also not in the list of allowed external modules: {self.extern_modules}", + name=name, + ) + cur = cur.children[atom] + if isinstance(cur, _ExternNode): + module = self.modules[name] = importlib.import_module(name) + + if compat_mapping := EXTERN_IMPORT_COMPAT_NAME_MAPPING.get(name): + for old_name, new_name in compat_mapping.items(): + module.__dict__.setdefault(old_name, new_name) + + return module + return self._make_module( + name, + cur.source_file, # type: ignore[attr-defined] + isinstance(cur, _PackageNode), + parent, + ) + + def _compile_source(self, fullpath: str, mangled_filename: str): + source = self.zip_reader.get_record(fullpath) + source = _normalize_line_endings(source) + return compile(source, mangled_filename, "exec", dont_inherit=True) + + # note: named `get_source` so that linecache can find the source + # when this is the __loader__ of a module. + def get_source(self, module_name) -> str: + # linecache calls `get_source` with the `module.__name__` as the argument, so we must demangle it here. + module = self.import_module(demangle(module_name)) + return self.zip_reader.get_record(demangle(module.__file__)).decode("utf-8") + + # note: named `get_resource_reader` so that importlib.resources can find it. + # This is otherwise considered an internal method. + def get_resource_reader(self, fullname): + try: + package = self._get_package(fullname) + except ImportError: + return None + if package.__loader__ is not self: + return None + return _PackageResourceReader(self, fullname) + + def _install_on_parent(self, parent: str, name: str, module: types.ModuleType): + if not parent: + return + # Set the module as an attribute on its parent. + parent_module = self.modules[parent] + if parent_module.__loader__ is self: + setattr(parent_module, name.rpartition(".")[2], module) + + # note: copied from cpython's import code, with call to create module replaced with _make_module + def _do_find_and_load(self, name): + parent = name.rpartition(".")[0] + module_name_no_parent = name.rpartition(".")[-1] + if parent: + if parent not in self.modules: + self._gcd_import(parent) + # Crazy side-effects! + if name in self.modules: + return self.modules[name] + parent_module = self.modules[parent] + + try: + parent_module.__path__ # type: ignore[attr-defined] + + except AttributeError: + # when we attempt to import a package only containing pybinded files, + # the parent directory isn't always a package as defined by python, + # so we search if the package is actually there or not before calling the error. + if isinstance( + parent_module.__loader__, + importlib.machinery.ExtensionFileLoader, + ): + if name not in self.extern_modules: + msg = ( + _ERR_MSG + + "; {!r} is a c extension module which was not externed. C extension modules \ + need to be externed by the PackageExporter in order to be used as we do not support interning them.}." + ).format(name, name) + raise ModuleNotFoundError(msg, name=name) from None + if not isinstance( + parent_module.__dict__.get(module_name_no_parent), + types.ModuleType, + ): + msg = ( + _ERR_MSG + + "; {!r} is a c extension package which does not contain {!r}." + ).format(name, parent, name) + raise ModuleNotFoundError(msg, name=name) from None + else: + msg = (_ERR_MSG + "; {!r} is not a package").format(name, parent) + raise ModuleNotFoundError(msg, name=name) from None + + module = self._load_module(name, parent) + + self._install_on_parent(parent, name, module) + + return module + + # note: copied from cpython's import code + def _find_and_load(self, name): + module = self.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return self._do_find_and_load(name) + + if module is None: + message = f"import of {name} halted; None in sys.modules" + raise ModuleNotFoundError(message, name=name) + + # To handle https://github.com/pytorch/pytorch/issues/57490, where std's + # creation of fake submodules via the hacking of sys.modules is not import + # friendly + if name == "os": + self.modules["os.path"] = cast(Any, module).path + elif name == "typing": + if sys.version_info < (3, 13): + self.modules["typing.io"] = cast(Any, module).io + self.modules["typing.re"] = cast(Any, module).re + + return module + + def _gcd_import(self, name, package=None, level=0): + """Import and return the module based on its name, the package the call is + being made from, and the level adjustment. + + This function represents the greatest common denominator of functionality + between import_module and __import__. This includes setting __package__ if + the loader did not. + + """ + _sanity_check(name, package, level) + if level > 0: + name = _resolve_name(name, package, level) + + return self._find_and_load(name) + + # note: copied from cpython's import code + def _handle_fromlist(self, module, fromlist, *, recursive=False): + """Figure out what __import__ should return. + + The import_ parameter is a callable which takes the name of module to + import. It is required to decouple the function from assuming importlib's + import implementation is desired. + + """ + module_name = demangle(module.__name__) + # The hell that is fromlist ... + # If a package was imported, try to import stuff from fromlist. + if hasattr(module, "__path__"): + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module_name + ".__all__" + else: + where = "``from list''" + raise TypeError( + f"Item in {where} must be str, not {type(x).__name__}" + ) + elif x == "*": + if not recursive and hasattr(module, "__all__"): + self._handle_fromlist(module, module.__all__, recursive=True) + elif not hasattr(module, x): + from_name = f"{module_name}.{x}" + try: + self._gcd_import(from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if ( + exc.name == from_name + and self.modules.get(from_name, _NEEDS_LOADING) is not None + ): + continue + raise + return module + + def __import__(self, name, globals=None, locals=None, fromlist=(), level=0): + if level == 0: + module = self._gcd_import(name) + else: + globals_ = globals if globals is not None else {} + package = _calc___package__(globals_) + module = self._gcd_import(name, package, level) + if not fromlist: + # Return up to the first dot in 'name'. This is complicated by the fact + # that 'name' may be relative. + if level == 0: + return self._gcd_import(name.partition(".")[0]) + elif not name: + return module + else: + # Figure out where to slice the module's name up to the first dot + # in 'name'. + cut_off = len(name) - len(name.partition(".")[0]) + # Slice end needs to be positive to alleviate need to special-case + # when ``'.' not in name``. + module_name = demangle(module.__name__) + return self.modules[module_name[: len(module_name) - cut_off]] + else: + return self._handle_fromlist(module, fromlist) + + def _get_package(self, package): + """Take a package name or module object and return the module. + + If a name, the module is imported. If the passed or imported module + object is not a package, raise an exception. + """ + if hasattr(package, "__spec__"): + if package.__spec__.submodule_search_locations is None: + raise TypeError(f"{package.__spec__.name!r} is not a package") + else: + return package + else: + module = self.import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError(f"{package!r} is not a package") + else: + return module + + def _zipfile_path(self, package, resource=None): + package = self._get_package(package) + assert package.__loader__ is self + name = demangle(package.__name__) + if resource is not None: + resource = _normalize_path(resource) + return f"{name.replace('.', '/')}/{resource}" + else: + return f"{name.replace('.', '/')}" + + def _get_or_create_package(self, atoms: list[str]) -> "_PackageNode | _ExternNode": + cur = self.root + for i, atom in enumerate(atoms): + node = cur.children.get(atom, None) + if node is None: + node = cur.children[atom] = _PackageNode(None) + if isinstance(node, _ExternNode): + return node + if isinstance(node, _ModuleNode): + name = ".".join(atoms[:i]) + raise ImportError( + f"inconsistent module structure. module {name} is not a package, but has submodules" + ) + assert isinstance(node, _PackageNode) + cur = node + return cur + + def _add_file(self, filename: str): + """Assembles a Python module out of the given file. Will ignore files in the .data directory. + + Args: + filename (str): the name of the file inside of the package archive to be added + """ + *prefix, last = filename.split("/") + if len(prefix) > 1 and prefix[0] == ".data": + return + package = self._get_or_create_package(prefix) + if isinstance(package, _ExternNode): + raise ImportError( + f"inconsistent module structure. package contains a module file {filename}" + f" that is a subpackage of a module marked external." + ) + if last == "__init__.py": + package.source_file = filename + elif last.endswith(".py"): + package_name = last[: -len(".py")] + package.children[package_name] = _ModuleNode(filename) + + def _add_extern(self, extern_name: str): + *prefix, last = extern_name.split(".") + package = self._get_or_create_package(prefix) + if isinstance(package, _ExternNode): + return # the shorter extern covers this extern case + package.children[last] = _ExternNode() + + +_NEEDS_LOADING = object() +_ERR_MSG_PREFIX = "No module named " +_ERR_MSG = _ERR_MSG_PREFIX + "{!r}" + + +class _PathNode: + __slots__ = [] + + +class _PackageNode(_PathNode): + def __init__(self, source_file: str | None): + self.source_file = source_file + self.children: dict[str, _PathNode] = {} + + +class _ModuleNode(_PathNode): + __slots__ = ["source_file"] + + def __init__(self, source_file: str): + self.source_file = source_file + + +class _ExternNode(_PathNode): + pass + + +# A private global registry of all modules that have been package-imported. +_package_imported_modules: WeakValueDictionary = WeakValueDictionary() + +# `inspect` by default only looks in `sys.modules` to find source files for classes. +# Patch it to check our private registry of package-imported modules as well. +_orig_getfile = inspect.getfile + + +def _patched_getfile(object): + if inspect.isclass(object): + if object.__module__ in _package_imported_modules: + return _package_imported_modules[object.__module__].__file__ + return _orig_getfile(object) + + +inspect.getfile = _patched_getfile + + +class _PackageResourceReader: + """Private class used to support PackageImporter.get_resource_reader(). + + Confirms to the importlib.abc.ResourceReader interface. Allowed to access + the innards of PackageImporter. + """ + + def __init__(self, importer, fullname): + self.importer = importer + self.fullname = fullname + + def open_resource(self, resource): + from io import BytesIO + + return BytesIO(self.importer.load_binary(self.fullname, resource)) + + def resource_path(self, resource): + # The contract for resource_path is that it either returns a concrete + # file system path or raises FileNotFoundError. + if isinstance( + self.importer.zip_reader, DirectoryReader + ) and self.importer.zip_reader.has_record( + os.path.join(self.fullname, resource) + ): + return os.path.join( + self.importer.zip_reader.directory, self.fullname, resource + ) + raise FileNotFoundError + + def is_resource(self, name): + path = self.importer._zipfile_path(self.fullname, name) + return self.importer.zip_reader.has_record(path) + + def contents(self): + from pathlib import Path + + filename = self.fullname.replace(".", "/") + + fullname_path = Path(self.importer._zipfile_path(self.fullname)) + files = self.importer.zip_reader.get_all_records() + subdirs_seen = set() + for filename in files: + try: + relative = Path(filename).relative_to(fullname_path) + except ValueError: + continue + # If the path of the file (which is relative to the top of the zip + # namespace), relative to the package given when the resource + # reader was created, has a parent, then it's a name in a + # subdirectory and thus we skip it. + parent_name = relative.parent.name + if len(parent_name) == 0: + yield relative.name + elif parent_name not in subdirs_seen: + subdirs_seen.add(parent_name) + yield parent_name